Merge pull request #835 from balena-io/os-migration-backup

Add the ability to restore volumes from a backup.tgz in the data part…

.gitignore

@@ -9,8 +9,10 @@
 tools/dind/config/
 tools/dind/config.json*
 tools/dind/apps.json
+tools/dind/backup.tgz
 test/data/config*.json
 test/data/*.sqlite
 test/data/led_file
 /coverage/
 report.xml
+.DS_Store

Makefile

@@ -99,6 +99,10 @@ ifeq ($(MOUNT_NODE_MODULES), true)
 	SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/../../node_modules:/resin-supervisor/node_modules
 endif
 
+ifeq ($(MOUNT_BACKUP), true)
+	SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/backup.tgz:/mnt/data/backup.tgz.mounted
+endif
+
 ifdef TARGET_COMPONENT
 	DOCKER_TARGET_COMPONENT := "--target=${TARGET_COMPONENT}"
 else

dindctl

@@ -11,23 +11,24 @@
 # Usage: dindctl action [options]
 #
 # Actions:
-# 	build					build local supervisor image. By default it will be balena/amd64-supervisor:master, you can override the tag with --tag.
-# 	run [options]			build dind host container, run it (with name balena_supervisor_1), which will include the specified supervisor image and run it.
-# 	buildrun [options]		run 'build' and then immediately 'run' the built container.
-# 	refresh					recompile sources in './src' and restart supervisor container on dind host - requires --mount-dist in order to work properly.
-# 	logs [-f]				print out supervisor log files - use '-f' to follow instead, or any other arguments you'd send to journalctl.
-# 	stop					stop dind supervisor host container.
+#   build                   build local supervisor image. By default it will be balena/amd64-supervisor:master, you can override the tag with --tag.
+#   run [options]           build dind host container, run it (with name balena_supervisor_1), which will include the specified supervisor image and run it.
+#   buildrun [options]      run 'build' and then immediately 'run' the built container.
+#   refresh                 recompile sources in './src' and restart supervisor container on dind host - requires --mount-dist in order to work properly.
+#   logs [-f]               print out supervisor log files - use '-f' to follow instead, or any other arguments you'd send to journalctl.
+#   stop                    stop dind supervisor host container.
 # Options:
-# 	--arch | -a  [arch]		architecture of the supervisor to build (default: amd64 )
-# 	--image | -i [image]	image name for supervisor image to build/use ( default: balena/$ARCH-supervisor:master )
-# 	--dind-image [image]	image to use for the resinos-in-container host (default: resin/resinos:2.12.5_rev1-intel-nuc)
-# 	--dind-container [name]	container name suffix for the dind host container ( default: "supervisor", which will produce a container named resinos-in-container-supervisor)
-# 	--mount-dist			bind-mount './dist/' (where webpack stores the built js) from local development environment into supervisor container.
-# 	--mount-nm				bind-mount './node_modules/' from local development environment into supervisor container.
-# 	--preload | -p			use tools/dind/apps.json to preload an application image into the dind host.
-# 	--config | -c [file]	path to config.json, relative to tools/dind ( default: config.json )
-# 	--tag | -t [tag]		for the "build" action, specify the tag to build (default: master)
-# 	--no-clean				for the "stop" action, skip removing the data, boot and state volumes
+#   --arch | -a [arch]      architecture of the supervisor to build (default: amd64 )
+#   --image | -i [image]    image name for supervisor image to build/use ( default: balena/$ARCH-supervisor:master )
+#   --dind-image [image]    image to use for the resinos-in-container host (default: resin/resinos:2.12.5_rev1-intel-nuc)
+#   --dind-container [name] container name suffix for the dind host container ( default: "supervisor", which will produce a container named resinos-in-container-supervisor)
+#   --mount-dist            bind-mount './dist/' (where webpack stores the built js) from local development environment into supervisor container.
+#   --mount-nm              bind-mount './node_modules/' from local development environment into supervisor container.
+#   --mount-backup          bind-mount './tools/dind/backup.tgz' to simulate a migration backup.
+#   --preload | -p          use tools/dind/apps.json to preload an application image into the dind host.
+#   --config | -c [file]    path to config.json, relative to tools/dind ( default: config.json )
+#   --tag | -t [tag]        for the "build" action, specify the tag to build (default: master)
+#   --no-clean              for the "stop" action, skip removing the data, boot and state volumes
 #
 # See README.md for examples.
 #
@@ -44,7 +45,7 @@ SUPERVISOR_BASE_DIR="${DIR}"
 
 ARCH="amd64"
 SUPERVISOR_IMAGE="balena/${ARCH}-supervisor:master"
-DIND_IMAGE="resin/resinos:2.12.5_rev1-intel-nuc"
+DIND_IMAGE="resin/resinos:2.27.0_rev1-intel-nuc"
 MOUNT_DIST="false"
 MOUNT_NODE_MODULES="false"
 CONTAINER_NAME="supervisor"
@@ -53,6 +54,7 @@ OPTIMIZE="true"
 CONFIG_FILENAME="config.json"
 TAG="master"
 CLEAN_VOLUMES="true"
+MOUNT_BACKUP="false"
 
 function showHelp {
 	cat $THIS_FILE | awk '{if(/^#/)print;else exit}' | tail -n +2 | sed 's/\#//' | sed 's|dindctl|'$THIS_FILE'|'
@@ -68,6 +70,9 @@ function parseOptions {
 			--mount-nm)
 				MOUNT_NODE_MODULES="true"
 				;;
+			--mount-backup)
+				MOUNT_BACKUP="true"
+				;;
 			-p|--preload)
 				PRELOADED_IMAGE="true"
 				;;
@@ -151,6 +156,7 @@ function runDind {
 		SUPERVISOR_IMAGE="$SUPERVISOR_IMAGE" \
 		MOUNT_DIST="$MOUNT_DIST" \
 		MOUNT_NODE_MODULES="$MOUNT_NODE_MODULES" \
+		MOUNT_BACKUP="$MOUNT_BACKUP" \
 		PRELOADED_IMAGE="$PRELOADED_IMAGE" \
 		CONTAINER_NAME="$CONTAINER_NAME" \
 		CONFIG_FILENAME="$CONFIG_FILENAME" \

package.json

@@ -78,6 +78,7 @@
     "resin-register-device": "^3.0.0",
     "balena-sync": "^10.0.0",
     "resumable-request": "^2.0.0",
+    "rimraf": "^2.6.2",
     "rwlock": "^5.0.0",
     "shell-quote": "^1.6.1",
     "ts-loader": "^3.5.0",

src/api-binder.coffee

@@ -7,8 +7,11 @@ deviceRegister = require 'resin-register-device'
 express = require 'express'
 bodyParser = require 'body-parser'
 Lock = require 'rwlock'
+path = require 'path'
 { request, requestOpts } = require './lib/request'
 { checkTruthy, checkInt } = require './lib/validation'
+{ pathExistsOnHost } = require './lib/fs-utils'
+constants = require './lib/constants'
 
 DuplicateUuidError = (err) ->
 	_.startsWith(err.message, '"uuid" must be unique')
@@ -83,6 +86,21 @@ module.exports = class APIBinder
 				passthrough: passthrough
 			@cachedBalenaApi = @balenaApi.clone({}, cache: {})
 
+	loadBackupFromMigration: (retryDelay) =>
+		pathExistsOnHost(path.join('mnt/data', constants.migrationBackupFile))
+		.then (exists) =>
+			if !exists
+				return
+			console.log('Migration backup detected')
+			@getTargetState()
+			.then (targetState) =>
+				@deviceState.restoreBackup(targetState)
+		.catch (err) =>
+			console.log('Error restoring migration backup, retrying: ', err)
+			Promise.delay(retryDelay)
+			.then =>
+				@loadBackupFromMigration(retryDelay)
+
 	start: =>
 		@config.getMany([ 'apiEndpoint', 'offlineMode', 'bootstrapRetryDelay' ])
 		.then ({ apiEndpoint, offlineMode, bootstrapRetryDelay }) =>
@@ -109,6 +127,8 @@ module.exports = class APIBinder
 			.then =>
 				console.log('Starting current state report')
 				@startCurrentStateReport()
+			.then =>
+				@loadBackupFromMigration(bootstrapRetryDelay)
 			.then =>
 				@readyForUpdates = true
 				console.log('Starting target state poll')

src/application-manager.coffee

@@ -13,6 +13,7 @@ Docker = require './lib/docker-utils'
 updateLock = require './lib/update-lock'
 { checkTruthy, checkInt, checkString } = require './lib/validation'
 { NotFoundError } = require './lib/errors'
+{ pathExistsOnHost } = require './lib/fs-utils'
 
 ServiceManager = require './compose/service-manager'
 { Service } = require './compose/service'
@@ -46,11 +47,6 @@ fetchAction = (service) ->
 		serviceId: service.serviceId
 	}
 
-pathExistsOnHost = (p) ->
-	fs.statAsync(path.join(constants.rootMountPoint, p))
-	.return(true)
-	.catchReturn(false)
-
 # TODO: implement additional v2 endpoints
 # Some v1 endpoins only work for single-container apps as they assume the app has a single service.
 createApplicationManagerRouter = (applications) ->

src/compose/volumes.coffee

@@ -76,15 +76,19 @@ module.exports = class Volumes
 
 	createFromLegacy: (appId) =>
 		name = defaultLegacyVolume()
-		@create({ name, appId })
+		legacyPath = path.join(constants.rootMountPoint, 'mnt/data/resin-data', appId.toString())
+		@createFromPath({ name, appId }, legacyPath)
+		.catch (err) =>
+			@logger.logSystemMessage("Warning: could not migrate legacy /data volume: #{err.message}", { error: err }, 'Volume migration error')
+
+	# oldPath must be a path inside /mnt/data
+	createFromPath: ({ name, config = {}, appId }, oldPath) =>
+		@create({ name, config, appId })
 		.get('handle')
 		.then (v) ->
 			# Convert the path to be of the same mountpoint so that rename can work
 			volumePath = path.join(constants.rootMountPoint, 'mnt/data', v.Mountpoint.split(path.sep).slice(3)...)
-			legacyPath = path.join(constants.rootMountPoint, 'mnt/data/resin-data', appId.toString())
-			safeRename(legacyPath, volumePath)
-		.catch (err) =>
-			@logger.logSystemMessage("Warning: could not migrate legacy /data volume: #{err.message}", { error: err }, 'Volume migration error')
+			safeRename(oldPath, volumePath)
 
 	remove: ({ name, appId }) ->
 		@logger.logSystemEvent(logTypes.removeVolume, { volume: { name } })

src/device-state.coffee

@@ -7,6 +7,10 @@ express = require 'express'
 bodyParser = require 'body-parser'
 hostConfig = require './host-config'
 network = require './network'
+execAsync = Promise.promisify(require('child_process').exec)
+mkdirp = Promise.promisify(require('mkdirp'))
+path = require 'path'
+rimraf = Promise.promisify(require('rimraf'))
 
 constants = require './lib/constants'
 validation = require './lib/validation'
@@ -406,6 +410,43 @@ module.exports = class DeviceState extends EventEmitter
 			apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId')
 			return { apps, config: deviceConf }
 
+	restoreBackup: (targetState) =>
+		@setTarget(targetState)
+		.then =>
+			appId = _.keys(targetState.local.apps)[0]
+			if !appId?
+				throw new Error('No appId in target state')
+			volumes = targetState.local.apps[appId].volumes
+			backupPath = path.join(constants.rootMountPoint, 'mnt/data/backup')
+			rimraf(backupPath) # We clear this path in case it exists from an incomplete run of this function
+			.then ->
+				mkdirp(backupPath)
+			.then ->
+				execAsync("tar -xzf backup.tgz -C #{backupPath} .", cwd: path.join(constants.rootMountPoint, 'mnt/data'))
+			.then ->
+				fs.readdirAsync(backupPath)
+			.then (dirContents) =>
+				Promise.mapSeries dirContents, (volumeName) =>
+					fs.statAsync(path.join(backupPath, volumeName))
+					.then (s) =>
+						if !s.isDirectory()
+							throw new Error("Invalid backup: #{volumeName} is not a directory")
+						if volumes[volumeName]?
+							console.log("Creating volume #{volumeName} from backup")
+							# If the volume exists (from a previous incomplete run of this restoreBackup), we delete it first
+							@applications.volumes.get({ appId, name: volumeName })
+							.then =>
+								@applications.volumes.remove({ appId, name: volumeName })
+							.catch(NotFoundError, _.noop)
+							.then =>
+								@applications.volumes.createFromPath({ appId, name: volumeName, config: volumes[volumeName] }, path.join(backupPath, volumeName))
+						else
+							throw new Error("Invalid backup: #{volumeName} is present in backup but not in target state")
+			.then ->
+				rimraf(backupPath)
+			.then ->
+				rimraf(path.join(constants.rootMountPoint, 'mnt/data', constants.migrationBackupFile))
+
 	loadTargetFromFile: (appsPath) ->
 		console.log('Attempting to load preloaded apps...')
 		appsPath ?= constants.appsJsonPath

src/lib/constants.ts

@@ -49,6 +49,7 @@ const constants = {
 	},
 	bootBlockDevice: '/dev/mmcblk0p1',
 	hostConfigVarPrefix: 'HOST_',
+	migrationBackupFile: 'backup.tgz',
 };
 
 if (process.env.DOCKER_HOST == null) {

src/lib/fs-utils.ts

@@ -1,6 +1,8 @@
 import * as Bluebird from 'bluebird';
 import { fs } from 'mz';
 import * as path from 'path';
+import * as constants from './constants';
+import { ENOENT } from './errors';
 
 export function writeAndSyncFile(path: string, data: string): Bluebird<void> {
 	return Bluebird.resolve(fs.open(path, 'w')).then(fd => {
@@ -22,3 +24,9 @@ export function safeRename(src: string, dest: string): Bluebird<void> {
 		.tap(fs.fsync)
 		.then(fs.close);
 }
+
+export function pathExistsOnHost(p: string): Bluebird<boolean> {
+	return Bluebird.resolve(fs.stat(path.join(constants.rootMountPoint, p)))
+		.return(true)
+		.catchReturn(ENOENT, false);
+}

tools/dind/start-resin-supervisor

@@ -23,6 +23,10 @@ if [ -d /resin-supervisor/node_modules ]; then
     EXTRA_MOUNTS="${EXTRA_MOUNTS} -v /resin-supervisor/node_modules:/usr/src/app/node_modules"
 fi
 
+if [ -f /mnt/data/backup.tgz.mounted ]; then
+  cp /mnt/data/backup.tgz.mounted /mnt/data/backup.tgz
+fi
+
 SUPERVISOR_IMAGE_ID=$(balena inspect --format='{{.Id}}' $SUPERVISOR_IMAGE:$SUPERVISOR_TAG)
 SUPERVISOR_CONTAINER_IMAGE_ID=$(balena inspect --format='{{.Image}}' resin_supervisor || echo "")