Merge pull request #835 from balena-io/os-migration-backup

Add the ability to restore volumes from a backup.tgz in the data part…
This commit is contained in:
Pablo Carranza Vélez 2018-12-12 20:24:20 -03:00 committed by GitHub
commit cbdba686f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1083 additions and 2263 deletions

2
.gitignore vendored
View File

@ -9,8 +9,10 @@
tools/dind/config/ tools/dind/config/
tools/dind/config.json* tools/dind/config.json*
tools/dind/apps.json tools/dind/apps.json
tools/dind/backup.tgz
test/data/config*.json test/data/config*.json
test/data/*.sqlite test/data/*.sqlite
test/data/led_file test/data/led_file
/coverage/ /coverage/
report.xml report.xml
.DS_Store

View File

@ -99,6 +99,10 @@ ifeq ($(MOUNT_NODE_MODULES), true)
SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/../../node_modules:/resin-supervisor/node_modules SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/../../node_modules:/resin-supervisor/node_modules
endif endif
ifeq ($(MOUNT_BACKUP), true)
SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/backup.tgz:/mnt/data/backup.tgz.mounted
endif
ifdef TARGET_COMPONENT ifdef TARGET_COMPONENT
DOCKER_TARGET_COMPONENT := "--target=${TARGET_COMPONENT}" DOCKER_TARGET_COMPONENT := "--target=${TARGET_COMPONENT}"
else else

40
dindctl
View File

@ -11,23 +11,24 @@
# Usage: dindctl action [options] # Usage: dindctl action [options]
# #
# Actions: # Actions:
# build build local supervisor image. By default it will be balena/amd64-supervisor:master, you can override the tag with --tag. # build build local supervisor image. By default it will be balena/amd64-supervisor:master, you can override the tag with --tag.
# run [options] build dind host container, run it (with name balena_supervisor_1), which will include the specified supervisor image and run it. # run [options] build dind host container, run it (with name balena_supervisor_1), which will include the specified supervisor image and run it.
# buildrun [options] run 'build' and then immediately 'run' the built container. # buildrun [options] run 'build' and then immediately 'run' the built container.
# refresh recompile sources in './src' and restart supervisor container on dind host - requires --mount-dist in order to work properly. # refresh recompile sources in './src' and restart supervisor container on dind host - requires --mount-dist in order to work properly.
# logs [-f] print out supervisor log files - use '-f' to follow instead, or any other arguments you'd send to journalctl. # logs [-f] print out supervisor log files - use '-f' to follow instead, or any other arguments you'd send to journalctl.
# stop stop dind supervisor host container. # stop stop dind supervisor host container.
# Options: # Options:
# --arch | -a [arch] architecture of the supervisor to build (default: amd64 ) # --arch | -a [arch] architecture of the supervisor to build (default: amd64 )
# --image | -i [image] image name for supervisor image to build/use ( default: balena/$ARCH-supervisor:master ) # --image | -i [image] image name for supervisor image to build/use ( default: balena/$ARCH-supervisor:master )
# --dind-image [image] image to use for the resinos-in-container host (default: resin/resinos:2.12.5_rev1-intel-nuc) # --dind-image [image] image to use for the resinos-in-container host (default: resin/resinos:2.12.5_rev1-intel-nuc)
# --dind-container [name] container name suffix for the dind host container ( default: "supervisor", which will produce a container named resinos-in-container-supervisor) # --dind-container [name] container name suffix for the dind host container ( default: "supervisor", which will produce a container named resinos-in-container-supervisor)
# --mount-dist bind-mount './dist/' (where webpack stores the built js) from local development environment into supervisor container. # --mount-dist bind-mount './dist/' (where webpack stores the built js) from local development environment into supervisor container.
# --mount-nm bind-mount './node_modules/' from local development environment into supervisor container. # --mount-nm bind-mount './node_modules/' from local development environment into supervisor container.
# --preload | -p use tools/dind/apps.json to preload an application image into the dind host. # --mount-backup bind-mount './tools/dind/backup.tgz' to simulate a migration backup.
# --config | -c [file] path to config.json, relative to tools/dind ( default: config.json ) # --preload | -p use tools/dind/apps.json to preload an application image into the dind host.
# --tag | -t [tag] for the "build" action, specify the tag to build (default: master) # --config | -c [file] path to config.json, relative to tools/dind ( default: config.json )
# --no-clean for the "stop" action, skip removing the data, boot and state volumes # --tag | -t [tag] for the "build" action, specify the tag to build (default: master)
# --no-clean for the "stop" action, skip removing the data, boot and state volumes
# #
# See README.md for examples. # See README.md for examples.
# #
@ -44,7 +45,7 @@ SUPERVISOR_BASE_DIR="${DIR}"
ARCH="amd64" ARCH="amd64"
SUPERVISOR_IMAGE="balena/${ARCH}-supervisor:master" SUPERVISOR_IMAGE="balena/${ARCH}-supervisor:master"
DIND_IMAGE="resin/resinos:2.12.5_rev1-intel-nuc" DIND_IMAGE="resin/resinos:2.27.0_rev1-intel-nuc"
MOUNT_DIST="false" MOUNT_DIST="false"
MOUNT_NODE_MODULES="false" MOUNT_NODE_MODULES="false"
CONTAINER_NAME="supervisor" CONTAINER_NAME="supervisor"
@ -53,6 +54,7 @@ OPTIMIZE="true"
CONFIG_FILENAME="config.json" CONFIG_FILENAME="config.json"
TAG="master" TAG="master"
CLEAN_VOLUMES="true" CLEAN_VOLUMES="true"
MOUNT_BACKUP="false"
function showHelp { function showHelp {
cat $THIS_FILE | awk '{if(/^#/)print;else exit}' | tail -n +2 | sed 's/\#//' | sed 's|dindctl|'$THIS_FILE'|' cat $THIS_FILE | awk '{if(/^#/)print;else exit}' | tail -n +2 | sed 's/\#//' | sed 's|dindctl|'$THIS_FILE'|'
@ -68,6 +70,9 @@ function parseOptions {
--mount-nm) --mount-nm)
MOUNT_NODE_MODULES="true" MOUNT_NODE_MODULES="true"
;; ;;
--mount-backup)
MOUNT_BACKUP="true"
;;
-p|--preload) -p|--preload)
PRELOADED_IMAGE="true" PRELOADED_IMAGE="true"
;; ;;
@ -151,6 +156,7 @@ function runDind {
SUPERVISOR_IMAGE="$SUPERVISOR_IMAGE" \ SUPERVISOR_IMAGE="$SUPERVISOR_IMAGE" \
MOUNT_DIST="$MOUNT_DIST" \ MOUNT_DIST="$MOUNT_DIST" \
MOUNT_NODE_MODULES="$MOUNT_NODE_MODULES" \ MOUNT_NODE_MODULES="$MOUNT_NODE_MODULES" \
MOUNT_BACKUP="$MOUNT_BACKUP" \
PRELOADED_IMAGE="$PRELOADED_IMAGE" \ PRELOADED_IMAGE="$PRELOADED_IMAGE" \
CONTAINER_NAME="$CONTAINER_NAME" \ CONTAINER_NAME="$CONTAINER_NAME" \
CONFIG_FILENAME="$CONFIG_FILENAME" \ CONFIG_FILENAME="$CONFIG_FILENAME" \

3205
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -78,6 +78,7 @@
"resin-register-device": "^3.0.0", "resin-register-device": "^3.0.0",
"balena-sync": "^10.0.0", "balena-sync": "^10.0.0",
"resumable-request": "^2.0.0", "resumable-request": "^2.0.0",
"rimraf": "^2.6.2",
"rwlock": "^5.0.0", "rwlock": "^5.0.0",
"shell-quote": "^1.6.1", "shell-quote": "^1.6.1",
"ts-loader": "^3.5.0", "ts-loader": "^3.5.0",

View File

@ -7,8 +7,11 @@ deviceRegister = require 'resin-register-device'
express = require 'express' express = require 'express'
bodyParser = require 'body-parser' bodyParser = require 'body-parser'
Lock = require 'rwlock' Lock = require 'rwlock'
path = require 'path'
{ request, requestOpts } = require './lib/request' { request, requestOpts } = require './lib/request'
{ checkTruthy, checkInt } = require './lib/validation' { checkTruthy, checkInt } = require './lib/validation'
{ pathExistsOnHost } = require './lib/fs-utils'
constants = require './lib/constants'
DuplicateUuidError = (err) -> DuplicateUuidError = (err) ->
_.startsWith(err.message, '"uuid" must be unique') _.startsWith(err.message, '"uuid" must be unique')
@ -83,6 +86,21 @@ module.exports = class APIBinder
passthrough: passthrough passthrough: passthrough
@cachedBalenaApi = @balenaApi.clone({}, cache: {}) @cachedBalenaApi = @balenaApi.clone({}, cache: {})
loadBackupFromMigration: (retryDelay) =>
pathExistsOnHost(path.join('mnt/data', constants.migrationBackupFile))
.then (exists) =>
if !exists
return
console.log('Migration backup detected')
@getTargetState()
.then (targetState) =>
@deviceState.restoreBackup(targetState)
.catch (err) =>
console.log('Error restoring migration backup, retrying: ', err)
Promise.delay(retryDelay)
.then =>
@loadBackupFromMigration(retryDelay)
start: => start: =>
@config.getMany([ 'apiEndpoint', 'offlineMode', 'bootstrapRetryDelay' ]) @config.getMany([ 'apiEndpoint', 'offlineMode', 'bootstrapRetryDelay' ])
.then ({ apiEndpoint, offlineMode, bootstrapRetryDelay }) => .then ({ apiEndpoint, offlineMode, bootstrapRetryDelay }) =>
@ -109,6 +127,8 @@ module.exports = class APIBinder
.then => .then =>
console.log('Starting current state report') console.log('Starting current state report')
@startCurrentStateReport() @startCurrentStateReport()
.then =>
@loadBackupFromMigration(bootstrapRetryDelay)
.then => .then =>
@readyForUpdates = true @readyForUpdates = true
console.log('Starting target state poll') console.log('Starting target state poll')

View File

@ -13,6 +13,7 @@ Docker = require './lib/docker-utils'
updateLock = require './lib/update-lock' updateLock = require './lib/update-lock'
{ checkTruthy, checkInt, checkString } = require './lib/validation' { checkTruthy, checkInt, checkString } = require './lib/validation'
{ NotFoundError } = require './lib/errors' { NotFoundError } = require './lib/errors'
{ pathExistsOnHost } = require './lib/fs-utils'
ServiceManager = require './compose/service-manager' ServiceManager = require './compose/service-manager'
{ Service } = require './compose/service' { Service } = require './compose/service'
@ -46,11 +47,6 @@ fetchAction = (service) ->
serviceId: service.serviceId serviceId: service.serviceId
} }
pathExistsOnHost = (p) ->
fs.statAsync(path.join(constants.rootMountPoint, p))
.return(true)
.catchReturn(false)
# TODO: implement additional v2 endpoints # TODO: implement additional v2 endpoints
# Some v1 endpoins only work for single-container apps as they assume the app has a single service. # Some v1 endpoins only work for single-container apps as they assume the app has a single service.
createApplicationManagerRouter = (applications) -> createApplicationManagerRouter = (applications) ->

View File

@ -76,15 +76,19 @@ module.exports = class Volumes
createFromLegacy: (appId) => createFromLegacy: (appId) =>
name = defaultLegacyVolume() name = defaultLegacyVolume()
@create({ name, appId }) legacyPath = path.join(constants.rootMountPoint, 'mnt/data/resin-data', appId.toString())
@createFromPath({ name, appId }, legacyPath)
.catch (err) =>
@logger.logSystemMessage("Warning: could not migrate legacy /data volume: #{err.message}", { error: err }, 'Volume migration error')
# oldPath must be a path inside /mnt/data
createFromPath: ({ name, config = {}, appId }, oldPath) =>
@create({ name, config, appId })
.get('handle') .get('handle')
.then (v) -> .then (v) ->
# Convert the path to be of the same mountpoint so that rename can work # Convert the path to be of the same mountpoint so that rename can work
volumePath = path.join(constants.rootMountPoint, 'mnt/data', v.Mountpoint.split(path.sep).slice(3)...) volumePath = path.join(constants.rootMountPoint, 'mnt/data', v.Mountpoint.split(path.sep).slice(3)...)
legacyPath = path.join(constants.rootMountPoint, 'mnt/data/resin-data', appId.toString()) safeRename(oldPath, volumePath)
safeRename(legacyPath, volumePath)
.catch (err) =>
@logger.logSystemMessage("Warning: could not migrate legacy /data volume: #{err.message}", { error: err }, 'Volume migration error')
remove: ({ name, appId }) -> remove: ({ name, appId }) ->
@logger.logSystemEvent(logTypes.removeVolume, { volume: { name } }) @logger.logSystemEvent(logTypes.removeVolume, { volume: { name } })

View File

@ -7,6 +7,10 @@ express = require 'express'
bodyParser = require 'body-parser' bodyParser = require 'body-parser'
hostConfig = require './host-config' hostConfig = require './host-config'
network = require './network' network = require './network'
execAsync = Promise.promisify(require('child_process').exec)
mkdirp = Promise.promisify(require('mkdirp'))
path = require 'path'
rimraf = Promise.promisify(require('rimraf'))
constants = require './lib/constants' constants = require './lib/constants'
validation = require './lib/validation' validation = require './lib/validation'
@ -406,6 +410,43 @@ module.exports = class DeviceState extends EventEmitter
apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId') apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId')
return { apps, config: deviceConf } return { apps, config: deviceConf }
restoreBackup: (targetState) =>
@setTarget(targetState)
.then =>
appId = _.keys(targetState.local.apps)[0]
if !appId?
throw new Error('No appId in target state')
volumes = targetState.local.apps[appId].volumes
backupPath = path.join(constants.rootMountPoint, 'mnt/data/backup')
rimraf(backupPath) # We clear this path in case it exists from an incomplete run of this function
.then ->
mkdirp(backupPath)
.then ->
execAsync("tar -xzf backup.tgz -C #{backupPath} .", cwd: path.join(constants.rootMountPoint, 'mnt/data'))
.then ->
fs.readdirAsync(backupPath)
.then (dirContents) =>
Promise.mapSeries dirContents, (volumeName) =>
fs.statAsync(path.join(backupPath, volumeName))
.then (s) =>
if !s.isDirectory()
throw new Error("Invalid backup: #{volumeName} is not a directory")
if volumes[volumeName]?
console.log("Creating volume #{volumeName} from backup")
# If the volume exists (from a previous incomplete run of this restoreBackup), we delete it first
@applications.volumes.get({ appId, name: volumeName })
.then =>
@applications.volumes.remove({ appId, name: volumeName })
.catch(NotFoundError, _.noop)
.then =>
@applications.volumes.createFromPath({ appId, name: volumeName, config: volumes[volumeName] }, path.join(backupPath, volumeName))
else
throw new Error("Invalid backup: #{volumeName} is present in backup but not in target state")
.then ->
rimraf(backupPath)
.then ->
rimraf(path.join(constants.rootMountPoint, 'mnt/data', constants.migrationBackupFile))
loadTargetFromFile: (appsPath) -> loadTargetFromFile: (appsPath) ->
console.log('Attempting to load preloaded apps...') console.log('Attempting to load preloaded apps...')
appsPath ?= constants.appsJsonPath appsPath ?= constants.appsJsonPath

View File

@ -49,6 +49,7 @@ const constants = {
}, },
bootBlockDevice: '/dev/mmcblk0p1', bootBlockDevice: '/dev/mmcblk0p1',
hostConfigVarPrefix: 'HOST_', hostConfigVarPrefix: 'HOST_',
migrationBackupFile: 'backup.tgz',
}; };
if (process.env.DOCKER_HOST == null) { if (process.env.DOCKER_HOST == null) {

View File

@ -1,6 +1,8 @@
import * as Bluebird from 'bluebird'; import * as Bluebird from 'bluebird';
import { fs } from 'mz'; import { fs } from 'mz';
import * as path from 'path'; import * as path from 'path';
import * as constants from './constants';
import { ENOENT } from './errors';
export function writeAndSyncFile(path: string, data: string): Bluebird<void> { export function writeAndSyncFile(path: string, data: string): Bluebird<void> {
return Bluebird.resolve(fs.open(path, 'w')).then(fd => { return Bluebird.resolve(fs.open(path, 'w')).then(fd => {
@ -22,3 +24,9 @@ export function safeRename(src: string, dest: string): Bluebird<void> {
.tap(fs.fsync) .tap(fs.fsync)
.then(fs.close); .then(fs.close);
} }
export function pathExistsOnHost(p: string): Bluebird<boolean> {
return Bluebird.resolve(fs.stat(path.join(constants.rootMountPoint, p)))
.return(true)
.catchReturn(ENOENT, false);
}

View File

@ -23,6 +23,10 @@ if [ -d /resin-supervisor/node_modules ]; then
EXTRA_MOUNTS="${EXTRA_MOUNTS} -v /resin-supervisor/node_modules:/usr/src/app/node_modules" EXTRA_MOUNTS="${EXTRA_MOUNTS} -v /resin-supervisor/node_modules:/usr/src/app/node_modules"
fi fi
if [ -f /mnt/data/backup.tgz.mounted ]; then
cp /mnt/data/backup.tgz.mounted /mnt/data/backup.tgz
fi
SUPERVISOR_IMAGE_ID=$(balena inspect --format='{{.Id}}' $SUPERVISOR_IMAGE:$SUPERVISOR_TAG) SUPERVISOR_IMAGE_ID=$(balena inspect --format='{{.Id}}' $SUPERVISOR_IMAGE:$SUPERVISOR_TAG)
SUPERVISOR_CONTAINER_IMAGE_ID=$(balena inspect --format='{{.Image}}' resin_supervisor || echo "") SUPERVISOR_CONTAINER_IMAGE_ID=$(balena inspect --format='{{.Image}}' resin_supervisor || echo "")