Extract backup logic to migration

Part of device-state refactor
Fix the wrong usage of TargetState as DeviceApplicationState

Change-type: patch
Signed-off-by: Theodor Gherzan <theodor@balena.io>

package-lock.json

@@ -463,6 +463,16 @@
         "@types/tough-cookie": "*"
       }
     },
+    "@types/rimraf": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-2.0.3.tgz",
+      "integrity": "sha512-dZfyfL/u9l/oi984hEXdmAjX3JHry7TLWw43u1HQ8HhPv6KtfxnrZ3T/bleJ0GEvnk9t5sM7eePkgMqz3yBcGg==",
+      "dev": true,
+      "requires": {
+        "@types/glob": "*",
+        "@types/node": "*"
+      }
+    },
     "@types/rwlock": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/@types/rwlock/-/rwlock-5.0.2.tgz",

package.json

@@ -55,6 +55,7 @@
     "@types/mz": "0.0.32",
     "@types/node": "^10.12.17",
     "@types/request": "^2.48.1",
+    "@types/rimraf": "^2.0.3",
     "@types/rwlock": "^5.0.2",
     "@types/shell-quote": "^1.6.0",
     "@types/sinon": "^7.0.13",

src/api-binder.ts

@@ -4,17 +4,15 @@ import * as express from 'express';
 import { isLeft } from 'fp-ts/lib/Either';
 import * as t from 'io-ts';
 import * as _ from 'lodash';
-import * as Path from 'path';
 import { PinejsClientRequest, StatusError } from 'pinejs-client-request';
 import * as deviceRegister from 'resin-register-device';
 import * as url from 'url';
 
 import Config, { ConfigType } from './config';
 import Database from './db';
-import DeviceConfig from './device-config';
 import { EventTracker } from './event-tracker';
+import { loadBackupFromMigration } from './lib/migration';
 
-import * as constants from './lib/constants';
 import {
 	ContractValidationError,
 	ContractViolationError,
@@ -22,10 +20,9 @@ import {
 	ExchangeKeyError,
 	InternalInconsistencyError,
 } from './lib/errors';
-import { pathExistsOnHost } from './lib/fs-utils';
 import * as request from './lib/request';
 import { writeLock } from './lib/update-lock';
-import { DeviceApplicationState } from './types/state';
+import { DeviceApplicationState, TargetState } from './types/state';
 
 import log from './lib/supervisor-console';
 
@@ -73,14 +70,12 @@ export class APIBinder {
 	public router: express.Router;
 
 	private config: Config;
-	private deviceState: {
-		deviceConfig: DeviceConfig;
-		[key: string]: any;
-	};
+	private deviceState: DeviceState;
 	private eventTracker: EventTracker;
 	private logger: Logger;
 
 	public balenaApi: PinejsClientRequest | null = null;
+	// TODO{type}: Retype me when all types are sorted
 	private cachedBalenaApi: PinejsClientRequest | null = null;
 	private lastReportedState: DeviceApplicationState = {
 		local: {},
@@ -90,7 +85,7 @@ export class APIBinder {
 		local: {},
 		dependent: {},
 	};
-	private lastTarget: DeviceApplicationState = {};
+	private lastTarget: TargetState;
 	private lastTargetStateFetch = process.hrtime();
 	private reportPending = false;
 	private stateReportErrors = 0;
@@ -170,25 +165,6 @@ export class APIBinder {
 		this.cachedBalenaApi = this.balenaApi.clone({}, { cache: {} });
 	}
 
-	public async loadBackupFromMigration(retryDelay: number): Promise<void> {
-		try {
-			const exists = await pathExistsOnHost(
-				Path.join('mnt/data', constants.migrationBackupFile),
-			);
-			if (!exists) {
-				return;
-			}
-			log.info('Migration backup detected');
-			const targetState = await this.getTargetState();
-			await this.deviceState.restoreBackup(targetState);
-		} catch (err) {
-			log.error(`Error restoring migration backup, retrying: ${err}`);
-
-			await Bluebird.delay(retryDelay);
-			return this.loadBackupFromMigration(retryDelay);
-		}
-	}
-
 	public async start() {
 		const conf = await this.config.getMany([
 			'apiEndpoint',
@@ -228,7 +204,11 @@ export class APIBinder {
 		log.debug('Starting current state report');
 		await this.startCurrentStateReport();
 
-		await this.loadBackupFromMigration(bootstrapRetryDelay);
+		await loadBackupFromMigration(
+			this.deviceState,
+			await this.getTargetState(),
+			bootstrapRetryDelay,
+		);
 
 		this.readyForUpdates = true;
 		log.debug('Starting target state poll');
@@ -331,15 +311,12 @@ export class APIBinder {
 			registered_at: Math.floor(Date.now() / 1000),
 		});
 
-		return (
-			(await this.balenaApi
-				.post({ resource: 'device', body: device })
-				// TODO: Remove the `as number` when we fix the config typings
-				.timeout(conf.apiTimeout)) as Device
-		);
+		return (await this.balenaApi
+			.post({ resource: 'device', body: device })
+			.timeout(conf.apiTimeout)) as Device;
 	}
 
-	public async getTargetState(): Promise<DeviceApplicationState> {
+	public async getTargetState(): Promise<TargetState> {
 		const { uuid, apiEndpoint, apiTimeout } = await this.config.getMany([
 			'uuid',
 			'apiEndpoint',
@@ -363,9 +340,9 @@ export class APIBinder {
 			this.cachedBalenaApi.passthrough,
 		);
 
-		return await this.cachedBalenaApi
+		return (await this.cachedBalenaApi
 			._request(requestParams)
-			.timeout(apiTimeout);
+			.timeout(apiTimeout)) as TargetState;
 	}
 
 	// TODO: Once 100% typescript, change this to a native promise
@@ -546,6 +523,10 @@ export class APIBinder {
 					`Non-200 response from the API! Status code: ${e.statusCode} - message:`,
 					e,
 				);
+				log.error(
+					`Non-200 response from the API! Status code: ${e.statusCode} - message:`,
+					e,
+				);
 			} else {
 				throw e;
 			}

src/device-state.coffee

@@ -1,27 +1,18 @@
 Promise = require 'bluebird'
 _ = require 'lodash'
 EventEmitter = require 'events'
-fs = Promise.promisifyAll(require('fs'))
 express = require 'express'
 bodyParser = require 'body-parser'
 prettyMs = require 'pretty-ms'
 hostConfig = require './host-config'
 network = require './network'
-execAsync = Promise.promisify(require('child_process').exec)
-mkdirp = Promise.promisify(require('mkdirp'))
-path = require 'path'
-rimraf = Promise.promisify(require('rimraf'))
 
 constants = require './lib/constants'
 validation = require './lib/validation'
 systemd = require './lib/systemd'
 updateLock = require './lib/update-lock'
 { loadTargetFromFile } =  require './device-state/preload'
-{ singleToMulticontainerApp } = require './lib/migration'
-{
-	NotFoundError,
-	UpdatesLockedError
-} = require './lib/errors'
+{ UpdatesLockedError } = require './lib/errors'
 
 { DeviceConfig } = require './device-config'
 ApplicationManager = require './application-manager'
@@ -323,44 +314,6 @@ module.exports = class DeviceState extends EventEmitter
 		_.assign(@_currentVolatile, newState)
 		@emitAsync('change')
 
-	restoreBackup: (targetState) =>
-		@setTarget(targetState)
-		.then =>
-			appId = _.keys(targetState.local.apps)[0]
-			if !appId?
-				throw new Error('No appId in target state')
-			volumes = targetState.local.apps[appId].volumes
-			backupPath = path.join(constants.rootMountPoint, 'mnt/data/backup')
-			rimraf(backupPath) # We clear this path in case it exists from an incomplete run of this function
-			.then ->
-				mkdirp(backupPath)
-			.then ->
-				execAsync("tar -xzf backup.tgz -C #{backupPath}", cwd: path.join(constants.rootMountPoint, 'mnt/data'))
-			.then ->
-				fs.readdirAsync(backupPath)
-			.then (dirContents) =>
-				Promise.mapSeries dirContents, (volumeName) =>
-					fs.statAsync(path.join(backupPath, volumeName))
-					.then (s) =>
-						if !s.isDirectory()
-							throw new Error("Invalid backup: #{volumeName} is not a directory")
-						if volumes[volumeName]?
-							log.debug("Creating volume #{volumeName} from backup")
-							# If the volume exists (from a previous incomplete run of this restoreBackup), we delete it first
-							@applications.volumes.get({ appId, name: volumeName })
-							.then =>
-								@applications.volumes.get({ appId, name: volumeName }).then (volume) ->
-									volume.remove()
-							.catch(NotFoundError, _.noop)
-							.then =>
-								@applications.volumes.createFromPath({ appId, name: volumeName }, volumes[volumeName], path.join(backupPath, volumeName))
-						else
-							throw new Error("Invalid backup: #{volumeName} is present in backup but not in target state")
-			.then ->
-				rimraf(backupPath)
-			.then ->
-				rimraf(path.join(constants.rootMountPoint, 'mnt/data', constants.migrationBackupFile))
-
 	reboot: (force, skipLock) =>
 		@applications.stopAll({ force, skipLock })
 		.then =>

src/device-state.d.ts

@@ -17,6 +17,9 @@ class DeviceState extends EventEmitter {
 	public config: Config;
 	public eventTracker: EventTracker;
 
+	// FIXME: I should be removed once device-state is refactored
+	public connected: boolean;
+
 	public constructor(args: {
 		config: Config;
 		db: Database;
@@ -31,6 +34,8 @@ class DeviceState extends EventEmitter {
 	public setTarget(target: any): Promise<any>;
 	public triggerApplyTarget(opts: any): Promise<any>;
 	public reportCurrentState(state: any);
+	public getCurrentForComparison(): Promise<any>;
+	public getStatus(): Promise<any>;
 
 	public async init();
 }

src/lib/errors.ts

@@ -104,3 +104,4 @@ export class ContractViolationError extends TypedError {
 
 export class AppsJsonParseError extends TypedError {}
 export class DatabaseParseError extends TypedError {}
+export class BackupError extends TypedError {}

src/lib/migration.ts

@@ -1,15 +1,27 @@
+import * as Bluebird from 'bluebird';
 import * as _ from 'lodash';
+import * as mkdirp from 'mkdirp';
+import { child_process, fs } from 'mz';
+import * as path from 'path';
 import { PinejsClientRequest } from 'pinejs-client-request';
+import * as rimraf from 'rimraf';
+
+const mkdirpAsync = Bluebird.promisify(mkdirp);
+const rimrafAsync = Bluebird.promisify(rimraf);
 
 import ApplicationManager from '../application-manager';
 import Config from '../config';
 import Database, { Transaction } from '../db';
-import { DatabaseParseError, NotFoundError } from '../lib/errors';
+import DeviceState = require('../device-state');
+import * as constants from '../lib/constants';
+import { BackupError, DatabaseParseError, NotFoundError } from '../lib/errors';
+import { pathExistsOnHost } from '../lib/fs-utils';
 import { log } from '../lib/supervisor-console';
 import {
 	ApplicationDatabaseFormat,
 	AppsJsonFormat,
 	TargetApplication,
+	TargetState,
 } from '../types/state';
 
 export const defaultLegacyVolume = () => 'resin-data';
@@ -175,12 +187,12 @@ export async function normaliseLegacyDatabase(
 		const imageFromDocker = await application.docker
 			.getImage(service.image)
 			.inspect()
-			.catch(e => {
-				if (e instanceof NotFoundError) {
+			.catch(error => {
+				if (error instanceof NotFoundError) {
 					return;
 				}
 
-				throw e;
+				throw error;
 			});
 		const imagesFromDatabase = await db
 			.models('image')
@@ -252,3 +264,88 @@ export async function normaliseLegacyDatabase(
 		legacyAppsPresent: false,
 	});
 }
+
+export async function loadBackupFromMigration(
+	deviceState: DeviceState,
+	targetState: TargetState,
+	retryDelay: number,
+): Promise<void> {
+	try {
+		const exists = await pathExistsOnHost(
+			path.join('mnt/data', constants.migrationBackupFile),
+		);
+		if (!exists) {
+			return;
+		}
+		log.info('Migration backup detected');
+
+		await deviceState.setTarget(targetState);
+
+		// multi-app warning!
+		const appId = parseInt(_.keys(targetState.local?.apps)[0], 10);
+
+		if (isNaN(appId)) {
+			throw new BackupError('No appId in target state');
+		}
+
+		const volumes = targetState.local?.apps?.[appId].volumes;
+
+		const backupPath = path.join(constants.rootMountPoint, 'mnt/data/backup');
+		// We clear this path in case it exists from an incomplete run of this function
+		await rimrafAsync(backupPath);
+		await mkdirpAsync(backupPath);
+		await child_process.exec(`tar -xzf backup.tgz -C ${backupPath}`, {
+			cwd: path.join(constants.rootMountPoint, 'mnt/data'),
+		});
+
+		for (const volumeName of await fs.readdir(backupPath)) {
+			const statInfo = await fs.stat(path.join(backupPath, volumeName));
+
+			if (!statInfo.isDirectory()) {
+				throw new BackupError(
+					`Invalid backup: ${volumeName} is not a directory`,
+				);
+			}
+
+			if (volumes[volumeName] != null) {
+				log.debug(`Creating volume ${volumeName} from backup`);
+				// If the volume exists (from a previous incomplete run of this restoreBackup), we delete it first
+				await deviceState.applications.volumes
+					.get({ appId, name: volumeName })
+					.then(volume => {
+						return volume.remove();
+					})
+					.catch(error => {
+						if (error instanceof NotFoundError) {
+							return;
+						}
+						throw error;
+					});
+
+				await deviceState.applications.volumes.createFromPath(
+					{ appId, name: volumeName },
+					volumes[volumeName],
+					path.join(backupPath, volumeName),
+				);
+			} else {
+				throw new BackupError(
+					`Invalid backup: ${volumeName} is present in backup but not in target state`,
+				);
+			}
+		}
+
+		await rimrafAsync(backupPath);
+		await rimrafAsync(
+			path.join(
+				constants.rootMountPoint,
+				'mnt/data',
+				constants.migrationBackupFile,
+			),
+		);
+	} catch (err) {
+		log.error(`Error restoring migration backup, retrying: ${err}`);
+
+		await Bluebird.delay(retryDelay);
+		return loadBackupFromMigration(deviceState, targetState, retryDelay);
+	}
+}