Rework data migration to a usable condition

Change-type: patch
Signed-off-by: Ken Bannister <kb2ma@runbox.com>

src/api-binder/index.ts

@@ -164,7 +164,11 @@ export async function start() {
 		'target-state-update',
 		async (targetState, force, isFromApi) => {
 			try {
+				log.debug('Call setTarget from api-binder start()');
 				await deviceState.setTarget(targetState);
+				log.debug(
+					'triggerApplyTarget from target-state-update from api-binder start()',
+				);
 				deviceState.triggerApplyTarget({ force, isFromApi });
 			} catch (err) {
 				handleTargetUpdateError(err);
@@ -174,6 +178,9 @@ export async function start() {
 	// Apply new target state
 	TargetState.emitter.on('target-state-apply', (force, isFromApi) => {
 		try {
+			log.debug(
+				'triggerApplyTarget from target-state-apply from api-binder start()',
+			);
 			deviceState.triggerApplyTarget({ force, isFromApi });
 		} catch (err) {
 			handleTargetUpdateError(err);

src/device-state.ts

@@ -274,6 +274,7 @@ export async function loadInitialState() {
 	// Only apply target if we have received a target
 	// from the cloud or loaded from file
 	if (conf.targetStateSet || loadedFromFile) {
+		log.debug('triggerApplyTarget from loadInitialState');
 		triggerApplyTarget({ initial: true });
 	}
 }
@@ -847,6 +848,7 @@ export function triggerApplyTarget({
 				scheduledApply.force = force;
 			}
 		}
+		log.debug('triggerApplyTarget while applyInProgress');
 		return;
 	}
 	applyCancelled = false;

src/device-state/target-state.ts

@@ -139,6 +139,7 @@ export const update = async (
 		if (statusCode === 304 && cache?.etag != null) {
 			// There's no change so no need to update the cache
 			// only emit the target state if it hasn't been emitted yet
+			log.debug('emitTargetState from update 304');
 			cache.emitted = emitTargetState(cache, force, isFromApi);
 			return;
 		}
@@ -154,6 +155,7 @@ export const update = async (
 		};
 
 		// Emit the target state and update the cache
+		log.debug('emitTargetState from update');
 		cache.emitted = emitTargetState(cache, force, isFromApi);
 	}).finally(() => {
 		lastFetch = process.hrtime();

src/lib/migration.ts

@@ -2,9 +2,11 @@ import * as _ from 'lodash';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 
+import { App } from '../compose/app';
 import * as volumeManager from '../compose/volume-manager';
+import { Volume } from '../compose/volume';
 import * as deviceState from '../device-state';
-import { TargetState } from '../types';
+import { InstancedDeviceState, TargetState } from '../types';
 import * as constants from './constants';
 import { BackupError, isNotFoundError } from './errors';
 import { exec, exists, mkdirp, unlinkAll } from './fs-utils';
@@ -12,6 +14,31 @@ import { log } from './supervisor-console';
 import { pathOnData } from './host-utils';
 import { setTimeout } from 'timers/promises';
 
+/**
+ * Collects volumes with a provided name from apps in the device state. Supports multi-app.
+ */
+function collectVolumes(
+	name: string,
+	localState: InstancedDeviceState,
+): Volume[] {
+	const vols = [] as Volume[];
+
+	const apps = Object.values(localState.local.apps) as App[];
+	for (const app of apps) {
+		vols.push(...app.volumes.filter((vol) => vol.name === name));
+	}
+	return vols;
+}
+
+/**
+ * Loads application data from a migration backup file (likely /mnt/data/backup.tgz)
+ * into data volumes. We expect the backup file to be an archive of a filesystem
+ * where each of the the top level entries must be a directory. Each top level entry
+ * must identify a volume in the provided target state.
+ *
+ * Note: targetState parameter is not necessary. This function is called only after
+ * targetState already has been set.
+ */
 export async function loadBackupFromMigration(
 	targetState: TargetState,
 	retryDelay: number,
@@ -22,29 +49,27 @@ export async function loadBackupFromMigration(
 		}
 		log.info('Migration backup detected');
 
+		// remove this line; see function comment
 		await deviceState.setTarget(targetState);
+		const localState = await deviceState.getTarget();
+		// log.debug(`localState: ${JSON.stringify(localState)}`);
 
-		// TODO: this code is only single-app compatible
-		const [uuid] = Object.keys(targetState.local?.apps);
-
-		if (!!uuid) {
+		// Verify at least one app is present.
+		if (!Object.keys(localState.local.apps)) {
 			throw new BackupError('No apps in the target state');
 		}
 
-		const { id: appId } = targetState.local?.apps[uuid];
-		const [release] = Object.values(targetState.local?.apps[uuid].releases);
-
-		const volumes = release?.volumes ?? {};
-
 		const backupPath = pathOnData('backup');
 		// We clear this path in case it exists from an incomplete run of this function
 		await unlinkAll(backupPath);
 		await mkdirp(backupPath);
+		log.debug('About to extract backup to /mnt/data/backup');
 		await exec(`tar -xzf backup.tgz -C ${backupPath}`, {
 			cwd: pathOnData(),
 		});
 
 		for (const volumeName of await fs.readdir(backupPath)) {
+			log.debug(`processing backup dir: ${volumeName}`);
 			const statInfo = await fs.stat(path.join(backupPath, volumeName));
 
 			if (!statInfo.isDirectory()) {
@@ -53,31 +78,36 @@ export async function loadBackupFromMigration(
 				);
 			}
 
-			if (volumes[volumeName] != null) {
-				log.debug(`Creating volume ${volumeName} from backup`);
-				// If the volume exists (from a previous incomplete run of this restoreBackup), we delete it first
-				await volumeManager
-					.get({ appId, name: volumeName })
-					.then((volume) => {
-						return volume.remove();
-					})
-					.catch((e: unknown) => {
-						if (isNotFoundError(e)) {
-							return;
-						}
-						throw e;
-					});
-
-				await volumeManager.createFromPath(
-					{ appId, name: volumeName },
-					volumes[volumeName],
-					path.join(backupPath, volumeName),
-				);
-			} else {
+			const volumes = collectVolumes(volumeName, localState);
+			if (!volumes) {
 				throw new BackupError(
 					`Invalid backup: ${volumeName} is present in backup but not in target state`,
 				);
 			}
+			if (volumes.length > 1) {
+				throw new BackupError(
+					`Invalid backup: ${volumeName} ambiguous; found in more than one app`,
+				);
+			}
+			log.debug(`Creating volume ${volumeName} from backup`);
+			// If the volume exists (from a previous incomplete run of this restoreBackup), we delete it first
+			await volumeManager
+				.get({ appId: volumes[0].appId, name: volumeName })
+				.then((volume) => {
+					return volume.remove();
+				})
+				.catch((e: unknown) => {
+					if (isNotFoundError(e)) {
+						return;
+					}
+					throw e;
+				});
+
+			await volumeManager.createFromPath(
+				{ appId: volumes[0].appId, name: volumeName },
+				volumes[0].config,
+				path.join(backupPath, volumeName),
+			);
 		}
 
 		await unlinkAll(backupPath);