Merge pull request #1133 from balena-io/legacy

Extract normaliseLegacy as normalise to migration module

src/device-state.coffee

@@ -160,105 +160,6 @@ module.exports = class DeviceState extends EventEmitter
 			applyTargetHealthy = conf.unmanaged or !@applyInProgress or @applications.fetchesInProgress > 0 or cycleTimeWithinInterval
 			return applyTargetHealthy
 
-	migrateLegacyApps: (balenaApi) =>
-		log.info('Migrating ids for legacy app...')
-		@db.models('app').select()
-		.then (apps) =>
-			if apps.length == 0
-				log.debug('No app to migrate')
-				return
-			Promise.map apps, (app) =>
-				services = JSON.parse(app.services)
-				# Check there's a main service, with legacy-container set
-				if services.length != 1
-					log.debug("App doesn't have a single service, ignoring")
-					return
-				service = services[0]
-				if !service.labels['io.resin.legacy-container'] and !service.labels['io.balena.legacy-container']
-					log.debug('Service is not marked as legacy, ignoring')
-					return
-				log.debug("Getting release #{app.commit} for app #{app.appId} from API")
-				balenaApi.get(
-					resource: 'release'
-					options:
-						$filter:
-							belongs_to__application: app.appId
-							commit: app.commit
-							status: 'success'
-						$expand:
-							contains__image:
-								$expand: 'image'
-				)
-				.then (releasesFromAPI) =>
-					if releasesFromAPI.length == 0
-						log.warn("No compatible releases found in API, removing #{app.appId} from target state")
-						return @db.models('app').where({ appId: app.appId }).del()
-					release = releasesFromAPI[0]
-					releaseId = release.id
-					image = release.contains__image[0].image[0]
-					imageId = image.id
-					serviceId = image.is_a_build_of__service.__id
-					imageUrl = image.is_stored_at__image_location
-					if image.content_hash
-						imageUrl += "@#{image.content_hash}"
-					log.debug("Found a release with releaseId #{releaseId}, imageId #{imageId}, serviceId #{serviceId}\nImage location is #{imageUrl}")
-					Promise.join(
-						@applications.docker.getImage(service.image).inspect().catchReturn(NotFoundError, null)
-						@db.models('image').where(name: service.image).select()
-						(imageFromDocker, imagesFromDB) =>
-							@db.transaction (trx) ->
-								Promise.try ->
-									if imagesFromDB.length > 0
-										log.debug('Deleting existing image entry in db')
-										trx('image').where(name: service.image).del()
-									else
-										log.debug('No image in db to delete')
-								.then ->
-									if imageFromDocker?
-										log.debug('Inserting fixed image entry in db')
-										newImage = {
-											name: imageUrl,
-											appId: app.appId,
-											serviceId: serviceId,
-											serviceName: service.serviceName,
-											imageId: imageId,
-											releaseId: releaseId,
-											dependent: 0
-											dockerImageId: imageFromDocker.Id
-										}
-										trx('image').insert(newImage)
-									else
-										log.debug('Image is not downloaded, so not saving it to db')
-								.then ->
-									service.image = imageUrl
-									service.serviceID = serviceId
-									service.imageId = imageId
-									service.releaseId = releaseId
-									delete service.labels['io.resin.legacy-container']
-									delete service.labels['io.balena.legacy-container']
-									app.services = JSON.stringify([ service ])
-									app.releaseId = releaseId
-									log.debug('Updating app entry in db')
-									log.success('Successfully migrated legacy applciation')
-									trx('app').update(app).where({ appId: app.appId })
-					)
-
-	normaliseLegacy: (balenaApi) =>
-		# When legacy apps are present, we kill their containers and migrate their /data to a named volume
-		# We also need to get the releaseId, serviceId, imageId and updated image URL
-		@migrateLegacyApps(balenaApi)
-		.then =>
-			log.debug('Killing legacy containers')
-			@applications.services.killAllLegacy()
-		.then =>
-			log.debug('Migrating legacy app volumes')
-			@applications.getTargetApps()
-			.then(_.keys)
-		.map (appId) =>
-			@applications.volumes.createFromLegacy(appId)
-		.then =>
-			@config.set({ legacyAppsPresent: 'false' })
-
 	init: ->
 		@config.on 'change', (changedConfig) =>
 			if changedConfig.loggingEnabled?

src/lib/errors.ts

@@ -103,3 +103,4 @@ export class ContractViolationError extends TypedError {
 }
 
 export class AppsJsonParseError extends TypedError {}
+export class DatabaseParseError extends TypedError {}

src/lib/migration.ts

@@ -1,6 +1,16 @@
 import * as _ from 'lodash';
+import { PinejsClientRequest } from 'pinejs-client-request';
 
-import { AppsJsonFormat, TargetApplication } from '../types/state';
+import ApplicationManager from '../application-manager';
+import Config from '../config';
+import Database, { Transaction } from '../db';
+import { DatabaseParseError, NotFoundError } from '../lib/errors';
+import { log } from '../lib/supervisor-console';
+import {
+	ApplicationDatabaseFormat,
+	AppsJsonFormat,
+	TargetApplication,
+} from '../types/state';
 
 export const defaultLegacyVolume = () => 'resin-data';
 
@@ -82,3 +92,167 @@ export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat {
 	const apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId');
 	return { apps, config: deviceConfig } as AppsJsonFormat;
 }
+
+export async function normaliseLegacyDatabase(
+	config: Config,
+	application: ApplicationManager,
+	db: Database,
+	balenaApi: PinejsClientRequest,
+) {
+	// When legacy apps are present, we kill their containers and migrate their /data to a named volume
+	log.info('Migrating ids for legacy app...');
+
+	const apps: ApplicationDatabaseFormat = await db.models('app').select();
+
+	if (apps.length === 0) {
+		log.debug('No app to migrate');
+		return;
+	}
+
+	for (const app of apps) {
+		let services: Array<TargetApplication['services']['']>;
+
+		try {
+			services = JSON.parse(app.services);
+		} catch (e) {
+			throw new DatabaseParseError(e);
+		}
+
+		// Check there's a main service, with legacy-container set
+		if (services.length !== 1) {
+			log.debug("App doesn't have a single service, ignoring");
+			return;
+		}
+
+		const service = services[0];
+		if (
+			!service.labels['io.resin.legacy-container'] &&
+			!service.labels['io.balena.legacy-container']
+		) {
+			log.debug('Service is not marked as legacy, ignoring');
+			return;
+		}
+
+		log.debug(`Getting release ${app.commit} for app ${app.appId} from API`);
+		const releases = (await balenaApi.get({
+			resource: 'release',
+			options: {
+				$filter: {
+					belongs_to__application: app.appId,
+					commit: app.commit,
+					status: 'success',
+				},
+				$expand: {
+					contains__image: {
+						$expand: 'image',
+					},
+				},
+			},
+		})) as Array<Dictionary<any>>;
+
+		if (releases.length === 0) {
+			log.warn(
+				`No compatible releases found in API, removing ${
+					app.appId
+				} from target state`,
+			);
+			await db
+				.models('app')
+				.where({ appId: app.appId })
+				.del();
+		}
+
+		// We need to get the release.id, serviceId, image.id and updated imageUrl
+		const release = releases[0];
+		const image = release.contains__image[0].image[0];
+		const serviceId = image.is_a_build_of__service.__id;
+		const imageUrl = !image.content_hash
+			? image.is_stored_at__image_location
+			: `${image.is_stored_at__image_location}@${image.content_hash}`;
+
+		log.debug(
+			`Found a release with releaseId ${release.id}, imageId ${
+				image.id
+			}, serviceId ${serviceId}\nImage location is ${imageUrl}`,
+		);
+
+		const imageFromDocker = await application.docker
+			.getImage(service.image)
+			.inspect()
+			.catch(e => {
+				if (e instanceof NotFoundError) {
+					return;
+				}
+
+				throw e;
+			});
+		const imagesFromDatabase = await db
+			.models('image')
+			.where({ name: service.image })
+			.select();
+
+		await db.transaction(async (trx: Transaction) => {
+			try {
+				if (imagesFromDatabase.length > 0) {
+					log.debug('Deleting existing image entry in db');
+					await trx('image')
+						.where({ name: service.image })
+						.del();
+				} else {
+					log.debug('No image in db to delete');
+				}
+			} finally {
+				if (imageFromDocker != null) {
+					log.debug('Inserting fixed image entry in db');
+					await trx('image').insert({
+						name: imageUrl,
+						appId: app.appId,
+						serviceId,
+						serviceName: service.serviceName,
+						imageId: image.id,
+						releaseId: release.id,
+						dependent: 0,
+						dockerImageId: imageFromDocker.Id,
+					});
+				} else {
+					log.debug('Image is not downloaded, so not saving it to db');
+				}
+
+				delete service.labels['io.resin.legacy-container'];
+				delete service.labels['io.balena.legacy-container'];
+
+				Object.assign(app, {
+					services: JSON.stringify([
+						Object.assign(service, {
+							image: imageUrl,
+							serviceID: serviceId,
+							imageId: image.id,
+							releaseId: release.id,
+						}),
+					]),
+					releaseId: release.id,
+				});
+
+				log.debug('Updating app entry in db');
+				log.success('Successfully migrated legacy application');
+				await trx('app')
+					.update(app)
+					.where({ appId: app.appId });
+			}
+		});
+	}
+
+	log.debug('Killing legacy containers');
+	await application.services.killAllLegacy();
+	log.debug('Migrating legacy app volumes');
+
+	const targetApps = await application.getTargetApps();
+
+	for (const appId of _.keys(targetApps)) {
+		await application.volumes.createFromLegacy(parseInt(appId, 10));
+	}
+
+	await config.set({
+		legacyAppsPresent: false,
+	});
+}

src/supervisor.ts

@@ -2,6 +2,7 @@ import APIBinder from './api-binder';
 import Config, { ConfigKey } from './config';
 import Database from './db';
 import EventTracker from './event-tracker';
+import { normaliseLegacyDatabase } from './lib/migration';
 import Logger from './logger';
 import SupervisorAPI from './supervisor-api';
 
@@ -97,9 +98,14 @@ export class Supervisor {
 		});
 
 		this.logger.logSystemMessage('Supervisor starting', {}, 'Supervisor start');
-		if (conf.legacyAppsPresent) {
+		if (conf.legacyAppsPresent && this.apiBinder.balenaApi != null) {
 			log.info('Legacy app detected, running migration');
-			this.deviceState.normaliseLegacy(this.apiBinder.balenaApi);
+			await normaliseLegacyDatabase(
+				this.deviceState.config,
+				this.deviceState.applications,
+				this.db,
+				this.apiBinder.balenaApi,
+			);
 		}
 
 		await this.deviceState.init();