Merge pull request #1132 from balena-io/extract-preload

Extract loadTargetFromFile into it's own module

package-lock.json

@@ -1972,6 +1972,17 @@
         "dns-txt": "^2.0.2",
         "multicast-dns": "git+https://github.com/resin-io-modules/multicast-dns.git#listen-on-all-interfaces",
         "multicast-dns-service-types": "^1.1.0"
+      },
+      "dependencies": {
+        "multicast-dns": {
+          "version": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
+          "from": "git+https://github.com/resin-io-modules/multicast-dns.git#listen-on-all-interfaces",
+          "dev": true,
+          "requires": {
+            "dns-packet": "^1.0.1",
+            "thunky": "^0.1.0"
+          }
+        }
       }
     },
     "brace-expansion": {
@@ -8235,15 +8246,6 @@
       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
       "dev": true
     },
-    "multicast-dns": {
-      "version": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
-      "from": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
-      "dev": true,
-      "requires": {
-        "dns-packet": "^1.0.1",
-        "thunky": "^0.1.0"
-      }
-    },
     "multicast-dns-service-types": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz",
@@ -11501,9 +11503,9 @@
       }
     },
     "typescript": {
-      "version": "3.5.1",
+      "version": "3.7.2",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.2.tgz",
-      "integrity": "sha512-64HkdiRv1yYZsSe4xC1WVgamNigVYjlssIoaH2HcZF0+ijsk5YK2g0G34w9wJkze8+5ow4STd22AynfO6ZYYLw==",
+      "integrity": "sha512-ml7V7JfiN2Xwvcer+XAf2csGO1bPBdRbFCkYBczNZggrBZ9c7G3riSUeJmqEU5uOtXNPMhE3n+R4FA/3YOAWOQ==",
       "dev": true
     },
     "uglify-js": {

package.json

@@ -118,7 +118,7 @@
     "ts-loader": "^5.3.0",
     "ts-node": "^8.3.0",
     "typed-error": "^2.0.0",
-    "typescript": "^3.5.1",
+    "typescript": "^3.7.0",
     "webpack": "^4.25.0",
     "webpack-cli": "^3.1.2",
     "winston": "^3.2.1"

src/application-manager.d.ts

@@ -6,7 +6,7 @@ import { EventTracker } from './event-tracker';
 import { Logger } from './logger';
 import { DeviceApplicationState } from './types/state';
 
-import Images from './compose/images';
+import ImageManager, { Image } from './compose/images';
 import ServiceManager from './compose/service-manager';
 import DB from './db';
 
@@ -53,7 +53,7 @@ export class ApplicationManager extends EventEmitter {
 	public networks: NetworkManager;
 	public config: Config;
 	public db: DB;
-	public images: Images;
+	public images: ImageManager;
 
 	public proxyvisor: any;
 
@@ -81,6 +81,7 @@ export class ApplicationManager extends EventEmitter {
 	public stopAll(opts: { force?: boolean; skipLock?: boolean }): Promise<void>;
 
 	public serviceNameFromId(serviceId: number): Bluebird<string>;
+	public imageForService(svc: any): Promise<Image>;
 }
 
 export default ApplicationManager;

src/compose/images.ts

@@ -476,7 +476,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
 		);
 	}
 
-	private normalise(imageName: string): Bluebird<string> {
+	public normalise(imageName: string): Bluebird<string> {
 		return this.docker.normaliseImageName(imageName);
 	}
 

src/device-state.coffee

@@ -16,10 +16,9 @@ constants = require './lib/constants'
 validation = require './lib/validation'
 systemd = require './lib/systemd'
 updateLock = require './lib/update-lock'
+{ loadTargetFromFile } =  require './device-state/preload'
 { singleToMulticontainerApp } = require './lib/migration'
 {
-	ENOENT,
-	EISDIR,
 	NotFoundError,
 	UpdatesLockedError
 } = require './lib/errors'
@@ -301,7 +300,7 @@ module.exports = class DeviceState extends EventEmitter
 				@applications.getTargetApps()
 			.then (targetApps) =>
 				if !conf.provisioned or (_.isEmpty(targetApps) and !conf.targetStateSet)
-					@loadTargetFromFile()
+					loadTargetFromFile(null, this)
 					.finally =>
 						@config.set({ targetStateSet: 'true' })
 				else
@@ -423,14 +422,6 @@ module.exports = class DeviceState extends EventEmitter
 		_.assign(@_currentVolatile, newState)
 		@emitAsync('change')
 
-	_convertLegacyAppsJson: (appsArray) ->
-		Promise.try ->
-			deviceConf = _.reduce(appsArray, (conf, app) ->
-				return _.merge({}, conf, app.config)
-			, {})
-			apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId')
-			return { apps, config: deviceConf }
-
 	restoreBackup: (targetState) =>
 		@setTarget(targetState)
 		.then =>
@@ -469,73 +460,6 @@ module.exports = class DeviceState extends EventEmitter
 			.then ->
 				rimraf(path.join(constants.rootMountPoint, 'mnt/data', constants.migrationBackupFile))
 
-	loadTargetFromFile: (appsPath) ->
-		log.info('Attempting to load preloaded apps...')
-		appsPath ?= constants.appsJsonPath
-		fs.readFileAsync(appsPath, 'utf8')
-		.then(JSON.parse)
-		.then (stateFromFile) =>
-			if _.isArray(stateFromFile)
-				# This is a legacy apps.json
-				log.debug('Legacy apps.json detected')
-				return @_convertLegacyAppsJson(stateFromFile)
-			else
-				return stateFromFile
-		.then (stateFromFile) =>
-			commitToPin = null
-			appToPin = null
-			if !_.isEmpty(stateFromFile)
-				images = _.flatMap stateFromFile.apps, (app, appId) =>
-					# multi-app warning!
-					# The following will need to be changed once running multiple applications is possible
-					commitToPin = app.commit
-					appToPin = appId
-					_.map app.services, (service, serviceId) =>
-						svc = {
-							imageName: service.image
-							serviceName: service.serviceName
-							imageId: service.imageId
-							serviceId
-							releaseId: app.releaseId
-							appId
-						}
-						return @applications.imageForService(svc)
-				Promise.map images, (img) =>
-					@applications.images.normalise(img.name)
-					.then (name) =>
-						img.name = name
-						@applications.images.save(img)
-				.then =>
-					@deviceConfig.getCurrent()
-				.then (deviceConf) =>
-					@deviceConfig.formatConfigKeys(stateFromFile.config)
-					.then (formattedConf) =>
-						stateFromFile.config = _.defaults(formattedConf, deviceConf)
-						stateFromFile.name ?= ''
-						@setTarget({
-							local: stateFromFile
-						})
-				.then =>
-					log.success('Preloading complete')
-					if stateFromFile.pinDevice
-						# multi-app warning!
-						# The following will need to be changed once running multiple applications is possible
-						log.debug('Device will be pinned')
-						if commitToPin? and appToPin?
-							@config.set
-								pinDevice: {
-									commit: commitToPin,
-									app: parseInt(appToPin, 10),
-								}
-		# Ensure that this is actually a file, and not an empty path
-		# It can be an empty path because if the file does not exist
-		# on host, the docker daemon creates an empty directory when
-		# the bind mount is added
-		.catch ENOENT, EISDIR, ->
-			log.debug('No apps.json file present, skipping preload')
-		.catch (err) =>
-			@eventTracker.track('Loading preloaded apps failed', { error: err })
-
 	reboot: (force, skipLock) =>
 		@applications.stopAll({ force, skipLock })
 		.then =>

src/device-state.d.ts

@@ -14,6 +14,8 @@ class DeviceState extends EventEmitter {
 	public applications: ApplicationManager;
 	public router: Router;
 	public deviceConfig: DeviceConfig;
+	public config: Config;
+	public eventTracker: EventTracker;
 
 	public constructor(args: {
 		config: Config;

src/device-state/preload.ts

@@ -0,0 +1,115 @@
+import * as _ from 'lodash';
+import { fs } from 'mz';
+
+import { Image } from '../compose/images';
+import DeviceState = require('../device-state');
+
+import constants = require('../lib/constants');
+import { AppsJsonParseError, EISDIR, ENOENT } from '../lib/errors';
+import log from '../lib/supervisor-console';
+
+import { convertLegacyAppsJson } from '../lib/migration';
+import { AppsJsonFormat } from '../types/state';
+
+export async function loadTargetFromFile(
+	appsPath: Nullable<string>,
+	deviceState: DeviceState,
+): Promise<void> {
+	log.info('Attempting to load any preloaded applications');
+	if (!appsPath) {
+		appsPath = constants.appsJsonPath;
+	}
+
+	try {
+		const content = await fs.readFile(appsPath, 'utf8');
+
+		// It's either a target state or it's a list of legacy
+		// style application definitions, we reconcile this below
+		let stateFromFile: AppsJsonFormat | any[];
+		try {
+			stateFromFile = JSON.parse(content);
+		} catch (e) {
+			throw new AppsJsonParseError(e);
+		}
+
+		if (_.isArray(stateFromFile)) {
+			log.debug('Detected a legacy apps.json, converting...');
+			stateFromFile = convertLegacyAppsJson(stateFromFile);
+		}
+		const preloadState = stateFromFile as AppsJsonFormat;
+
+		let commitToPin: string | undefined;
+		let appToPin: string | undefined;
+
+		if (_.isEmpty(preloadState)) {
+			return;
+		}
+
+		const images: Image[] = [];
+		const appIds = _.keys(preloadState.apps);
+		for (const appId of appIds) {
+			const app = preloadState.apps[appId];
+			// Multi-app warning!
+			// The following will need to be changed once running
+			// multiple applications is possible
+			commitToPin = app.commit;
+			appToPin = appId;
+			const serviceIds = _.keys(app.services);
+			for (const serviceId of serviceIds) {
+				const service = app.services[serviceId];
+				const svc = {
+					imageName: service.image,
+					serviceName: service.serviceName,
+					imageId: service.imageId,
+					serviceId,
+					releaseId: app.releaseId,
+					appId,
+				};
+				images.push(await deviceState.applications.imageForService(svc));
+			}
+		}
+
+		for (const image of images) {
+			const name = await deviceState.applications.images.normalise(image.name);
+			image.name = name;
+			await deviceState.applications.images.save(image);
+		}
+
+		const deviceConf = await deviceState.deviceConfig.getCurrent();
+		const formattedConf = await deviceState.deviceConfig.formatConfigKeys(
+			preloadState.config,
+		);
+		preloadState.config = { ...formattedConf, ...deviceConf };
+		const localState = { local: { name: '', ...preloadState } };
+
+		await deviceState.setTarget(localState);
+
+		log.success('Preloading complete');
+		if (stateFromFile.pinDevice) {
+			// Multi-app warning!
+			// The following will need to be changed once running
+			// multiple applications is possible
+			if (commitToPin != null && appToPin != null) {
+				log.debug('Device will be pinned');
+				await deviceState.config.set({
+					pinDevice: {
+						commit: commitToPin,
+						app: parseInt(appToPin, 10),
+					},
+				});
+			}
+		}
+	} catch (e) {
+		// Ensure that this is actually a file, and not an empty path
+		// It can be an empty path because if the file does not exist
+		// on host, the docker daemon creates an empty directory when
+		// the bind mount is added
+		if (ENOENT(e) || EISDIR(e)) {
+			log.debug('No apps.json file present, skipping preload');
+		} else {
+			deviceState.eventTracker.track('Loading preloaded apps failed', {
+				error: e,
+			});
+		}
+	}
+}

src/lib/errors.ts

@@ -101,3 +101,5 @@ export class ContractViolationError extends TypedError {
 		);
 	}
 }
+
+export class AppsJsonParseError extends TypedError {}

src/lib/migration.ts

@@ -1,10 +1,12 @@
 import * as _ from 'lodash';
 
-import { Application } from '../types/application';
+import { AppsJsonFormat, TargetApplication } from '../types/state';
 
 export const defaultLegacyVolume = () => 'resin-data';
 
-export function singleToMulticontainerApp(app: Dictionary<any>): Application {
+export function singleToMulticontainerApp(
+	app: Dictionary<any>,
+): TargetApplication & { appId: string } {
 	const environment: Dictionary<string> = {};
 	for (const key in app.env) {
 		if (!/^RESIN_/.test(key)) {
@@ -14,15 +16,15 @@ export function singleToMulticontainerApp(app: Dictionary<any>): Application {
 
 	const { appId } = app;
 	const conf = app.config != null ? app.config : {};
-	const newApp = new Application();
+	const newApp: TargetApplication & { appId: string } = {
-	_.assign(newApp, {
+		appId: appId.toString(),
-		appId,
 		commit: app.commit,
 		name: app.name,
 		releaseId: 1,
 		networks: {},
 		volumes: {},
-	});
+		services: {},
+	};
 	const defaultVolume = exports.defaultLegacyVolume();
 	newApp.volumes[defaultVolume] = {};
 	const updateStrategy =
@@ -67,3 +69,16 @@ export function singleToMulticontainerApp(app: Dictionary<any>): Application {
 	};
 	return newApp;
 }
+
+export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat {
+	const deviceConfig = _.reduce(
+		appsArray,
+		(conf, app) => {
+			return _.merge({}, conf, app.config);
+		},
+		{},
+	);
+
+	const apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId');
+	return { apps, config: deviceConfig } as AppsJsonFormat;
+}

src/types/application.ts

@@ -1,12 +0,0 @@
-import { ServiceComposeConfig } from '../compose/types/service';
-
-export class Application {
-	public appId: number;
-	public commit: string;
-	public name: string;
-	public releaseId: number;
-	public networks: Dictionary<any>;
-	public volumes: Dictionary<any>;
-
-	public services: Dictionary<ServiceComposeConfig>;
-}

src/types/state.ts

@@ -1,3 +1,7 @@
+import { ComposeNetworkConfig } from '../compose/types/network';
+import { ServiceComposeConfig } from '../compose/types/service';
+import { ComposeVolumeConfig } from '../compose/volume';
+
 export interface DeviceApplicationState {
 	local?: {
 		config?: Dictionary<string>;
@@ -17,3 +21,52 @@ export interface DeviceApplicationState {
 	dependent?: any;
 	commit?: string;
 }
+
+// TODO: Define this with io-ts so we can perform validation
+// on the target state from the api, local mode, and preload
+export interface TargetState {
+	local: {
+		name: string;
+		config: Dictionary<string>;
+		apps: {
+			[appId: string]: {
+				name: string;
+				commit: string;
+				releaseId: number;
+				services: {
+					[serviceId: string]: {
+						labels: Dictionary<string>;
+						imageId: number;
+						serviceName: string;
+						image: string;
+						running: boolean;
+						environment: Dictionary<string>;
+					} & ServiceComposeConfig;
+				};
+				volumes: Dictionary<Partial<ComposeVolumeConfig>>;
+				networks: Dictionary<Partial<ComposeNetworkConfig>>;
+			};
+		};
+	};
+	// TODO: Correctly type this once dependent devices are
+	// actually properly supported
+	dependent: Dictionary<any>;
+}
+
+export type LocalTargetState = TargetState['local'];
+export type TargetApplications = LocalTargetState['apps'];
+export type TargetApplication = LocalTargetState['apps'][0];
+export type AppsJsonFormat = Omit<TargetState['local'], 'name'> & {
+	pinDevice?: boolean;
+};
+
+export type ApplicationDatabaseFormat = Array<{
+	appId: number;
+	commit: string;
+	name: string;
+	source: string;
+	releaseId: number;
+	services: string;
+	networks: string;
+	volumes: string;
+}>;

test/05-device-state.spec.ts

@@ -14,6 +14,8 @@ import { RPiConfigBackend } from '../src/config/backend';
 import DB from '../src/db';
 import DeviceState = require('../src/device-state');
 
+import { loadTargetFromFile } from '../src/device-state/preload';
+
 import Service from '../src/compose/service';
 
 const mockedInitialConfig = {
@@ -260,8 +262,9 @@ describe('deviceState', () => {
 		);
 
 		try {
-			await deviceState.loadTargetFromFile(
+			await loadTargetFromFile(
 				process.env.ROOT_MOUNTPOINT + '/apps.json',
+				deviceState,
 			);
 			const targetState = await deviceState.getTarget();
 
@@ -288,21 +291,22 @@ describe('deviceState', () => {
 		stub(deviceState.deviceConfig, 'getCurrent').returns(
 			Promise.resolve(mockedInitialConfig),
 		);
-		deviceState
+		loadTargetFromFile(
-			.loadTargetFromFile(process.env.ROOT_MOUNTPOINT + '/apps-pin.json')
+			process.env.ROOT_MOUNTPOINT + '/apps-pin.json',
-			.then(() => {
+			deviceState,
-				(deviceState as any).applications.images.save.restore();
+		).then(() => {
-				(deviceState as any).deviceConfig.getCurrent.restore();
+			(deviceState as any).applications.images.save.restore();
+			(deviceState as any).deviceConfig.getCurrent.restore();
 
-				config.get('pinDevice').then(pinned => {
+			config.get('pinDevice').then(pinned => {
-					expect(pinned)
+				expect(pinned)
-						.to.have.property('app')
+					.to.have.property('app')
-						.that.equals(1234);
+					.that.equals(1234);
-					expect(pinned)
+				expect(pinned)
-						.to.have.property('commit')
+					.to.have.property('commit')
-						.that.equals('abcdef');
+					.that.equals('abcdef');
-				});
 			});
+		});
 	});
 
 	it('emits a change event when a new state is reported', () => {