From 09a8231fdee467b8b726269a2ef1bc8338733a19 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Wed, 6 Nov 2019 16:52:28 +0000 Subject: [PATCH] Extract loadTargetFromFile function to preload module Change-type: patch Signed-off-by: Cameron Diver --- src/application-manager.d.ts | 5 +- src/compose/images.ts | 2 +- src/device-state.coffee | 80 +----------------------- src/device-state.d.ts | 2 + src/device-state/preload.ts | 115 +++++++++++++++++++++++++++++++++++ src/lib/errors.ts | 2 + src/lib/migration.ts | 27 ++++++-- test/05-device-state.spec.ts | 32 +++++----- 8 files changed, 164 insertions(+), 101 deletions(-) create mode 100644 src/device-state/preload.ts diff --git a/src/application-manager.d.ts b/src/application-manager.d.ts index fa78c19d..bab14a73 100644 --- a/src/application-manager.d.ts +++ b/src/application-manager.d.ts @@ -6,7 +6,7 @@ import { EventTracker } from './event-tracker'; import { Logger } from './logger'; import { DeviceApplicationState } from './types/state'; -import Images from './compose/images'; +import ImageManager, { Image } from './compose/images'; import ServiceManager from './compose/service-manager'; import DB from './db'; @@ -53,7 +53,7 @@ export class ApplicationManager extends EventEmitter { public networks: NetworkManager; public config: Config; public db: DB; - public images: Images; + public images: ImageManager; public proxyvisor: any; @@ -81,6 +81,7 @@ export class ApplicationManager extends EventEmitter { public stopAll(opts: { force?: boolean; skipLock?: boolean }): Promise; public serviceNameFromId(serviceId: number): Bluebird; + public imageForService(svc: any): Promise; } export default ApplicationManager; diff --git a/src/compose/images.ts b/src/compose/images.ts index 7dbca49e..3bfdc503 100644 --- a/src/compose/images.ts +++ b/src/compose/images.ts @@ -476,7 +476,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) { ); } - private normalise(imageName: string): Bluebird { + public normalise(imageName: string): Bluebird { return this.docker.normaliseImageName(imageName); } diff --git a/src/device-state.coffee b/src/device-state.coffee index 89ab51aa..6d4e636c 100644 --- a/src/device-state.coffee +++ b/src/device-state.coffee @@ -16,10 +16,9 @@ constants = require './lib/constants' validation = require './lib/validation' systemd = require './lib/systemd' updateLock = require './lib/update-lock' +{ loadTargetFromFile } = require './device-state/preload' { singleToMulticontainerApp } = require './lib/migration' { - ENOENT, - EISDIR, NotFoundError, UpdatesLockedError } = require './lib/errors' @@ -301,7 +300,7 @@ module.exports = class DeviceState extends EventEmitter @applications.getTargetApps() .then (targetApps) => if !conf.provisioned or (_.isEmpty(targetApps) and !conf.targetStateSet) - @loadTargetFromFile() + loadTargetFromFile(null, this) .finally => @config.set({ targetStateSet: 'true' }) else @@ -423,14 +422,6 @@ module.exports = class DeviceState extends EventEmitter _.assign(@_currentVolatile, newState) @emitAsync('change') - _convertLegacyAppsJson: (appsArray) -> - Promise.try -> - deviceConf = _.reduce(appsArray, (conf, app) -> - return _.merge({}, conf, app.config) - , {}) - apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId') - return { apps, config: deviceConf } - restoreBackup: (targetState) => @setTarget(targetState) .then => @@ -469,73 +460,6 @@ module.exports = class DeviceState extends EventEmitter .then -> rimraf(path.join(constants.rootMountPoint, 'mnt/data', constants.migrationBackupFile)) - loadTargetFromFile: (appsPath) -> - log.info('Attempting to load preloaded apps...') - appsPath ?= constants.appsJsonPath - fs.readFileAsync(appsPath, 'utf8') - .then(JSON.parse) - .then (stateFromFile) => - if _.isArray(stateFromFile) - # This is a legacy apps.json - log.debug('Legacy apps.json detected') - return @_convertLegacyAppsJson(stateFromFile) - else - return stateFromFile - .then (stateFromFile) => - commitToPin = null - appToPin = null - if !_.isEmpty(stateFromFile) - images = _.flatMap stateFromFile.apps, (app, appId) => - # multi-app warning! - # The following will need to be changed once running multiple applications is possible - commitToPin = app.commit - appToPin = appId - _.map app.services, (service, serviceId) => - svc = { - imageName: service.image - serviceName: service.serviceName - imageId: service.imageId - serviceId - releaseId: app.releaseId - appId - } - return @applications.imageForService(svc) - Promise.map images, (img) => - @applications.images.normalise(img.name) - .then (name) => - img.name = name - @applications.images.save(img) - .then => - @deviceConfig.getCurrent() - .then (deviceConf) => - @deviceConfig.formatConfigKeys(stateFromFile.config) - .then (formattedConf) => - stateFromFile.config = _.defaults(formattedConf, deviceConf) - stateFromFile.name ?= '' - @setTarget({ - local: stateFromFile - }) - .then => - log.success('Preloading complete') - if stateFromFile.pinDevice - # multi-app warning! - # The following will need to be changed once running multiple applications is possible - log.debug('Device will be pinned') - if commitToPin? and appToPin? - @config.set - pinDevice: { - commit: commitToPin, - app: parseInt(appToPin, 10), - } - # Ensure that this is actually a file, and not an empty path - # It can be an empty path because if the file does not exist - # on host, the docker daemon creates an empty directory when - # the bind mount is added - .catch ENOENT, EISDIR, -> - log.debug('No apps.json file present, skipping preload') - .catch (err) => - @eventTracker.track('Loading preloaded apps failed', { error: err }) - reboot: (force, skipLock) => @applications.stopAll({ force, skipLock }) .then => diff --git a/src/device-state.d.ts b/src/device-state.d.ts index 8f42b58b..737a00f8 100644 --- a/src/device-state.d.ts +++ b/src/device-state.d.ts @@ -14,6 +14,8 @@ class DeviceState extends EventEmitter { public applications: ApplicationManager; public router: Router; public deviceConfig: DeviceConfig; + public config: Config; + public eventTracker: EventTracker; public constructor(args: { config: Config; diff --git a/src/device-state/preload.ts b/src/device-state/preload.ts new file mode 100644 index 00000000..bdf11488 --- /dev/null +++ b/src/device-state/preload.ts @@ -0,0 +1,115 @@ +import * as _ from 'lodash'; +import { fs } from 'mz'; + +import { Image } from '../compose/images'; +import DeviceState = require('../device-state'); + +import constants = require('../lib/constants'); +import { AppsJsonParseError, EISDIR, ENOENT } from '../lib/errors'; +import log from '../lib/supervisor-console'; + +import { convertLegacyAppsJson } from '../lib/migration'; +import { AppsJsonFormat } from '../types/state'; + +export async function loadTargetFromFile( + appsPath: Nullable, + deviceState: DeviceState, +): Promise { + log.info('Attempting to load any preloaded applications'); + if (!appsPath) { + appsPath = constants.appsJsonPath; + } + + try { + const content = await fs.readFile(appsPath, 'utf8'); + + // It's either a target state or it's a list of legacy + // style application definitions, we reconcile this below + let stateFromFile: AppsJsonFormat | any[]; + try { + stateFromFile = JSON.parse(content); + } catch (e) { + throw new AppsJsonParseError(e); + } + + if (_.isArray(stateFromFile)) { + log.debug('Detected a legacy apps.json, converting...'); + stateFromFile = convertLegacyAppsJson(stateFromFile); + } + const preloadState = stateFromFile as AppsJsonFormat; + + let commitToPin: string | undefined; + let appToPin: string | undefined; + + if (_.isEmpty(preloadState)) { + return; + } + + const images: Image[] = []; + const appIds = _.keys(preloadState.apps); + for (const appId of appIds) { + const app = preloadState.apps[appId]; + // Multi-app warning! + // The following will need to be changed once running + // multiple applications is possible + commitToPin = app.commit; + appToPin = appId; + const serviceIds = _.keys(app.services); + for (const serviceId of serviceIds) { + const service = app.services[serviceId]; + const svc = { + imageName: service.image, + serviceName: service.serviceName, + imageId: service.imageId, + serviceId, + releaseId: app.releaseId, + appId, + }; + images.push(await deviceState.applications.imageForService(svc)); + } + } + + for (const image of images) { + const name = await deviceState.applications.images.normalise(image.name); + image.name = name; + await deviceState.applications.images.save(image); + } + + const deviceConf = await deviceState.deviceConfig.getCurrent(); + const formattedConf = await deviceState.deviceConfig.formatConfigKeys( + preloadState.config, + ); + preloadState.config = { ...formattedConf, ...deviceConf }; + const localState = { local: { name: '', ...preloadState } }; + + await deviceState.setTarget(localState); + + log.success('Preloading complete'); + if (stateFromFile.pinDevice) { + // Multi-app warning! + // The following will need to be changed once running + // multiple applications is possible + if (commitToPin != null && appToPin != null) { + log.debug('Device will be pinned'); + await deviceState.config.set({ + pinDevice: { + commit: commitToPin, + app: parseInt(appToPin, 10), + }, + }); + } + } + } catch (e) { + // Ensure that this is actually a file, and not an empty path + // It can be an empty path because if the file does not exist + // on host, the docker daemon creates an empty directory when + // the bind mount is added + if (ENOENT(e) || EISDIR(e)) { + log.debug('No apps.json file present, skipping preload'); + } else { + deviceState.eventTracker.track('Loading preloaded apps failed', { + error: e, + }); + } + } +} diff --git a/src/lib/errors.ts b/src/lib/errors.ts index b153750f..55e73853 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -101,3 +101,5 @@ export class ContractViolationError extends TypedError { ); } } + +export class AppsJsonParseError extends TypedError {} diff --git a/src/lib/migration.ts b/src/lib/migration.ts index a0f5895e..b44321a8 100644 --- a/src/lib/migration.ts +++ b/src/lib/migration.ts @@ -1,10 +1,12 @@ import * as _ from 'lodash'; -import { Application } from '../types/application'; +import { AppsJsonFormat, TargetApplication } from '../types/state'; export const defaultLegacyVolume = () => 'resin-data'; -export function singleToMulticontainerApp(app: Dictionary): Application { +export function singleToMulticontainerApp( + app: Dictionary, +): TargetApplication & { appId: string } { const environment: Dictionary = {}; for (const key in app.env) { if (!/^RESIN_/.test(key)) { @@ -14,15 +16,15 @@ export function singleToMulticontainerApp(app: Dictionary): Application { const { appId } = app; const conf = app.config != null ? app.config : {}; - const newApp = new Application(); - _.assign(newApp, { - appId, + const newApp: TargetApplication & { appId: string } = { + appId: appId.toString(), commit: app.commit, name: app.name, releaseId: 1, networks: {}, volumes: {}, - }); + services: {}, + }; const defaultVolume = exports.defaultLegacyVolume(); newApp.volumes[defaultVolume] = {}; const updateStrategy = @@ -67,3 +69,16 @@ export function singleToMulticontainerApp(app: Dictionary): Application { }; return newApp; } + +export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat { + const deviceConfig = _.reduce( + appsArray, + (conf, app) => { + return _.merge({}, conf, app.config); + }, + {}, + ); + + const apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId'); + return { apps, config: deviceConfig } as AppsJsonFormat; +} diff --git a/test/05-device-state.spec.ts b/test/05-device-state.spec.ts index 83c75917..985002da 100644 --- a/test/05-device-state.spec.ts +++ b/test/05-device-state.spec.ts @@ -14,6 +14,8 @@ import { RPiConfigBackend } from '../src/config/backend'; import DB from '../src/db'; import DeviceState = require('../src/device-state'); +import { loadTargetFromFile } from '../src/device-state/preload'; + import Service from '../src/compose/service'; const mockedInitialConfig = { @@ -260,8 +262,9 @@ describe('deviceState', () => { ); try { - await deviceState.loadTargetFromFile( + await loadTargetFromFile( process.env.ROOT_MOUNTPOINT + '/apps.json', + deviceState, ); const targetState = await deviceState.getTarget(); @@ -288,21 +291,22 @@ describe('deviceState', () => { stub(deviceState.deviceConfig, 'getCurrent').returns( Promise.resolve(mockedInitialConfig), ); - deviceState - .loadTargetFromFile(process.env.ROOT_MOUNTPOINT + '/apps-pin.json') - .then(() => { - (deviceState as any).applications.images.save.restore(); - (deviceState as any).deviceConfig.getCurrent.restore(); + loadTargetFromFile( + process.env.ROOT_MOUNTPOINT + '/apps-pin.json', + deviceState, + ).then(() => { + (deviceState as any).applications.images.save.restore(); + (deviceState as any).deviceConfig.getCurrent.restore(); - config.get('pinDevice').then(pinned => { - expect(pinned) - .to.have.property('app') - .that.equals(1234); - expect(pinned) - .to.have.property('commit') - .that.equals('abcdef'); - }); + config.get('pinDevice').then(pinned => { + expect(pinned) + .to.have.property('app') + .that.equals(1234); + expect(pinned) + .to.have.property('commit') + .that.equals('abcdef'); }); + }); }); it('emits a change event when a new state is reported', () => {