From 1d89174caf116236e856ee849369f35445f29d9d Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Wed, 6 Nov 2019 15:01:05 +0000 Subject: [PATCH 1/4] Upgrade typescript to 3.7 Change-type: patch Signed-off-by: Cameron Diver --- package-lock.json | 26 ++++++++++++++------------ package.json | 2 +- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index 381f683e..1e8226f8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1972,6 +1972,17 @@ "dns-txt": "^2.0.2", "multicast-dns": "git+https://github.com/resin-io-modules/multicast-dns.git#listen-on-all-interfaces", "multicast-dns-service-types": "^1.1.0" + }, + "dependencies": { + "multicast-dns": { + "version": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc", + "from": "git+https://github.com/resin-io-modules/multicast-dns.git#listen-on-all-interfaces", + "dev": true, + "requires": { + "dns-packet": "^1.0.1", + "thunky": "^0.1.0" + } + } } }, "brace-expansion": { @@ -8235,15 +8246,6 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", "dev": true }, - "multicast-dns": { - "version": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc", - "from": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc", - "dev": true, - "requires": { - "dns-packet": "^1.0.1", - "thunky": "^0.1.0" - } - }, "multicast-dns-service-types": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz", @@ -11501,9 +11503,9 @@ } }, "typescript": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.1.tgz", - "integrity": "sha512-64HkdiRv1yYZsSe4xC1WVgamNigVYjlssIoaH2HcZF0+ijsk5YK2g0G34w9wJkze8+5ow4STd22AynfO6ZYYLw==", + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.2.tgz", + "integrity": "sha512-ml7V7JfiN2Xwvcer+XAf2csGO1bPBdRbFCkYBczNZggrBZ9c7G3riSUeJmqEU5uOtXNPMhE3n+R4FA/3YOAWOQ==", "dev": true }, "uglify-js": { diff --git a/package.json b/package.json index 3f94b7e7..e0d8174d 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,7 @@ "ts-loader": "^5.3.0", "ts-node": "^8.3.0", "typed-error": "^2.0.0", - "typescript": "^3.5.1", + "typescript": "^3.7.0", "webpack": "^4.25.0", "webpack-cli": "^3.1.2", "winston": "^3.2.1" From fea80c52059752fe7d97ce9032828243355c9981 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Wed, 6 Nov 2019 16:51:46 +0000 Subject: [PATCH 2/4] Define TargetApplicationState in types and remove Application type Change-type: patch Signed-off-by: Cameron Diver --- src/types/application.ts | 12 ------------ src/types/state.ts | 42 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 12 deletions(-) delete mode 100644 src/types/application.ts diff --git a/src/types/application.ts b/src/types/application.ts deleted file mode 100644 index 3afac3ef..00000000 --- a/src/types/application.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ServiceComposeConfig } from '../compose/types/service'; - -export class Application { - public appId: number; - public commit: string; - public name: string; - public releaseId: number; - public networks: Dictionary; - public volumes: Dictionary; - - public services: Dictionary; -} diff --git a/src/types/state.ts b/src/types/state.ts index 6dd90100..bcff9cc9 100644 --- a/src/types/state.ts +++ b/src/types/state.ts @@ -1,3 +1,7 @@ +import { ComposeNetworkConfig } from '../compose/types/network'; +import { ServiceComposeConfig } from '../compose/types/service'; +import { ComposeVolumeConfig } from '../compose/volume'; + export interface DeviceApplicationState { local?: { config?: Dictionary; @@ -17,3 +21,41 @@ export interface DeviceApplicationState { dependent?: any; commit?: string; } + +// TODO: Define this with io-ts so we can perform validation +// on the target state from the api, local mode, and preload +export interface TargetState { + local: { + name: string; + config: Dictionary; + apps: { + [appId: string]: { + name: string; + commit: string; + releaseId: number; + services: { + [serviceId: string]: { + labels: Dictionary; + imageId: number; + serviceName: string; + image: string; + running: boolean; + environment: Dictionary; + } & ServiceComposeConfig; + }; + volumes: Dictionary>; + networks: Dictionary>; + }; + }; + }; + // TODO: Correctly type this once dependent devices are + // actually properly supported + dependent: Dictionary; +} + +export type LocalTargetState = TargetState['local']; +export type TargetApplications = LocalTargetState['apps']; +export type TargetApplication = LocalTargetState['apps'][0]; +export type AppsJsonFormat = Omit & { + pinDevice?: boolean; +}; From 09a8231fdee467b8b726269a2ef1bc8338733a19 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Wed, 6 Nov 2019 16:52:28 +0000 Subject: [PATCH 3/4] Extract loadTargetFromFile function to preload module Change-type: patch Signed-off-by: Cameron Diver --- src/application-manager.d.ts | 5 +- src/compose/images.ts | 2 +- src/device-state.coffee | 80 +----------------------- src/device-state.d.ts | 2 + src/device-state/preload.ts | 115 +++++++++++++++++++++++++++++++++++ src/lib/errors.ts | 2 + src/lib/migration.ts | 27 ++++++-- test/05-device-state.spec.ts | 32 +++++----- 8 files changed, 164 insertions(+), 101 deletions(-) create mode 100644 src/device-state/preload.ts diff --git a/src/application-manager.d.ts b/src/application-manager.d.ts index fa78c19d..bab14a73 100644 --- a/src/application-manager.d.ts +++ b/src/application-manager.d.ts @@ -6,7 +6,7 @@ import { EventTracker } from './event-tracker'; import { Logger } from './logger'; import { DeviceApplicationState } from './types/state'; -import Images from './compose/images'; +import ImageManager, { Image } from './compose/images'; import ServiceManager from './compose/service-manager'; import DB from './db'; @@ -53,7 +53,7 @@ export class ApplicationManager extends EventEmitter { public networks: NetworkManager; public config: Config; public db: DB; - public images: Images; + public images: ImageManager; public proxyvisor: any; @@ -81,6 +81,7 @@ export class ApplicationManager extends EventEmitter { public stopAll(opts: { force?: boolean; skipLock?: boolean }): Promise; public serviceNameFromId(serviceId: number): Bluebird; + public imageForService(svc: any): Promise; } export default ApplicationManager; diff --git a/src/compose/images.ts b/src/compose/images.ts index 7dbca49e..3bfdc503 100644 --- a/src/compose/images.ts +++ b/src/compose/images.ts @@ -476,7 +476,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) { ); } - private normalise(imageName: string): Bluebird { + public normalise(imageName: string): Bluebird { return this.docker.normaliseImageName(imageName); } diff --git a/src/device-state.coffee b/src/device-state.coffee index 89ab51aa..6d4e636c 100644 --- a/src/device-state.coffee +++ b/src/device-state.coffee @@ -16,10 +16,9 @@ constants = require './lib/constants' validation = require './lib/validation' systemd = require './lib/systemd' updateLock = require './lib/update-lock' +{ loadTargetFromFile } = require './device-state/preload' { singleToMulticontainerApp } = require './lib/migration' { - ENOENT, - EISDIR, NotFoundError, UpdatesLockedError } = require './lib/errors' @@ -301,7 +300,7 @@ module.exports = class DeviceState extends EventEmitter @applications.getTargetApps() .then (targetApps) => if !conf.provisioned or (_.isEmpty(targetApps) and !conf.targetStateSet) - @loadTargetFromFile() + loadTargetFromFile(null, this) .finally => @config.set({ targetStateSet: 'true' }) else @@ -423,14 +422,6 @@ module.exports = class DeviceState extends EventEmitter _.assign(@_currentVolatile, newState) @emitAsync('change') - _convertLegacyAppsJson: (appsArray) -> - Promise.try -> - deviceConf = _.reduce(appsArray, (conf, app) -> - return _.merge({}, conf, app.config) - , {}) - apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId') - return { apps, config: deviceConf } - restoreBackup: (targetState) => @setTarget(targetState) .then => @@ -469,73 +460,6 @@ module.exports = class DeviceState extends EventEmitter .then -> rimraf(path.join(constants.rootMountPoint, 'mnt/data', constants.migrationBackupFile)) - loadTargetFromFile: (appsPath) -> - log.info('Attempting to load preloaded apps...') - appsPath ?= constants.appsJsonPath - fs.readFileAsync(appsPath, 'utf8') - .then(JSON.parse) - .then (stateFromFile) => - if _.isArray(stateFromFile) - # This is a legacy apps.json - log.debug('Legacy apps.json detected') - return @_convertLegacyAppsJson(stateFromFile) - else - return stateFromFile - .then (stateFromFile) => - commitToPin = null - appToPin = null - if !_.isEmpty(stateFromFile) - images = _.flatMap stateFromFile.apps, (app, appId) => - # multi-app warning! - # The following will need to be changed once running multiple applications is possible - commitToPin = app.commit - appToPin = appId - _.map app.services, (service, serviceId) => - svc = { - imageName: service.image - serviceName: service.serviceName - imageId: service.imageId - serviceId - releaseId: app.releaseId - appId - } - return @applications.imageForService(svc) - Promise.map images, (img) => - @applications.images.normalise(img.name) - .then (name) => - img.name = name - @applications.images.save(img) - .then => - @deviceConfig.getCurrent() - .then (deviceConf) => - @deviceConfig.formatConfigKeys(stateFromFile.config) - .then (formattedConf) => - stateFromFile.config = _.defaults(formattedConf, deviceConf) - stateFromFile.name ?= '' - @setTarget({ - local: stateFromFile - }) - .then => - log.success('Preloading complete') - if stateFromFile.pinDevice - # multi-app warning! - # The following will need to be changed once running multiple applications is possible - log.debug('Device will be pinned') - if commitToPin? and appToPin? - @config.set - pinDevice: { - commit: commitToPin, - app: parseInt(appToPin, 10), - } - # Ensure that this is actually a file, and not an empty path - # It can be an empty path because if the file does not exist - # on host, the docker daemon creates an empty directory when - # the bind mount is added - .catch ENOENT, EISDIR, -> - log.debug('No apps.json file present, skipping preload') - .catch (err) => - @eventTracker.track('Loading preloaded apps failed', { error: err }) - reboot: (force, skipLock) => @applications.stopAll({ force, skipLock }) .then => diff --git a/src/device-state.d.ts b/src/device-state.d.ts index 8f42b58b..737a00f8 100644 --- a/src/device-state.d.ts +++ b/src/device-state.d.ts @@ -14,6 +14,8 @@ class DeviceState extends EventEmitter { public applications: ApplicationManager; public router: Router; public deviceConfig: DeviceConfig; + public config: Config; + public eventTracker: EventTracker; public constructor(args: { config: Config; diff --git a/src/device-state/preload.ts b/src/device-state/preload.ts new file mode 100644 index 00000000..bdf11488 --- /dev/null +++ b/src/device-state/preload.ts @@ -0,0 +1,115 @@ +import * as _ from 'lodash'; +import { fs } from 'mz'; + +import { Image } from '../compose/images'; +import DeviceState = require('../device-state'); + +import constants = require('../lib/constants'); +import { AppsJsonParseError, EISDIR, ENOENT } from '../lib/errors'; +import log from '../lib/supervisor-console'; + +import { convertLegacyAppsJson } from '../lib/migration'; +import { AppsJsonFormat } from '../types/state'; + +export async function loadTargetFromFile( + appsPath: Nullable, + deviceState: DeviceState, +): Promise { + log.info('Attempting to load any preloaded applications'); + if (!appsPath) { + appsPath = constants.appsJsonPath; + } + + try { + const content = await fs.readFile(appsPath, 'utf8'); + + // It's either a target state or it's a list of legacy + // style application definitions, we reconcile this below + let stateFromFile: AppsJsonFormat | any[]; + try { + stateFromFile = JSON.parse(content); + } catch (e) { + throw new AppsJsonParseError(e); + } + + if (_.isArray(stateFromFile)) { + log.debug('Detected a legacy apps.json, converting...'); + stateFromFile = convertLegacyAppsJson(stateFromFile); + } + const preloadState = stateFromFile as AppsJsonFormat; + + let commitToPin: string | undefined; + let appToPin: string | undefined; + + if (_.isEmpty(preloadState)) { + return; + } + + const images: Image[] = []; + const appIds = _.keys(preloadState.apps); + for (const appId of appIds) { + const app = preloadState.apps[appId]; + // Multi-app warning! + // The following will need to be changed once running + // multiple applications is possible + commitToPin = app.commit; + appToPin = appId; + const serviceIds = _.keys(app.services); + for (const serviceId of serviceIds) { + const service = app.services[serviceId]; + const svc = { + imageName: service.image, + serviceName: service.serviceName, + imageId: service.imageId, + serviceId, + releaseId: app.releaseId, + appId, + }; + images.push(await deviceState.applications.imageForService(svc)); + } + } + + for (const image of images) { + const name = await deviceState.applications.images.normalise(image.name); + image.name = name; + await deviceState.applications.images.save(image); + } + + const deviceConf = await deviceState.deviceConfig.getCurrent(); + const formattedConf = await deviceState.deviceConfig.formatConfigKeys( + preloadState.config, + ); + preloadState.config = { ...formattedConf, ...deviceConf }; + const localState = { local: { name: '', ...preloadState } }; + + await deviceState.setTarget(localState); + + log.success('Preloading complete'); + if (stateFromFile.pinDevice) { + // Multi-app warning! + // The following will need to be changed once running + // multiple applications is possible + if (commitToPin != null && appToPin != null) { + log.debug('Device will be pinned'); + await deviceState.config.set({ + pinDevice: { + commit: commitToPin, + app: parseInt(appToPin, 10), + }, + }); + } + } + } catch (e) { + // Ensure that this is actually a file, and not an empty path + // It can be an empty path because if the file does not exist + // on host, the docker daemon creates an empty directory when + // the bind mount is added + if (ENOENT(e) || EISDIR(e)) { + log.debug('No apps.json file present, skipping preload'); + } else { + deviceState.eventTracker.track('Loading preloaded apps failed', { + error: e, + }); + } + } +} diff --git a/src/lib/errors.ts b/src/lib/errors.ts index b153750f..55e73853 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -101,3 +101,5 @@ export class ContractViolationError extends TypedError { ); } } + +export class AppsJsonParseError extends TypedError {} diff --git a/src/lib/migration.ts b/src/lib/migration.ts index a0f5895e..b44321a8 100644 --- a/src/lib/migration.ts +++ b/src/lib/migration.ts @@ -1,10 +1,12 @@ import * as _ from 'lodash'; -import { Application } from '../types/application'; +import { AppsJsonFormat, TargetApplication } from '../types/state'; export const defaultLegacyVolume = () => 'resin-data'; -export function singleToMulticontainerApp(app: Dictionary): Application { +export function singleToMulticontainerApp( + app: Dictionary, +): TargetApplication & { appId: string } { const environment: Dictionary = {}; for (const key in app.env) { if (!/^RESIN_/.test(key)) { @@ -14,15 +16,15 @@ export function singleToMulticontainerApp(app: Dictionary): Application { const { appId } = app; const conf = app.config != null ? app.config : {}; - const newApp = new Application(); - _.assign(newApp, { - appId, + const newApp: TargetApplication & { appId: string } = { + appId: appId.toString(), commit: app.commit, name: app.name, releaseId: 1, networks: {}, volumes: {}, - }); + services: {}, + }; const defaultVolume = exports.defaultLegacyVolume(); newApp.volumes[defaultVolume] = {}; const updateStrategy = @@ -67,3 +69,16 @@ export function singleToMulticontainerApp(app: Dictionary): Application { }; return newApp; } + +export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat { + const deviceConfig = _.reduce( + appsArray, + (conf, app) => { + return _.merge({}, conf, app.config); + }, + {}, + ); + + const apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId'); + return { apps, config: deviceConfig } as AppsJsonFormat; +} diff --git a/test/05-device-state.spec.ts b/test/05-device-state.spec.ts index 83c75917..985002da 100644 --- a/test/05-device-state.spec.ts +++ b/test/05-device-state.spec.ts @@ -14,6 +14,8 @@ import { RPiConfigBackend } from '../src/config/backend'; import DB from '../src/db'; import DeviceState = require('../src/device-state'); +import { loadTargetFromFile } from '../src/device-state/preload'; + import Service from '../src/compose/service'; const mockedInitialConfig = { @@ -260,8 +262,9 @@ describe('deviceState', () => { ); try { - await deviceState.loadTargetFromFile( + await loadTargetFromFile( process.env.ROOT_MOUNTPOINT + '/apps.json', + deviceState, ); const targetState = await deviceState.getTarget(); @@ -288,21 +291,22 @@ describe('deviceState', () => { stub(deviceState.deviceConfig, 'getCurrent').returns( Promise.resolve(mockedInitialConfig), ); - deviceState - .loadTargetFromFile(process.env.ROOT_MOUNTPOINT + '/apps-pin.json') - .then(() => { - (deviceState as any).applications.images.save.restore(); - (deviceState as any).deviceConfig.getCurrent.restore(); + loadTargetFromFile( + process.env.ROOT_MOUNTPOINT + '/apps-pin.json', + deviceState, + ).then(() => { + (deviceState as any).applications.images.save.restore(); + (deviceState as any).deviceConfig.getCurrent.restore(); - config.get('pinDevice').then(pinned => { - expect(pinned) - .to.have.property('app') - .that.equals(1234); - expect(pinned) - .to.have.property('commit') - .that.equals('abcdef'); - }); + config.get('pinDevice').then(pinned => { + expect(pinned) + .to.have.property('app') + .that.equals(1234); + expect(pinned) + .to.have.property('commit') + .that.equals('abcdef'); }); + }); }); it('emits a change event when a new state is reported', () => { From 053e111626330ab4cc7a7dc545c94aa0674405ee Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Wed, 6 Nov 2019 17:50:07 +0000 Subject: [PATCH 4/4] Define the database type of the application Change-type: patch Signed-off-by: Cameron Diver --- src/types/state.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/types/state.ts b/src/types/state.ts index bcff9cc9..bcb2978e 100644 --- a/src/types/state.ts +++ b/src/types/state.ts @@ -59,3 +59,14 @@ export type TargetApplication = LocalTargetState['apps'][0]; export type AppsJsonFormat = Omit & { pinDevice?: boolean; }; + +export type ApplicationDatabaseFormat = Array<{ + appId: number; + commit: string; + name: string; + source: string; + releaseId: number; + services: string; + networks: string; + volumes: string; +}>;