From cf79ed8cdb96431a5a5fbe94885c1e96680df75c Mon Sep 17 00:00:00 2001 From: Theodor Gherzan Date: Wed, 6 Nov 2019 18:45:53 +0000 Subject: [PATCH] Extract normaliseLegacy as normalise to migration module Change-type: patch Signed-off-by: Theodor Gherzan --- src/device-state.coffee | 99 ---------------------- src/lib/errors.ts | 1 + src/lib/migration.ts | 176 +++++++++++++++++++++++++++++++++++++++- src/supervisor.ts | 10 ++- 4 files changed, 184 insertions(+), 102 deletions(-) diff --git a/src/device-state.coffee b/src/device-state.coffee index 6d4e636c..cfbc707c 100644 --- a/src/device-state.coffee +++ b/src/device-state.coffee @@ -160,105 +160,6 @@ module.exports = class DeviceState extends EventEmitter applyTargetHealthy = conf.unmanaged or !@applyInProgress or @applications.fetchesInProgress > 0 or cycleTimeWithinInterval return applyTargetHealthy - migrateLegacyApps: (balenaApi) => - log.info('Migrating ids for legacy app...') - @db.models('app').select() - .then (apps) => - if apps.length == 0 - log.debug('No app to migrate') - return - Promise.map apps, (app) => - services = JSON.parse(app.services) - # Check there's a main service, with legacy-container set - if services.length != 1 - log.debug("App doesn't have a single service, ignoring") - return - service = services[0] - if !service.labels['io.resin.legacy-container'] and !service.labels['io.balena.legacy-container'] - log.debug('Service is not marked as legacy, ignoring') - return - log.debug("Getting release #{app.commit} for app #{app.appId} from API") - balenaApi.get( - resource: 'release' - options: - $filter: - belongs_to__application: app.appId - commit: app.commit - status: 'success' - $expand: - contains__image: - $expand: 'image' - ) - .then (releasesFromAPI) => - if releasesFromAPI.length == 0 - log.warn("No compatible releases found in API, removing #{app.appId} from target state") - return @db.models('app').where({ appId: app.appId }).del() - release = releasesFromAPI[0] - releaseId = release.id - image = release.contains__image[0].image[0] - imageId = image.id - serviceId = image.is_a_build_of__service.__id - imageUrl = image.is_stored_at__image_location - if image.content_hash - imageUrl += "@#{image.content_hash}" - log.debug("Found a release with releaseId #{releaseId}, imageId #{imageId}, serviceId #{serviceId}\nImage location is #{imageUrl}") - Promise.join( - @applications.docker.getImage(service.image).inspect().catchReturn(NotFoundError, null) - @db.models('image').where(name: service.image).select() - (imageFromDocker, imagesFromDB) => - @db.transaction (trx) -> - Promise.try -> - if imagesFromDB.length > 0 - log.debug('Deleting existing image entry in db') - trx('image').where(name: service.image).del() - else - log.debug('No image in db to delete') - .then -> - if imageFromDocker? - log.debug('Inserting fixed image entry in db') - newImage = { - name: imageUrl, - appId: app.appId, - serviceId: serviceId, - serviceName: service.serviceName, - imageId: imageId, - releaseId: releaseId, - dependent: 0 - dockerImageId: imageFromDocker.Id - } - trx('image').insert(newImage) - else - log.debug('Image is not downloaded, so not saving it to db') - .then -> - service.image = imageUrl - service.serviceID = serviceId - service.imageId = imageId - service.releaseId = releaseId - delete service.labels['io.resin.legacy-container'] - delete service.labels['io.balena.legacy-container'] - app.services = JSON.stringify([ service ]) - app.releaseId = releaseId - log.debug('Updating app entry in db') - log.success('Successfully migrated legacy applciation') - trx('app').update(app).where({ appId: app.appId }) - ) - - normaliseLegacy: (balenaApi) => - # When legacy apps are present, we kill their containers and migrate their /data to a named volume - # We also need to get the releaseId, serviceId, imageId and updated image URL - @migrateLegacyApps(balenaApi) - .then => - log.debug('Killing legacy containers') - @applications.services.killAllLegacy() - .then => - log.debug('Migrating legacy app volumes') - @applications.getTargetApps() - .then(_.keys) - .map (appId) => - @applications.volumes.createFromLegacy(appId) - .then => - @config.set({ legacyAppsPresent: 'false' }) - init: -> @config.on 'change', (changedConfig) => if changedConfig.loggingEnabled? diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 55e73853..947bf71d 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -103,3 +103,4 @@ export class ContractViolationError extends TypedError { } export class AppsJsonParseError extends TypedError {} +export class DatabaseParseError extends TypedError {} diff --git a/src/lib/migration.ts b/src/lib/migration.ts index b44321a8..7d9098a6 100644 --- a/src/lib/migration.ts +++ b/src/lib/migration.ts @@ -1,6 +1,16 @@ import * as _ from 'lodash'; +import { PinejsClientRequest } from 'pinejs-client-request'; -import { AppsJsonFormat, TargetApplication } from '../types/state'; +import ApplicationManager from '../application-manager'; +import Config from '../config'; +import Database, { Transaction } from '../db'; +import { DatabaseParseError, NotFoundError } from '../lib/errors'; +import { log } from '../lib/supervisor-console'; +import { + ApplicationDatabaseFormat, + AppsJsonFormat, + TargetApplication, +} from '../types/state'; export const defaultLegacyVolume = () => 'resin-data'; @@ -82,3 +92,167 @@ export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat { const apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId'); return { apps, config: deviceConfig } as AppsJsonFormat; } + +export async function normaliseLegacyDatabase( + config: Config, + application: ApplicationManager, + db: Database, + balenaApi: PinejsClientRequest, +) { + // When legacy apps are present, we kill their containers and migrate their /data to a named volume + log.info('Migrating ids for legacy app...'); + + const apps: ApplicationDatabaseFormat = await db.models('app').select(); + + if (apps.length === 0) { + log.debug('No app to migrate'); + return; + } + + for (const app of apps) { + let services: Array; + + try { + services = JSON.parse(app.services); + } catch (e) { + throw new DatabaseParseError(e); + } + + // Check there's a main service, with legacy-container set + if (services.length !== 1) { + log.debug("App doesn't have a single service, ignoring"); + return; + } + + const service = services[0]; + if ( + !service.labels['io.resin.legacy-container'] && + !service.labels['io.balena.legacy-container'] + ) { + log.debug('Service is not marked as legacy, ignoring'); + return; + } + + log.debug(`Getting release ${app.commit} for app ${app.appId} from API`); + const releases = (await balenaApi.get({ + resource: 'release', + options: { + $filter: { + belongs_to__application: app.appId, + commit: app.commit, + status: 'success', + }, + $expand: { + contains__image: { + $expand: 'image', + }, + }, + }, + })) as Array>; + + if (releases.length === 0) { + log.warn( + `No compatible releases found in API, removing ${ + app.appId + } from target state`, + ); + await db + .models('app') + .where({ appId: app.appId }) + .del(); + } + + // We need to get the release.id, serviceId, image.id and updated imageUrl + const release = releases[0]; + const image = release.contains__image[0].image[0]; + const serviceId = image.is_a_build_of__service.__id; + const imageUrl = !image.content_hash + ? image.is_stored_at__image_location + : `${image.is_stored_at__image_location}@${image.content_hash}`; + + log.debug( + `Found a release with releaseId ${release.id}, imageId ${ + image.id + }, serviceId ${serviceId}\nImage location is ${imageUrl}`, + ); + + const imageFromDocker = await application.docker + .getImage(service.image) + .inspect() + .catch(e => { + if (e instanceof NotFoundError) { + return; + } + + throw e; + }); + const imagesFromDatabase = await db + .models('image') + .where({ name: service.image }) + .select(); + + await db.transaction(async (trx: Transaction) => { + try { + if (imagesFromDatabase.length > 0) { + log.debug('Deleting existing image entry in db'); + await trx('image') + .where({ name: service.image }) + .del(); + } else { + log.debug('No image in db to delete'); + } + } finally { + if (imageFromDocker != null) { + log.debug('Inserting fixed image entry in db'); + await trx('image').insert({ + name: imageUrl, + appId: app.appId, + serviceId, + serviceName: service.serviceName, + imageId: image.id, + releaseId: release.id, + dependent: 0, + dockerImageId: imageFromDocker.Id, + }); + } else { + log.debug('Image is not downloaded, so not saving it to db'); + } + + delete service.labels['io.resin.legacy-container']; + delete service.labels['io.balena.legacy-container']; + + Object.assign(app, { + services: JSON.stringify([ + Object.assign(service, { + image: imageUrl, + serviceID: serviceId, + imageId: image.id, + releaseId: release.id, + }), + ]), + releaseId: release.id, + }); + + log.debug('Updating app entry in db'); + log.success('Successfully migrated legacy application'); + await trx('app') + .update(app) + .where({ appId: app.appId }); + } + }); + } + + log.debug('Killing legacy containers'); + await application.services.killAllLegacy(); + log.debug('Migrating legacy app volumes'); + + const targetApps = await application.getTargetApps(); + + for (const appId of _.keys(targetApps)) { + await application.volumes.createFromLegacy(parseInt(appId, 10)); + } + + await config.set({ + legacyAppsPresent: false, + }); +} diff --git a/src/supervisor.ts b/src/supervisor.ts index 85d872cb..75bfe2ae 100644 --- a/src/supervisor.ts +++ b/src/supervisor.ts @@ -2,6 +2,7 @@ import APIBinder from './api-binder'; import Config, { ConfigKey } from './config'; import Database from './db'; import EventTracker from './event-tracker'; +import { normaliseLegacyDatabase } from './lib/migration'; import Logger from './logger'; import SupervisorAPI from './supervisor-api'; @@ -97,9 +98,14 @@ export class Supervisor { }); this.logger.logSystemMessage('Supervisor starting', {}, 'Supervisor start'); - if (conf.legacyAppsPresent) { + if (conf.legacyAppsPresent && this.apiBinder.balenaApi != null) { log.info('Legacy app detected, running migration'); - this.deviceState.normaliseLegacy(this.apiBinder.balenaApi); + await normaliseLegacyDatabase( + this.deviceState.config, + this.deviceState.applications, + this.db, + this.apiBinder.balenaApi, + ); } await this.deviceState.init();