mirror of
https://github.com/balena-os/balena-supervisor.git
synced 2024-12-22 15:02:23 +00:00
Clean up migration from legacy target state format
Creates `lib/legacy.ts` and `device-state/legacy.ts` to deal with migration from legacy target states (single container and v2) for all apps and for apps.json respectively
This commit is contained in:
parent
7425d1110b
commit
1edd060143
@ -7,7 +7,6 @@ import { NotFoundError, InternalInconsistencyError } from '../lib/errors';
|
|||||||
import { safeRename } from '../lib/fs-utils';
|
import { safeRename } from '../lib/fs-utils';
|
||||||
import { docker } from '../lib/docker-utils';
|
import { docker } from '../lib/docker-utils';
|
||||||
import * as LogTypes from '../lib/log-types';
|
import * as LogTypes from '../lib/log-types';
|
||||||
import { defaultLegacyVolume } from '../lib/migration';
|
|
||||||
import log from '../lib/supervisor-console';
|
import log from '../lib/supervisor-console';
|
||||||
import * as logger from '../logger';
|
import * as logger from '../logger';
|
||||||
import { ResourceRecreationAttemptError } from './errors';
|
import { ResourceRecreationAttemptError } from './errors';
|
||||||
@ -78,25 +77,6 @@ export async function remove(volume: Volume) {
|
|||||||
await volume.remove();
|
await volume.remove();
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function createFromLegacy(appId: number): Promise<Volume | void> {
|
|
||||||
const name = defaultLegacyVolume();
|
|
||||||
const legacyPath = Path.join(
|
|
||||||
constants.rootMountPoint,
|
|
||||||
'mnt/data/resin-data',
|
|
||||||
appId.toString(),
|
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await createFromPath({ name, appId }, {}, legacyPath);
|
|
||||||
} catch (e) {
|
|
||||||
logger.logSystemMessage(
|
|
||||||
`Warning: could not migrate legacy /data volume: ${e.message}`,
|
|
||||||
{ error: e },
|
|
||||||
'Volume migration error',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function createFromPath(
|
export async function createFromPath(
|
||||||
{ name, appId }: VolumeNameOpts,
|
{ name, appId }: VolumeNameOpts,
|
||||||
config: Partial<VolumeConfig>,
|
config: Partial<VolumeConfig>,
|
||||||
|
116
src/device-state/legacy.ts
Normal file
116
src/device-state/legacy.ts
Normal file
@ -0,0 +1,116 @@
|
|||||||
|
import * as _ from 'lodash';
|
||||||
|
import { fromV2TargetApps, TargetAppsV2 } from '../lib/legacy';
|
||||||
|
import { AppsJsonFormat, TargetApp, TargetRelease } from '../types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts a single app from single container format into
|
||||||
|
* multi-container, multi-app format (v3)
|
||||||
|
*
|
||||||
|
* This function doesn't pull ids from the cloud, but uses dummy values,
|
||||||
|
* letting the normaliseLegacyDatabase() method perform the normalization
|
||||||
|
*/
|
||||||
|
function singleToMulticontainerApp(
|
||||||
|
app: Dictionary<any>,
|
||||||
|
): TargetApp & { uuid: string } {
|
||||||
|
const environment: Dictionary<string> = {};
|
||||||
|
for (const key in app.env) {
|
||||||
|
if (!/^RESIN_/.test(key)) {
|
||||||
|
environment[key] = app.env[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const { appId } = app;
|
||||||
|
|
||||||
|
const release: TargetRelease = {
|
||||||
|
id: 1,
|
||||||
|
networks: {},
|
||||||
|
volumes: {},
|
||||||
|
services: {},
|
||||||
|
};
|
||||||
|
const conf = app.config != null ? app.config : {};
|
||||||
|
const newApp: TargetApp & { uuid: string } = {
|
||||||
|
id: appId,
|
||||||
|
uuid: 'user-app',
|
||||||
|
name: app.name,
|
||||||
|
class: 'fleet',
|
||||||
|
releases: {
|
||||||
|
[app.commit]: release,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const defaultVolume = exports.defaultLegacyVolume();
|
||||||
|
release.volumes[defaultVolume] = {};
|
||||||
|
const updateStrategy =
|
||||||
|
conf['RESIN_SUPERVISOR_UPDATE_STRATEGY'] != null
|
||||||
|
? conf['RESIN_SUPERVISOR_UPDATE_STRATEGY']
|
||||||
|
: 'download-then-kill';
|
||||||
|
const handoverTimeout =
|
||||||
|
conf['RESIN_SUPERVISOR_HANDOVER_TIMEOUT'] != null
|
||||||
|
? conf['RESIN_SUPERVISOR_HANDOVER_TIMEOUT']
|
||||||
|
: '';
|
||||||
|
const restartPolicy =
|
||||||
|
conf['RESIN_APP_RESTART_POLICY'] != null
|
||||||
|
? conf['RESIN_APP_RESTART_POLICY']
|
||||||
|
: 'always';
|
||||||
|
release.services = {
|
||||||
|
main: {
|
||||||
|
id: 1,
|
||||||
|
image_id: 1,
|
||||||
|
image: app.imageId,
|
||||||
|
labels: {
|
||||||
|
'io.resin.features.kernel-modules': '1',
|
||||||
|
'io.resin.features.firmware': '1',
|
||||||
|
'io.resin.features.dbus': '1',
|
||||||
|
'io.resin.features.supervisor-api': '1',
|
||||||
|
'io.resin.features.resin-api': '1',
|
||||||
|
'io.resin.update.strategy': updateStrategy,
|
||||||
|
'io.resin.update.handover-timeout': handoverTimeout,
|
||||||
|
'io.resin.legacy-container': '1',
|
||||||
|
},
|
||||||
|
environment,
|
||||||
|
running: true,
|
||||||
|
composition: {
|
||||||
|
restart: restartPolicy,
|
||||||
|
privileged: true,
|
||||||
|
networkMode: 'host',
|
||||||
|
volumes: [`${defaultVolume}:/data`],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
return newApp;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts an apps.json from single container to multi-app (v3) format.
|
||||||
|
*/
|
||||||
|
export function fromLegacyAppsJson(appsArray: any[]): AppsJsonFormat {
|
||||||
|
const deviceConfig = _.reduce(
|
||||||
|
appsArray,
|
||||||
|
(conf, app) => {
|
||||||
|
return _.merge({}, conf, app.config);
|
||||||
|
},
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
|
const apps = _.keyBy(
|
||||||
|
_.map(appsArray, singleToMulticontainerApp),
|
||||||
|
'uuid',
|
||||||
|
) as Dictionary<TargetApp>;
|
||||||
|
return { apps, config: deviceConfig } as AppsJsonFormat;
|
||||||
|
}
|
||||||
|
|
||||||
|
type AppsJsonV2 = {
|
||||||
|
config: {
|
||||||
|
[varName: string]: string;
|
||||||
|
};
|
||||||
|
apps: TargetAppsV2;
|
||||||
|
pinDevice?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function fromV2AppsJson(
|
||||||
|
appsJson: AppsJsonV2,
|
||||||
|
): Promise<AppsJsonFormat> {
|
||||||
|
const { config: conf, apps, pinDevice } = appsJson;
|
||||||
|
|
||||||
|
const v3apps = await fromV2TargetApps(apps);
|
||||||
|
return { config: conf, apps: v3apps, ...(pinDevice && { pinDevice }) };
|
||||||
|
}
|
@ -16,7 +16,7 @@ import {
|
|||||||
} from '../lib/errors';
|
} from '../lib/errors';
|
||||||
import log from '../lib/supervisor-console';
|
import log from '../lib/supervisor-console';
|
||||||
|
|
||||||
import { convertLegacyAppsJson, convertV2toV3AppsJson } from '../lib/migration';
|
import { fromLegacyAppsJson, fromV2AppsJson } from './legacy';
|
||||||
import { AppsJsonFormat } from '../types/state';
|
import { AppsJsonFormat } from '../types/state';
|
||||||
import * as fsUtils from '../lib/fs-utils';
|
import * as fsUtils from '../lib/fs-utils';
|
||||||
import { isLeft } from 'fp-ts/lib/Either';
|
import { isLeft } from 'fp-ts/lib/Either';
|
||||||
@ -42,7 +42,7 @@ export async function loadTargetFromFile(appsPath: string): Promise<boolean> {
|
|||||||
|
|
||||||
if (_.isArray(stateFromFile)) {
|
if (_.isArray(stateFromFile)) {
|
||||||
log.debug('Detected a legacy apps.json, converting...');
|
log.debug('Detected a legacy apps.json, converting...');
|
||||||
stateFromFile = convertLegacyAppsJson(stateFromFile as any[]);
|
stateFromFile = fromLegacyAppsJson(stateFromFile as any[]);
|
||||||
}
|
}
|
||||||
|
|
||||||
// if apps.json apps are keyed by numeric ids, then convert to v3 target state
|
// if apps.json apps are keyed by numeric ids, then convert to v3 target state
|
||||||
@ -51,7 +51,7 @@ export async function loadTargetFromFile(appsPath: string): Promise<boolean> {
|
|||||||
(appId) => !isNaN(parseInt(appId, 10)),
|
(appId) => !isNaN(parseInt(appId, 10)),
|
||||||
)
|
)
|
||||||
) {
|
) {
|
||||||
stateFromFile = await convertV2toV3AppsJson(stateFromFile as any);
|
stateFromFile = await fromV2AppsJson(stateFromFile as any);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check that transformed apps.json has the correct format
|
// Check that transformed apps.json has the correct format
|
||||||
|
345
src/lib/legacy.ts
Normal file
345
src/lib/legacy.ts
Normal file
@ -0,0 +1,345 @@
|
|||||||
|
import * as _ from 'lodash';
|
||||||
|
|
||||||
|
import * as path from 'path';
|
||||||
|
import * as apiBinder from '../api-binder';
|
||||||
|
import * as config from '../config';
|
||||||
|
import * as db from '../db';
|
||||||
|
import * as volumeManager from '../compose/volume-manager';
|
||||||
|
import * as serviceManager from '../compose/service-manager';
|
||||||
|
import * as deviceState from '../device-state';
|
||||||
|
import * as applicationManager from '../compose/application-manager';
|
||||||
|
import {
|
||||||
|
DatabaseParseError,
|
||||||
|
NotFoundError,
|
||||||
|
InternalInconsistencyError,
|
||||||
|
} from '../lib/errors';
|
||||||
|
import * as constants from '../lib/constants';
|
||||||
|
import { docker } from '../lib/docker-utils';
|
||||||
|
import { log } from '../lib/supervisor-console';
|
||||||
|
import Volume from '../compose/volume';
|
||||||
|
import * as logger from '../logger';
|
||||||
|
import type {
|
||||||
|
DatabaseApp,
|
||||||
|
DatabaseService,
|
||||||
|
} from '../device-state/target-state-cache';
|
||||||
|
|
||||||
|
import { TargetApp, TargetApps } from '../types';
|
||||||
|
|
||||||
|
const defaultLegacyVolume = () => 'resin-data';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a docker volume from the legacy data directory
|
||||||
|
*/
|
||||||
|
export async function createVolumeFromLegacyData(
|
||||||
|
appId: number,
|
||||||
|
): Promise<Volume | void> {
|
||||||
|
const name = defaultLegacyVolume();
|
||||||
|
const legacyPath = path.join(
|
||||||
|
constants.rootMountPoint,
|
||||||
|
'mnt/data/resin-data',
|
||||||
|
appId.toString(),
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await volumeManager.createFromPath({ name, appId }, {}, legacyPath);
|
||||||
|
} catch (e) {
|
||||||
|
logger.logSystemMessage(
|
||||||
|
`Warning: could not migrate legacy /data volume: ${e.message}`,
|
||||||
|
{ error: e },
|
||||||
|
'Volume migration error',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets proper database ids from the cloud for the app and app services
|
||||||
|
*/
|
||||||
|
export async function normaliseLegacyDatabase() {
|
||||||
|
await apiBinder.initialized;
|
||||||
|
await deviceState.initialized;
|
||||||
|
|
||||||
|
if (apiBinder.balenaApi == null) {
|
||||||
|
throw new InternalInconsistencyError(
|
||||||
|
'API binder is not initialized correctly',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// When legacy apps are present, we kill their containers and migrate their /data to a named volume
|
||||||
|
log.info('Migrating ids for legacy app...');
|
||||||
|
|
||||||
|
const apps: DatabaseApp[] = await db.models('app').select();
|
||||||
|
|
||||||
|
if (apps.length === 0) {
|
||||||
|
log.debug('No app to migrate');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const app of apps) {
|
||||||
|
let services: DatabaseService[];
|
||||||
|
|
||||||
|
try {
|
||||||
|
services = JSON.parse(app.services);
|
||||||
|
} catch (e) {
|
||||||
|
throw new DatabaseParseError(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check there's a main service, with legacy-container set
|
||||||
|
if (services.length !== 1) {
|
||||||
|
log.debug("App doesn't have a single service, ignoring");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const service = services[0];
|
||||||
|
if (
|
||||||
|
!service.labels['io.resin.legacy-container'] &&
|
||||||
|
!service.labels['io.balena.legacy-container']
|
||||||
|
) {
|
||||||
|
log.debug('Service is not marked as legacy, ignoring');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug(`Getting release ${app.commit} for app ${app.appId} from API`);
|
||||||
|
const releases = await apiBinder.balenaApi.get({
|
||||||
|
resource: 'release',
|
||||||
|
options: {
|
||||||
|
$filter: {
|
||||||
|
belongs_to__application: app.appId,
|
||||||
|
commit: app.commit,
|
||||||
|
status: 'success',
|
||||||
|
},
|
||||||
|
$expand: {
|
||||||
|
contains__image: {
|
||||||
|
$expand: 'image',
|
||||||
|
},
|
||||||
|
belongs_to__application: {
|
||||||
|
$select: ['uuid'],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (releases.length === 0) {
|
||||||
|
log.warn(
|
||||||
|
`No compatible releases found in API, removing ${app.appId} from target state`,
|
||||||
|
);
|
||||||
|
await db.models('app').where({ appId: app.appId }).del();
|
||||||
|
}
|
||||||
|
|
||||||
|
// We need to get the app.uuid, release.id, serviceId, image.id and updated imageUrl
|
||||||
|
const release = releases[0];
|
||||||
|
const uuid = release.belongs_to__application[0].uuid;
|
||||||
|
const image = release.contains__image[0].image[0];
|
||||||
|
const serviceId = image.is_a_build_of__service.__id;
|
||||||
|
const imageUrl = !image.content_hash
|
||||||
|
? image.is_stored_at__image_location
|
||||||
|
: `${image.is_stored_at__image_location}@${image.content_hash}`;
|
||||||
|
|
||||||
|
log.debug(
|
||||||
|
`Found a release with releaseId ${release.id}, imageId ${image.id}, serviceId ${serviceId}\nImage location is ${imageUrl}`,
|
||||||
|
);
|
||||||
|
|
||||||
|
const imageFromDocker = await docker
|
||||||
|
.getImage(service.image)
|
||||||
|
.inspect()
|
||||||
|
.catch((error) => {
|
||||||
|
if (error instanceof NotFoundError) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
const imagesFromDatabase = await db
|
||||||
|
.models('image')
|
||||||
|
.where({ name: service.image })
|
||||||
|
.select();
|
||||||
|
|
||||||
|
await db.transaction(async (trx: db.Transaction) => {
|
||||||
|
try {
|
||||||
|
if (imagesFromDatabase.length > 0) {
|
||||||
|
log.debug('Deleting existing image entry in db');
|
||||||
|
await trx('image').where({ name: service.image }).del();
|
||||||
|
} else {
|
||||||
|
log.debug('No image in db to delete');
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
if (imageFromDocker != null) {
|
||||||
|
log.debug('Inserting fixed image entry in db');
|
||||||
|
await trx('image').insert({
|
||||||
|
name: imageUrl,
|
||||||
|
appId: app.appId,
|
||||||
|
appUuid: uuid,
|
||||||
|
serviceId,
|
||||||
|
serviceName: service.serviceName,
|
||||||
|
imageId: image.id,
|
||||||
|
releaseId: release.id,
|
||||||
|
commit: app.commit,
|
||||||
|
dependent: 0,
|
||||||
|
dockerImageId: imageFromDocker.Id,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
log.debug('Image is not downloaded, so not saving it to db');
|
||||||
|
}
|
||||||
|
|
||||||
|
delete service.labels['io.resin.legacy-container'];
|
||||||
|
delete service.labels['io.balena.legacy-container'];
|
||||||
|
|
||||||
|
Object.assign(app, {
|
||||||
|
services: JSON.stringify([
|
||||||
|
Object.assign(service, {
|
||||||
|
appId: app.appId,
|
||||||
|
appUuid: uuid,
|
||||||
|
image: imageUrl,
|
||||||
|
serviceId,
|
||||||
|
imageId: image.id,
|
||||||
|
releaseId: release.id,
|
||||||
|
commit: app.commit,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
uuid,
|
||||||
|
releaseId: release.id,
|
||||||
|
class: 'fleet',
|
||||||
|
});
|
||||||
|
|
||||||
|
log.debug('Updating app entry in db');
|
||||||
|
log.success('Successfully migrated legacy application');
|
||||||
|
await trx('app').update(app).where({ appId: app.appId });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug('Killing legacy containers');
|
||||||
|
await serviceManager.killAllLegacy();
|
||||||
|
log.debug('Migrating legacy app volumes');
|
||||||
|
|
||||||
|
await applicationManager.initialized;
|
||||||
|
const targetApps = await applicationManager.getTargetApps();
|
||||||
|
|
||||||
|
for (const app of Object.values(targetApps)) {
|
||||||
|
await createVolumeFromLegacyData(app.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
await config.set({
|
||||||
|
legacyAppsPresent: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export type TargetAppsV2 = {
|
||||||
|
[id: string]: {
|
||||||
|
name: string;
|
||||||
|
commit: string;
|
||||||
|
releaseId: number;
|
||||||
|
services: { [id: string]: any };
|
||||||
|
volumes: { [name: string]: any };
|
||||||
|
networks: { [name: string]: any };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert v2 to v3 target apps. If local is false
|
||||||
|
* it will query the API to get the app uuid
|
||||||
|
*/
|
||||||
|
export async function fromV2TargetApps(
|
||||||
|
apps: TargetAppsV2,
|
||||||
|
local = false,
|
||||||
|
): Promise<TargetApps> {
|
||||||
|
await apiBinder.initialized;
|
||||||
|
await deviceState.initialized;
|
||||||
|
|
||||||
|
if (apiBinder.balenaApi == null) {
|
||||||
|
throw new InternalInconsistencyError(
|
||||||
|
'API binder is not initialized correctly',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const { balenaApi } = apiBinder;
|
||||||
|
const getUUIDFromAPI = async (appId: number) => {
|
||||||
|
const appDetails = await balenaApi.get({
|
||||||
|
resource: 'application',
|
||||||
|
id: appId,
|
||||||
|
options: {
|
||||||
|
$select: ['uuid'],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!appDetails || !appDetails.uuid) {
|
||||||
|
throw new InternalInconsistencyError(
|
||||||
|
`No app with id ${appId} found on the API.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return appDetails.uuid;
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
(
|
||||||
|
await Promise.all(
|
||||||
|
Object.keys(apps).map(
|
||||||
|
async (id): Promise<[string, TargetApp]> => {
|
||||||
|
const appId = parseInt(id, 10);
|
||||||
|
const app = apps[appId];
|
||||||
|
|
||||||
|
// If local mode just use id as uuid
|
||||||
|
const uuid = local ? id : await getUUIDFromAPI(appId);
|
||||||
|
|
||||||
|
const releases = app.commit
|
||||||
|
? {
|
||||||
|
[app.commit]: {
|
||||||
|
id: app.releaseId,
|
||||||
|
services: Object.keys(app.services)
|
||||||
|
.map((serviceId) => {
|
||||||
|
const {
|
||||||
|
imageId,
|
||||||
|
serviceName,
|
||||||
|
image,
|
||||||
|
environment,
|
||||||
|
labels,
|
||||||
|
running,
|
||||||
|
serviceId: _serviceId,
|
||||||
|
contract,
|
||||||
|
...composition
|
||||||
|
} = app.services[serviceId];
|
||||||
|
|
||||||
|
return [
|
||||||
|
serviceName,
|
||||||
|
{
|
||||||
|
id: serviceId,
|
||||||
|
image_id: imageId,
|
||||||
|
image,
|
||||||
|
environment,
|
||||||
|
labels,
|
||||||
|
running,
|
||||||
|
contract,
|
||||||
|
composition,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
})
|
||||||
|
.reduce(
|
||||||
|
(res, [serviceName, svc]) => ({
|
||||||
|
...res,
|
||||||
|
[serviceName]: svc,
|
||||||
|
}),
|
||||||
|
{},
|
||||||
|
),
|
||||||
|
volumes: app.volumes,
|
||||||
|
networks: app.networks,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {};
|
||||||
|
|
||||||
|
return [
|
||||||
|
uuid,
|
||||||
|
{
|
||||||
|
id: appId,
|
||||||
|
name: app.name,
|
||||||
|
class: 'fleet',
|
||||||
|
releases,
|
||||||
|
} as TargetApp,
|
||||||
|
];
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
// Key by uuid
|
||||||
|
.reduce((res, [uuid, app]) => ({ ...res, [uuid]: app }), {})
|
||||||
|
);
|
||||||
|
}
|
@ -6,376 +6,14 @@ import * as rimraf from 'rimraf';
|
|||||||
|
|
||||||
const rimrafAsync = Bluebird.promisify(rimraf);
|
const rimrafAsync = Bluebird.promisify(rimraf);
|
||||||
|
|
||||||
import * as apiBinder from '../api-binder';
|
|
||||||
import * as config from '../config';
|
|
||||||
import * as db from '../db';
|
|
||||||
import * as volumeManager from '../compose/volume-manager';
|
import * as volumeManager from '../compose/volume-manager';
|
||||||
import * as serviceManager from '../compose/service-manager';
|
|
||||||
import * as deviceState from '../device-state';
|
import * as deviceState from '../device-state';
|
||||||
import * as applicationManager from '../compose/application-manager';
|
|
||||||
import * as constants from '../lib/constants';
|
import * as constants from '../lib/constants';
|
||||||
import {
|
import { BackupError, NotFoundError } from '../lib/errors';
|
||||||
BackupError,
|
|
||||||
DatabaseParseError,
|
|
||||||
NotFoundError,
|
|
||||||
InternalInconsistencyError,
|
|
||||||
} from '../lib/errors';
|
|
||||||
import { docker } from '../lib/docker-utils';
|
|
||||||
import { exec, pathExistsOnHost, mkdirp } from '../lib/fs-utils';
|
import { exec, pathExistsOnHost, mkdirp } from '../lib/fs-utils';
|
||||||
import { log } from '../lib/supervisor-console';
|
import { log } from '../lib/supervisor-console';
|
||||||
import {
|
|
||||||
AppsJsonFormat,
|
|
||||||
TargetApp,
|
|
||||||
TargetState,
|
|
||||||
TargetRelease,
|
|
||||||
} from '../types';
|
|
||||||
import type {
|
|
||||||
DatabaseApp,
|
|
||||||
DatabaseService,
|
|
||||||
} from '../device-state/target-state-cache';
|
|
||||||
|
|
||||||
export const defaultLegacyVolume = () => 'resin-data';
|
import { TargetState } from '../types';
|
||||||
|
|
||||||
function singleToMulticontainerApp(
|
|
||||||
app: Dictionary<any>,
|
|
||||||
): TargetApp & { uuid: string } {
|
|
||||||
const environment: Dictionary<string> = {};
|
|
||||||
for (const key in app.env) {
|
|
||||||
if (!/^RESIN_/.test(key)) {
|
|
||||||
environment[key] = app.env[key];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const { appId } = app;
|
|
||||||
|
|
||||||
const release: TargetRelease = {
|
|
||||||
id: 1,
|
|
||||||
networks: {},
|
|
||||||
volumes: {},
|
|
||||||
services: {},
|
|
||||||
};
|
|
||||||
const conf = app.config != null ? app.config : {};
|
|
||||||
const newApp: TargetApp & { uuid: string } = {
|
|
||||||
id: appId,
|
|
||||||
uuid: 'user-app',
|
|
||||||
name: app.name,
|
|
||||||
class: 'fleet',
|
|
||||||
releases: {
|
|
||||||
[app.commit]: release,
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const defaultVolume = exports.defaultLegacyVolume();
|
|
||||||
release.volumes[defaultVolume] = {};
|
|
||||||
const updateStrategy =
|
|
||||||
conf['RESIN_SUPERVISOR_UPDATE_STRATEGY'] != null
|
|
||||||
? conf['RESIN_SUPERVISOR_UPDATE_STRATEGY']
|
|
||||||
: 'download-then-kill';
|
|
||||||
const handoverTimeout =
|
|
||||||
conf['RESIN_SUPERVISOR_HANDOVER_TIMEOUT'] != null
|
|
||||||
? conf['RESIN_SUPERVISOR_HANDOVER_TIMEOUT']
|
|
||||||
: '';
|
|
||||||
const restartPolicy =
|
|
||||||
conf['RESIN_APP_RESTART_POLICY'] != null
|
|
||||||
? conf['RESIN_APP_RESTART_POLICY']
|
|
||||||
: 'always';
|
|
||||||
release.services = {
|
|
||||||
main: {
|
|
||||||
id: 1,
|
|
||||||
image_id: 1,
|
|
||||||
image: app.imageId,
|
|
||||||
labels: {
|
|
||||||
'io.resin.features.kernel-modules': '1',
|
|
||||||
'io.resin.features.firmware': '1',
|
|
||||||
'io.resin.features.dbus': '1',
|
|
||||||
'io.resin.features.supervisor-api': '1',
|
|
||||||
'io.resin.features.resin-api': '1',
|
|
||||||
'io.resin.update.strategy': updateStrategy,
|
|
||||||
'io.resin.update.handover-timeout': handoverTimeout,
|
|
||||||
'io.resin.legacy-container': '1',
|
|
||||||
},
|
|
||||||
environment,
|
|
||||||
running: true,
|
|
||||||
composition: {
|
|
||||||
restart: restartPolicy,
|
|
||||||
privileged: true,
|
|
||||||
networkMode: 'host',
|
|
||||||
volumes: [`${defaultVolume}:/data`],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
return newApp;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat {
|
|
||||||
const deviceConfig = _.reduce(
|
|
||||||
appsArray,
|
|
||||||
(conf, app) => {
|
|
||||||
return _.merge({}, conf, app.config);
|
|
||||||
},
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
|
|
||||||
const apps = _.keyBy(
|
|
||||||
_.map(appsArray, singleToMulticontainerApp),
|
|
||||||
'uuid',
|
|
||||||
) as Dictionary<TargetApp>;
|
|
||||||
return { apps, config: deviceConfig } as AppsJsonFormat;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function normaliseLegacyDatabase() {
|
|
||||||
await apiBinder.initialized;
|
|
||||||
await deviceState.initialized;
|
|
||||||
|
|
||||||
if (apiBinder.balenaApi == null) {
|
|
||||||
throw new InternalInconsistencyError(
|
|
||||||
'API binder is not initialized correctly',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// When legacy apps are present, we kill their containers and migrate their /data to a named volume
|
|
||||||
log.info('Migrating ids for legacy app...');
|
|
||||||
|
|
||||||
const apps: DatabaseApp[] = await db.models('app').select();
|
|
||||||
|
|
||||||
if (apps.length === 0) {
|
|
||||||
log.debug('No app to migrate');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const app of apps) {
|
|
||||||
let services: DatabaseService[];
|
|
||||||
|
|
||||||
try {
|
|
||||||
services = JSON.parse(app.services);
|
|
||||||
} catch (e) {
|
|
||||||
throw new DatabaseParseError(e);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check there's a main service, with legacy-container set
|
|
||||||
if (services.length !== 1) {
|
|
||||||
log.debug("App doesn't have a single service, ignoring");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const service = services[0];
|
|
||||||
if (
|
|
||||||
!service.labels['io.resin.legacy-container'] &&
|
|
||||||
!service.labels['io.balena.legacy-container']
|
|
||||||
) {
|
|
||||||
log.debug('Service is not marked as legacy, ignoring');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug(`Getting release ${app.commit} for app ${app.appId} from API`);
|
|
||||||
const releases = await apiBinder.balenaApi.get({
|
|
||||||
resource: 'release',
|
|
||||||
options: {
|
|
||||||
$filter: {
|
|
||||||
belongs_to__application: app.appId,
|
|
||||||
commit: app.commit,
|
|
||||||
status: 'success',
|
|
||||||
},
|
|
||||||
$expand: {
|
|
||||||
contains__image: {
|
|
||||||
$expand: 'image',
|
|
||||||
},
|
|
||||||
belongs_to__application: {
|
|
||||||
$select: ['uuid'],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (releases.length === 0) {
|
|
||||||
log.warn(
|
|
||||||
`No compatible releases found in API, removing ${app.appId} from target state`,
|
|
||||||
);
|
|
||||||
await db.models('app').where({ appId: app.appId }).del();
|
|
||||||
}
|
|
||||||
|
|
||||||
// We need to get the app.uuid, release.id, serviceId, image.id and updated imageUrl
|
|
||||||
const release = releases[0];
|
|
||||||
const uuid = release.belongs_to__application[0].uuid;
|
|
||||||
const image = release.contains__image[0].image[0];
|
|
||||||
const serviceId = image.is_a_build_of__service.__id;
|
|
||||||
const imageUrl = !image.content_hash
|
|
||||||
? image.is_stored_at__image_location
|
|
||||||
: `${image.is_stored_at__image_location}@${image.content_hash}`;
|
|
||||||
|
|
||||||
log.debug(
|
|
||||||
`Found a release with releaseId ${release.id}, imageId ${image.id}, serviceId ${serviceId}\nImage location is ${imageUrl}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const imageFromDocker = await docker
|
|
||||||
.getImage(service.image)
|
|
||||||
.inspect()
|
|
||||||
.catch((error) => {
|
|
||||||
if (error instanceof NotFoundError) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
throw error;
|
|
||||||
});
|
|
||||||
const imagesFromDatabase = await db
|
|
||||||
.models('image')
|
|
||||||
.where({ name: service.image })
|
|
||||||
.select();
|
|
||||||
|
|
||||||
await db.transaction(async (trx: db.Transaction) => {
|
|
||||||
try {
|
|
||||||
if (imagesFromDatabase.length > 0) {
|
|
||||||
log.debug('Deleting existing image entry in db');
|
|
||||||
await trx('image').where({ name: service.image }).del();
|
|
||||||
} else {
|
|
||||||
log.debug('No image in db to delete');
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
if (imageFromDocker != null) {
|
|
||||||
log.debug('Inserting fixed image entry in db');
|
|
||||||
await trx('image').insert({
|
|
||||||
name: imageUrl,
|
|
||||||
appId: app.appId,
|
|
||||||
appUuid: uuid,
|
|
||||||
serviceId,
|
|
||||||
serviceName: service.serviceName,
|
|
||||||
imageId: image.id,
|
|
||||||
releaseId: release.id,
|
|
||||||
commit: app.commit,
|
|
||||||
dependent: 0,
|
|
||||||
dockerImageId: imageFromDocker.Id,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
log.debug('Image is not downloaded, so not saving it to db');
|
|
||||||
}
|
|
||||||
|
|
||||||
delete service.labels['io.resin.legacy-container'];
|
|
||||||
delete service.labels['io.balena.legacy-container'];
|
|
||||||
|
|
||||||
Object.assign(app, {
|
|
||||||
services: JSON.stringify([
|
|
||||||
Object.assign(service, {
|
|
||||||
appId: app.appId,
|
|
||||||
appUuid: uuid,
|
|
||||||
image: imageUrl,
|
|
||||||
serviceId,
|
|
||||||
imageId: image.id,
|
|
||||||
releaseId: release.id,
|
|
||||||
commit: app.commit,
|
|
||||||
}),
|
|
||||||
]),
|
|
||||||
uuid,
|
|
||||||
class: 'fleet',
|
|
||||||
releaseId: release.id,
|
|
||||||
});
|
|
||||||
|
|
||||||
log.debug('Updating app entry in db');
|
|
||||||
log.success('Successfully migrated legacy application');
|
|
||||||
await trx('app').update(app).where({ appId: app.appId });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Killing legacy containers');
|
|
||||||
await serviceManager.killAllLegacy();
|
|
||||||
log.debug('Migrating legacy app volumes');
|
|
||||||
|
|
||||||
await applicationManager.initialized;
|
|
||||||
const targetApps = await applicationManager.getTargetApps();
|
|
||||||
|
|
||||||
for (const app of Object.values(targetApps)) {
|
|
||||||
await volumeManager.createFromLegacy(app.id);
|
|
||||||
}
|
|
||||||
|
|
||||||
await config.set({
|
|
||||||
legacyAppsPresent: false,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
type AppsJsonV2 = {
|
|
||||||
config: {
|
|
||||||
[varName: string]: string;
|
|
||||||
};
|
|
||||||
apps: {
|
|
||||||
[id: string]: {
|
|
||||||
name: string;
|
|
||||||
commit: string;
|
|
||||||
releaseId: number;
|
|
||||||
services: { [id: string]: any };
|
|
||||||
volumes: { [name: string]: any };
|
|
||||||
networks: { [name: string]: any };
|
|
||||||
};
|
|
||||||
};
|
|
||||||
pinDevice?: boolean;
|
|
||||||
};
|
|
||||||
|
|
||||||
export async function convertV2toV3AppsJson(
|
|
||||||
appsJson: AppsJsonV2,
|
|
||||||
): Promise<AppsJsonFormat> {
|
|
||||||
const { config: conf, apps, pinDevice } = appsJson;
|
|
||||||
|
|
||||||
await apiBinder.initialized;
|
|
||||||
await deviceState.initialized;
|
|
||||||
|
|
||||||
if (apiBinder.balenaApi == null) {
|
|
||||||
throw new InternalInconsistencyError(
|
|
||||||
'API binder is not initialized correctly',
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const { balenaApi } = apiBinder;
|
|
||||||
|
|
||||||
const v3apps = (
|
|
||||||
await Promise.all(
|
|
||||||
Object.keys(apps).map(
|
|
||||||
async (id): Promise<[string, TargetApp]> => {
|
|
||||||
const appId = parseInt(id, 10);
|
|
||||||
const app = apps[appId];
|
|
||||||
|
|
||||||
const appDetails = await balenaApi.get({
|
|
||||||
resource: 'application',
|
|
||||||
id: appId,
|
|
||||||
options: {
|
|
||||||
$select: ['uuid'],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!appDetails || appDetails.length === 0) {
|
|
||||||
throw new InternalInconsistencyError(
|
|
||||||
`No app with id ${appId} found on the API. Skipping apps.json migration`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const { uuid } = appDetails[0];
|
|
||||||
|
|
||||||
const releases = app.commit
|
|
||||||
? {
|
|
||||||
[app.commit]: {
|
|
||||||
id: app.releaseId,
|
|
||||||
services: app.services,
|
|
||||||
volumes: app.volumes,
|
|
||||||
networks: app.networks,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
: {};
|
|
||||||
|
|
||||||
return [
|
|
||||||
uuid,
|
|
||||||
{
|
|
||||||
id: appId,
|
|
||||||
name: app.name,
|
|
||||||
class: 'fleet',
|
|
||||||
releases,
|
|
||||||
} as TargetApp,
|
|
||||||
];
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
// Key by uuid
|
|
||||||
.reduce((res, [uuid, app]) => ({ ...res, [uuid]: app }), {});
|
|
||||||
|
|
||||||
return { config: conf, apps: v3apps, ...(pinDevice && { pinDevice }) };
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function loadBackupFromMigration(
|
export async function loadBackupFromMigration(
|
||||||
targetState: TargetState,
|
targetState: TargetState,
|
||||||
|
@ -4,7 +4,7 @@ import * as config from './config';
|
|||||||
import * as deviceState from './device-state';
|
import * as deviceState from './device-state';
|
||||||
import * as eventTracker from './event-tracker';
|
import * as eventTracker from './event-tracker';
|
||||||
import { intialiseContractRequirements } from './lib/contracts';
|
import { intialiseContractRequirements } from './lib/contracts';
|
||||||
import { normaliseLegacyDatabase } from './lib/migration';
|
import { normaliseLegacyDatabase } from './lib/legacy';
|
||||||
import * as osRelease from './lib/os-release';
|
import * as osRelease from './lib/os-release';
|
||||||
import * as logger from './logger';
|
import * as logger from './logger';
|
||||||
import SupervisorAPI from './supervisor-api';
|
import SupervisorAPI from './supervisor-api';
|
||||||
|
Loading…
Reference in New Issue
Block a user