ExternalVendorCode/balena-supervisor
1
0
Fork 0
mirror of https://github.com/balena-os/balena-supervisor.git synced 2025-01-29 15:44:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for GET v3 target state

Browse Source 
This change updates types and database format in order to allow
receiving the new format of the target state from the cloud and allow
applications to keep working.

This change also updates metadata in the containers, meaning services
will need to be restarted on supervisor update

Change-type: major
This commit is contained in:
Felipe Lalanne 2021-08-03 23:12:47 +00:00
parent ccae1f7cb8
commit 7425d1110b
32 changed files with 1827 additions and 1074 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/api-binder.ts
View File

@ -353,7 +353,12 @@ async function pinDevice({ app, commit }: DevicePinInfo) {
// We force a fresh get to make sure we have the latest state
// and can guarantee we don't clash with any already reported config
const targetConfigUnformatted = (await TargetState.get())?.local?.config;
const uuid = await config.get('uuid');
if (!uuid) {
throw new InternalInconsistencyError('No uuid for local device');
}
const targetConfigUnformatted = (await TargetState.get())?.[uuid]?.config;
if (targetConfigUnformatted == null) {
throw new InternalInconsistencyError(
'Attempt to report initial state with malformed target state',
@ -389,10 +394,12 @@ async function reportInitialEnv(
);
}
const targetConfigUnformatted = _.get(
await TargetState.get(),
'local.config',
);
const uuid = await config.get('uuid');
if (!uuid) {
throw new InternalInconsistencyError('No uuid for local device');
}
const targetConfigUnformatted = (await TargetState.get())?.[uuid]?.config;
if (targetConfigUnformatted == null) {
throw new InternalInconsistencyError(
'Attempt to report initial state with malformed target state',

97
src/compose/application-manager.ts
View File

@ -36,9 +36,8 @@ import {
TargetApps,
DeviceStatus,
DeviceReportFields,
TargetState,
} from '../types/state';
import { checkTruthy, checkInt } from '../lib/validation';
import { checkTruthy } from '../lib/validation';
import { Proxyvisor } from '../proxyvisor';
import { EventEmitter } from 'events';
@ -448,7 +447,6 @@ export async function executeStep(
// FIXME: This shouldn't be in this module
export async function setTarget(
apps: TargetApps,
dependent: TargetState['dependent'],
source: string,
maybeTrx?: Transaction,
) {
@ -467,10 +465,9 @@ export async function setTarget(
// Currently this will only happen if the release
// which would replace it fails a contract
// validation check
_.map(apps, (_v, appId) => checkInt(appId)),
Object.values(apps).map(({ id: appId }) => appId),
)
.del();
await proxyvisor.setTargetInTransaction(dependent, trx);
};
// We look at the container contracts here, as if we
@ -487,18 +484,29 @@ export async function setTarget(
const filteredApps = _.cloneDeep(apps);
_.each(
fulfilledContracts,
({ valid, unmetServices, fulfilledServices, unmetAndOptional }, appId) => {
(
{ valid, unmetServices, fulfilledServices, unmetAndOptional },
appUuid,
) => {
if (!valid) {
contractViolators[apps[appId].name] = unmetServices;
return delete filteredApps[appId];
contractViolators[apps[appUuid].name] = unmetServices;
return delete filteredApps[appUuid];
} else {
// valid is true, but we could still be missing
// some optional containers, and need to filter
// these out of the target state
filteredApps[appId].services = _.pickBy(
filteredApps[appId].services,
({ serviceName }) => fulfilledServices.includes(serviceName),
);
const [releaseUuid] = Object.keys(filteredApps[appUuid].releases);
if (releaseUuid) {
const services =
filteredApps[appUuid].releases[releaseUuid].services ?? {};
filteredApps[appUuid].releases[releaseUuid].services = _.pick(
services,
Object.keys(services).filter((serviceName) =>
fulfilledServices.includes(serviceName),
),
);
}
if (unmetAndOptional.length !== 0) {
return reportOptionalContainers(unmetAndOptional);
}
@ -527,17 +535,20 @@ export async function getTargetApps(): Promise<TargetApps> {
// the instances throughout the supervisor. The target state is derived from
// the database entries anyway, so these two things should never be different
// (except for the volatile state)
_.each(apps, (app) => {
if (!_.isEmpty(app.services)) {
app.services = _.mapValues(app.services, (svc) => {
if (svc.imageId && targetVolatilePerImageId[svc.imageId] != null) {
return { ...svc, ...targetVolatilePerImageId };
}
return svc;
});
}
});
//
_.each(apps, (app) =>
// There should only be a single release but is a simpler option
_.each(app.releases, (release) => {
if (!_.isEmpty(release.services)) {
release.services = _.mapValues(release.services, (svc) => {
if (svc.image_id && targetVolatilePerImageId[svc.image_id] != null) {
return { ...svc, ...targetVolatilePerImageId };
}
return svc;
});
}
}),
);
return apps;
}
@ -562,19 +573,28 @@ export function getDependentTargets() {
return proxyvisor.getTarget();
}
/**
* This is only used by the API. Do not use as the use of serviceIds is getting
* deprecated
*
* @deprecated
*/
export async function serviceNameFromId(serviceId: number) {
// We get the target here as it shouldn't matter, and getting the target is cheaper
const targets = await getTargetApps();
for (const appId of Object.keys(targets)) {
const app = targets[parseInt(appId, 10)];
const service = _.find(app.services, { serviceId });
if (service?.serviceName === null) {
throw new InternalInconsistencyError(
`Could not find a service name for id: ${serviceId}`,
);
const targetApps = await getTargetApps();
for (const { releases } of Object.values(targetApps)) {
const [release] = Object.values(releases);
const services = release?.services ?? {};
const serviceName = Object.keys(services).find(
(svcName) => services[svcName].id === serviceId,
);
if (!!serviceName) {
return serviceName;
}
return service!.serviceName;
}
throw new InternalInconsistencyError(
`Could not find a service for id: ${serviceId}`,
);
@ -622,15 +642,6 @@ function saveAndRemoveImages(
availableImages: imageManager.Image[],
localMode: boolean,
): CompositionStep[] {
const imageForService = (service: Service): imageManager.Image => ({
name: service.imageName!,
appId: service.appId,
serviceId: service.serviceId!,
serviceName: service.serviceName!,
imageId: service.imageId!,
releaseId: service.releaseId!,
dependent: 0,
});
type ImageWithoutID = Omit<imageManager.Image, 'dockerImageId' | 'id'>;
// imagesToRemove: images that
@ -666,7 +677,7 @@ function saveAndRemoveImages(
) as imageManager.Image[];
const targetServices = Object.values(target).flatMap((app) => app.services);
const targetImages = targetServices.map(imageForService);
const targetImages = targetServices.map(imageManager.imageFromService);
const availableAndUnused = _.filter(
availableWithoutIds,
@ -735,7 +746,7 @@ function saveAndRemoveImages(
// services
!targetServices.some(
(svc) =>
imageManager.isSameImage(img, imageForService(svc)) &&
imageManager.isSameImage(img, imageManager.imageFromService(svc)) &&
svc.config.labels['io.balena.update.strategy'] ===
'delete-then-download',
),

31
src/compose/images.ts
View File

@ -29,14 +29,29 @@ interface FetchProgressEvent {
export interface Image {
id?: number;
// image registry/repo@digest or registry/repo:tag
/**
* image [registry/]repo@digest or [registry/]repo:tag
*/
name: string;
/**
* @deprecated to be removed in target state v4
*/
appId: number;
appUuid: string;
/**
* @deprecated to be removed in target state v4
*/
serviceId: number;
serviceName: string;
// Id from balena api
/**
* @deprecated to be removed in target state v4
*/
imageId: number;
/**
* @deprecated to be removed in target state v4
*/
releaseId: number;
commit: string;
dependent: number;
dockerImageId?: string;
status?: 'Downloading' | 'Downloaded' | 'Deleting';
@ -151,17 +166,26 @@ function reportEvent(event: 'start' | 'update' | 'finish', state: Image) {
type ServiceInfo = Pick<
Service,
'imageName' | 'appId' | 'serviceId' | 'serviceName' | 'imageId' | 'releaseId'
| 'imageName'
| 'appId'
| 'serviceId'
| 'serviceName'
| 'imageId'
| 'releaseId'
| 'appUuid'
| 'commit'
>;
export function imageFromService(service: ServiceInfo): Image {
// We know these fields are defined because we create these images from target state
return {
name: service.imageName!,
appId: service.appId,
appUuid: service.appUuid!,
serviceId: service.serviceId!,
serviceName: service.serviceName!,
imageId: service.imageId!,
releaseId: service.releaseId!,
commit: service.commit!,
dependent: 0,
};
}
@ -747,6 +771,7 @@ function format(image: Image): Partial<Omit<Image, 'id'>> {
serviceName: null,
imageId: null,
releaseId: null,
commit: null,
dependent: 0,
dockerImageId: null,
})

56
src/compose/service.ts
View File

@ -44,6 +44,7 @@ export type ServiceStatus =
export class Service {
public appId: number;
public appUuid?: string;
public imageId: number;
public config: ServiceConfig;
public serviceName: string;
@ -111,7 +112,10 @@ export class Service {
): Promise<Service> {
const service = new Service();
appConfig = ComposeUtils.camelCaseConfig(appConfig);
appConfig = {
...appConfig,
composition: ComposeUtils.camelCaseConfig(appConfig.composition || {}),
};
if (!appConfig.appId) {
throw new InternalInconsistencyError('No app id for service');
@ -124,27 +128,33 @@ export class Service {
// Separate the application information from the docker
// container configuration
service.imageId = parseInt(appConfig.imageId, 10);
delete appConfig.imageId;
service.serviceName = appConfig.serviceName;
delete appConfig.serviceName;
service.appId = appId;
delete appConfig.appId;
service.releaseId = parseInt(appConfig.releaseId, 10);
delete appConfig.releaseId;
service.serviceId = parseInt(appConfig.serviceId, 10);
delete appConfig.serviceId;
service.imageName = appConfig.image;
service.dependsOn = appConfig.dependsOn || null;
delete appConfig.dependsOn;
service.createdAt = appConfig.createdAt;
delete appConfig.createdAt;
service.commit = appConfig.commit;
delete appConfig.commit;
service.appUuid = appConfig.appUuid;
delete appConfig.contract;
// dependsOn is used by other parts of the step
// calculation so we delete it from the composition
service.dependsOn = appConfig.composition?.dependsOn || null;
delete appConfig.composition?.dependsOn;
// Get remaining fields from appConfig
const { image, running, labels, environment, composition } = appConfig;
// Get rid of any extra values and report them to the user
const config = sanitiseComposeConfig(appConfig);
const config = sanitiseComposeConfig({
image,
running,
...composition,
// Ensure the top level label and environment definition is used
labels: { ...(composition?.labels ?? {}), ...labels },
environment: { ...(composition?.environment ?? {}), ...environment },
});
// Process some values into the correct format, delete them from
// the original object, and add them to the defaults object below
@ -265,6 +275,7 @@ export class Service {
config.environment || {},
options,
service.appId || 0,
service.appUuid!,
service.serviceName || '',
),
);
@ -275,6 +286,7 @@ export class Service {
service.appId || 0,
service.serviceId || 0,
service.serviceName || '',
service.appUuid!, // appUuid will always exist on the target state
),
);
@ -614,6 +626,7 @@ export class Service {
);
}
svc.appId = appId;
svc.appUuid = svc.config.labels['io.balena.app-uuid'];
svc.serviceName = svc.config.labels['io.balena.service-name'];
svc.serviceId = parseInt(svc.config.labels['io.balena.service-id'], 10);
if (Number.isNaN(svc.serviceId)) {
@ -957,6 +970,7 @@ export class Service {
environment: { [envVarName: string]: string } | null | undefined,
options: DeviceMetadata,
appId: number,
appUuid: string,
serviceName: string,
): { [envVarName: string]: string } {
const defaultEnv: { [envVarName: string]: string } = {};
@ -966,6 +980,7 @@ export class Service {
_.mapKeys(
{
APP_ID: appId.toString(),
APP_UUID: appUuid,
APP_NAME: options.appName,
SERVICE_NAME: serviceName,
DEVICE_UUID: options.uuid,
@ -1071,13 +1086,18 @@ export class Service {
appId: number,
serviceId: number,
serviceName: string,
appUuid: string,
): { [labelName: string]: string } {
let newLabels = _.defaults(labels, {
'io.balena.supervised': 'true',
'io.balena.app-id': appId.toString(),
'io.balena.service-id': serviceId.toString(),
'io.balena.service-name': serviceName,
});
let newLabels = {
...labels,
...{
'io.balena.supervised': 'true',
'io.balena.app-id': appId.toString(),
'io.balena.service-id': serviceId.toString(),
'io.balena.service-name': serviceName,
'io.balena.app-uuid': appUuid,
},
};
const imageLabels = _.get(imageInfo, 'Config.Labels', {});
newLabels = _.defaults(newLabels, imageLabels);

18
src/device-api/v2.ts
View File

@ -557,19 +557,17 @@ export function createV2Api(router: Router) {
router.get('/v2/cleanup-volumes', async (req: AuthorizedRequest, res) => {
const targetState = await applicationManager.getTargetApps();
const referencedVolumes: string[] = [];
_.each(targetState, (app, appId) => {
const referencedVolumes = Object.values(targetState)
// if this app isn't in scope of the request, do not cleanup it's volumes
if (!req.auth.isScoped({ apps: [parseInt(appId, 10)] })) {
return;
}
_.each(app.volumes, (_volume, volumeName) => {
referencedVolumes.push(
Volume.generateDockerName(parseInt(appId, 10), volumeName),
.filter((app) => req.auth.isScoped({ apps: [app.id] }))
.flatMap((app) => {
const [release] = Object.values(app.releases);
// Return a list of the volume names
return Object.keys(release?.volumes ?? {}).map((volumeName) =>
Volume.generateDockerName(app.id, volumeName),
);
});
});
await volumeManager.removeOrphanedVolumes(referencedVolumes);
res.json({
status: 'success',

32
src/device-state.ts
View File

@ -479,27 +479,29 @@ export async function setTarget(target: TargetState, localSource?: boolean) {
globalEventBus.getInstance().emit('targetStateChanged', target);
const apiEndpoint = await config.get('apiEndpoint');
const { uuid, apiEndpoint } = await config.getMany([
'uuid',
'apiEndpoint',
'name',
]);
if (!uuid || !target[uuid]) {
throw new Error(
`Expected target state for local device with uuid '${uuid}'.`,
);
}
const localTarget = target[uuid];
await usingWriteLockTarget(async () => {
await db.transaction(async (trx) => {
await config.set({ name: target.local.name }, trx);
await deviceConfig.setTarget(target.local.config, trx);
await config.set({ name: localTarget.name }, trx);
await deviceConfig.setTarget(localTarget.config, trx);
if (localSource || apiEndpoint == null || apiEndpoint === '') {
await applicationManager.setTarget(
target.local.apps,
target.dependent,
'local',
trx,
);
await applicationManager.setTarget(localTarget.apps, 'local', trx);
} else {
await applicationManager.setTarget(
target.local.apps,
target.dependent,
apiEndpoint,
trx,
);
await applicationManager.setTarget(localTarget.apps, apiEndpoint, trx);
}
await config.set({ targetStateSet: true }, trx);
});

143
src/device-state/db-format.ts
View File

@ -1,7 +1,8 @@
import * as _ from 'lodash';
import * as db from '../db';
import * as targetStateCache from '../device-state/target-state-cache';
import * as targetStateCache from './target-state-cache';
import { DatabaseApp, DatabaseService } from './target-state-cache';
import App from '../compose/app';
import * as images from '../compose/images';
@ -10,9 +11,9 @@ import {
InstancedAppState,
TargetApp,
TargetApps,
TargetRelease,
TargetService,
} from '../types/state';
import { checkInt } from '../lib/validation';
type InstancedApp = InstancedAppState[0];
@ -37,72 +38,112 @@ export async function getApps(): Promise<InstancedAppState> {
}
export async function setApps(
apps: { [appId: number]: TargetApp },
apps: TargetApps,
source: string,
trx?: db.Transaction,
) {
const dbApps = await Promise.all(
Object.keys(apps).map(async (appIdStr) => {
const appId = checkInt(appIdStr)!;
const dbApps = Object.keys(apps).map((uuid) => {
const { id: appId, ...app } = apps[uuid];
const app = apps[appId];
const services = await Promise.all(
_.map(app.services, async (s, sId) => ({
...s,
appId,
releaseId: app.releaseId,
serviceId: checkInt(sId),
commit: app.commit,
image: await images.normalise(s.image),
})),
);
// Get the first uuid
const [releaseUuid] = Object.keys(app.releases);
const release = releaseUuid
? app.releases[releaseUuid]
: ({} as TargetRelease);
const services = Object.keys(release.services ?? {}).map((serviceName) => {
const { id: releaseId } = release;
const { id: serviceId, image_id: imageId, ...service } = release.services[
serviceName
];
return {
...service,
appId,
source,
commit: app.commit,
name: app.name,
releaseId: app.releaseId,
services: JSON.stringify(services),
networks: JSON.stringify(app.networks ?? {}),
volumes: JSON.stringify(app.volumes ?? {}),
appUuid: uuid,
releaseId,
commit: releaseUuid,
imageId,
serviceId,
serviceName,
image: images.normalise(service.image),
};
}),
);
});
return {
appId,
uuid,
source,
class: app.class,
name: app.name,
...(releaseUuid && { releaseId: release.id, commit: releaseUuid }),
services: JSON.stringify(services),
networks: JSON.stringify(release.networks ?? {}),
volumes: JSON.stringify(release.volumes ?? {}),
};
});
await targetStateCache.setTargetApps(dbApps, trx);
}
/**
* Create target state from database state
*/
export async function getTargetJson(): Promise<TargetApps> {
const dbApps = await getDBEntry();
const apps: TargetApps = {};
await Promise.all(
dbApps.map(async (app) => {
const parsedServices = JSON.parse(app.services);
const services = _(parsedServices)
.keyBy('serviceId')
.mapValues(
(svc: TargetService) => _.omit(svc, 'commit') as TargetService,
)
.value();
return dbApps
.map(({ source, uuid, releaseId, commit: releaseUuid, ...app }): [
string,
TargetApp,
] => {
const services = (JSON.parse(app.services) as DatabaseService[])
.map(({ serviceName, serviceId, imageId, ...service }): [
string,
TargetService,
] => [
serviceName,
{
id: serviceId,
image_id: imageId,
..._.omit(service, ['appId', 'appUuid', 'commit', 'releaseId']),
} as TargetService,
])
// Map by serviceName
.reduce(
(svcs, [serviceName, s]) => ({
...svcs,
[serviceName]: s,
}),
{},
);
apps[app.appId] = {
// We remove the id as this is the supervisor database id, and the
// source is internal and isn't used except for when we fetch the target
// state
..._.omit(app, ['id', 'source']),
services,
networks: JSON.parse(app.networks),
volumes: JSON.parse(app.volumes),
// We can add this cast because it's required in the db
} as TargetApp;
}),
);
return apps;
const releases = releaseUuid
? {
[releaseUuid]: {
id: releaseId,
services,
networks: JSON.parse(app.networks),
volumes: JSON.parse(app.volumes),
} as TargetRelease,
}
: {};
return [
uuid,
{
id: app.appId,
name: app.name,
class: app.class,
releases,
},
];
})
.reduce((apps, [uuid, app]) => ({ ...apps, [uuid]: app }), {});
}
function getDBEntry(): Promise<targetStateCache.DatabaseApp[]>;
function getDBEntry(appId: number): Promise<targetStateCache.DatabaseApp>;
function getDBEntry(): Promise<DatabaseApp[]>;
function getDBEntry(appId: number): Promise<DatabaseApp>;
async function getDBEntry(appId?: number) {
await targetStateCache.initialized;

113
src/device-state/preload.ts
View File

@ -6,14 +6,21 @@ import * as deviceState from '../device-state';
import * as config from '../config';
import * as deviceConfig from '../device-config';
import * as eventTracker from '../event-tracker';
import * as images from '../compose/images';
import * as imageManager from '../compose/images';
import { AppsJsonParseError, EISDIR, ENOENT } from '../lib/errors';
import {
AppsJsonParseError,
EISDIR,
ENOENT,
InternalInconsistencyError,
} from '../lib/errors';
import log from '../lib/supervisor-console';
import { convertLegacyAppsJson } from '../lib/migration';
import { convertLegacyAppsJson, convertV2toV3AppsJson } from '../lib/migration';
import { AppsJsonFormat } from '../types/state';
import * as fsUtils from '../lib/fs-utils';
import { isLeft } from 'fp-ts/lib/Either';
import Reporter from 'io-ts-reporters';
export function appsJsonBackup(appsPath: string) {
return `${appsPath}.preloaded`;
@ -37,51 +44,78 @@ export async function loadTargetFromFile(appsPath: string): Promise<boolean> {
log.debug('Detected a legacy apps.json, converting...');
stateFromFile = convertLegacyAppsJson(stateFromFile as any[]);
}
const preloadState = stateFromFile as AppsJsonFormat;
let commitToPin: string | undefined;
let appToPin: string | undefined;
// if apps.json apps are keyed by numeric ids, then convert to v3 target state
if (
Object.keys(stateFromFile.apps || {}).some(
(appId) => !isNaN(parseInt(appId, 10)),
)
) {
stateFromFile = await convertV2toV3AppsJson(stateFromFile as any);
}
if (_.isEmpty(preloadState)) {
// Check that transformed apps.json has the correct format
const decodedAppsJson = AppsJsonFormat.decode(stateFromFile);
if (isLeft(decodedAppsJson)) {
throw new AppsJsonParseError(
['Invalid apps.json.']
.concat(Reporter.report(decodedAppsJson))
.join('\n'),
);
}
// If decoding apps.json succeeded then preloadState will have the right format
const preloadState = decodedAppsJson.right;
if (_.isEmpty(preloadState.config) && _.isEmpty(preloadState.apps)) {
return false;
}
const imgs: Image[] = [];
const appIds = _.keys(preloadState.apps);
for (const appId of appIds) {
const app = preloadState.apps[appId];
// Multi-app warning!
// The following will need to be changed once running
// multiple applications is possible
commitToPin = app.commit;
appToPin = appId;
const serviceIds = _.keys(app.services);
for (const serviceId of serviceIds) {
const service = app.services[serviceId];
const svc = {
imageName: service.image,
serviceName: service.serviceName,
imageId: service.imageId,
serviceId: parseInt(serviceId, 10),
releaseId: app.releaseId,
appId: parseInt(appId, 10),
};
imgs.push(imageFromService(svc));
}
const uuid = await config.get('uuid');
if (!uuid) {
throw new InternalInconsistencyError(
`No uuid found for the local device`,
);
}
const imgs: Image[] = Object.keys(preloadState.apps)
.map((appUuid) => {
const app = preloadState.apps[appUuid];
const [releaseUuid] = Object.keys(app.releases);
const release = app.releases[releaseUuid] ?? {};
const services = release?.services ?? {};
return Object.keys(services).map((serviceName) => {
const service = services[serviceName];
const svc = {
imageName: service.image,
serviceName,
imageId: service.image_id,
serviceId: service.id,
releaseId: release.id,
commit: releaseUuid,
appId: app.id,
appUuid,
};
return imageFromService(svc);
});
})
.reduce((res, images) => res.concat(images), []);
for (const image of imgs) {
const name = images.normalise(image.name);
const name = imageManager.normalise(image.name);
image.name = name;
await images.save(image);
await imageManager.save(image);
}
const deviceConf = await deviceConfig.getCurrent();
const formattedConf = deviceConfig.formatConfigKeys(preloadState.config);
preloadState.config = { ...formattedConf, ...deviceConf };
const localState = {
local: { name: '', ...preloadState },
dependent: { apps: {}, devices: {} },
[uuid]: {
name: '',
config: { ...formattedConf, ...deviceConf },
apps: preloadState.apps,
},
};
await deviceState.setTarget(localState);
@ -89,13 +123,17 @@ export async function loadTargetFromFile(appsPath: string): Promise<boolean> {
if (preloadState.pinDevice) {
// Multi-app warning!
// The following will need to be changed once running
// multiple applications is possible
// multiple applications is possible.
// For now, just select the first app with 'fleet' class (there can be only one)
const [appToPin] = Object.values(preloadState.apps).filter(
(app) => app.class === 'fleet',
);
const [commitToPin] = Object.keys(appToPin?.releases ?? {});
if (commitToPin != null && appToPin != null) {
log.debug('Device will be pinned');
await config.set({
pinDevice: {
commit: commitToPin,
app: parseInt(appToPin, 10),
app: appToPin.id,
},
});
}
@ -109,6 +147,7 @@ export async function loadTargetFromFile(appsPath: string): Promise<boolean> {
if (ENOENT(e) || EISDIR(e)) {
log.debug('No apps.json file present, skipping preload');
} else {
log.debug(e.message);
eventTracker.track('Loading preloaded apps failed', {
error: e,
});

33
src/device-state/target-state-cache.ts
View File

@ -2,20 +2,43 @@ import * as _ from 'lodash';
import * as config from '../config';
import * as db from '../db';
import { TargetAppClass } from '../types';
// We omit the id (which does appear in the db) in this type, as we don't use it
// at all, and we can use the below type for both insertion and retrieval.
export interface DatabaseApp {
name: string;
/**
* @deprecated to be removed in target state v4
*/
releaseId?: number;
commit?: string;
/**
* @deprecated to be removed in target state v4
*/
appId: number;
uuid: string;
services: string;
networks: string;
volumes: string;
source: string;
class: TargetAppClass;
}
export type DatabaseApps = DatabaseApp[];
export type DatabaseService = {
appId: string;
appUuid: string;
releaseId: number;
commit: string;
serviceName: string;
serviceId: number;
imageId: number;
image: string;
labels: { [key: string]: string };
environment: { [key: string]: string };
composition?: { [key: string]: any };
};
/*
* This module is a wrapper around the database fetching and retrieving of
@ -26,7 +49,7 @@ export type DatabaseApps = DatabaseApp[];
* accesses the target state for every log line. This can very quickly cause
* serious memory problems and database connection timeouts.
*/
let targetState: DatabaseApps | undefined;
let targetState: DatabaseApp[] | undefined;
export const initialized = (async () => {
await db.initialized;
@ -53,7 +76,7 @@ export async function getTargetApp(
return _.find(targetState, (app) => app.appId === appId);
}
export async function getTargetApps(): Promise<DatabaseApps> {
export async function getTargetApps(): Promise<DatabaseApp[]> {
if (targetState == null) {
const { apiEndpoint, localMode } = await config.getMany([
'apiEndpoint',
@ -67,7 +90,7 @@ export async function getTargetApps(): Promise<DatabaseApps> {
}
export async function setTargetApps(
apps: DatabaseApps,
apps: DatabaseApp[],
trx?: db.Transaction,
): Promise<void> {
// We can't cache the value here, as it could be for a
@ -75,6 +98,6 @@ export async function setTargetApps(
targetState = undefined;
await Promise.all(
apps.map((app) => db.upsertModel('app', app, { appId: app.appId }, trx)),
apps.map((app) => db.upsertModel('app', app, { uuid: app.uuid }, trx)),
);
}

2
src/device-state/target-state.ts
View File

@ -135,7 +135,7 @@ export const update = async (
);
}
const endpoint = url.resolve(apiEndpoint, `/device/v2/${uuid}/state`);
const endpoint = url.resolve(apiEndpoint, `/device/v3/${uuid}/state`);
const request = await getRequestInstance();
const params: CoreOptions = {

103
src/lib/contracts.ts
View File

@ -1,27 +1,16 @@
import { isLeft } from 'fp-ts/lib/Either';
import * as t from 'io-ts';
import { reporter } from 'io-ts-reporters';
import Reporter from 'io-ts-reporters';
import * as _ from 'lodash';
import { Blueprint, Contract, ContractObject } from '@balena/contrato';
import { ContractValidationError, InternalInconsistencyError } from './errors';
import { checkTruthy } from './validation';
import { TargetApps } from '../types';
export { ContractObject };
// TODO{type}: When target and current state are correctly
// defined, replace this
interface AppWithContracts {
services: {
[key: string]: {
serviceName: string;
contract?: ContractObject;
labels?: Dictionary<string>;
};
};
}
export interface ApplicationContractResult {
valid: boolean;
unmetServices: string[];
@ -194,7 +183,7 @@ export function validateContract(contract: unknown): boolean {
const result = contractObjectValidator.decode(contract);
if (isLeft(result)) {
throw new Error(reporter(result).join('\n'));
throw new Error(Reporter.report(result).join('\n'));
}
const requirementVersions = contractRequirementVersions;
@ -208,46 +197,66 @@ export function validateContract(contract: unknown): boolean {
return true;
}
export function validateTargetContracts(
apps: Dictionary<AppWithContracts>,
apps: TargetApps,
): Dictionary<ApplicationContractResult> {
const appsFulfilled: Dictionary<ApplicationContractResult> = {};
return Object.keys(apps)
.map((appUuid): [string, ApplicationContractResult] => {
const app = apps[appUuid];
const [release] = Object.values(app.releases);
const serviceContracts = Object.keys(release?.services ?? [])
.map((serviceName) => {
const service = release.services[serviceName];
const { contract } = service;
if (contract) {
try {
// Validate the contract syntax
validateContract(contract);
for (const appId of _.keys(apps)) {
const app = apps[appId];
const serviceContracts: ServiceContracts = {};
return {
serviceName,
contract,
optional: checkTruthy(
service.labels?.['io.balena.features.optional'],
),
};
} catch (e) {
throw new ContractValidationError(serviceName, e.message);
}
}
for (const svcId of _.keys(app.services)) {
const svc = app.services[svcId];
// Return a default contract for the service if no contract is defined
return { serviceName, contract: undefined, optional: false };
})
// map by serviceName
.reduce(
(contracts, { serviceName, ...serviceContract }) => ({
...contracts,
[serviceName]: serviceContract,
}),
{} as ServiceContracts,
);
if (svc.contract) {
try {
validateContract(svc.contract);
serviceContracts[svc.serviceName] = {
contract: svc.contract,
optional: checkTruthy(svc.labels?.['io.balena.features.optional']),
};
} catch (e) {
throw new ContractValidationError(svc.serviceName, e.message);
}
} else {
serviceContracts[svc.serviceName] = {
contract: undefined,
optional: false,
};
if (Object.keys(serviceContracts).length > 0) {
// Validate service contracts if any
return [appUuid, containerContractsFulfilled(serviceContracts)];
}
if (!_.isEmpty(serviceContracts)) {
appsFulfilled[appId] = containerContractsFulfilled(serviceContracts);
} else {
appsFulfilled[appId] = {
// Return success if no services are found
return [
appUuid,
{
valid: true,
fulfilledServices: _.map(app.services, 'serviceName'),
fulfilledServices: Object.keys(release?.services ?? []),
unmetAndOptional: [],
unmetServices: [],
};
}
}
}
return appsFulfilled;
},
];
})
.reduce(
(result, [appUuid, contractFulfilled]) => ({
...result,
[appUuid]: contractFulfilled,
}),
{} as Dictionary<ApplicationContractResult>,
);
}

186
src/lib/migration.ts
View File

@ -23,15 +23,22 @@ import {
import { docker } from '../lib/docker-utils';
import { exec, pathExistsOnHost, mkdirp } from '../lib/fs-utils';
import { log } from '../lib/supervisor-console';
import type { AppsJsonFormat, TargetApp, TargetState } from '../types/state';
import type { DatabaseApp } from '../device-state/target-state-cache';
import { ShortString } from '../types';
import {
AppsJsonFormat,
TargetApp,
TargetState,
TargetRelease,
} from '../types';
import type {
DatabaseApp,
DatabaseService,
} from '../device-state/target-state-cache';
export const defaultLegacyVolume = () => 'resin-data';
export function singleToMulticontainerApp(
function singleToMulticontainerApp(
app: Dictionary<any>,
): TargetApp & { appId: string } {
): TargetApp & { uuid: string } {
const environment: Dictionary<string> = {};
for (const key in app.env) {
if (!/^RESIN_/.test(key)) {
@ -40,18 +47,25 @@ export function singleToMulticontainerApp(
}
const { appId } = app;
const conf = app.config != null ? app.config : {};
const newApp: TargetApp & { appId: string } = {
appId: appId.toString(),
commit: app.commit,
name: app.name,
releaseId: 1,
const release: TargetRelease = {
id: 1,
networks: {},
volumes: {},
services: {},
};
const conf = app.config != null ? app.config : {};
const newApp: TargetApp & { uuid: string } = {
id: appId,
uuid: 'user-app',
name: app.name,
class: 'fleet',
releases: {
[app.commit]: release,
},
};
const defaultVolume = exports.defaultLegacyVolume();
newApp.volumes[defaultVolume] = {};
release.volumes[defaultVolume] = {};
const updateStrategy =
conf['RESIN_SUPERVISOR_UPDATE_STRATEGY'] != null
? conf['RESIN_SUPERVISOR_UPDATE_STRATEGY']
@ -64,19 +78,11 @@ export function singleToMulticontainerApp(
conf['RESIN_APP_RESTART_POLICY'] != null
? conf['RESIN_APP_RESTART_POLICY']
: 'always';
newApp.services = {
// Disable the next line, as this *has* to be a string
// tslint:disable-next-line
'1': {
appId,
serviceName: 'main' as ShortString,
imageId: 1,
commit: app.commit,
releaseId: 1,
release.services = {
main: {
id: 1,
image_id: 1,
image: app.imageId,
privileged: true,
networkMode: 'host',
volumes: [`${defaultVolume}:/data`],
labels: {
'io.resin.features.kernel-modules': '1',
'io.resin.features.firmware': '1',
@ -88,8 +94,13 @@ export function singleToMulticontainerApp(
'io.resin.legacy-container': '1',
},
environment,
restart: restartPolicy,
running: true,
composition: {
restart: restartPolicy,
privileged: true,
networkMode: 'host',
volumes: [`${defaultVolume}:/data`],
},
},
};
return newApp;
@ -104,7 +115,10 @@ export function convertLegacyAppsJson(appsArray: any[]): AppsJsonFormat {
{},
);
const apps = _.keyBy(_.map(appsArray, singleToMulticontainerApp), 'appId');
const apps = _.keyBy(
_.map(appsArray, singleToMulticontainerApp),
'uuid',
) as Dictionary<TargetApp>;
return { apps, config: deviceConfig } as AppsJsonFormat;
}
@ -129,7 +143,7 @@ export async function normaliseLegacyDatabase() {
}
for (const app of apps) {
let services: Array<TargetApp['services']['']>;
let services: DatabaseService[];
try {
services = JSON.parse(app.services);
@ -165,6 +179,9 @@ export async function normaliseLegacyDatabase() {
contains__image: {
$expand: 'image',
},
belongs_to__application: {
$select: ['uuid'],
},
},
},
});
@ -176,8 +193,9 @@ export async function normaliseLegacyDatabase() {
await db.models('app').where({ appId: app.appId }).del();
}
// We need to get the release.id, serviceId, image.id and updated imageUrl
// We need to get the app.uuid, release.id, serviceId, image.id and updated imageUrl
const release = releases[0];
const uuid = release.belongs_to__application[0].uuid;
const image = release.contains__image[0].image[0];
const serviceId = image.is_a_build_of__service.__id;
const imageUrl = !image.content_hash
@ -217,10 +235,12 @@ export async function normaliseLegacyDatabase() {
await trx('image').insert({
name: imageUrl,
appId: app.appId,
appUuid: uuid,
serviceId,
serviceName: service.serviceName,
imageId: image.id,
releaseId: release.id,
commit: app.commit,
dependent: 0,
dockerImageId: imageFromDocker.Id,
});
@ -234,12 +254,17 @@ export async function normaliseLegacyDatabase() {
Object.assign(app, {
services: JSON.stringify([
Object.assign(service, {
appId: app.appId,
appUuid: uuid,
image: imageUrl,
serviceID: serviceId,
serviceId,
imageId: image.id,
releaseId: release.id,
commit: app.commit,
}),
]),
uuid,
class: 'fleet',
releaseId: release.id,
});
@ -257,8 +282,8 @@ export async function normaliseLegacyDatabase() {
await applicationManager.initialized;
const targetApps = await applicationManager.getTargetApps();
for (const appId of _.keys(targetApps)) {
await volumeManager.createFromLegacy(parseInt(appId, 10));
for (const app of Object.values(targetApps)) {
await volumeManager.createFromLegacy(app.id);
}
await config.set({
@ -266,6 +291,92 @@ export async function normaliseLegacyDatabase() {
});
}
type AppsJsonV2 = {
config: {
[varName: string]: string;
};
apps: {
[id: string]: {
name: string;
commit: string;
releaseId: number;
services: { [id: string]: any };
volumes: { [name: string]: any };
networks: { [name: string]: any };
};
};
pinDevice?: boolean;
};
export async function convertV2toV3AppsJson(
appsJson: AppsJsonV2,
): Promise<AppsJsonFormat> {
const { config: conf, apps, pinDevice } = appsJson;
await apiBinder.initialized;
await deviceState.initialized;
if (apiBinder.balenaApi == null) {
throw new InternalInconsistencyError(
'API binder is not initialized correctly',
);
}
const { balenaApi } = apiBinder;
const v3apps = (
await Promise.all(
Object.keys(apps).map(
async (id): Promise<[string, TargetApp]> => {
const appId = parseInt(id, 10);
const app = apps[appId];
const appDetails = await balenaApi.get({
resource: 'application',
id: appId,
options: {
$select: ['uuid'],
},
});
if (!appDetails || appDetails.length === 0) {
throw new InternalInconsistencyError(
`No app with id ${appId} found on the API. Skipping apps.json migration`,
);
}
const { uuid } = appDetails[0];
const releases = app.commit
? {
[app.commit]: {
id: app.releaseId,
services: app.services,
volumes: app.volumes,
networks: app.networks,
},
}
: {};
return [
uuid,
{
id: appId,
name: app.name,
class: 'fleet',
releases,
} as TargetApp,
];
},
),
)
)
// Key by uuid
.reduce((res, [uuid, app]) => ({ ...res, [uuid]: app }), {});
return { config: conf, apps: v3apps, ...(pinDevice && { pinDevice }) };
}
export async function loadBackupFromMigration(
targetState: TargetState,
retryDelay: number,
@ -281,14 +392,17 @@ export async function loadBackupFromMigration(
await deviceState.setTarget(targetState);
// multi-app warning!
const appId = parseInt(_.keys(targetState.local?.apps)[0], 10);
// TODO: this code is only single-app compatible
const [uuid] = Object.keys(targetState.local?.apps);
if (isNaN(appId)) {
throw new BackupError('No appId in target state');
if (!!uuid) {
throw new BackupError('No apps in the target state');
}
const volumes = targetState.local?.apps?.[appId].volumes;
const { id: appId } = targetState.local?.apps[uuid];
const [release] = Object.values(targetState.local?.apps[uuid].releases);
const volumes = release?.volumes ?? {};
const backupPath = path.join(constants.rootMountPoint, 'mnt/data/backup');
// We clear this path in case it exists from an incomplete run of this function

56
src/migrations/M00008.js Normal file
View File

@ -0,0 +1,56 @@
export async function up(knex) {
await knex.schema.table('app', (table) => {
table.string('uuid');
table.unique('uuid');
table.string('class').defaultTo('fleet');
table.boolean('isHost').defaultTo(false);
});
await knex.schema.table('image', (table) => {
table.string('appUuid');
table.string('commit');
});
const legacyAppsPresent = await knex('config')
.where({ key: 'legacyAppsPresent' })
.select('value');
// If there are legacy apps we let the database normalization function
// populate the correct values
if (legacyAppsPresent && legacyAppsPresent.length > 0) {
return;
}
// Otherwise delete cloud target apps and images in the database so they can get repopulated
// with the uuid from the target state. Removing the `targetStateSet` configuration ensures that
// the supervisor will maintain the current state and will only apply the new target once it gets
// a new cloud copy, which should include the proper metadata
await knex('image').del();
await knex('app').whereNot({ source: 'local' }).del();
await knex('config').where({ key: 'targetStateSet' }).del();
const apps = await knex('app').select();
// For remaining local apps, if any, the appUuid is not that relevant, so just
// use appId to prevent the app from getting uninstalled. Adding the appUuid will restart
// the app though
await Promise.all(
apps.map((app) => {
const services = JSON.parse(app.services).map((svc) => ({
...svc,
appUuid: app.appId.toString(),
}));
return knex('app')
.where({ id: app.id })
.update({
uuid: app.appId.toString(),
services: JSON.stringify(services),
});
}),
);
}
export function down() {
throw new Error('Not Implemented');
}

51
src/types/basic.ts
View File

@ -1,6 +1,6 @@
import * as t from 'io-ts';
import { chain } from 'fp-ts/lib/Either';
import { pipe } from 'fp-ts/function';
import { chain, fold, isRight, left, right, Either } from 'fp-ts/lib/Either';
import { pipe, flow } from 'fp-ts/function';
/**
* A short string is a non null string between
@ -184,3 +184,50 @@ export const DeviceName = new t.Type<string, string>(
);
export type DeviceName = t.TypeOf<typeof DeviceName>;
/**
* Creates a record type that checks for constraints on the record elements
*/
const restrictedRecord = <
K extends t.Mixed,
V extends t.Mixed,
R extends { [key in t.TypeOf<K>]: t.TypeOf<V> }
>(
k: K,
v: V,
test: (i: R) => Either<string, R>,
name = 'RestrictedRecord',
) => {
return new t.Type<R>(
name,
(i): i is R => t.record(k, v).is(i) && isRight(test(i as R)),
(i, c) =>
pipe(
// pipe takes the first result and passes it through rest of the function arguments
t.record(k, v).validate(i, c), // validate that the element is a proper record first (returns an Either)
chain(
// chain takes a function (a) => Either and converts it into a function (Either) => (Either)
flow(
// flow creates a function composition
test, // receives a record and returns Either<string,R>
fold((m) => t.failure(i, c, m), t.success), // fold converts Either<string,R> to an Either<Errors, R>
),
),
),
t.identity,
);
};
export const nonEmptyRecord = <K extends t.Mixed, V extends t.Mixed>(
k: K,
v: V,
) =>
restrictedRecord(
k,
v,
(o) =>
Object.keys(o).length > 0
? right(o)
: left('must have at least 1 element'),
'NonEmptyRecord',
);

224
src/types/state.ts
View File

@ -2,17 +2,16 @@ import * as t from 'io-ts';
// TODO: move all these exported types to ../compose/types
import { ComposeNetworkConfig } from '../compose/types/network';
import { ServiceComposeConfig } from '../compose/types/service';
import { ComposeVolumeConfig } from '../compose/volume';
import {
DockerName,
EnvVarObject,
LabelObject,
StringIdentifier,
NumericIdentifier,
ShortString,
DeviceName,
nonEmptyRecord,
} from './basic';
import App from '../compose/app';
@ -107,10 +106,23 @@ const fromType = <T extends object>(name: string) =>
t.identity,
);
// Alias short string to UUID so code reads more clearly
export const UUID = ShortString;
/**
* A target service has docker image, a set of environment variables
* and labels as well as one or more configurations
*/
export const TargetService = t.intersection([
t.type({
serviceName: DockerName,
imageId: NumericIdentifier,
/**
* @deprecated to be removed in state v4
*/
id: NumericIdentifier,
/**
* @deprecated to be removed in state v4
*/
image_id: NumericIdentifier,
image: ShortString,
environment: EnvVarObject,
labels: LabelObject,
@ -118,112 +130,164 @@ export const TargetService = t.intersection([
t.partial({
running: withDefault(t.boolean, true),
contract: t.record(t.string, t.unknown),
// This will not be validated
// TODO: convert ServiceComposeConfig to a io-ts type
composition: t.record(t.string, t.unknown),
}),
// This will not be validated
// TODO: convert ServiceComposeConfig to a io-ts type
fromType<ServiceComposeConfig>('ServiceComposition'),
]);
export type TargetService = t.TypeOf<typeof TargetService>;
/**
* Target state release format
*/
export const TargetRelease = t.type({
/**
* @deprecated to be removed in state v4
*/
id: NumericIdentifier,
services: withDefault(t.record(DockerName, TargetService), {}),
volumes: withDefault(
t.record(
DockerName,
// TargetVolume format will NOT be validated
// TODO: convert ComposeVolumeConfig to a io-ts type
fromType<Partial<ComposeVolumeConfig>>('Volume'),
),
{},
),
networks: withDefault(
t.record(
DockerName,
// TargetNetwork format will NOT be validated
// TODO: convert ComposeVolumeConfig to a io-ts type
fromType<Partial<ComposeNetworkConfig>>('Network'),
),
{},
),
});
export type TargetRelease = t.TypeOf<typeof TargetRelease>;
export const TargetAppClass = t.union([
t.literal('fleet'),
t.literal('app'),
t.literal('block'),
]);
export type TargetAppClass = t.TypeOf<typeof TargetAppClass>;
/**
* A target app is composed by a release and a collection of volumes and
* networks.
*/
const TargetApp = t.intersection(
[
t.type({
/**
* @deprecated to be removed in state v4
*/
id: NumericIdentifier,
name: ShortString,
services: withDefault(t.record(StringIdentifier, TargetService), {}),
volumes: withDefault(
t.record(
DockerName,
// TargetVolume format will NOT be validated
// TODO: convert ComposeVolumeConfig to a io-ts type
fromType<Partial<ComposeVolumeConfig>>('Volume'),
),
{},
),
networks: withDefault(
t.record(
DockerName,
// TargetNetwork format will NOT be validated
// TODO: convert ComposeVolumeConfig to a io-ts type
fromType<Partial<ComposeNetworkConfig>>('Network'),
),
{},
),
// There should be only one fleet class app in the target state but we
// are not validating that here
class: withDefault(TargetAppClass, 'fleet'),
// TODO: target release must have at most one value. Should we validate?
releases: withDefault(t.record(UUID, TargetRelease), {}),
}),
t.partial({
commit: ShortString,
releaseId: NumericIdentifier,
parent_app: UUID,
is_host: t.boolean,
}),
],
'App',
);
export type TargetApp = t.TypeOf<typeof TargetApp>;
export const TargetApps = t.record(StringIdentifier, TargetApp);
export const TargetApps = t.record(UUID, TargetApp);
export type TargetApps = t.TypeOf<typeof TargetApps>;
const DependentApp = t.intersection(
[
t.type({
name: ShortString,
parentApp: NumericIdentifier,
config: EnvVarObject,
}),
t.partial({
releaseId: NumericIdentifier,
imageId: NumericIdentifier,
commit: ShortString,
image: ShortString,
}),
],
'DependentApp',
);
const DependentDevice = t.type(
{
name: ShortString, // device uuid
apps: t.record(
StringIdentifier,
t.type({ config: EnvVarObject, environment: EnvVarObject }),
),
},
'DependentDevice',
);
// Although the original types for dependent apps and dependent devices was a dictionary,
// proxyvisor.js accepts both a dictionary and an array. Unfortunately
// the CLI sends an array, thus the types need to accept both
const DependentApps = t.union([
t.array(DependentApp),
t.record(StringIdentifier, DependentApp),
]);
const DependentDevices = t.union([
t.array(DependentDevice),
t.record(StringIdentifier, DependentDevice),
]);
export const TargetState = t.type({
local: t.type({
/**
* A device has a name, config and collection of apps
*/
const TargetDevice = t.intersection([
t.type({
name: DeviceName,
config: EnvVarObject,
apps: TargetApps,
}),
dependent: t.type({
apps: DependentApps,
devices: DependentDevices,
t.partial({
parent_device: UUID,
}),
});
]);
export type TargetDevice = t.TypeOf<typeof TargetDevice>;
/**
* Target state is a collection of devices one local device
* (with uuid matching the one in config.json) and zero or more dependent
* devices
*
*
* When all io-ts types are composed, the final type of the target state
* is the one given by the following description
* ```
* {
* [uuid: string]: {
* name: string;
* parent_device?: string;
* config?: {
* [varName: string]: string;
* };
* apps: {
* [uuid: string]: {
* // @deprecated to be removed in state v4
* id: number;
* name: string;
* class: 'fleet' | 'block' | 'app';
* parent_app?: string;
* is_host?: boolean;
* releases?: {
* [uuid: string]: {
* // @deprecated to be removed in state v4
* id: number;
* services?: {
* [name: string]: {
* // @deprecated to be removed in state v4
* id: number;
* // @deprecated to be removed in state v4
* image_id: number;
* image: string;
* // defaults to true if undefined
* running?: boolean;
* environment: {
* [varName: string]: string;
* };
* labels: {
* [labelName: string]: string;
* };
* contract?: AnyObject;
* composition?: ServiceComposition;
* };
* };
* volumes?: AnyObject;
* networks?: AnyObject;
* };
* };
* };
* };
* };
* }
* ```
*/
export const TargetState = t.record(UUID, TargetDevice);
export type TargetState = t.TypeOf<typeof TargetState>;
const TargetAppWithRelease = t.intersection([
TargetApp,
t.type({ commit: t.string, releaseId: NumericIdentifier }),
t.type({ releases: nonEmptyRecord(UUID, TargetRelease) }),
]);
const AppsJsonFormat = t.intersection([
export const AppsJsonFormat = t.intersection([
t.type({
config: EnvVarObject,
apps: t.record(StringIdentifier, TargetAppWithRelease),
config: withDefault(EnvVarObject, {}),
apps: withDefault(t.record(UUID, TargetAppWithRelease), {}),
}),
t.partial({ pinDevice: t.boolean }),
]);

531
test/05-device-state.spec.ts
View File

@ -1,12 +1,10 @@
import * as _ from 'lodash';
import { SinonStub, stub } from 'sinon';
import { expect } from 'chai';
import * as sinon from 'sinon';
import { StatusCodeError, UpdatesLockedError } from '../src/lib/errors';
import prepare = require('./lib/prepare');
import * as dockerUtils from '../src/lib/docker-utils';
import * as config from '../src/config';
import * as images from '../src/compose/images';
import * as imageManager from '../src/compose/images';
import { ConfigTxt } from '../src/config/backends/config-txt';
import * as deviceState from '../src/device-state';
import * as deviceConfig from '../src/device-config';
@ -18,6 +16,10 @@ import Service from '../src/compose/service';
import { intialiseContractRequirements } from '../src/lib/contracts';
import * as updateLock from '../src/lib/update-lock';
import * as fsUtils from '../src/lib/fs-utils';
import { TargetState } from '../src/types';
import * as dbHelper from './lib/db-helper';
import log from '../src/lib/supervisor-console';
const mockedInitialConfig = {
RESIN_SUPERVISOR_CONNECTIVITY_CHECK: 'true',
@ -35,147 +37,37 @@ const mockedInitialConfig = {
RESIN_SUPERVISOR_VPN_CONTROL: 'true',
};
const testTarget2 = {
local: {
name: 'aDeviceWithDifferentName',
config: {
RESIN_HOST_CONFIG_gpu_mem: '512',
},
apps: {
'1234': {
name: 'superapp',
commit: 'afafafa',
releaseId: 2,
services: {
'23': {
serviceName: 'aservice',
imageId: 12345,
image: 'registry2.resin.io/superapp/edfabc',
environment: {
FOO: 'bar',
},
labels: {},
},
'24': {
serviceName: 'anotherService',
imageId: 12346,
image: 'registry2.resin.io/superapp/afaff',
environment: {
FOO: 'bro',
},
labels: {},
},
},
volumes: {},
networks: {},
},
},
},
dependent: { apps: {}, devices: {} },
};
const testTargetWithDefaults2 = {
local: {
name: 'aDeviceWithDifferentName',
config: {
HOST_CONFIG_gpu_mem: '512',
HOST_FIREWALL_MODE: 'off',
HOST_DISCOVERABILITY: 'true',
SUPERVISOR_CONNECTIVITY_CHECK: 'true',
SUPERVISOR_DELTA: 'false',
SUPERVISOR_DELTA_APPLY_TIMEOUT: '0',
SUPERVISOR_DELTA_REQUEST_TIMEOUT: '30000',
SUPERVISOR_DELTA_RETRY_COUNT: '30',
SUPERVISOR_DELTA_RETRY_INTERVAL: '10000',
SUPERVISOR_DELTA_VERSION: '2',
SUPERVISOR_INSTANT_UPDATE_TRIGGER: 'true',
SUPERVISOR_LOCAL_MODE: 'false',
SUPERVISOR_LOG_CONTROL: 'true',
SUPERVISOR_OVERRIDE_LOCK: 'false',
SUPERVISOR_POLL_INTERVAL: '60000',
SUPERVISOR_VPN_CONTROL: 'true',
SUPERVISOR_PERSISTENT_LOGGING: 'false',
},
apps: {
'1234': {
appId: 1234,
name: 'superapp',
commit: 'afafafa',
releaseId: 2,
services: [
_.merge(
{ appId: 1234, serviceId: 23, releaseId: 2 },
_.clone(testTarget2.local.apps['1234'].services['23']),
),
_.merge(
{ appId: 1234, serviceId: 24, releaseId: 2 },
_.clone(testTarget2.local.apps['1234'].services['24']),
),
],
volumes: {},
networks: {},
},
},
},
};
const testTargetInvalid = {
local: {
name: 'aDeviceWithDifferentName',
config: {
RESIN_HOST_CONFIG_gpu_mem: '512',
},
apps: {
1234: {
appId: '1234',
name: 'superapp',
commit: 'afafafa',
releaseId: '2',
config: {},
services: {
23: {
serviceId: '23',
serviceName: 'aservice',
imageId: '12345',
image: 'registry2.resin.io/superapp/edfabc',
config: {},
environment: {
' FOO': 'bar',
},
labels: {},
},
24: {
serviceId: '24',
serviceName: 'anotherService',
imageId: '12346',
image: 'registry2.resin.io/superapp/afaff',
config: {},
environment: {
FOO: 'bro',
},
labels: {},
},
},
},
},
},
dependent: { apps: {}, devices: {} },
};
describe('deviceState', () => {
let source: string;
const originalImagesSave = images.save;
const originalImagesInspect = images.inspectByName;
describe('device-state', () => {
const originalImagesSave = imageManager.save;
const originalImagesInspect = imageManager.inspectByName;
const originalGetCurrent = deviceConfig.getCurrent;
let testDb: dbHelper.TestDatabase;
before(async () => {
await prepare();
testDb = await dbHelper.createDB();
await config.initialized;
// Prevent side effects from changes in config
sinon.stub(config, 'on');
// Set the device uuid
await config.set({ uuid: 'local' });
await deviceState.initialized;
source = await config.get('apiEndpoint');
// disable log output during testing
sinon.stub(log, 'debug');
sinon.stub(log, 'warn');
sinon.stub(log, 'info');
sinon.stub(log, 'event');
sinon.stub(log, 'success');
stub(Service as any, 'extendEnvVars').callsFake((env) => {
// TODO: all these stubs are internal implementation details of
// deviceState, we should refactor deviceState to use dependency
// injection instead of initializing everything in memory
sinon.stub(Service as any, 'extendEnvVars').callsFake((env) => {
env['ADDITIONAL_ENV_VAR'] = 'foo';
return env;
});
@ -185,20 +77,20 @@ describe('deviceState', () => {
deviceType: 'intel-nuc',
});
stub(dockerUtils, 'getNetworkGateway').returns(
Promise.resolve('172.17.0.1'),
);
sinon
.stub(dockerUtils, 'getNetworkGateway')
.returns(Promise.resolve('172.17.0.1'));
// @ts-expect-error Assigning to a RO property
images.cleanImageData = () => {
imageManager.cleanImageData = () => {
console.log('Cleanup database called');
};
// @ts-expect-error Assigning to a RO property
images.save = () => Promise.resolve();
imageManager.save = () => Promise.resolve();
// @ts-expect-error Assigning to a RO property
images.inspectByName = () => {
imageManager.inspectByName = () => {
const err: StatusCodeError = new Error();
err.statusCode = 404;
return Promise.reject(err);
@ -211,20 +103,27 @@ describe('deviceState', () => {
deviceConfig.getCurrent = async () => mockedInitialConfig;
});
after(() => {
after(async () => {
(Service as any).extendEnvVars.restore();
(dockerUtils.getNetworkGateway as sinon.SinonStub).restore();
// @ts-expect-error Assigning to a RO property
images.save = originalImagesSave;
imageManager.save = originalImagesSave;
// @ts-expect-error Assigning to a RO property
images.inspectByName = originalImagesInspect;
imageManager.inspectByName = originalImagesInspect;
// @ts-expect-error Assigning to a RO property
deviceConfig.getCurrent = originalGetCurrent;
try {
await testDb.destroy();
} catch (e) {
/* noop */
}
sinon.restore();
});
beforeEach(async () => {
await prepare();
afterEach(async () => {
await testDb.reset();
});
it('loads a target state from an apps.json file and saves it as target state, then returns it', async () => {
@ -233,6 +132,11 @@ describe('deviceState', () => {
const targetState = await deviceState.getTarget();
expect(await fsUtils.exists(appsJsonBackup(appsJson))).to.be.true;
expect(targetState)
.to.have.property('local')
.that.has.property('config')
.that.has.property('HOST_CONFIG_gpu_mem')
.that.equals('256');
expect(targetState)
.to.have.property('local')
.that.has.property('apps')
@ -308,45 +212,140 @@ describe('deviceState', () => {
});
it('emits a change event when a new state is reported', (done) => {
// TODO: where is the test on this test?
deviceState.once('change', done);
deviceState.reportCurrentState({ someStateDiff: 'someValue' } as any);
});
it.skip('writes the target state to the db with some extra defaults', async () => {
const testTarget = _.cloneDeep(testTargetWithDefaults2);
it('writes the target state to the db with some extra defaults', async () => {
await deviceState.setTarget({
local: {
name: 'aDeviceWithDifferentName',
config: {
RESIN_HOST_CONFIG_gpu_mem: '512',
},
apps: {
myapp: {
id: 1234,
name: 'superapp',
class: 'fleet',
releases: {
afafafa: {
id: 2,
services: {
aservice: {
id: 23,
image_id: 12345,
image: 'registry2.resin.io/superapp/edfabc',
environment: {
FOO: 'bar',
},
labels: {},
},
anotherService: {
id: 24,
image_id: 12346,
image: 'registry2.resin.io/superapp/afaff',
environment: {
FOO: 'bro',
},
labels: {},
},
},
volumes: {},
networks: {},
},
},
},
},
},
} as TargetState);
const targetState = await deviceState.getTarget();
const services: Service[] = [];
for (const service of testTarget.local.apps['1234'].services) {
const imageName = images.normalise(service.image);
service.image = imageName;
(service as any).imageName = imageName;
services.push(
await Service.fromComposeObject(service, {
appName: 'supertest',
} as any),
);
}
expect(targetState)
.to.have.property('local')
.that.has.property('config')
.that.has.property('HOST_CONFIG_gpu_mem')
.that.equals('512');
(testTarget as any).local.apps['1234'].services = _.keyBy(
services,
'serviceId',
);
(testTarget as any).local.apps['1234'].source = source;
await deviceState.setTarget(testTarget2);
const target = await deviceState.getTarget();
expect(JSON.parse(JSON.stringify(target))).to.deep.equal(
JSON.parse(JSON.stringify(testTarget)),
);
expect(targetState)
.to.have.property('local')
.that.has.property('apps')
.that.has.property('1234')
.that.is.an('object');
const app = targetState.local.apps[1234];
expect(app).to.have.property('appName').that.equals('superapp');
expect(app).to.have.property('commit').that.equals('afafafa');
expect(app).to.have.property('services').that.is.an('array').with.length(2);
expect(app.services[0])
.to.have.property('config')
.that.has.property('image')
.that.equals('registry2.resin.io/superapp/edfabc:latest');
expect(app.services[0].config)
.to.have.property('environment')
.that.has.property('FOO')
.that.equals('bar');
expect(app.services[1])
.to.have.property('config')
.that.has.property('image')
.that.equals('registry2.resin.io/superapp/afaff:latest');
expect(app.services[1].config)
.to.have.property('environment')
.that.has.property('FOO')
.that.equals('bro');
});
it('does not allow setting an invalid target state', () => {
expect(deviceState.setTarget(testTargetInvalid as any)).to.be.rejected;
// v2 state should be rejected
expect(
deviceState.setTarget({
local: {
name: 'aDeviceWithDifferentName',
config: {
RESIN_HOST_CONFIG_gpu_mem: '512',
},
apps: {
1234: {
appId: '1234',
name: 'superapp',
commit: 'afafafa',
releaseId: '2',
config: {},
services: {
23: {
serviceId: '23',
serviceName: 'aservice',
imageId: '12345',
image: 'registry2.resin.io/superapp/edfabc',
environment: {
' FOO': 'bar',
},
labels: {},
},
24: {
serviceId: '24',
serviceName: 'anotherService',
imageId: '12346',
image: 'registry2.resin.io/superapp/afaff',
environment: {
FOO: 'bro',
},
labels: {},
},
},
},
},
},
dependent: { apps: {}, devices: {} },
} as any),
).to.be.rejected;
});
it('allows triggering applying the target state', (done) => {
const applyTargetStub = stub(deviceState, 'applyTarget').returns(
Promise.resolve(),
);
const applyTargetStub = sinon
.stub(deviceState, 'applyTarget')
.returns(Promise.resolve());
deviceState.triggerApplyTarget({ force: true });
expect(applyTargetStub).to.not.be.called;
@ -361,6 +360,178 @@ describe('deviceState', () => {
}, 1000);
});
it('accepts a target state with an valid contract', async () => {
await deviceState.setTarget({
local: {
name: 'aDeviceWithDifferentName',
config: {},
apps: {
myapp: {
id: 1234,
name: 'superapp',
class: 'fleet',
releases: {
one: {
id: 2,
services: {
valid: {
id: 23,
image_id: 12345,
image: 'registry2.resin.io/superapp/valid',
environment: {},
labels: {},
},
alsoValid: {
id: 24,
image_id: 12346,
image: 'registry2.resin.io/superapp/afaff',
contract: {
type: 'sw.container',
slug: 'supervisor-version',
name: 'Enforce supervisor version',
requires: [
{
type: 'sw.supervisor',
version: '>=11.0.0',
},
],
},
environment: {},
labels: {},
},
},
volumes: {},
networks: {},
},
},
},
},
},
} as TargetState);
const targetState = await deviceState.getTarget();
const app = targetState.local.apps[1234];
expect(app).to.have.property('appName').that.equals('superapp');
expect(app).to.have.property('commit').that.equals('one');
// Only a single service should be on the target state
expect(app).to.have.property('services').that.is.an('array').with.length(2);
expect(app.services[1])
.that.has.property('serviceName')
.that.equals('alsoValid');
});
it('accepts a target state with an invalid contract for an optional container', async () => {
await deviceState.setTarget({
local: {
name: 'aDeviceWithDifferentName',
config: {},
apps: {
myapp: {
id: 1234,
name: 'superapp',
class: 'fleet',
releases: {
one: {
id: 2,
services: {
valid: {
id: 23,
image_id: 12345,
image: 'registry2.resin.io/superapp/valid',
environment: {},
labels: {},
},
invalidButOptional: {
id: 24,
image_id: 12346,
image: 'registry2.resin.io/superapp/afaff',
contract: {
type: 'sw.container',
slug: 'supervisor-version',
name: 'Enforce supervisor version',
requires: [
{
type: 'sw.supervisor',
version: '>=12.0.0',
},
],
},
environment: {},
labels: {
'io.balena.features.optional': 'true',
},
},
},
volumes: {},
networks: {},
},
},
},
},
},
} as TargetState);
const targetState = await deviceState.getTarget();
const app = targetState.local.apps[1234];
expect(app).to.have.property('appName').that.equals('superapp');
expect(app).to.have.property('commit').that.equals('one');
// Only a single service should be on the target state
expect(app).to.have.property('services').that.is.an('array').with.length(1);
expect(app.services[0])
.that.has.property('serviceName')
.that.equals('valid');
});
it('rejects a target state with invalid contract and non optional service', async () => {
await expect(
deviceState.setTarget({
local: {
name: 'aDeviceWithDifferentName',
config: {},
apps: {
myapp: {
id: 1234,
name: 'superapp',
class: 'fleet',
releases: {
one: {
id: 2,
services: {
valid: {
id: 23,
image_id: 12345,
image: 'registry2.resin.io/superapp/valid',
environment: {},
labels: {},
},
invalid: {
id: 24,
image_id: 12346,
image: 'registry2.resin.io/superapp/afaff',
contract: {
type: 'sw.container',
slug: 'supervisor-version',
name: 'Enforce supervisor version',
requires: [
{
type: 'sw.supervisor',
version: '>=12.0.0',
},
],
},
environment: {},
labels: {},
},
},
volumes: {},
networks: {},
},
},
},
},
},
} as TargetState),
).to.be.rejected;
});
// TODO: There is no easy way to test this behaviour with the current
// interface of device-state. We should really think about the device-state
// interface to allow this flexibility (and to avoid having to change module
@ -373,9 +544,9 @@ describe('deviceState', () => {
it('prevents reboot or shutdown when HUP rollback breadcrumbs are present', async () => {
const testErrMsg = 'Waiting for Host OS updates to finish';
stub(updateLock, 'abortIfHUPInProgress').throws(
new UpdatesLockedError(testErrMsg),
);
sinon
.stub(updateLock, 'abortIfHUPInProgress')
.throws(new UpdatesLockedError(testErrMsg));
await expect(deviceState.reboot())
.to.eventually.be.rejectedWith(testErrMsg)
@ -384,6 +555,6 @@ describe('deviceState', () => {
.to.eventually.be.rejectedWith(testErrMsg)
.and.be.an.instanceOf(UpdatesLockedError);
(updateLock.abortIfHUPInProgress as SinonStub).restore();
(updateLock.abortIfHUPInProgress as sinon.SinonStub).restore();
});
});

202
test/06-validation.spec.ts
View File

@ -258,9 +258,11 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
'1234': {
abcd: {
id: 1234,
name: 'something',
services: {},
class: 'fleet',
releases: {},
},
}),
),
@ -269,11 +271,15 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
'1234': {
abcd: {
id: 1234,
name: 'something',
releaseId: 123,
commit: 'bar',
services: {},
releases: {
bar: {
id: 123,
services: {},
},
},
},
}),
),
@ -283,21 +289,25 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
'1234': {
abcd: {
id: 1234,
name: 'something',
releaseId: 123,
commit: 'bar',
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: { MY_SERVICE_ENV_VAR: '123' },
labels: { 'io.balena.features.supervisor-api': 'true' },
releases: {
bar: {
id: 123,
services: {
bazbaz: {
id: 45,
image_id: 34,
image: 'foo',
environment: { MY_SERVICE_ENV_VAR: '123' },
labels: { 'io.balena.features.supervisor-api': 'true' },
},
},
volumes: {},
networks: {},
},
},
volumes: {},
networks: {},
},
}),
),
@ -309,17 +319,23 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
'1234': {
abcd: {
id: 1234,
name: 'something',
releaseId: 123,
commit: 'bar',
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: { ' baz': 'bat' },
labels: {},
releases: {
bar: {
id: 123,
services: {
bazbaz: {
id: 45,
image_id: 34,
image: 'foo',
environment: { ' aaa': '123' },
labels: {},
},
},
volumes: {},
networks: {},
},
},
},
@ -332,17 +348,23 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
'1234': {
abcd: {
id: 1234,
name: 'something',
releaseId: 123,
commit: 'bar',
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: {},
labels: { ' not a valid #name': 'label value' },
releases: {
bar: {
id: 123,
services: {
bazbaz: {
id: 45,
image_id: 34,
image: 'foo',
environment: {},
labels: { ' not a valid #name': 'label value' },
},
},
volumes: {},
networks: {},
},
},
},
@ -355,40 +377,35 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
boo: {
abcd: {
id: 'booo',
name: 'something',
releaseId: 123,
commit: 'bar',
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: {},
labels: {},
},
},
releases: {},
},
}),
),
).to.be.false;
});
it('rejects a commit that is too long', () => {
it('rejects a release uuid that is too long', () => {
expect(
isRight(
TargetApps.decode({
boo: {
abcd: {
id: '123',
name: 'something',
releaseId: 123,
commit: 'a'.repeat(256),
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: {},
labels: {},
releases: {
['a'.repeat(256)]: {
id: 1234,
services: {
bazbaz: {
id: 45,
image_id: 34,
image: 'foo',
environment: {},
labels: {},
},
},
},
},
},
@ -400,17 +417,21 @@ describe('validation', () => {
expect(
isRight(
TargetApps.decode({
boo: {
abcd: {
id: '123',
name: 'something',
releaseId: 123,
commit: 'a'.repeat(256),
services: {
'45': {
serviceName: ' not a valid name',
imageId: 34,
image: 'foo',
environment: {},
labels: {},
releases: {
aaaa: {
id: 1234,
services: {
' not a valid name': {
id: 45,
image_id: 34,
image: 'foo',
environment: {},
labels: {},
},
},
},
},
},
@ -420,44 +441,17 @@ describe('validation', () => {
});
});
it('rejects a commit that is too long', () => {
expect(
isRight(
TargetApps.decode({
boo: {
name: 'something',
releaseId: 123,
commit: 'a'.repeat(256),
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: {},
labels: {},
},
},
},
}),
),
).to.be.false;
});
it('rejects app with an invalid releaseId', () => {
expect(
isRight(
TargetApps.decode({
boo: {
abcd: {
id: '123',
name: 'something',
releaseId: '123aaa',
commit: 'bar',
services: {
'45': {
serviceName: 'bazbaz',
imageId: 34,
image: 'foo',
environment: {},
labels: {},
releases: {
aaaa: {
id: 'boooo',
services: {},
},
},
},

387
test/28-db-format.spec.ts
View File

@ -1,166 +1,287 @@
import { expect } from 'chai';
import * as _ from 'lodash';
import prepare = require('./lib/prepare');
import { isRight } from 'fp-ts/lib/Either';
import * as sinon from 'sinon';
import App from '../src/compose/app';
import Network from '../src/compose/network';
import * as config from '../src/config';
import * as dbFormat from '../src/device-state/db-format';
import * as targetStateCache from '../src/device-state/target-state-cache';
import * as images from '../src/compose/images';
import log from '../src/lib/supervisor-console';
import { TargetApps } from '../src/types/state';
import * as dbHelper from './lib/db-helper';
import { withMockerode } from './lib/mockerode';
import App from '../src/compose/app';
import Service from '../src/compose/service';
import Network from '../src/compose/network';
import { TargetApp } from '../src/types/state';
function getDefaultNetworks(appId: number) {
function getDefaultNetwork(appId: number) {
return {
default: Network.fromComposeObject('default', appId, {}),
};
}
describe('DB Format', () => {
const originalInspect = images.inspectByName;
describe('db-format', () => {
let testDb: dbHelper.TestDatabase;
let apiEndpoint: string;
before(async () => {
await prepare();
await config.initialized;
await targetStateCache.initialized;
testDb = await dbHelper.createDB();
await config.initialized;
// Prevent side effects from changes in config
sinon.stub(config, 'on');
// TargetStateCache checks the API endpoint to
// store and invalidate the cache
// TODO: this is an implementation detail that
// should not be part of the test suite. We need to change
// the target state architecture for this
apiEndpoint = await config.get('apiEndpoint');
// Setup some mocks
// @ts-expect-error Assigning to a RO property
images.inspectByName = () => {
const error = new Error();
// @ts-ignore
error.statusCode = 404;
return Promise.reject(error);
};
await targetStateCache.setTargetApps([
{
appId: 1,
commit: 'abcdef',
name: 'test-app',
source: apiEndpoint,
releaseId: 123,
services: '[]',
networks: '[]',
volumes: '[]',
},
{
appId: 2,
commit: 'abcdef2',
name: 'test-app2',
source: apiEndpoint,
releaseId: 1232,
services: JSON.stringify([
{
serviceName: 'test-service',
image: 'test-image',
imageId: 5,
environment: {
TEST_VAR: 'test-string',
},
tty: true,
appId: 2,
releaseId: 1232,
serviceId: 567,
commit: 'abcdef2',
},
]),
networks: '[]',
volumes: '[]',
},
]);
// disable log output during testing
sinon.stub(log, 'debug');
sinon.stub(log, 'warn');
sinon.stub(log, 'info');
sinon.stub(log, 'event');
sinon.stub(log, 'success');
});
after(async () => {
await prepare();
try {
await testDb.destroy();
} catch (e) {
/* noop */
}
sinon.restore();
});
// @ts-expect-error Assigning to a RO property
images.inspectByName = originalInspect;
afterEach(async () => {
await testDb.reset();
});
it('converts target apps into the database format', async () => {
await dbFormat.setApps(
{
deadbeef: {
id: 1,
name: 'test-app',
class: 'fleet',
releases: {
one: {
id: 1,
services: {
ubuntu: {
id: 1,
image_id: 1,
image: 'ubuntu:latest',
environment: {},
labels: { 'my-label': 'true' },
composition: {
command: ['sleep', 'infinity'],
},
},
},
volumes: {},
networks: {},
},
},
},
},
'local',
);
const [app] = await testDb.models('app').where({ uuid: 'deadbeef' });
expect(app).to.not.be.undefined;
expect(app.name).to.equal('test-app');
expect(app.releaseId).to.equal(1);
expect(app.commit).to.equal('one');
expect(app.appId).to.equal(1);
expect(app.source).to.equal('local');
expect(app.uuid).to.equal('deadbeef');
expect(app.isHost).to.equal(0);
expect(app.services).to.equal(
'[{"image":"ubuntu:latest","environment":{},"labels":{"my-label":"true"},"composition":{"command":["sleep","infinity"]},"appId":1,"appUuid":"deadbeef","releaseId":1,"commit":"one","imageId":1,"serviceId":1,"serviceName":"ubuntu"}]',
);
expect(app.volumes).to.equal('{}');
expect(app.networks).to.equal('{}');
});
it('should retrieve a single app from the database', async () => {
const app = await dbFormat.getApp(1);
expect(app).to.be.an.instanceOf(App);
expect(app).to.have.property('appId').that.equals(1);
expect(app).to.have.property('commit').that.equals('abcdef');
expect(app).to.have.property('appName').that.equals('test-app');
expect(app)
.to.have.property('source')
.that.deep.equals(await config.get('apiEndpoint'));
expect(app).to.have.property('services').that.deep.equals([]);
expect(app).to.have.property('volumes').that.deep.equals({});
expect(app)
.to.have.property('networks')
.that.deep.equals(getDefaultNetworks(1));
});
await dbFormat.setApps(
{
deadbeef: {
id: 1,
name: 'test-app',
class: 'fleet',
releases: {
one: {
id: 1,
services: {
ubuntu: {
id: 1,
image_id: 1,
image: 'ubuntu:latest',
environment: {},
labels: { 'my-label': 'true' },
composition: {
command: ['sleep', 'infinity'],
},
},
},
volumes: {},
networks: {},
},
},
},
},
apiEndpoint,
);
it('should correctly build services from the database', async () => {
const app = await dbFormat.getApp(2);
expect(app).to.have.property('services').that.is.an('array');
const services = _.keyBy(app.services, 'serviceId');
expect(Object.keys(services)).to.deep.equal(['567']);
// getApp creates a new app instance which requires a docker instance
// withMockerode mocks engine
await withMockerode(async () => {
const app = await dbFormat.getApp(1);
expect(app).to.be.an.instanceOf(App);
expect(app).to.have.property('appId').that.equals(1);
expect(app).to.have.property('commit').that.equals('one');
expect(app).to.have.property('appName').that.equals('test-app');
expect(app).to.have.property('source').that.equals(apiEndpoint);
expect(app).to.have.property('services').that.has.lengthOf(1);
expect(app).to.have.property('volumes').that.deep.equals({});
expect(app)
.to.have.property('networks')
.that.deep.equals(getDefaultNetwork(1));
const service = services[567];
expect(service).to.be.instanceof(Service);
// Don't do a deep equals here as a bunch of other properties are added that are
// tested elsewhere
expect(service.config)
.to.have.property('environment')
.that.has.property('TEST_VAR')
.that.equals('test-string');
expect(service.config).to.have.property('tty').that.equals(true);
expect(service).to.have.property('imageName').that.equals('test-image');
expect(service).to.have.property('imageId').that.equals(5);
const [service] = app.services;
expect(service).to.have.property('appId').that.equals(1);
expect(service).to.have.property('serviceId').that.equals(1);
expect(service).to.have.property('imageId').that.equals(1);
expect(service).to.have.property('releaseId').that.equals(1);
expect(service.config)
.to.have.property('image')
.that.equals('ubuntu:latest');
expect(service.config)
.to.have.property('labels')
.that.deep.includes({ 'my-label': 'true' });
expect(service.config)
.to.have.property('command')
.that.deep.equals(['sleep', 'infinity']);
});
});
it('should retrieve multiple apps from the database', async () => {
const apps = await dbFormat.getApps();
expect(Object.keys(apps)).to.have.length(2).and.deep.equal(['1', '2']);
await dbFormat.setApps(
{
deadbeef: {
id: 1,
name: 'test-app',
class: 'fleet',
releases: {
one: {
id: 1,
services: {
ubuntu: {
id: 1,
image_id: 1,
image: 'ubuntu:latest',
environment: {},
labels: {},
composition: {
command: ['sleep', 'infinity'],
},
},
},
volumes: {},
networks: {},
},
},
},
deadc0de: {
id: 2,
name: 'other-app',
class: 'app',
releases: {
two: {
id: 2,
services: {},
volumes: {},
networks: {},
},
},
},
},
apiEndpoint,
);
await withMockerode(async () => {
const apps = Object.values(await dbFormat.getApps());
expect(apps).to.have.lengthOf(2);
const [app, otherapp] = apps;
expect(app).to.be.an.instanceOf(App);
expect(app).to.have.property('appId').that.equals(1);
expect(app).to.have.property('commit').that.equals('one');
expect(app).to.have.property('appName').that.equals('test-app');
expect(app).to.have.property('source').that.equals(apiEndpoint);
expect(app).to.have.property('services').that.has.lengthOf(1);
expect(app).to.have.property('volumes').that.deep.equals({});
expect(app)
.to.have.property('networks')
.that.deep.equals(getDefaultNetwork(1));
expect(otherapp).to.have.property('appId').that.equals(2);
expect(otherapp).to.have.property('commit').that.equals('two');
expect(otherapp).to.have.property('appName').that.equals('other-app');
});
});
it('should write target states to the database', async () => {
const target = await import('./data/state-endpoints/simple.json');
const dbApps: { [appId: number]: TargetApp } = {};
dbApps[1234] = {
...target.local.apps[1234],
it('should retrieve app target state from database', async () => {
const srcApps: TargetApps = {
deadbeef: {
id: 1,
name: 'test-app',
class: 'fleet',
releases: {
one: {
id: 1,
services: {
ubuntu: {
id: 1,
image_id: 1,
image: 'ubuntu:latest',
environment: {},
labels: { 'my-label': 'true' },
composition: {
command: ['sleep', 'infinity'],
},
},
},
volumes: {},
networks: {},
},
},
},
deadc0de: {
id: 2,
name: 'other-app',
class: 'app',
releases: {
two: {
id: 2,
services: {},
volumes: {},
networks: {},
},
},
},
};
await dbFormat.setApps(dbApps, apiEndpoint);
await dbFormat.setApps(srcApps, apiEndpoint);
const app = await dbFormat.getApp(1234);
expect(app).to.have.property('appName').that.equals('pi4test');
expect(app).to.have.property('services').that.is.an('array');
expect(_.keyBy(app.services, 'serviceId'))
.to.have.property('482141')
.that.has.property('serviceName')
.that.equals('main');
});
it('should add default and missing fields when retreiving from the database', async () => {
const originalImagesInspect = images.inspectByName;
try {
// @ts-expect-error Assigning a RO property
images.inspectByName = () =>
Promise.resolve({
Config: { Cmd: ['someCommand'], Entrypoint: ['theEntrypoint'] },
});
const app = await dbFormat.getApp(2);
const conf =
app.services[parseInt(Object.keys(app.services)[0], 10)].config;
expect(conf)
.to.have.property('entrypoint')
.that.deep.equals(['theEntrypoint']);
expect(conf)
.to.have.property('command')
.that.deep.equals(['someCommand']);
} finally {
// @ts-expect-error Assigning a RO property
images.inspectByName = originalImagesInspect;
}
// getApp creates a new app instance which requires a docker instance
// withMockerode mocks engine
await withMockerode(async () => {
const result = await dbFormat.getTargetJson();
expect(
isRight(TargetApps.decode(result)),
'resulting target apps is a valid TargetApps object',
);
expect(result).to.deep.equal(srcApps);
});
});
});

10
test/34-firewall.spec.ts
View File

@ -161,8 +161,10 @@ describe('Host Firewall', function () {
await targetStateCache.setTargetApps([
{
appId: 2,
uuid: 'myapp',
commit: 'abcdef2',
name: 'test-app2',
class: 'fleet',
source: apiEndpoint,
releaseId: 1232,
services: JSON.stringify([
@ -213,24 +215,28 @@ describe('Host Firewall', function () {
await targetStateCache.setTargetApps([
{
appId: 2,
uuid: 'myapp',
commit: 'abcdef2',
name: 'test-app2',
source: apiEndpoint,
class: 'fleet',
releaseId: 1232,
services: JSON.stringify([
{
serviceName: 'test-service',
networkMode: 'host',
image: 'test-image',
imageId: 5,
environment: {
TEST_VAR: 'test-string',
},
tty: true,
appId: 2,
releaseId: 1232,
serviceId: 567,
commit: 'abcdef2',
composition: {
tty: true,
network_mode: 'host',
},
},
]),
networks: '[]',

49
test/data/apps-pin.json
View File

@ -1,26 +1,29 @@
{
"name": "aDevice",
"config": {
"RESIN_HOST_CONFIG_gpu_mem": "256",
"RESIN_HOST_LOG_TO_DISPLAY": "0"
},
"apps": {
"1234": {
"name": "superapp",
"commit": "abcdef",
"releaseId": 1,
"services": {
"23": {
"imageId": 12345,
"serviceName": "someservice",
"image": "registry2.resin.io/superapp/abcdef",
"labels": {
"io.resin.something": "bar"
},
"environment": {}
}
}
}
},
"config": {
"RESIN_HOST_CONFIG_gpu_mem": "256",
"RESIN_HOST_LOG_TO_DISPLAY": "0"
},
"apps": {
"myapp": {
"id": 1234,
"name": "superapp",
"releases": {
"abcdef": {
"id": 1,
"services": {
"someservice": {
"id": 123,
"image_id": 12345,
"image": "registry2.resin.io/superapp/abcdef",
"labels": {
"io.resin.something": "bar"
},
"environment": {}
}
}
}
}
}
},
"pinDevice": true
}

49
test/data/apps.json
View File

@ -1,25 +1,28 @@
{
"name": "aDevice",
"config": {
"RESIN_HOST_CONFIG_gpu_mem": "256",
"RESIN_HOST_LOG_TO_DISPLAY": "0"
},
"apps": {
"1234": {
"name": "superapp",
"commit": "abcdef",
"releaseId": 1,
"services": {
"23": {
"imageId": 12345,
"serviceName": "someservice",
"image": "registry2.resin.io/superapp/abcdef",
"labels": {
"io.resin.something": "bar"
},
"environment": {}
}
}
}
}
"config": {
"RESIN_HOST_CONFIG_gpu_mem": "256",
"RESIN_HOST_LOG_TO_DISPLAY": "0"
},
"apps": {
"myapp": {
"id": 1234,
"name": "superapp",
"releases": {
"abcdef": {
"id": 1,
"services": {
"someservice": {
"id": 123,
"image_id": 12345,
"image": "registry2.resin.io/superapp/abcdef",
"labels": {
"io.resin.something": "bar"
},
"environment": {}
}
}
}
}
}
}
}

5
test/data/docker-states/entrypoint/compose.json
View File

@ -6,9 +6,12 @@
"environment": {},
"labels": {},
"appId": 1011165,
"appUuid": "aaaaaaaa",
"releaseId": 597007,
"serviceId": 43697,
"commit": "ff300a701054ac15281de1f9c0e84b8c",
"imageName": "registry2.resin.io/v2/bf9c649a5ac2fe147bbe350875042388@sha256:3a5c17b715b4f8265539c1a006dd1abdd2ff3b758aa23df99f77c792f40c3d43",
"tty": true
"composition": {
"tty": true
}
}

3
test/data/docker-states/entrypoint/inspect.json
View File

@ -142,6 +142,7 @@
"StdinOnce": false,
"Env": [
"RESIN_APP_ID=1011165",
"RESIN_APP_UUID=aaaaaaaa",
"RESIN_APP_NAME=supervisortest",
"RESIN_SERVICE_NAME=main",
"RESIN_DEVICE_UUID=a7feb967fac7f559ccf2a006a36bcf5d",
@ -152,6 +153,7 @@
"RESIN_SERVICE_KILL_ME_PATH=/tmp/balena/handover-complete",
"RESIN=1",
"BALENA_APP_ID=1011165",
"BALENA_APP_UUID=aaaaaaaa",
"BALENA_APP_NAME=supervisortest",
"BALENA_SERVICE_NAME=main",
"BALENA_DEVICE_UUID=a7feb967fac7f559ccf2a006a36bcf5d",
@ -181,6 +183,7 @@
"OnBuild": null,
"Labels": {
"io.resin.app-id": "1011165",
"io.balena.app-uuid": "aaaaaaaa",
"io.resin.service-id": "43697",
"io.resin.service-name": "main",
"io.resin.supervised": "true"

7
test/data/docker-states/network-mode-service/compose.json
View File

@ -4,10 +4,13 @@
"image": "sha256:f9e0fa6e3e68caedbcbb4ef35d5a8dce2a8d33e39cc94115d567800f25d826f4",
"running": true,
"appId": 1011165,
"appUuid": "aaaaaaaa",
"releaseId": 572579,
"serviceId": 43697,
"commit": "b14730d691467ab0f448a308af6bf839",
"imageName": "registry2.resin.io/v2/8ddbe4a22e881f06def0f31400bfb6de@sha256:09b0db9e71cead5f91107fc9254b1af7088444cc6da55afa2da595940f72a34a",
"tty": true,
"network_mode": "service: test"
"composition": {
"tty": true,
"network_mode": "service: test"
}
}

3
test/data/docker-states/network-mode-service/inspect.json
View File

@ -142,6 +142,7 @@
"StdinOnce": false,
"Env": [
"RESIN_APP_ID=1011165",
"RESIN_APP_UUID=aaaaaaaa",
"RESIN_APP_NAME=supervisortest",
"RESIN_SERVICE_NAME=main",
"RESIN_DEVICE_UUID=7dadabd4edec3067948d5952c2f2f26f",
@ -152,6 +153,7 @@
"RESIN_SERVICE_KILL_ME_PATH=/tmp/balena/handover-complete",
"RESIN=1",
"BALENA_APP_ID=1011165",
"BALENA_APP_UUID=aaaaaaaa",
"BALENA_APP_NAME=supervisortest",
"BALENA_SERVICE_NAME=main",
"BALENA_DEVICE_UUID=7dadabd4edec3067948d5952c2f2f26f",
@ -179,6 +181,7 @@
"OnBuild": null,
"Labels": {
"io.resin.app-id": "1011165",
"io.balena.app-uuid": "aaaaaaaa",
"io.resin.architecture": "armv7hf",
"io.resin.device-type": "raspberry-pi2",
"io.resin.qemu.version": "2.9.0.resin1-arm",

5
test/data/docker-states/simple/compose.json
View File

@ -4,9 +4,12 @@
"image": "sha256:f9e0fa6e3e68caedbcbb4ef35d5a8dce2a8d33e39cc94115d567800f25d826f4",
"running": true,
"appId": 1011165,
"appUuid": "aaaaaaaa",
"releaseId": 572579,
"serviceId": 43697,
"commit": "b14730d691467ab0f448a308af6bf839",
"imageName": "registry2.resin.io/v2/8ddbe4a22e881f06def0f31400bfb6de@sha256:09b0db9e71cead5f91107fc9254b1af7088444cc6da55afa2da595940f72a34a",
"tty": true
"composition": {
"tty": true
}
}

3
test/data/docker-states/simple/inspect.json
View File

@ -142,6 +142,7 @@
"StdinOnce": false,
"Env": [
"RESIN_APP_ID=1011165",
"RESIN_APP_UUID=aaaaaaaa",
"RESIN_APP_NAME=supervisortest",
"RESIN_SERVICE_NAME=main",
"RESIN_DEVICE_UUID=7dadabd4edec3067948d5952c2f2f26f",
@ -152,6 +153,7 @@
"RESIN_SERVICE_KILL_ME_PATH=/tmp/balena/handover-complete",
"RESIN=1",
"BALENA_APP_ID=1011165",
"BALENA_APP_UUID=aaaaaaaa",
"BALENA_APP_NAME=supervisortest",
"BALENA_SERVICE_NAME=main",
"BALENA_DEVICE_UUID=7dadabd4edec3067948d5952c2f2f26f",
@ -179,6 +181,7 @@
"OnBuild": null,
"Labels": {
"io.resin.app-id": "1011165",
"io.balena.app-uuid": "aaaaaaaa",
"io.resin.architecture": "armv7hf",
"io.resin.device-type": "raspberry-pi2",
"io.resin.qemu.version": "2.9.0.resin1-arm",

56
test/data/state-endpoints/simple.json
View File

@ -1,56 +0,0 @@
{
"local": {
"name": "lingering-frost",
"config": {
"RESIN_SUPERVISOR_DELTA_VERSION": "3",
"RESIN_SUPERVISOR_NATIVE_LOGGER": "true",
"RESIN_HOST_CONFIG_arm_64bit": "1",
"RESIN_HOST_CONFIG_disable_splash": "1",
"RESIN_HOST_CONFIG_dtoverlay": "\"vc4-fkms-v3d\"",
"RESIN_HOST_CONFIG_dtparam": "\"i2c_arm=on\",\"spi=on\",\"audio=on\"",
"RESIN_HOST_CONFIG_enable_uart": "1",
"RESIN_HOST_CONFIG_gpu_mem": "16",
"RESIN_SUPERVISOR_DELTA": "1",
"RESIN_SUPERVISOR_POLL_INTERVAL": "900000"
},
"apps": {
"1234": {
"name": "pi4test",
"commit": "d0b7b1d5353c4a1d9d411614caf827f5",
"releaseId": 1405939,
"services": {
"482141": {
"privileged": true,
"tty": true,
"restart": "always",
"network_mode": "host",
"volumes": [
"resin-data:/data"
],
"labels": {
"io.resin.features.dbus": "1",
"io.resin.features.firmware": "1",
"io.resin.features.kernel-modules": "1",
"io.resin.features.resin-api": "1",
"io.resin.features.supervisor-api": "1"
},
"imageId": 2339002,
"serviceName": "main",
"image": "registry2.balena-cloud.com/v2/f5aff5560e1fb6740a868bfe2e8a4684@sha256:9cd1d09aad181b98067dac95e08f121c3af16426f078c013a485c41a63dc035c",
"running": true,
"environment": {}
}
},
"volumes": {
"resin-data": {}
},
"networks": {}
}
}
},
"dependent": {
"apps": {},
"devices": {}
}
}

56
test/src/compose/app.spec.ts
View File

@ -55,6 +55,7 @@ async function createService(
appId,
serviceName,
commit,
running: true,
...conf,
},
options,
@ -248,7 +249,7 @@ describe('compose/app', () => {
const current = createApp({
services: [
await createService({
volumes: ['test-volume:/data'],
composition: { volumes: ['test-volume:/data'] },
}),
],
volumes: [Volume.fromComposeObject('test-volume', 1, {})],
@ -256,7 +257,7 @@ describe('compose/app', () => {
const target = createApp({
services: [
await createService({
volumes: ['test-volume:/data'],
composition: { volumes: ['test-volume:/data'] },
}),
],
volumes: [
@ -291,7 +292,7 @@ describe('compose/app', () => {
it('should not output a kill step for a service which is already stopping when changing a volume', async () => {
const service = await createService({
volumes: ['test-volume:/data'],
composition: { volumes: ['test-volume:/data'] },
});
service.status = 'Stopping';
const current = createApp({
@ -314,8 +315,8 @@ describe('compose/app', () => {
it('should generate the correct step sequence for a volume purge request', async () => {
const service = await createService({
volumes: ['db-volume:/data'],
image: 'test-image',
composition: { volumes: ['db-volume:/data'] },
});
const volume = Volume.fromComposeObject('db-volume', service.appId, {});
const contextWithImages = {
@ -516,7 +517,11 @@ describe('compose/app', () => {
it('should kill dependencies of networks before removing', async () => {
const current = createApp({
services: [await createService({ networks: { 'test-network': {} } })],
services: [
await createService({
composition: { networks: { 'test-network': {} } },
}),
],
networks: [Network.fromComposeObject('test-network', 1, {})],
});
const target = createApp({
@ -535,11 +540,19 @@ describe('compose/app', () => {
it('should kill dependencies of networks before changing config', async () => {
const current = createApp({
services: [await createService({ networks: { 'test-network': {} } })],
services: [
await createService({
composition: { networks: { 'test-network': {} } },
}),
],
networks: [Network.fromComposeObject('test-network', 1, {})],
});
const target = createApp({
services: [await createService({ networks: { 'test-network': {} } })],
services: [
await createService({
composition: { networks: { 'test-network': {} } },
}),
],
networks: [
Network.fromComposeObject('test-network', 1, {
labels: { test: 'test' },
@ -731,7 +744,7 @@ describe('compose/app', () => {
const current = createApp({
services: [
await createService(
{ restart: 'no', running: false },
{ composition: { restart: 'no' }, running: false },
{ state: { containerId: 'run_once' } },
),
],
@ -746,7 +759,7 @@ describe('compose/app', () => {
const target = createApp({
services: [
await createService(
{ restart: 'no', running: false },
{ composition: { restart: 'no' }, running: false },
{ state: { containerId: 'run_once' } },
),
],
@ -779,9 +792,9 @@ describe('compose/app', () => {
const target = createApp({
services: [
await createService({
privileged: true,
appId: 1,
serviceName: 'main',
composition: { privileged: true },
}),
],
networks: [defaultNetwork],
@ -851,7 +864,9 @@ describe('compose/app', () => {
await createService({
appId: 1,
serviceName: 'main',
dependsOn: ['dep'],
composition: {
depends_on: ['dep'],
},
}),
await createService({
appId: 1,
@ -1022,10 +1037,12 @@ describe('compose/app', () => {
services: [
await createService({
image: 'main-image',
dependsOn: ['dep'],
appId: 1,
serviceName: 'main',
commit: 'old-release',
composition: {
depends_on: ['dep'],
},
}),
await createService({
image: 'dep-image',
@ -1040,10 +1057,12 @@ describe('compose/app', () => {
services: [
await createService({
image: 'main-image-2',
dependsOn: ['dep'],
appId: 1,
serviceName: 'main',
commit: 'new-release',
composition: {
depends_on: ['dep'],
},
}),
await createService({
image: 'dep-image-2',
@ -1134,7 +1153,9 @@ describe('compose/app', () => {
services: [
await createService({
labels,
privileged: true,
composition: {
privileged: true,
},
}),
],
isTarget: true,
@ -1150,7 +1171,12 @@ describe('compose/app', () => {
it('should not start a service when a network it depends on is not ready', async () => {
const current = createApp({ networks: [defaultNetwork] });
const target = createApp({
services: [await createService({ networks: ['test'], appId: 1 })],
services: [
await createService({
composition: { networks: ['test'] },
appId: 1,
}),
],
networks: [defaultNetwork, Network.fromComposeObject('test', 1, {})],
isTarget: true,
});

90
test/src/compose/application-manager.spec.ts
View File

@ -20,6 +20,7 @@ const DEFAULT_NETWORK = Network.fromComposeObject('default', 1, {});
async function createService(
{
appId = 1,
appUuid = 'app-uuid',
serviceName = 'main',
commit = 'main-commit',
...conf
@ -29,6 +30,7 @@ async function createService(
const svc = await Service.fromComposeObject(
{
appId,
appUuid,
serviceName,
commit,
// db ids should not be used for target state calculation, but images
@ -52,23 +54,26 @@ async function createService(
function createImage(
{
appId = 1,
dependent = 0,
appUuid = 'app-uuid',
name = 'test-image',
serviceName = 'test',
serviceName = 'main',
commit = 'main-commit',
...extra
} = {} as Partial<Image>,
) {
return {
appId,
dependent,
appUuid,
name,
serviceName,
commit,
// db ids should not be used for target state calculation, but images
// are compared using _.isEqual so leaving this here to have image comparisons
// match
imageId: 1,
releaseId: 1,
serviceId: 1,
dependent: 0,
...extra,
} as Image;
}
@ -130,7 +135,7 @@ function createCurrentState({
volumes = [] as Volume[],
images = services.map((s) => ({
// Infer images from services by default
dockerImageId: s.config.image,
dockerImageId: s.dockerImageId,
...imageManager.imageFromService(s),
})) as Image[],
downloading = [] as string[],
@ -363,12 +368,15 @@ describe('compose/application-manager', () => {
containerIdsByAppId,
} = createCurrentState({
services: [
await createService({
image: 'image-old',
labels,
appId: 1,
commit: 'old-release',
}),
await createService(
{
image: 'image-old',
labels,
appId: 1,
commit: 'old-release',
},
{ options: { imageInfo: { Id: 'sha256:image-old-id' } } },
),
],
networks: [DEFAULT_NETWORK],
});
@ -414,12 +422,15 @@ describe('compose/application-manager', () => {
containerIdsByAppId,
} = createCurrentState({
services: [
await createService({
image: 'image-old',
labels,
appId: 1,
commit: 'old-release',
}),
await createService(
{
image: 'image-old',
labels,
appId: 1,
commit: 'old-release',
},
{ options: { imageInfo: { Id: 'sha256:image-old-id' } } },
),
],
networks: [DEFAULT_NETWORK],
});
@ -499,10 +510,12 @@ describe('compose/application-manager', () => {
services: [
await createService({
image: 'main-image',
dependsOn: ['dep'],
appId: 1,
commit: 'new-release',
serviceName: 'main',
composition: {
depends_on: ['dep'],
},
}),
await createService({
image: 'dep-image',
@ -523,10 +536,12 @@ describe('compose/application-manager', () => {
} = createCurrentState({
services: [
await createService({
dependsOn: ['dep'],
appId: 1,
commit: 'old-release',
serviceName: 'main',
composition: {
depends_on: ['dep'],
},
}),
await createService({
appId: 1,
@ -566,14 +581,18 @@ describe('compose/application-manager', () => {
services: [
await createService({
image: 'main-image',
dependsOn: ['dep'],
appId: 1,
appUuid: 'app-uuid',
commit: 'new-release',
serviceName: 'main',
composition: {
depends_on: ['dep'],
},
}),
await createService({
image: 'dep-image',
appId: 1,
appUuid: 'app-uuid',
commit: 'new-release',
serviceName: 'dep',
}),
@ -591,13 +610,17 @@ describe('compose/application-manager', () => {
} = createCurrentState({
services: [
await createService({
dependsOn: ['dep'],
appId: 1,
appUuid: 'app-uuid',
commit: 'old-release',
serviceName: 'main',
composition: {
depends_on: ['dep'],
},
}),
await createService({
appId: 1,
appUuid: 'app-uuid',
commit: 'old-release',
serviceName: 'dep',
}),
@ -607,13 +630,17 @@ describe('compose/application-manager', () => {
// Both images have been downloaded
createImage({
appId: 1,
appUuid: 'app-uuid',
name: 'main-image',
serviceName: 'main',
commit: 'new-release',
}),
createImage({
appId: 1,
appUuid: 'app-uuid',
name: 'dep-image',
serviceName: 'dep',
commit: 'new-release',
}),
],
});
@ -647,9 +674,11 @@ describe('compose/application-manager', () => {
services: [
await createService({
image: 'main-image',
dependsOn: ['dep'],
serviceName: 'main',
commit: 'new-release',
composition: {
depends_on: ['dep'],
},
}),
await createService({
image: 'dep-image',
@ -673,14 +702,14 @@ describe('compose/application-manager', () => {
images: [
// Both images have been downloaded
createImage({
appId: 1,
name: 'main-image',
serviceName: 'main',
commit: 'new-release',
}),
createImage({
appId: 1,
name: 'dep-image',
serviceName: 'dep',
commit: 'new-release',
}),
],
});
@ -711,9 +740,11 @@ describe('compose/application-manager', () => {
services: [
await createService({
image: 'main-image',
dependsOn: ['dep'],
serviceName: 'main',
commit: 'new-release',
composition: {
depends_on: ['dep'],
},
}),
await createService({
image: 'dep-image',
@ -743,14 +774,14 @@ describe('compose/application-manager', () => {
images: [
// Both images have been downloaded
createImage({
appId: 1,
name: 'main-image',
serviceName: 'main',
commit: 'new-release',
}),
createImage({
appId: 1,
name: 'dep-image',
serviceName: 'dep',
commit: 'new-release',
}),
],
});
@ -793,7 +824,6 @@ describe('compose/application-manager', () => {
images: [
// Image has been downloaded
createImage({
appId: 1,
name: 'main-image',
serviceName: 'main',
}),
@ -1159,12 +1189,14 @@ describe('compose/application-manager', () => {
running: true,
image: 'main-image-1',
appId: 1,
appUuid: 'app-one',
commit: 'commit-for-app-1',
}),
await createService({
running: true,
image: 'main-image-2',
appId: 2,
appUuid: 'app-two',
commit: 'commit-for-app-2',
}),
],
@ -1192,12 +1224,16 @@ describe('compose/application-manager', () => {
createImage({
name: 'main-image-1',
appId: 1,
appUuid: 'app-one',
serviceName: 'main',
commit: 'commit-for-app-1',
}),
createImage({
name: 'main-image-2',
appId: 2,
appUuid: 'app-two',
serviceName: 'main',
commit: 'commit-for-app-2',
}),
],
});

172
test/src/compose/images.spec.ts
View File

@ -7,6 +7,26 @@ import * as sinon from 'sinon';
import log from '../../../src/lib/supervisor-console';
// TODO: this code is duplicated in multiple tests
// create a test module with all helper functions like this
function createDBImage(
{
appId = 1,
name = 'test-image',
serviceName = 'test',
dependent = 0,
...extra
} = {} as Partial<imageManager.Image>,
) {
return {
appId,
dependent,
name,
serviceName,
...extra,
} as imageManager.Image;
}
describe('compose/images', () => {
let testDb: dbHelper.TestDatabase;
before(async () => {
@ -36,19 +56,12 @@ describe('compose/images', () => {
});
it('finds image by matching digest on the database', async () => {
const dbImage = {
id: 246,
const dbImage = createDBImage({
name:
'registry2.balena-cloud.com/v2/aaaaa@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650325,
serviceName: 'app_1',
imageId: 2693229,
releaseId: 1524186,
dependent: 0,
dockerImageId:
'sha256:f1154d76c731f04711e5856b6e6858730e3023d9113124900ac65c2ccc90e8e7',
};
});
await testDb.models('image').insert([dbImage]);
const images = [
@ -72,8 +85,8 @@ describe('compose/images', () => {
await expect(mockerode.getImage(dbImage.name).inspect()).to.be.rejected;
// Looking up the image by id should succeed
await expect(mockerode.getImage(dbImage.dockerImageId).inspect()).to.not
.be.rejected;
await expect(mockerode.getImage(dbImage.dockerImageId!).inspect()).to
.not.be.rejected;
// The image is found
expect(await imageManager.inspectByName(dbImage.name))
@ -126,18 +139,11 @@ describe('compose/images', () => {
});
it('finds image by tag on the database', async () => {
const dbImage = {
id: 246,
const dbImage = createDBImage({
name: 'some-image:some-tag',
appId: 1658654,
serviceId: 650325,
serviceName: 'app_1',
imageId: 2693229,
releaseId: 1524186,
dependent: 0,
dockerImageId:
'sha256:f1154d76c731f04711e5856b6e6858730e3023d9113124900ac65c2ccc90e8e7',
};
});
await testDb.models('image').insert([dbImage]);
const images = [
@ -245,53 +251,29 @@ describe('compose/images', () => {
it('returns all images in both the database and the engine', async () => {
await testDb.models('image').insert([
{
id: 1,
createDBImage({
name: 'first-image-name:first-image-tag',
appId: 1,
serviceId: 1,
serviceName: 'app_1',
imageId: 1,
releaseId: 1,
dependent: 0,
dockerImageId: 'sha256:first-image-id',
},
{
id: 2,
}),
createDBImage({
name: 'second-image-name:second-image-tag',
appId: 2,
serviceId: 2,
serviceName: 'app_2',
imageId: 2,
releaseId: 2,
dependent: 0,
dockerImageId: 'sha256:second-image-id',
},
{
id: 3,
}),
createDBImage({
name:
'registry2.balena-cloud.com/v2/three@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf558',
appId: 3,
serviceId: 3,
serviceName: 'app_3',
imageId: 3,
releaseId: 3,
dependent: 0,
// Third image has different name but same docker id
dockerImageId: 'sha256:second-image-id',
},
{
id: 4,
}),
createDBImage({
name: 'fourth-image-name:fourth-image-tag',
appId: 4,
serviceId: 4,
serviceName: 'app_4',
imageId: 4,
releaseId: 4,
dependent: 0,
// The fourth image exists on the engine but with the wrong id
dockerImageId: 'sha256:fourth-image-id',
},
}),
]);
const images = [
@ -336,16 +318,9 @@ describe('compose/images', () => {
it('removes a single legacy db images without dockerImageId', async () => {
// Legacy images don't have a dockerImageId so they are queried by name
const imageToRemove = {
id: 246,
const imageToRemove = createDBImage({
name: 'image-name:image-tag',
appId: 1658654,
serviceId: 650325,
serviceName: 'app_1',
imageId: 2693229,
releaseId: 1524186,
dependent: 0,
};
});
await testDb.models('image').insert([imageToRemove]);
@ -405,34 +380,20 @@ describe('compose/images', () => {
it('removes image from DB and engine when there is a single DB image with matching name', async () => {
// Newer image
const imageToRemove = {
id: 246,
const imageToRemove = createDBImage({
name:
'registry2.balena-cloud.com/v2/one@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650325,
serviceName: 'app_1',
imageId: 2693229,
releaseId: 1524186,
dependent: 0,
dockerImageId: 'sha256:image-id-one',
};
});
// Insert images into the db
await testDb.models('image').insert([
imageToRemove,
{
id: 247,
createDBImage({
name:
'registry2.balena-cloud.com/v2/two@sha256:12345a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650331,
serviceName: 'app_2',
imageId: 2693230,
releaseId: 1524186,
dependent: 0,
dockerImageId: 'sha256:image-id-two',
},
}),
]);
// Engine image state
@ -507,33 +468,18 @@ describe('compose/images', () => {
});
it('removes the requested image even when there are multiple DB images with same docker ID', async () => {
const imageToRemove = {
id: 246,
const imageToRemove = createDBImage({
name:
'registry2.balena-cloud.com/v2/one@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650325,
serviceName: 'app_1',
imageId: 2693229,
releaseId: 1524186,
dependent: 0,
dockerImageId: 'sha256:image-id-one',
};
});
const imageWithSameDockerImageId = {
id: 247,
const imageWithSameDockerImageId = createDBImage({
name:
'registry2.balena-cloud.com/v2/two@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650331,
serviceName: 'app_2',
imageId: 2693230,
releaseId: 1524186,
dependent: 0,
// Same imageId
dockerImageId: 'sha256:image-id-one',
};
});
// Insert images into the db
await testDb.models('image').insert([
@ -547,7 +493,7 @@ describe('compose/images', () => {
// The image to remove
createImage(
{
Id: imageToRemove.dockerImageId,
Id: imageToRemove.dockerImageId!,
},
{
References: [imageToRemove.name, imageWithSameDockerImageId.name],
@ -570,7 +516,7 @@ describe('compose/images', () => {
// Check that the image is on the engine
// really checking mockerode behavior
await expect(
mockerode.getImage(imageToRemove.dockerImageId).inspect(),
mockerode.getImage(imageToRemove.dockerImageId!).inspect(),
'image exists on the engine before the test',
).to.not.be.rejected;
@ -607,32 +553,18 @@ describe('compose/images', () => {
});
it('removes image from DB by tag when deltas are being used', async () => {
const imageToRemove = {
id: 246,
const imageToRemove = createDBImage({
name:
'registry2.balena-cloud.com/v2/one@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650325,
serviceName: 'app_1',
imageId: 2693229,
releaseId: 1524186,
dependent: 0,
dockerImageId: 'sha256:image-one-id',
};
});
const imageWithSameDockerImageId = {
id: 247,
const imageWithSameDockerImageId = createDBImage({
name:
'registry2.balena-cloud.com/v2/two@sha256:2c969a1ba1c6bc10df53481f48c6a74dbd562cfb41ba58f81beabd03facf5582',
appId: 1658654,
serviceId: 650331,
serviceName: 'app_2',
imageId: 2693230,
releaseId: 1524186,
dependent: 0,
// Same docker id
dockerImageId: 'sha256:image-one-id',
};
});
// Insert images into the db
await testDb.models('image').insert([
@ -646,7 +578,7 @@ describe('compose/images', () => {
// The image to remove
createImage(
{
Id: imageToRemove.dockerImageId,
Id: imageToRemove.dockerImageId!,
},
{
References: [
@ -663,7 +595,7 @@ describe('compose/images', () => {
async (mockerode) => {
// Check that the image is on the engine
await expect(
mockerode.getImage(imageToRemove.dockerImageId).inspect(),
mockerode.getImage(imageToRemove.dockerImageId!).inspect(),
'image can be found by id before the test',
).to.not.be.rejected;

111
test/src/compose/service.spec.ts
View File

@ -60,6 +60,7 @@ describe('compose/service', () => {
};
const service = {
appId: '23',
appUuid: 'deadbeef',
releaseId: 2,
serviceId: 3,
imageId: 4,
@ -78,6 +79,7 @@ describe('compose/service', () => {
FOO: 'bar',
A_VARIABLE: 'ITS_VALUE',
RESIN_APP_ID: '23',
RESIN_APP_UUID: 'deadbeef',
RESIN_APP_NAME: 'awesomeApp',
RESIN_DEVICE_UUID: '1234',
RESIN_DEVICE_ARCH: 'amd64',
@ -88,6 +90,7 @@ describe('compose/service', () => {
RESIN_SERVICE_KILL_ME_PATH: '/tmp/balena/handover-complete',
RESIN: '1',
BALENA_APP_ID: '23',
BALENA_APP_UUID: 'deadbeef',
BALENA_APP_NAME: 'awesomeApp',
BALENA_DEVICE_UUID: '1234',
BALENA_DEVICE_ARCH: 'amd64',
@ -127,8 +130,10 @@ describe('compose/service', () => {
releaseId: 2,
serviceId: 3,
imageId: 4,
expose: [1000, '243/udp'],
ports: ['2344', '2345:2354', '2346:2367/udp'],
composition: {
expose: [1000, '243/udp'],
ports: ['2344', '2345:2354', '2346:2367/udp'],
},
},
{
imageInfo: {
@ -183,8 +188,10 @@ describe('compose/service', () => {
releaseId: 2,
serviceId: 3,
imageId: 4,
expose: [1000, '243/udp'],
ports: ['1000-1003:2000-2003'],
composition: {
expose: [1000, '243/udp'],
ports: ['1000-1003:2000-2003'],
},
},
{ appName: 'test' } as any,
);
@ -236,7 +243,9 @@ describe('compose/service', () => {
releaseId: 2,
serviceId: 3,
imageId: 4,
ports: ['5-65536:5-65536/tcp', '5-65536:5-65536/udp'],
composition: {
ports: ['5-65536:5-65536/tcp', '5-65536:5-65536/udp'],
},
},
{ appName: 'test' } as any,
);
@ -250,7 +259,9 @@ describe('compose/service', () => {
appId: 123456,
serviceId: 123456,
serviceName: 'test',
ports: ['80:80', '100:100'],
composition: {
ports: ['80:80', '100:100'],
},
},
{ appName: 'test' } as any,
);
@ -266,12 +277,14 @@ describe('compose/service', () => {
appId: 123,
serviceId: 123,
serviceName: 'test',
volumes: [
'vol1:vol2',
'vol3 :/usr/src/app',
'vol4: /usr/src/app',
'vol5 : vol6',
],
composition: {
volumes: [
'vol1:vol2',
'vol3 :/usr/src/app',
'vol4: /usr/src/app',
'vol5 : vol6',
],
},
},
{ appName: 'test' } as any,
);
@ -296,7 +309,9 @@ describe('compose/service', () => {
appId: 123456,
serviceId: 123456,
serviceName: 'foobar',
mem_limit: memLimit,
composition: {
mem_limit: memLimit,
},
},
{ appName: 'test' } as any,
);
@ -381,7 +396,9 @@ describe('compose/service', () => {
appId: 123456,
serviceId: 123456,
serviceName: 'foobar',
workingDir: workdir,
composition: {
workingDir: workdir,
},
},
{ appName: 'test' } as any,
);
@ -414,7 +431,9 @@ describe('compose/service', () => {
appId: 123456,
serviceId: 123456,
serviceName: 'test',
networks,
composition: {
networks,
},
},
{ appName: 'test' } as any,
);
@ -473,7 +492,9 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
dns: ['8.8.8.8', '1.1.1.1'],
composition: {
dns: ['8.8.8.8', '1.1.1.1'],
},
},
{ appName: 'test' } as any,
);
@ -482,7 +503,9 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
dns: ['8.8.8.8', '1.1.1.1'],
composition: {
dns: ['8.8.8.8', '1.1.1.1'],
},
},
{ appName: 'test' } as any,
);
@ -493,7 +516,9 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
dns: ['1.1.1.1', '8.8.8.8'],
composition: {
dns: ['1.1.1.1', '8.8.8.8'],
},
},
{ appName: 'test' } as any,
);
@ -506,7 +531,9 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
volumes: ['abcdef', 'ghijk'],
composition: {
volumes: ['abcdef', 'ghijk'],
},
},
{ appName: 'test' } as any,
);
@ -515,7 +542,9 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
volumes: ['abcdef', 'ghijk'],
composition: {
volumes: ['abcdef', 'ghijk'],
},
},
{ appName: 'test' } as any,
);
@ -526,7 +555,9 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
volumes: ['ghijk', 'abcdef'],
composition: {
volumes: ['ghijk', 'abcdef'],
},
},
{ appName: 'test' } as any,
);
@ -539,8 +570,10 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
volumes: ['abcdef', 'ghijk'],
dns: ['8.8.8.8', '1.1.1.1'],
composition: {
volumes: ['abcdef', 'ghijk'],
dns: ['8.8.8.8', '1.1.1.1'],
},
},
{ appName: 'test' } as any,
);
@ -549,8 +582,10 @@ describe('compose/service', () => {
appId: 1,
serviceId: 1,
serviceName: 'test',
volumes: ['ghijk', 'abcdef'],
dns: ['8.8.8.8', '1.1.1.1'],
composition: {
volumes: ['ghijk', 'abcdef'],
dns: ['8.8.8.8', '1.1.1.1'],
},
},
{ appName: 'test' } as any,
);
@ -951,7 +986,9 @@ describe('compose/service', () => {
releaseId: 2,
serviceId: 3,
imageId: 4,
network_mode: 'service: test',
composition: {
network_mode: 'service: test',
},
},
{ appName: 'test' } as any,
);
@ -965,8 +1002,10 @@ describe('compose/service', () => {
releaseId: 2,
serviceId: 3,
imageId: 4,
depends_on: ['another_service'],
network_mode: 'service: test',
composition: {
depends_on: ['another_service'],
network_mode: 'service: test',
},
},
{ appName: 'test' } as any,
);
@ -982,7 +1021,9 @@ describe('compose/service', () => {
releaseId: 2,
serviceId: 3,
imageId: 4,
network_mode: 'service: test',
composition: {
network_mode: 'service: test',
},
},
{ appName: 'test' } as any,
);
@ -1039,11 +1080,13 @@ describe('compose/service', () => {
appId: 123,
serviceId: 123,
serviceName: 'test',
securityOpt: [
'label=user:USER',
'label=user:ROLE',
'seccomp=unconfined',
],
composition: {
securityOpt: [
'label=user:USER',
'label=user:ROLE',
'seccomp=unconfined',
],
},
},
{ appName: 'test' } as any,
);