ExternalVendorCode/balena-supervisor
1
0
Fork 0
mirror of https://github.com/balena-os/balena-supervisor.git synced 2025-05-05 10:32:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1351 from balena-io/singleton-docker

Browse Source 
Make docker module a singleton
This commit is contained in:
bulldozer-balena[bot] 2020-06-02 19:08:55 +00:00 committed by GitHub
commit 96e55585ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 430 additions and 489 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/application-manager.d.ts vendored
View File

@ -25,7 +25,6 @@ import {
import Network from './compose/network';
import Service from './compose/service';
import Volume from './compose/volume';
import DockerUtils from './lib/docker-utils';
declare interface Options {
force?: boolean;
@ -52,7 +51,6 @@ class ApplicationManager extends EventEmitter {
public deviceState: DeviceState;
public eventTracker: EventTracker;
public apiBinder: APIBinder;
public docker: DockerUtils;
public services: ServiceManager;
public volumes: VolumeManager;

19
src/application-manager.js
View File

@ -11,7 +11,8 @@ import { log } from './lib/supervisor-console';
import * as config from './config';
import { validateTargetContracts } from './lib/contracts';
import { DockerUtils as Docker } from './lib/docker-utils';
import { docker } from './lib/docker-utils';
import * as dockerUtils from './lib/docker-utils';
import { LocalModeManager } from './local-mode';
import * as updateLock from './lib/update-lock';
import { checkTruthy, checkInt, checkString } from './lib/validation';
@ -172,30 +173,24 @@ export class ApplicationManager extends EventEmitter {
this.eventTracker = eventTracker;
this.deviceState = deviceState;
this.apiBinder = apiBinder;
this.docker = new Docker();
this.images = new Images({
docker: this.docker,
logger: this.logger,
});
this.services = new ServiceManager({
docker: this.docker,
logger: this.logger,
});
this.networks = new NetworkManager({
docker: this.docker,
logger: this.logger,
});
this.volumes = new VolumeManager({
docker: this.docker,
logger: this.logger,
});
this.proxyvisor = new Proxyvisor({
logger: this.logger,
docker: this.docker,
images: this.images,
applications: this,
});
this.localModeManager = new LocalModeManager(this.docker, this.logger);
this.localModeManager = new LocalModeManager(this.logger);
this.timeSpentFetching = 0;
this.fetchesInProgress = 0;
this._targetVolatilePerImageId = {};
@ -257,9 +252,7 @@ export class ApplicationManager extends EventEmitter {
})
.then(() => {
const cleanup = () => {
return this.docker
.listContainers({ all: true })
.then((containers) => {
return docker.listContainers({ all: true }).then((containers) => {
return this.logger.clearOutOfDateDBLogs(_.map(containers, 'Id'));
});
};
@ -1061,14 +1054,12 @@ export class ApplicationManager extends EventEmitter {
createTargetVolume(name, appId, volume) {
return Volume.fromComposeObject(name, appId, volume, {
docker: this.docker,
logger: this.logger,
});
}
createTargetNetwork(name, appId, network) {
return Network.fromComposeObject(name, appId, network, {
docker: this.docker,
logger: this.logger,
});
}
@ -1076,7 +1067,7 @@ export class ApplicationManager extends EventEmitter {
normaliseAndExtendAppFromDB(app) {
return Promise.join(
config.get('extendedEnvOptions'),
this.docker
dockerUtils
.getNetworkGateway(constants.supervisorNetworkInterface)
.catch(() => '127.0.0.1'),
Promise.props({

43
src/compose/images.ts
View File

@ -8,9 +8,11 @@ import * as db from '../db';
import * as constants from '../lib/constants';
import {
DeltaFetchOptions,
DockerUtils,
FetchOptions,
docker,
dockerToolbelt,
} from '../lib/docker-utils';
import * as dockerUtils from '../lib/docker-utils';
import { DeltaStillProcessingError, NotFoundError } from '../lib/errors';
import * as LogTypes from '../lib/log-types';
import * as validation from '../lib/validation';
@ -26,7 +28,6 @@ interface ImageEvents {
type ImageEventEmitter = StrictEventEmitter<EventEmitter, ImageEvents>;
interface ImageConstructOpts {
docker: DockerUtils;
logger: Logger;
}
@ -56,7 +57,6 @@ type NormalisedDockerImage = Docker.ImageInfo & {
};
export class Images extends (EventEmitter as new () => ImageEventEmitter) {
private docker: DockerUtils;
private logger: Logger;
public appUpdatePollInterval: number;
@ -72,7 +72,6 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
public constructor(opts: ImageConstructOpts) {
super();
this.docker = opts.docker;
this.logger = opts.logger;
}
@ -202,7 +201,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
cb: (dockerImages: NormalisedDockerImage[], composeImages: Image[]) => T,
) {
const [normalisedImages, dbImages] = await Promise.all([
Bluebird.map(this.docker.listImages({ digests: true }), async (image) => {
Bluebird.map(docker.listImages({ digests: true }), async (image) => {
const newImage = _.clone(image) as NormalisedDockerImage;
newImage.NormalisedRepoTags = await this.getNormalisedTags(image);
return newImage;
@ -337,8 +336,8 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
supervisorImage,
usedImageIds,
] = await Promise.all([
this.docker.getRegistryAndName(constants.supervisorImage),
this.docker.getImage(constants.supervisorImage).inspect(),
dockerToolbelt.getRegistryAndName(constants.supervisorImage),
docker.getImage(constants.supervisorImage).inspect(),
db
.models('image')
.select('dockerImageId')
@ -366,7 +365,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
);
};
const dockerImages = await this.docker.listImages({ digests: true });
const dockerImages = await docker.listImages({ digests: true });
for (const image of dockerImages) {
// Cleanup should remove truly dangling images (i.e dangling and with no digests)
if (Images.isDangling(image) && !_.includes(usedImageIds, image.Id)) {
@ -377,7 +376,9 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
) {
// We also remove images from the supervisor repository with a different tag
for (const tag of image.RepoTags) {
const imageNameComponents = await this.docker.getRegistryAndName(tag);
const imageNameComponents = await dockerToolbelt.getRegistryAndName(
tag,
);
if (isSupervisorRepoTag(imageNameComponents)) {
images.push(image.Id);
}
@ -400,7 +401,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
imageName: string,
): Promise<Docker.ImageInspectInfo> {
try {
return await this.docker.getImage(imageName).inspect();
return await docker.getImage(imageName).inspect();
} catch (e) {
if (NotFoundError(e)) {
const digest = imageName.split('@')[1];
@ -418,7 +419,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
for (const image of imagesFromDb) {
if (image.dockerImageId != null) {
return await this.docker.getImage(image.dockerImageId).inspect();
return await docker.getImage(image.dockerImageId).inspect();
}
}
}
@ -435,7 +436,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
for (const image of images) {
log.debug(`Cleaning up ${image}`);
try {
await this.docker.getImage(image).remove({ force: true });
await docker.getImage(image).remove({ force: true });
delete this.imageCleanupFailures[image];
} catch (e) {
this.logger.logSystemMessage(
@ -459,7 +460,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
}
public normalise(imageName: string): Bluebird<string> {
return this.docker.normaliseImageName(imageName);
return dockerToolbelt.normaliseImageName(imageName);
}
private static isDangling(image: Docker.ImageInfo): boolean {
@ -496,7 +497,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
if (img.dockerImageId == null) {
// Legacy image from before we started using dockerImageId, so we try to remove it
// by name
await this.docker.getImage(img.name).remove({ force: true });
await docker.getImage(img.name).remove({ force: true });
removed = true;
} else {
const imagesFromDb = await db
@ -512,11 +513,11 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
_.merge(_.clone(image), { status: 'Deleting' }),
);
this.logger.logSystemEvent(LogTypes.deleteImage, { image });
this.docker.getImage(img.dockerImageId).remove({ force: true });
docker.getImage(img.dockerImageId).remove({ force: true });
removed = true;
} else if (!Images.hasDigest(img.name)) {
// Image has a regular tag, so we might have to remove unnecessary tags
const dockerImage = await this.docker
const dockerImage = await docker
.getImage(img.dockerImageId)
.inspect();
const differentTags = _.reject(imagesFromDb, { name: img.name });
@ -528,7 +529,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
_.some(differentTags, { name: t }),
)
) {
await this.docker.getImage(img.name).remove({ noprune: true });
await docker.getImage(img.name).remove({ noprune: true });
}
removed = false;
} else {
@ -589,7 +590,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
const srcImage = await this.inspectByName(deltaOpts.deltaSource);
deltaOpts.deltaSourceId = srcImage.Id;
const id = await this.docker.fetchDeltaWithProgress(
const id = await dockerUtils.fetchDeltaWithProgress(
image.name,
deltaOpts,
onProgress,
@ -597,8 +598,8 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
);
if (!Images.hasDigest(image.name)) {
const { repo, tag } = await this.docker.getRepoAndTag(image.name);
await this.docker.getImage(id).tag({ repo, tag });
const { repo, tag } = await dockerUtils.getRepoAndTag(image.name);
await docker.getImage(id).tag({ repo, tag });
}
return id;
@ -610,7 +611,7 @@ export class Images extends (EventEmitter as new () => ImageEventEmitter) {
onProgress: (evt: FetchProgressEvent) => void,
): Promise<string> {
this.logger.logSystemEvent(LogTypes.downloadImage, { image });
return this.docker.fetchImageWithProgress(image.name, opts, onProgress);
return dockerUtils.fetchImageWithProgress(image.name, opts, onProgress);
}
// TODO: find out if imageId can actually be null

28
src/compose/network-manager.ts
View File

@ -3,7 +3,7 @@ import * as _ from 'lodash';
import { fs } from 'mz';
import * as constants from '../lib/constants';
import Docker from '../lib/docker-utils';
import { docker } from '../lib/docker-utils';
import { ENOENT, NotFoundError } from '../lib/errors';
import logTypes = require('../lib/log-types');
import { Logger } from '../logger';
@ -13,23 +13,20 @@ import log from '../lib/supervisor-console';
import { ResourceRecreationAttemptError } from './errors';
export class NetworkManager {
private docker: Docker;
private logger: Logger;
constructor(opts: NetworkOptions) {
this.docker = opts.docker;
this.logger = opts.logger;
}
public getAll(): Bluebird<Network[]> {
return this.getWithBothLabels().map((network: { Name: string }) => {
return this.docker
return docker
.getNetwork(network.Name)
.inspect()
.then((net) => {
return Network.fromDockerNetwork(
{
docker: this.docker,
logger: this.logger,
},
net,
@ -43,13 +40,10 @@ export class NetworkManager {
}
public async get(network: { name: string; appId: number }): Promise<Network> {
const dockerNet = await this.docker
const dockerNet = await docker
.getNetwork(Network.generateDockerName(network.appId, network.name))
.inspect();
return Network.fromDockerNetwork(
{ docker: this.docker, logger: this.logger },
dockerNet,
);
return Network.fromDockerNetwork({ logger: this.logger }, dockerNet);
}
public async create(network: Network) {
@ -89,7 +83,7 @@ export class NetworkManager {
fs.stat(`/sys/class/net/${constants.supervisorNetworkInterface}`),
)
.then(() => {
return this.docker
return docker
.getNetwork(constants.supervisorNetworkInterface)
.inspect();
})
@ -108,16 +102,16 @@ export class NetworkManager {
public ensureSupervisorNetwork(): Bluebird<void> {
const removeIt = () => {
return Bluebird.resolve(
this.docker.getNetwork(constants.supervisorNetworkInterface).remove(),
docker.getNetwork(constants.supervisorNetworkInterface).remove(),
).then(() => {
return this.docker
return docker
.getNetwork(constants.supervisorNetworkInterface)
.inspect();
});
};
return Bluebird.resolve(
this.docker.getNetwork(constants.supervisorNetworkInterface).inspect(),
docker.getNetwork(constants.supervisorNetworkInterface).inspect(),
)
.then((net) => {
if (
@ -138,7 +132,7 @@ export class NetworkManager {
.catch(NotFoundError, () => {
log.debug(`Creating ${constants.supervisorNetworkInterface} network`);
return Bluebird.resolve(
this.docker.createNetwork({
docker.createNetwork({
Name: constants.supervisorNetworkInterface,
Options: {
'com.docker.network.bridge.name':
@ -160,12 +154,12 @@ export class NetworkManager {
private getWithBothLabels() {
return Bluebird.join(
this.docker.listNetworks({
docker.listNetworks({
filters: {
label: ['io.resin.supervised'],
},
}),
this.docker.listNetworks({
docker.listNetworks({
filters: {
label: ['io.balena.supervised'],
},

9
src/compose/network.ts
View File

@ -1,7 +1,7 @@
import * as Bluebird from 'bluebird';
import * as _ from 'lodash';
import Docker from '../lib/docker-utils';
import { docker } from '../lib/docker-utils';
import { InvalidAppIdError } from '../lib/errors';
import logTypes = require('../lib/log-types');
import { checkInt } from '../lib/validation';
@ -22,7 +22,6 @@ import {
} from './errors';
export interface NetworkOptions {
docker: Docker;
logger: Logger;
}
@ -31,11 +30,9 @@ export class Network {
public name: string;
public config: NetworkConfig;
private docker: Docker;
private logger: Logger;
private constructor(opts: NetworkOptions) {
this.docker = opts.docker;
this.logger = opts.logger;
}
@ -145,7 +142,7 @@ export class Network {
network: { name: this.name },
});
return await this.docker.createNetwork(this.toDockerConfig());
return await docker.createNetwork(this.toDockerConfig());
}
public toDockerConfig(): DockerNetworkConfig {
@ -191,7 +188,7 @@ export class Network {
});
return Bluebird.resolve(
this.docker
docker
.getNetwork(Network.generateDockerName(this.appId, this.name))
.remove(),
).tapCatch((error) => {

40
src/compose/service-manager.ts
View File

@ -8,7 +8,7 @@ import { fs } from 'mz';
import StrictEventEmitter from 'strict-event-emitter-types';
import * as config from '../config';
import Docker from '../lib/docker-utils';
import { docker } from '../lib/docker-utils';
import Logger from '../logger';
import { PermissiveNumber } from '../config/types';
@ -26,7 +26,6 @@ import { serviceNetworksToDockerNetworks } from './utils';
import log from '../lib/supervisor-console';
interface ServiceConstructOpts {
docker: Docker;
logger: Logger;
}
@ -44,7 +43,6 @@ interface KillOpts {
}
export class ServiceManager extends (EventEmitter as new () => ServiceManagerEventEmitter) {
private docker: Docker;
private logger: Logger;
// Whether a container has died, indexed by ID
@ -56,7 +54,6 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
public constructor(opts: ServiceConstructOpts) {
super();
this.docker = opts.docker;
this.logger = opts.logger;
}
@ -68,7 +65,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
const services = await Bluebird.map(containers, async (container) => {
try {
const serviceInspect = await this.docker
const serviceInspect = await docker
.getContainer(container.Id)
.inspect();
const service = Service.fromDockerContainer(serviceInspect);
@ -138,7 +135,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
public async getByDockerContainerId(
containerId: string,
): Promise<Service | null> {
const container = await this.docker.getContainer(containerId).inspect();
const container = await docker.getContainer(containerId).inspect();
if (
container.Config.Labels['io.balena.supervised'] == null &&
container.Config.Labels['io.resin.supervised'] == null
@ -159,7 +156,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
);
}
await this.docker.getContainer(svc.containerId).rename({
await docker.getContainer(svc.containerId).rename({
name: `${service.serviceName}_${metadata.imageId}_${metadata.releaseId}`,
});
}
@ -180,10 +177,10 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
// Containers haven't been normalized (this is an updated supervisor)
// so we need to stop and remove them
const supervisorImageId = (
await this.docker.getImage(constants.supervisorImage).inspect()
await docker.getImage(constants.supervisorImage).inspect()
).Id;
for (const container of await this.docker.listContainers({ all: true })) {
for (const container of await docker.listContainers({ all: true })) {
if (container.ImageID !== supervisorImageId) {
await this.killContainer(container.Id, {
serviceName: 'legacy',
@ -212,7 +209,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
}
try {
await this.docker
await docker
.getContainer(existingService.containerId)
.remove({ v: true });
} catch (e) {
@ -244,7 +241,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
`No containerId provided for service ${service.serviceName} in ServiceManager.updateMetadata. Service: ${service}`,
);
}
return this.docker.getContainer(existing.containerId);
return docker.getContainer(existing.containerId);
} catch (e) {
if (!NotFoundError(e)) {
this.logger.logSystemEvent(LogTypes.installServiceError, {
@ -280,12 +277,12 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
this.logger.logSystemEvent(LogTypes.installService, { service });
this.reportNewStatus(mockContainerId, service, 'Installing');
const container = await this.docker.createContainer(conf);
const container = await docker.createContainer(conf);
service.containerId = container.id;
await Promise.all(
_.map((nets || {}).EndpointsConfig, (endpointConfig, name) =>
this.docker.getNetwork(name).connect({
docker.getNetwork(name).connect({
Container: container.id,
EndpointConfig: endpointConfig,
}),
@ -366,7 +363,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
);
}
this.logger.attach(this.docker, container.id, { serviceId, imageId });
this.logger.attach(container.id, { serviceId, imageId });
if (!alreadyStarted) {
this.logger.logSystemEvent(LogTypes.startServiceSuccess, { service });
@ -389,10 +386,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
this.listening = true;
const listen = async () => {
const stream = await this.docker.getEvents({
// Remove the as any once
// https://github.com/DefinitelyTyped/DefinitelyTyped/pull/43100
// is merged and released
const stream = await docker.getEvents({
filters: { type: ['container'] } as any,
});
@ -434,7 +428,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
`serviceId and imageId not defined for service: ${service.serviceName} in ServiceManager.listenToEvents`,
);
}
this.logger.attach(this.docker, data.id, {
this.logger.attach(data.id, {
serviceId,
imageId,
});
@ -487,7 +481,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
`containerId not defined for service: ${service.serviceName} in ServiceManager.attachToRunning`,
);
}
this.logger.attach(this.docker, service.containerId, {
this.logger.attach(service.containerId, {
serviceId,
imageId,
});
@ -544,7 +538,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
this.reportNewStatus(containerId, service, 'Stopping');
}
const containerObj = this.docker.getContainer(containerId);
const containerObj = docker.getContainer(containerId);
const killPromise = Bluebird.resolve(containerObj.stop())
.then(() => {
if (removeContainer) {
@ -605,7 +599,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
labelList: string[],
): Promise<Dockerode.ContainerInfo[]> {
const listWithPrefix = (prefix: string) =>
this.docker.listContainers({
docker.listContainers({
all: true,
filters: {
label: _.map(labelList, (v) => `${prefix}${v}`),
@ -627,7 +621,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
`No containerId provided for service ${service.serviceName} in ServiceManager.prepareForHandover. Service: ${service}`,
);
}
const container = this.docker.getContainer(svc.containerId);
const container = docker.getContainer(svc.containerId);
await container.update({ RestartPolicy: {} });
return await container.rename({
name: `old_${service.serviceName}_${service.imageId}_${service.imageId}_${service.releaseId}`,

34
src/compose/volume-manager.ts
View File

@ -1,10 +1,11 @@
import * as Docker from 'dockerode';
import * as _ from 'lodash';
import * as Path from 'path';
import { VolumeInspectInfo } from 'dockerode';
import constants = require('../lib/constants');
import { NotFoundError } from '../lib/errors';
import { safeRename } from '../lib/fs-utils';
import { docker } from '../lib/docker-utils';
import * as LogTypes from '../lib/log-types';
import { defaultLegacyVolume } from '../lib/migration';
import Logger from '../logger';
@ -12,7 +13,6 @@ import { ResourceRecreationAttemptError } from './errors';
import Volume, { VolumeConfig } from './volume';
export interface VolumeMangerConstructOpts {
docker: Docker;
logger: Logger;
}
@ -22,30 +22,23 @@ export interface VolumeNameOpts {
}
export class VolumeManager {
private docker: Docker;
private logger: Logger;
public constructor(opts: VolumeMangerConstructOpts) {
this.docker = opts.docker;
this.logger = opts.logger;
}
public async get({ name, appId }: VolumeNameOpts): Promise<Volume> {
return Volume.fromDockerVolume(
{ docker: this.docker, logger: this.logger },
await this.docker
.getVolume(Volume.generateDockerName(appId, name))
.inspect(),
{ logger: this.logger },
await docker.getVolume(Volume.generateDockerName(appId, name)).inspect(),
);
}
public async getAll(): Promise<Volume[]> {
const volumeInspect = await this.listWithBothLabels();
return volumeInspect.map((inspect) =>
Volume.fromDockerVolume(
{ logger: this.logger, docker: this.docker },
inspect,
),
Volume.fromDockerVolume({ logger: this.logger }, inspect),
);
}
@ -111,11 +104,10 @@ export class VolumeManager {
): Promise<Volume> {
const volume = Volume.fromComposeObject(name, appId, config, {
logger: this.logger,
docker: this.docker,
});
await this.create(volume);
const inspect = await this.docker
const inspect = await docker
.getVolume(Volume.generateDockerName(volume.appId, volume.name))
.inspect();
@ -139,8 +131,8 @@ export class VolumeManager {
// *all* containers. This means we don't remove
// something that's part of a sideloaded container
const [dockerContainers, dockerVolumes] = await Promise.all([
this.docker.listContainers(),
this.docker.listVolumes(),
docker.listContainers(),
docker.listVolumes(),
]);
const containerVolumes = _(dockerContainers)
@ -160,17 +152,15 @@ export class VolumeManager {
// in the target state
referencedVolumes,
);
await Promise.all(
volumesToRemove.map((v) => this.docker.getVolume(v).remove()),
);
await Promise.all(volumesToRemove.map((v) => docker.getVolume(v).remove()));
}
private async listWithBothLabels(): Promise<Docker.VolumeInspectInfo[]> {
private async listWithBothLabels(): Promise<VolumeInspectInfo[]> {
const [legacyResponse, currentResponse] = await Promise.all([
this.docker.listVolumes({
docker.listVolumes({
filters: { label: ['io.resin.supervised'] },
}),
this.docker.listVolumes({
docker.listVolumes({
filters: { label: ['io.balena.supervised'] },
}),
]);

8
src/compose/volume.ts
View File

@ -4,6 +4,7 @@ import isEqual = require('lodash/isEqual');
import omitBy = require('lodash/omitBy');
import constants = require('../lib/constants');
import { docker } from '../lib/docker-utils';
import { InternalInconsistencyError } from '../lib/errors';
import * as LogTypes from '../lib/log-types';
import { LabelObject } from '../lib/types';
@ -12,7 +13,6 @@ import * as ComposeUtils from './utils';
export interface VolumeConstructOpts {
logger: Logger;
docker: Docker;
}
export interface VolumeConfig {
@ -29,7 +29,6 @@ export interface ComposeVolumeConfig {
export class Volume {
private logger: Logger;
private docker: Docker;
private constructor(
public name: string,
@ -38,7 +37,6 @@ export class Volume {
opts: VolumeConstructOpts,
) {
this.logger = opts.logger;
this.docker = opts.docker;
}
public static fromDockerVolume(
@ -100,7 +98,7 @@ export class Volume {
this.logger.logSystemEvent(LogTypes.createVolume, {
volume: { name: this.name },
});
await this.docker.createVolume({
await docker.createVolume({
Name: Volume.generateDockerName(this.appId, this.name),
Labels: this.config.labels,
Driver: this.config.driver,
@ -114,7 +112,7 @@ export class Volume {
});
try {
await this.docker
await docker
.getVolume(Volume.generateDockerName(this.appId, this.name))
.remove();
} catch (e) {

120
src/lib/docker-utils.ts
View File

@ -42,20 +42,20 @@ interface ImageNameParts {
// (10 mins)
const DELTA_TOKEN_TIMEOUT = 10 * 60 * 1000;
export class DockerUtils extends DockerToolbelt {
public dockerProgress: DockerProgress;
export const docker = new Dockerode();
export const dockerToolbelt = new DockerToolbelt(undefined);
export const dockerProgress = new DockerProgress({
dockerToolbelt,
});
public constructor(opts?: Dockerode.DockerOptions) {
super(opts);
this.dockerProgress = new DockerProgress({ dockerToolbelt: this });
}
public async getRepoAndTag(
export async function getRepoAndTag(
image: string,
): Promise<{ repo: string; tag: string }> {
const { registry, imageName, tagName } = await this.getRegistryAndName(
image,
);
): Promise<{ repo: string; tag: string }> {
const {
registry,
imageName,
tagName,
} = await dockerToolbelt.getRegistryAndName(image);
let repoName = imageName;
@ -64,14 +64,14 @@ export class DockerUtils extends DockerToolbelt {
}
return { repo: repoName, tag: tagName };
}
}
public async fetchDeltaWithProgress(
export async function fetchDeltaWithProgress(
imgDest: string,
deltaOpts: DeltaFetchOptions,
onProgress: ProgressCallback,
serviceName: string,
): Promise<string> {
): Promise<string> {
const deltaSourceId =
deltaOpts.deltaSourceId != null
? deltaOpts.deltaSourceId
@ -86,7 +86,7 @@ export class DockerUtils extends DockerToolbelt {
logFn(
`Unsupported delta version: ${deltaOpts.deltaVersion}. Falling back to regular pull`,
);
return await this.fetchImageWithProgress(imgDest, deltaOpts, onProgress);
return await fetchImageWithProgress(imgDest, deltaOpts, onProgress);
}
// We need to make sure that we're not trying to apply a
@ -95,30 +95,27 @@ export class DockerUtils extends DockerToolbelt {
// image pull
if (
deltaOpts.deltaVersion === 3 &&
(await DockerUtils.isV2DeltaImage(this, deltaOpts.deltaSourceId))
(await isV2DeltaImage(deltaOpts.deltaSourceId))
) {
logFn(
`Cannot create a delta from V2 to V3, falling back to regular pull`,
);
return await this.fetchImageWithProgress(imgDest, deltaOpts, onProgress);
logFn(`Cannot create a delta from V2 to V3, falling back to regular pull`);
return await fetchImageWithProgress(imgDest, deltaOpts, onProgress);
}
// Since the supevisor never calls this function with a source anymore,
// this should never happen, but w ehandle it anyway
if (deltaOpts.deltaSource == null) {
logFn('Falling back to regular pull due to lack of a delta source');
return this.fetchImageWithProgress(imgDest, deltaOpts, onProgress);
return fetchImageWithProgress(imgDest, deltaOpts, onProgress);
}
const docker = this;
logFn(`Starting delta to ${imgDest}`);
const [dstInfo, srcInfo] = await Promise.all([
this.getRegistryAndName(imgDest),
this.getRegistryAndName(deltaOpts.deltaSource),
dockerToolbelt.getRegistryAndName(imgDest),
dockerToolbelt.getRegistryAndName(deltaOpts.deltaSource),
]);
const token = await this.getAuthToken(srcInfo, dstInfo, deltaOpts);
const token = await getAuthToken(srcInfo, dstInfo, deltaOpts);
const opts: request.requestLib.CoreOptions = {
followRedirect: false,
@ -160,7 +157,7 @@ export class DockerUtils extends DockerToolbelt {
maxRetries: deltaOpts.deltaRetryCount,
retryInterval: deltaOpts.deltaRetryInterval,
};
id = await DockerUtils.applyRsyncDelta(
id = await applyRsyncDelta(
deltaSrc,
deltaUrl,
timeout,
@ -183,27 +180,15 @@ export class DockerUtils extends DockerToolbelt {
`Got an error when parsing delta server response for v3 delta: ${e}`,
);
}
id = await DockerUtils.applyBalenaDelta(
docker,
name,
token,
onProgress,
logFn,
);
id = await applyBalenaDelta(name, token, onProgress, logFn);
break;
default:
throw new Error(
`Unsupported delta version: ${deltaOpts.deltaVersion}`,
);
throw new Error(`Unsupported delta version: ${deltaOpts.deltaVersion}`);
}
} catch (e) {
if (e instanceof OutOfSyncError) {
logFn('Falling back to regular pull due to delta out of sync error');
return await this.fetchImageWithProgress(
imgDest,
deltaOpts,
onProgress,
);
return await this.fetchImageWithProgress(imgDest, deltaOpts, onProgress);
} else {
logFn(`Delta failed with ${e}`);
throw e;
@ -212,14 +197,14 @@ export class DockerUtils extends DockerToolbelt {
logFn(`Delta applied successfully`);
return id;
}
}
public async fetchImageWithProgress(
export async function fetchImageWithProgress(
image: string,
{ uuid, currentApiKey }: FetchOptions,
onProgress: ProgressCallback,
): Promise<string> {
const { registry } = await this.getRegistryAndName(image);
): Promise<string> {
const { registry } = await dockerToolbelt.getRegistryAndName(image);
const dockerOpts = {
authconfig: {
@ -229,11 +214,11 @@ export class DockerUtils extends DockerToolbelt {
},
};
await this.dockerProgress.pull(image, onProgress, dockerOpts);
return (await this.getImage(image).inspect()).Id;
}
await dockerProgress.pull(image, onProgress, dockerOpts);
return (await docker.getImage(image).inspect()).Id;
}
public async getImageEnv(id: string): Promise<EnvVarObject> {
export async function getImageEnv(id: string): Promise<EnvVarObject> {
const inspect = await this.getImage(id).inspect();
try {
@ -242,9 +227,9 @@ export class DockerUtils extends DockerToolbelt {
log.error('Error getting env from image', e);
return {};
}
}
}
public async getNetworkGateway(networkName: string): Promise<string> {
export async function getNetworkGateway(networkName: string): Promise<string> {
if (networkName === 'host') {
return '127.0.0.1';
}
@ -262,16 +247,16 @@ export class DockerUtils extends DockerToolbelt {
throw new InvalidNetGatewayError(
`Cannot determine network gateway for ${networkName}`,
);
}
}
private static applyRsyncDelta(
function applyRsyncDelta(
imgSrc: string,
deltaUrl: string,
applyTimeout: number,
opts: RsyncApplyOptions,
onProgress: ProgressCallback,
logFn: (str: string) => void,
): Promise<string> {
): Promise<string> {
logFn('Applying rsync delta...');
return new Promise(async (resolve, reject) => {
@ -306,15 +291,14 @@ export class DockerUtils extends DockerToolbelt {
}
});
});
}
}
private static async applyBalenaDelta(
docker: DockerUtils,
async function applyBalenaDelta(
deltaImg: string,
token: string | null,
onProgress: ProgressCallback,
logFn: (str: string) => void,
): Promise<string> {
): Promise<string> {
logFn('Applying balena delta...');
let auth: Dictionary<unknown> | undefined;
@ -327,14 +311,11 @@ export class DockerUtils extends DockerToolbelt {
};
}
await docker.dockerProgress.pull(deltaImg, onProgress, auth);
await dockerProgress.pull(deltaImg, onProgress, auth);
return (await docker.getImage(deltaImg).inspect()).Id;
}
}
public static async isV2DeltaImage(
docker: DockerUtils,
imageName: string,
): Promise<boolean> {
export async function isV2DeltaImage(imageName: string): Promise<boolean> {
const inspect = await docker.getImage(imageName).inspect();
// It's extremely unlikely that an image is valid if
@ -342,9 +323,9 @@ export class DockerUtils extends DockerToolbelt {
// For this reason, this is the method that we use to
// detect when an image is a v2 delta
return inspect.Size < 40 && inspect.VirtualSize < 40;
}
}
private getAuthToken = memoizee(
const getAuthToken = memoizee(
async (
srcInfo: ImageNameParts,
dstInfo: ImageNameParts,
@ -373,7 +354,4 @@ export class DockerUtils extends DockerToolbelt {
return token;
},
{ maxAge: DELTA_TOKEN_TIMEOUT, promise: true },
);
}
export default DockerUtils;
);

3
src/lib/migration.ts
View File

@ -15,6 +15,7 @@ import * as db from '../db';
import DeviceState from '../device-state';
import * as constants from '../lib/constants';
import { BackupError, DatabaseParseError, NotFoundError } from '../lib/errors';
import { docker } from '../lib/docker-utils';
import { pathExistsOnHost } from '../lib/fs-utils';
import { log } from '../lib/supervisor-console';
import {
@ -179,7 +180,7 @@ export async function normaliseLegacyDatabase(
`Found a release with releaseId ${release.id}, imageId ${image.id}, serviceId ${serviceId}\nImage location is ${imageUrl}`,
);
const imageFromDocker = await application.docker
const imageFromDocker = await docker
.getImage(service.image)
.inspect()
.catch((error) => {

23
src/local-mode.ts
View File

@ -1,10 +1,10 @@
import * as Bluebird from 'bluebird';
import * as Docker from 'dockerode';
import * as _ from 'lodash';
import * as config from './config';
import * as db from './db';
import * as constants from './lib/constants';
import { docker } from './lib/docker-utils';
import { SupervisorContainerNotFoundError } from './lib/errors';
import log from './lib/supervisor-console';
import { Logger } from './logger';
@ -71,7 +71,6 @@ const SUPERVISOR_CONTAINER_NAME_FALLBACK = 'resin_supervisor';
*/
export class LocalModeManager {
public constructor(
public docker: Docker,
public logger: Logger,
private containerId: string | undefined = constants.containerId,
) {}
@ -121,16 +120,14 @@ export class LocalModeManager {
// Query the engine to get currently running containers and installed images.
public async collectEngineSnapshot(): Promise<EngineSnapshotRecord> {
const containersPromise = this.docker
const containersPromise = docker
.listContainers()
.then((resp) => _.map(resp, 'Id'));
const imagesPromise = this.docker
.listImages()
.then((resp) => _.map(resp, 'Id'));
const volumesPromise = this.docker
const imagesPromise = docker.listImages().then((resp) => _.map(resp, 'Id'));
const volumesPromise = docker
.listVolumes()
.then((resp) => _.map(resp.Volumes, 'Name'));
const networksPromise = this.docker
const networksPromise = docker
.listNetworks()
.then((resp) => _.map(resp, 'Id'));
@ -149,7 +146,7 @@ export class LocalModeManager {
private async collectContainerResources(
nameOrId: string,
): Promise<EngineSnapshot> {
const inspectInfo = await this.docker.getContainer(nameOrId).inspect();
const inspectInfo = await docker.getContainer(nameOrId).inspect();
return new EngineSnapshot(
[inspectInfo.Id],
[inspectInfo.Image],
@ -236,25 +233,25 @@ export class LocalModeManager {
// Delete engine objects. We catch every deletion error, so that we can attempt other objects deletions.
await Bluebird.map(objects.containers, (cId) => {
return this.docker
return docker
.getContainer(cId)
.remove({ force: true })
.catch((e) => log.error(`Unable to delete container ${cId}`, e));
});
await Bluebird.map(objects.images, (iId) => {
return this.docker
return docker
.getImage(iId)
.remove({ force: true })
.catch((e) => log.error(`Unable to delete image ${iId}`, e));
});
await Bluebird.map(objects.networks, (nId) => {
return this.docker
return docker
.getNetwork(nId)
.remove()
.catch((e) => log.error(`Unable to delete network ${nId}`, e));
});
await Bluebird.map(objects.volumes, (vId) => {
return this.docker
return docker
.getVolume(vId)
.remove()
.catch((e) => log.error(`Unable to delete volume ${vId}`, e));

4
src/logger.ts
View File

@ -4,7 +4,6 @@ import * as _ from 'lodash';
import * as config from './config';
import * as db from './db';
import { EventTracker } from './event-tracker';
import Docker from './lib/docker-utils';
import { LogType } from './lib/log-types';
import { writeLock } from './lib/update-lock';
import {
@ -159,7 +158,6 @@ export class Logger {
}
public attach(
docker: Docker,
containerId: string,
serviceInfo: { serviceId: number; imageId: number },
): Bluebird<void> {
@ -170,7 +168,7 @@ export class Logger {
}
return Bluebird.using(this.lock(containerId), async () => {
const logs = new ContainerLogs(containerId, docker);
const logs = new ContainerLogs(containerId);
this.containerLogs[containerId] = logs;
logs.on('error', (err) => {
log.error('Container log retrieval error', err);

6
src/logging/container.ts
View File

@ -4,7 +4,7 @@ import * as _ from 'lodash';
import * as Stream from 'stream';
import StrictEventEmitter from 'strict-event-emitter-types';
import Docker from '../lib/docker-utils';
import { docker } from '../lib/docker-utils';
export interface ContainerLog {
message: string;
@ -21,7 +21,7 @@ interface LogsEvents {
type LogsEventEmitter = StrictEventEmitter<EventEmitter, LogsEvents>;
export class ContainerLogs extends (EventEmitter as new () => LogsEventEmitter) {
public constructor(public containerId: string, private docker: Docker) {
public constructor(public containerId: string) {
super();
}
@ -34,7 +34,7 @@ export class ContainerLogs extends (EventEmitter as new () => LogsEventEmitter)
const stdoutLogOpts = { stdout: true, stderr: false, ...logOpts };
const stderrLogOpts = { stderr: true, stdout: false, ...logOpts };
const container = this.docker.getContainer(this.containerId);
const container = docker.getContainer(this.containerId);
const stdoutStream = await container.logs(stdoutLogOpts);
const stderrStream = await container.logs(stderrLogOpts);

6
src/proxyvisor.js
View File

@ -17,6 +17,7 @@ import * as url from 'url';
import { log } from './lib/supervisor-console';
import * as db from './db';
import * as config from './config';
import * as dockerUtils from './lib/docker-utils';
const mkdirpAsync = Promise.promisify(mkdirp);
@ -344,7 +345,7 @@ const createProxyvisorRouter = function (proxyvisor) {
};
export class Proxyvisor {
constructor({ logger, docker, images, applications }) {
constructor({ logger, images, applications }) {
this.bindToAPI = this.bindToAPI.bind(this);
this.executeStepAction = this.executeStepAction.bind(this);
this.getCurrentStates = this.getCurrentStates.bind(this);
@ -361,7 +362,6 @@ export class Proxyvisor {
this.sendDeleteHook = this.sendDeleteHook.bind(this);
this.sendUpdates = this.sendUpdates.bind(this);
this.logger = logger;
this.docker = docker;
this.images = images;
this.applications = applications;
this.acknowledgedState = {};
@ -904,7 +904,7 @@ export class Proxyvisor {
})
.then((parentApp) => {
return Promise.map(parentApp?.services ?? [], (service) => {
return this.docker.getImageEnv(service.image);
return dockerUtils.getImageEnv(service.image);
}).then(function (imageEnvs) {
const imageHookAddresses = _.map(
imageEnvs,

6
test/05-device-state.spec.ts
View File

@ -6,6 +6,7 @@ import { SinonSpy, SinonStub, spy, stub } from 'sinon';
import chai = require('./lib/chai-config');
import prepare = require('./lib/prepare');
import Log from '../src/lib/supervisor-console';
import * as dockerUtils from '../src/lib/docker-utils';
import * as config from '../src/config';
import { RPiConfigBackend } from '../src/config/backend';
import DeviceState from '../src/device-state';
@ -235,7 +236,7 @@ describe('deviceState', () => {
apiBinder: null as any,
});
stub(deviceState.applications.docker, 'getNetworkGateway').returns(
stub(dockerUtils, 'getNetworkGateway').returns(
Promise.resolve('172.17.0.1'),
);
@ -250,8 +251,7 @@ describe('deviceState', () => {
after(() => {
(Service as any).extendEnvVars.restore();
(deviceState.applications.docker
.getNetworkGateway as sinon.SinonStub).restore();
(dockerUtils.getNetworkGateway as sinon.SinonStub).restore();
(deviceState.applications.images
.inspectByName as sinon.SinonStub).restore();
});

16
test/14-application-manager.spec.ts
View File

@ -8,6 +8,7 @@ import Service from '../src/compose/service';
import Volume from '../src/compose/volume';
import DeviceState from '../src/device-state';
import EventTracker from '../src/event-tracker';
import * as dockerUtils from '../src/lib/docker-utils';
import chai = require('./lib/chai-config');
import prepare = require('./lib/prepare');
@ -148,13 +149,13 @@ describe('ApplicationManager', function () {
},
}),
);
stub(this.applications.docker, 'getNetworkGateway').returns(
stub(dockerUtils, 'getNetworkGateway').returns(
Bluebird.Promise.resolve('172.17.0.1'),
);
stub(this.applications.docker, 'listContainers').returns(
stub(dockerUtils.docker, 'listContainers').returns(
Bluebird.Promise.resolve([]),
);
stub(this.applications.docker, 'listImages').returns(
stub(dockerUtils.docker, 'listImages').returns(
Bluebird.Promise.resolve([]),
);
stub(Service as any, 'extendEnvVars').callsFake(function (env) {
@ -174,7 +175,6 @@ describe('ApplicationManager', function () {
appCloned.networks,
(config, name) => {
return Network.fromComposeObject(name, app.appId, config, {
docker: this.applications.docker,
logger: this.logger,
});
},
@ -235,8 +235,12 @@ describe('ApplicationManager', function () {
after(function () {
this.applications.images.inspectByName.restore();
this.applications.docker.getNetworkGateway.restore();
this.applications.docker.listContainers.restore();
// @ts-expect-error restore on non-stubbed type
dockerUtils.getNetworkGateway.restore();
// @ts-expect-error restore on non-stubbed type
dockerUtils.docker.listContainers.restore();
// @ts-expect-error restore on non-stubbed type
dockerUtils.docker.listImages.restore();
return (Service as any).extendEnvVars.restore();
});

6
test/18-startup.ts
View File

@ -4,7 +4,7 @@ import APIBinder from '../src/api-binder';
import { ApplicationManager } from '../src/application-manager';
import DeviceState from '../src/device-state';
import * as constants from '../src/lib/constants';
import { DockerUtils as Docker } from '../src/lib/docker-utils';
import { docker } from '../src/lib/docker-utils';
import { Supervisor } from '../src/supervisor';
import { expect } from './lib/chai-config';
@ -30,9 +30,7 @@ describe('Startup', () => {
deviceStateStub = stub(DeviceState.prototype as any, 'applyTarget').returns(
Promise.resolve(),
);
dockerStub = stub(Docker.prototype, 'listContainers').returns(
Promise.resolve([]),
);
dockerStub = stub(docker, 'listContainers').returns(Promise.resolve([]));
});
after(() => {

21
test/20-compose-volume.ts
View File

@ -1,5 +1,7 @@
import { expect } from 'chai';
import { stub } from 'sinon';
import { stub, SinonStub } from 'sinon';
import { docker } from '../src/lib/docker-utils';
import Volume from '../src/compose/volume';
import logTypes = require('../src/lib/log-types');
@ -8,13 +10,18 @@ const fakeLogger = {
logSystemMessage: stub(),
logSystemEvent: stub(),
};
const fakeDocker = {
createVolume: stub(),
};
const opts: any = { logger: fakeLogger, docker: fakeDocker };
const opts: any = { logger: fakeLogger };
describe('Compose volumes', () => {
let createVolumeStub: SinonStub;
before(() => {
createVolumeStub = stub(docker, 'createVolume');
});
after(() => {
createVolumeStub.restore();
});
describe('Parsing volumes', () => {
it('should correctly parse docker volumes', () => {
const volume = Volume.fromDockerVolume(opts, {
@ -122,7 +129,7 @@ describe('Compose volumes', () => {
describe('Generating docker options', () => {
afterEach(() => {
fakeDocker.createVolume.reset();
createVolumeStub.reset();
fakeLogger.logSystemEvent.reset();
fakeLogger.logSystemMessage.reset();
});
@ -143,7 +150,7 @@ describe('Compose volumes', () => {
await volume.create();
expect(
fakeDocker.createVolume.calledWith({
createVolumeStub.calledWith({
Labels: {
'my-label': 'test-label',
'io.balena.supervised': 'true',

19
test/23-local-mode.ts
View File

@ -4,6 +4,7 @@ import * as Docker from 'dockerode';
import * as sinon from 'sinon';
import * as db from '../src/db';
import { docker } from '../src/lib/docker-utils';
import LocalModeManager, {
EngineSnapshot,
EngineSnapshotRecord,
@ -13,7 +14,7 @@ import ShortStackError from './lib/errors';
describe('LocalModeManager', () => {
let localMode: LocalModeManager;
let dockerStub: sinon.SinonStubbedInstance<Docker>;
let dockerStub: sinon.SinonStubbedInstance<typeof docker>;
const supervisorContainerId = 'super-container-1';
@ -32,14 +33,14 @@ describe('LocalModeManager', () => {
before(async () => {
await db.initialized;
dockerStub = sinon.createStubInstance(Docker);
dockerStub = sinon.stub(docker);
const loggerStub = (sinon.createStubInstance(Logger) as unknown) as Logger;
localMode = new LocalModeManager(
(dockerStub as unknown) as Docker,
loggerStub,
supervisorContainerId,
);
localMode = new LocalModeManager(loggerStub, supervisorContainerId);
});
after(async () => {
sinon.restore();
});
describe('EngineSnapshot', () => {
@ -427,8 +428,4 @@ describe('LocalModeManager', () => {
});
});
});
after(async () => {
sinon.restore();
});
});

8
test/25-deltas.spec.ts
View File

@ -1,13 +1,11 @@
import { expect } from 'chai';
import { stub } from 'sinon';
import DockerUtils from '../src/lib/docker-utils';
const dockerUtils = new DockerUtils({});
import * as dockerUtils from '../src/lib/docker-utils';
describe('Deltas', () => {
it('should correctly detect a V2 delta', async () => {
const imageStub = stub(dockerUtils, 'getImage').returns({
const imageStub = stub(dockerUtils.docker, 'getImage').returns({
inspect: () => {
return Promise.resolve({
Id:
@ -99,7 +97,7 @@ describe('Deltas', () => {
},
} as any);
expect(await DockerUtils.isV2DeltaImage(dockerUtils, 'test')).to.be.true;
expect(await dockerUtils.isV2DeltaImage('test')).to.be.true;
expect(imageStub.callCount).to.equal(1);
imageStub.restore();
});