mirror of
https://github.com/balena-os/balena-supervisor.git
synced 2025-05-05 02:22:56 +00:00
Make service-manager module a singleton
Change-type: patch Signed-off-by: Cameron Diver <cameron@balena.io>
This commit is contained in:
parent
adaad786af
commit
0e8d92e08a
3
src/application-manager.d.ts
vendored
3
src/application-manager.d.ts
vendored
@ -7,7 +7,6 @@ import { ServiceAction } from './device-api/common';
|
|||||||
import { DeviceStatus, InstancedAppState } from './types/state';
|
import { DeviceStatus, InstancedAppState } from './types/state';
|
||||||
|
|
||||||
import type { Image } from './compose/images';
|
import type { Image } from './compose/images';
|
||||||
import ServiceManager from './compose/service-manager';
|
|
||||||
import DeviceState from './device-state';
|
import DeviceState from './device-state';
|
||||||
|
|
||||||
import { APIBinder } from './api-binder';
|
import { APIBinder } from './api-binder';
|
||||||
@ -45,8 +44,6 @@ class ApplicationManager extends EventEmitter {
|
|||||||
public deviceState: DeviceState;
|
public deviceState: DeviceState;
|
||||||
public apiBinder: APIBinder;
|
public apiBinder: APIBinder;
|
||||||
|
|
||||||
public services: ServiceManager;
|
|
||||||
|
|
||||||
public proxyvisor: any;
|
public proxyvisor: any;
|
||||||
public timeSpentFetching: number;
|
public timeSpentFetching: number;
|
||||||
public fetchesInProgress: number;
|
public fetchesInProgress: number;
|
||||||
|
@ -21,11 +21,11 @@ import {
|
|||||||
|
|
||||||
import * as dbFormat from './device-state/db-format';
|
import * as dbFormat from './device-state/db-format';
|
||||||
|
|
||||||
import { ServiceManager } from './compose/service-manager';
|
|
||||||
import * as Images from './compose/images';
|
import * as Images from './compose/images';
|
||||||
import { Network } from './compose/network';
|
import { Network } from './compose/network';
|
||||||
import * as networkManager from './compose/network-manager';
|
import * as networkManager from './compose/network-manager';
|
||||||
import * as volumeManager from './compose/volume-manager';
|
import * as volumeManager from './compose/volume-manager';
|
||||||
|
import * as serviceManager from './compose/service-manager';
|
||||||
import * as compositionSteps from './compose/composition-steps';
|
import * as compositionSteps from './compose/composition-steps';
|
||||||
|
|
||||||
import { Proxyvisor } from './proxyvisor';
|
import { Proxyvisor } from './proxyvisor';
|
||||||
@ -159,7 +159,6 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
this.deviceState = deviceState;
|
this.deviceState = deviceState;
|
||||||
this.apiBinder = apiBinder;
|
this.apiBinder = apiBinder;
|
||||||
|
|
||||||
this.services = new ServiceManager();
|
|
||||||
this.proxyvisor = new Proxyvisor({
|
this.proxyvisor = new Proxyvisor({
|
||||||
applications: this,
|
applications: this,
|
||||||
});
|
});
|
||||||
@ -171,7 +170,6 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
|
|
||||||
this.actionExecutors = compositionSteps.getExecutors({
|
this.actionExecutors = compositionSteps.getExecutors({
|
||||||
lockFn: this._lockingIfNecessary,
|
lockFn: this._lockingIfNecessary,
|
||||||
services: this.services,
|
|
||||||
applications: this,
|
applications: this,
|
||||||
callbacks: {
|
callbacks: {
|
||||||
containerStarted: (id) => {
|
containerStarted: (id) => {
|
||||||
@ -198,7 +196,7 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
);
|
);
|
||||||
this.router = createApplicationManagerRouter(this);
|
this.router = createApplicationManagerRouter(this);
|
||||||
Images.on('change', this.reportCurrentState);
|
Images.on('change', this.reportCurrentState);
|
||||||
this.services.on('change', this.reportCurrentState);
|
serviceManager.on('change', this.reportCurrentState);
|
||||||
}
|
}
|
||||||
|
|
||||||
reportCurrentState(data) {
|
reportCurrentState(data) {
|
||||||
@ -223,14 +221,14 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
// But also run it in on startup
|
// But also run it in on startup
|
||||||
await cleanup();
|
await cleanup();
|
||||||
await this.localModeManager.init();
|
await this.localModeManager.init();
|
||||||
await this.services.attachToRunning();
|
await serviceManager.attachToRunning();
|
||||||
await this.services.listenToEvents();
|
await serviceManager.listenToEvents();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns the status of applications and their services
|
// Returns the status of applications and their services
|
||||||
getStatus() {
|
getStatus() {
|
||||||
return Promise.join(
|
return Promise.join(
|
||||||
this.services.getStatus(),
|
serviceManager.getStatus(),
|
||||||
Images.getStatus(),
|
Images.getStatus(),
|
||||||
config.get('currentCommit'),
|
config.get('currentCommit'),
|
||||||
function (services, images, currentCommit) {
|
function (services, images, currentCommit) {
|
||||||
@ -362,7 +360,7 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
|
|
||||||
getCurrentForComparison() {
|
getCurrentForComparison() {
|
||||||
return Promise.join(
|
return Promise.join(
|
||||||
this.services.getAll(),
|
serviceManager.getAll(),
|
||||||
networkManager.getAll(),
|
networkManager.getAll(),
|
||||||
volumeManager.getAll(),
|
volumeManager.getAll(),
|
||||||
config.get('currentCommit'),
|
config.get('currentCommit'),
|
||||||
@ -372,7 +370,7 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
|
|
||||||
getCurrentApp(appId) {
|
getCurrentApp(appId) {
|
||||||
return Promise.join(
|
return Promise.join(
|
||||||
this.services.getAllByAppId(appId),
|
serviceManager.getAllByAppId(appId),
|
||||||
networkManager.getAllByAppId(appId),
|
networkManager.getAllByAppId(appId),
|
||||||
volumeManager.getAllByAppId(appId),
|
volumeManager.getAllByAppId(appId),
|
||||||
config.get('currentCommit'),
|
config.get('currentCommit'),
|
||||||
@ -1339,13 +1337,13 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
stopAll({ force = false, skipLock = false } = {}) {
|
stopAll({ force = false, skipLock = false } = {}) {
|
||||||
return Promise.resolve(this.services.getAll())
|
return Promise.resolve(serviceManager.getAll())
|
||||||
.map((service) => {
|
.map((service) => {
|
||||||
return this._lockingIfNecessary(
|
return this._lockingIfNecessary(
|
||||||
service.appId,
|
service.appId,
|
||||||
{ force, skipLock },
|
{ force, skipLock },
|
||||||
() => {
|
() => {
|
||||||
return this.services
|
return serviceManager
|
||||||
.kill(service, { removeContainer: false, wait: true })
|
.kill(service, { removeContainer: false, wait: true })
|
||||||
.then(() => {
|
.then(() => {
|
||||||
delete this._containerStarted[service.containerId];
|
delete this._containerStarted[service.containerId];
|
||||||
@ -1391,7 +1389,7 @@ export class ApplicationManager extends EventEmitter {
|
|||||||
if (intId == null) {
|
if (intId == null) {
|
||||||
throw new Error(`Invalid id: ${id}`);
|
throw new Error(`Invalid id: ${id}`);
|
||||||
}
|
}
|
||||||
containerIdsByAppId[intId] = this.services.getContainerIdMap(intId);
|
containerIdsByAppId[intId] = serviceManager.getContainerIdMap(intId);
|
||||||
});
|
});
|
||||||
|
|
||||||
return config.get('localMode').then((localMode) => {
|
return config.get('localMode').then((localMode) => {
|
||||||
|
@ -7,7 +7,7 @@ import type { Image } from './images';
|
|||||||
import * as images from './images';
|
import * as images from './images';
|
||||||
import Network from './network';
|
import Network from './network';
|
||||||
import Service from './service';
|
import Service from './service';
|
||||||
import ServiceManager from './service-manager';
|
import * as serviceManager from './service-manager';
|
||||||
import Volume from './volume';
|
import Volume from './volume';
|
||||||
|
|
||||||
import { checkTruthy } from '../lib/validation';
|
import { checkTruthy } from '../lib/validation';
|
||||||
@ -136,7 +136,6 @@ interface CompositionCallbacks {
|
|||||||
|
|
||||||
export function getExecutors(app: {
|
export function getExecutors(app: {
|
||||||
lockFn: LockingFn;
|
lockFn: LockingFn;
|
||||||
services: ServiceManager;
|
|
||||||
applications: ApplicationManager;
|
applications: ApplicationManager;
|
||||||
callbacks: CompositionCallbacks;
|
callbacks: CompositionCallbacks;
|
||||||
}) {
|
}) {
|
||||||
@ -150,7 +149,7 @@ export function getExecutors(app: {
|
|||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
const wait = _.get(step, ['options', 'wait'], false);
|
const wait = _.get(step, ['options', 'wait'], false);
|
||||||
await app.services.kill(step.current, {
|
await serviceManager.kill(step.current, {
|
||||||
removeContainer: false,
|
removeContainer: false,
|
||||||
wait,
|
wait,
|
||||||
});
|
});
|
||||||
@ -166,7 +165,7 @@ export function getExecutors(app: {
|
|||||||
skipLock: step.skipLock || _.get(step, ['options', 'skipLock']),
|
skipLock: step.skipLock || _.get(step, ['options', 'skipLock']),
|
||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
await app.services.kill(step.current);
|
await serviceManager.kill(step.current);
|
||||||
app.callbacks.containerKilled(step.current.containerId);
|
app.callbacks.containerKilled(step.current.containerId);
|
||||||
if (_.get(step, ['options', 'removeImage'])) {
|
if (_.get(step, ['options', 'removeImage'])) {
|
||||||
await images.removeByDockerId(step.current.config.image);
|
await images.removeByDockerId(step.current.config.image);
|
||||||
@ -177,7 +176,7 @@ export function getExecutors(app: {
|
|||||||
remove: async (step) => {
|
remove: async (step) => {
|
||||||
// Only called for dead containers, so no need to
|
// Only called for dead containers, so no need to
|
||||||
// take locks
|
// take locks
|
||||||
await app.services.remove(step.current);
|
await serviceManager.remove(step.current);
|
||||||
},
|
},
|
||||||
updateMetadata: (step) => {
|
updateMetadata: (step) => {
|
||||||
const skipLock =
|
const skipLock =
|
||||||
@ -190,7 +189,7 @@ export function getExecutors(app: {
|
|||||||
skipLock: skipLock || _.get(step, ['options', 'skipLock']),
|
skipLock: skipLock || _.get(step, ['options', 'skipLock']),
|
||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
await app.services.updateMetadata(step.current, step.target);
|
await serviceManager.updateMetadata(step.current, step.target);
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
@ -202,9 +201,9 @@ export function getExecutors(app: {
|
|||||||
skipLock: step.skipLock || _.get(step, ['options', 'skipLock']),
|
skipLock: step.skipLock || _.get(step, ['options', 'skipLock']),
|
||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
await app.services.kill(step.current, { wait: true });
|
await serviceManager.kill(step.current, { wait: true });
|
||||||
app.callbacks.containerKilled(step.current.containerId);
|
app.callbacks.containerKilled(step.current.containerId);
|
||||||
const container = await app.services.start(step.target);
|
const container = await serviceManager.start(step.target);
|
||||||
app.callbacks.containerStarted(container.id);
|
app.callbacks.containerStarted(container.id);
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
@ -216,7 +215,7 @@ export function getExecutors(app: {
|
|||||||
});
|
});
|
||||||
},
|
},
|
||||||
start: async (step) => {
|
start: async (step) => {
|
||||||
const container = await app.services.start(step.target);
|
const container = await serviceManager.start(step.target);
|
||||||
app.callbacks.containerStarted(container.id);
|
app.callbacks.containerStarted(container.id);
|
||||||
},
|
},
|
||||||
updateCommit: async (step) => {
|
updateCommit: async (step) => {
|
||||||
@ -230,7 +229,7 @@ export function getExecutors(app: {
|
|||||||
skipLock: step.skipLock || _.get(step, ['options', 'skipLock']),
|
skipLock: step.skipLock || _.get(step, ['options', 'skipLock']),
|
||||||
},
|
},
|
||||||
async () => {
|
async () => {
|
||||||
await app.services.handover(step.current, step.target);
|
await serviceManager.handover(step.current, step.target);
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
|
@ -32,39 +32,42 @@ type ServiceManagerEventEmitter = StrictEventEmitter<
|
|||||||
EventEmitter,
|
EventEmitter,
|
||||||
ServiceManagerEvents
|
ServiceManagerEvents
|
||||||
>;
|
>;
|
||||||
|
const events: ServiceManagerEventEmitter = new EventEmitter();
|
||||||
|
|
||||||
interface KillOpts {
|
interface KillOpts {
|
||||||
removeContainer?: boolean;
|
removeContainer?: boolean;
|
||||||
wait?: boolean;
|
wait?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ServiceManager extends (EventEmitter as new () => ServiceManagerEventEmitter) {
|
export const on: typeof events['on'] = events.on.bind(events);
|
||||||
// Whether a container has died, indexed by ID
|
export const once: typeof events['once'] = events.once.bind(events);
|
||||||
private containerHasDied: Dictionary<boolean> = {};
|
export const removeListener: typeof events['removeListener'] = events.removeListener.bind(
|
||||||
private listening = false;
|
events,
|
||||||
// Volatile state of containers, indexed by containerId (or random strings if
|
);
|
||||||
// we don't yet have an id)
|
export const removeAllListeners: typeof events['removeAllListeners'] = events.removeAllListeners.bind(
|
||||||
private volatileState: Dictionary<Partial<Service>> = {};
|
events,
|
||||||
|
);
|
||||||
|
|
||||||
public constructor() {
|
// Whether a container has died, indexed by ID
|
||||||
super();
|
const containerHasDied: Dictionary<boolean> = {};
|
||||||
}
|
let listening = false;
|
||||||
|
// Volatile state of containers, indexed by containerId (or random strings if
|
||||||
|
// we don't yet have an id)
|
||||||
|
const volatileState: Dictionary<Partial<Service>> = {};
|
||||||
|
|
||||||
public async getAll(
|
export async function getAll(
|
||||||
extraLabelFilters: string | string[] = [],
|
extraLabelFilters: string | string[] = [],
|
||||||
): Promise<Service[]> {
|
): Promise<Service[]> {
|
||||||
const filterLabels = ['supervised'].concat(extraLabelFilters);
|
const filterLabels = ['supervised'].concat(extraLabelFilters);
|
||||||
const containers = await this.listWithBothLabels(filterLabels);
|
const containers = await listWithBothLabels(filterLabels);
|
||||||
|
|
||||||
const services = await Bluebird.map(containers, async (container) => {
|
const services = await Bluebird.map(containers, async (container) => {
|
||||||
try {
|
try {
|
||||||
const serviceInspect = await docker
|
const serviceInspect = await docker.getContainer(container.Id).inspect();
|
||||||
.getContainer(container.Id)
|
|
||||||
.inspect();
|
|
||||||
const service = Service.fromDockerContainer(serviceInspect);
|
const service = Service.fromDockerContainer(serviceInspect);
|
||||||
// We know that the containerId is set below, because `fromDockerContainer`
|
// We know that the containerId is set below, because `fromDockerContainer`
|
||||||
// always sets it
|
// always sets it
|
||||||
const vState = this.volatileState[service.containerId!];
|
const vState = volatileState[service.containerId!];
|
||||||
if (vState != null && vState.status != null) {
|
if (vState != null && vState.status != null) {
|
||||||
service.status = vState.status;
|
service.status = vState.status;
|
||||||
}
|
}
|
||||||
@ -78,13 +81,13 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
});
|
});
|
||||||
|
|
||||||
return services.filter((s) => s != null) as Service[];
|
return services.filter((s) => s != null) as Service[];
|
||||||
}
|
}
|
||||||
|
|
||||||
public async get(service: Service) {
|
export async function get(service: Service) {
|
||||||
// Get the container ids for special network handling
|
// Get the container ids for special network handling
|
||||||
const containerIds = await this.getContainerIdMap(service.appId!);
|
const containerIds = await getContainerIdMap(service.appId!);
|
||||||
const services = (
|
const services = (
|
||||||
await this.getAll(`service-id=${service.serviceId}`)
|
await getAll(`service-id=${service.serviceId}`)
|
||||||
).filter((currentService) =>
|
).filter((currentService) =>
|
||||||
currentService.isEqualConfig(service, containerIds),
|
currentService.isEqualConfig(service, containerIds),
|
||||||
);
|
);
|
||||||
@ -97,11 +100,11 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
return services[0];
|
return services[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getStatus() {
|
export async function getStatus() {
|
||||||
const services = await this.getAll();
|
const services = await getAll();
|
||||||
const status = _.clone(this.volatileState);
|
const status = _.clone(volatileState);
|
||||||
|
|
||||||
for (const service of services) {
|
for (const service of services) {
|
||||||
if (service.containerId == null) {
|
if (service.containerId == null) {
|
||||||
@ -123,11 +126,11 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
}
|
}
|
||||||
|
|
||||||
return _.values(status);
|
return _.values(status);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getByDockerContainerId(
|
export async function getByDockerContainerId(
|
||||||
containerId: string,
|
containerId: string,
|
||||||
): Promise<Service | null> {
|
): Promise<Service | null> {
|
||||||
const container = await docker.getContainer(containerId).inspect();
|
const container = await docker.getContainer(containerId).inspect();
|
||||||
if (
|
if (
|
||||||
container.Config.Labels['io.balena.supervised'] == null &&
|
container.Config.Labels['io.balena.supervised'] == null &&
|
||||||
@ -136,13 +139,13 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return Service.fromDockerContainer(container);
|
return Service.fromDockerContainer(container);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async updateMetadata(
|
export async function updateMetadata(
|
||||||
service: Service,
|
service: Service,
|
||||||
metadata: { imageId: number; releaseId: number },
|
metadata: { imageId: number; releaseId: number },
|
||||||
) {
|
) {
|
||||||
const svc = await this.get(service);
|
const svc = await get(service);
|
||||||
if (svc.containerId == null) {
|
if (svc.containerId == null) {
|
||||||
throw new InternalInconsistencyError(
|
throw new InternalInconsistencyError(
|
||||||
`No containerId provided for service ${service.serviceName} in ServiceManager.updateMetadata. Service: ${service}`,
|
`No containerId provided for service ${service.serviceName} in ServiceManager.updateMetadata. Service: ${service}`,
|
||||||
@ -152,48 +155,47 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
await docker.getContainer(svc.containerId).rename({
|
await docker.getContainer(svc.containerId).rename({
|
||||||
name: `${service.serviceName}_${metadata.imageId}_${metadata.releaseId}`,
|
name: `${service.serviceName}_${metadata.imageId}_${metadata.releaseId}`,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public async handover(current: Service, target: Service) {
|
export async function handover(current: Service, target: Service) {
|
||||||
// We set the running container to not restart so that in case of a poweroff
|
// We set the running container to not restart so that in case of a poweroff
|
||||||
// it doesn't come back after boot.
|
// it doesn't come back after boot.
|
||||||
await this.prepareForHandover(current);
|
await prepareForHandover(current);
|
||||||
await this.start(target);
|
await start(target);
|
||||||
await this.waitToKill(
|
await waitToKill(
|
||||||
current,
|
current,
|
||||||
target.config.labels['io.balena.update.handover-timeout'],
|
target.config.labels['io.balena.update.handover-timeout'],
|
||||||
);
|
);
|
||||||
await this.kill(current);
|
await kill(current);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async killAllLegacy(): Promise<void> {
|
export async function killAllLegacy(): Promise<void> {
|
||||||
// Containers haven't been normalized (this is an updated supervisor)
|
// Containers haven't been normalized (this is an updated supervisor)
|
||||||
// so we need to stop and remove them
|
|
||||||
const supervisorImageId = (
|
const supervisorImageId = (
|
||||||
await docker.getImage(constants.supervisorImage).inspect()
|
await docker.getImage(constants.supervisorImage).inspect()
|
||||||
).Id;
|
).Id;
|
||||||
|
|
||||||
for (const container of await docker.listContainers({ all: true })) {
|
for (const container of await docker.listContainers({ all: true })) {
|
||||||
if (container.ImageID !== supervisorImageId) {
|
if (container.ImageID !== supervisorImageId) {
|
||||||
await this.killContainer(container.Id, {
|
await killContainer(container.Id, {
|
||||||
serviceName: 'legacy',
|
serviceName: 'legacy',
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public kill(service: Service, opts: KillOpts = {}) {
|
export function kill(service: Service, opts: KillOpts = {}) {
|
||||||
if (service.containerId == null) {
|
if (service.containerId == null) {
|
||||||
throw new InternalInconsistencyError(
|
throw new InternalInconsistencyError(
|
||||||
`Attempt to kill container without containerId! Service :${service}`,
|
`Attempt to kill container without containerId! Service :${service}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return this.killContainer(service.containerId, service, opts);
|
return killContainer(service.containerId, service, opts);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async remove(service: Service) {
|
export async function remove(service: Service) {
|
||||||
logger.logSystemEvent(LogTypes.removeDeadService, { service });
|
logger.logSystemEvent(LogTypes.removeDeadService, { service });
|
||||||
const existingService = await this.get(service);
|
const existingService = await get(service);
|
||||||
|
|
||||||
if (existingService.containerId == null) {
|
if (existingService.containerId == null) {
|
||||||
throw new InternalInconsistencyError(
|
throw new InternalInconsistencyError(
|
||||||
@ -202,9 +204,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await docker
|
await docker.getContainer(existingService.containerId).remove({ v: true });
|
||||||
.getContainer(existingService.containerId)
|
|
||||||
.remove({ v: true });
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (!NotFoundError(e)) {
|
if (!NotFoundError(e)) {
|
||||||
logger.logSystemEvent(LogTypes.removeDeadServiceError, {
|
logger.logSystemEvent(LogTypes.removeDeadServiceError, {
|
||||||
@ -214,21 +214,21 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
public getAllByAppId(appId: number) {
|
export function getAllByAppId(appId: number) {
|
||||||
return this.getAll(`app-id=${appId}`);
|
return getAll(`app-id=${appId}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async stopAllByAppId(appId: number) {
|
export async function stopAllByAppId(appId: number) {
|
||||||
for (const app of await this.getAllByAppId(appId)) {
|
for (const app of await getAllByAppId(appId)) {
|
||||||
await this.kill(app, { removeContainer: false });
|
await kill(app, { removeContainer: false });
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async create(service: Service) {
|
export async function create(service: Service) {
|
||||||
const mockContainerId = config.newUniqueKey();
|
const mockContainerId = config.newUniqueKey();
|
||||||
try {
|
try {
|
||||||
const existing = await this.get(service);
|
const existing = await get(service);
|
||||||
if (existing.containerId == null) {
|
if (existing.containerId == null) {
|
||||||
throw new InternalInconsistencyError(
|
throw new InternalInconsistencyError(
|
||||||
`No containerId provided for service ${service.serviceName} in ServiceManager.updateMetadata. Service: ${service}`,
|
`No containerId provided for service ${service.serviceName} in ServiceManager.updateMetadata. Service: ${service}`,
|
||||||
@ -258,17 +258,15 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
'Attempt to start a service without an existing application ID',
|
'Attempt to start a service without an existing application ID',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
const serviceContainerIds = await this.getContainerIdMap(service.appId);
|
const serviceContainerIds = await getContainerIdMap(service.appId);
|
||||||
const conf = service.toDockerContainer({
|
const conf = service.toDockerContainer({
|
||||||
deviceName,
|
deviceName,
|
||||||
containerIds: serviceContainerIds,
|
containerIds: serviceContainerIds,
|
||||||
});
|
});
|
||||||
const nets = serviceNetworksToDockerNetworks(
|
const nets = serviceNetworksToDockerNetworks(service.extraNetworksToJoin());
|
||||||
service.extraNetworksToJoin(),
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.logSystemEvent(LogTypes.installService, { service });
|
logger.logSystemEvent(LogTypes.installService, { service });
|
||||||
this.reportNewStatus(mockContainerId, service, 'Installing');
|
reportNewStatus(mockContainerId, service, 'Installing');
|
||||||
|
|
||||||
const container = await docker.createContainer(conf);
|
const container = await docker.createContainer(conf);
|
||||||
service.containerId = container.id;
|
service.containerId = container.id;
|
||||||
@ -285,22 +283,22 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
logger.logSystemEvent(LogTypes.installServiceSuccess, { service });
|
logger.logSystemEvent(LogTypes.installServiceSuccess, { service });
|
||||||
return container;
|
return container;
|
||||||
} finally {
|
} finally {
|
||||||
this.reportChange(mockContainerId);
|
reportChange(mockContainerId);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public async start(service: Service) {
|
export async function start(service: Service) {
|
||||||
let alreadyStarted = false;
|
let alreadyStarted = false;
|
||||||
let containerId: string | null = null;
|
let containerId: string | null = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const container = await this.create(service);
|
const container = await create(service);
|
||||||
containerId = container.id;
|
containerId = container.id;
|
||||||
logger.logSystemEvent(LogTypes.startService, { service });
|
logger.logSystemEvent(LogTypes.startService, { service });
|
||||||
|
|
||||||
this.reportNewStatus(containerId, service, 'Starting');
|
reportNewStatus(containerId, service, 'Starting');
|
||||||
|
|
||||||
let remove = false;
|
let shouldRemove = false;
|
||||||
let err: Error | undefined;
|
let err: Error | undefined;
|
||||||
try {
|
try {
|
||||||
await container.start();
|
await container.start();
|
||||||
@ -309,10 +307,8 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
// definitely an int for comparison reasons
|
// definitely an int for comparison reasons
|
||||||
const maybeStatusCode = PermissiveNumber.decode(e.statusCode);
|
const maybeStatusCode = PermissiveNumber.decode(e.statusCode);
|
||||||
if (isLeft(maybeStatusCode)) {
|
if (isLeft(maybeStatusCode)) {
|
||||||
remove = true;
|
shouldRemove = true;
|
||||||
err = new Error(
|
err = new Error(`Could not parse status code from docker error: ${e}`);
|
||||||
`Could not parse status code from docker error: ${e}`,
|
|
||||||
);
|
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
const statusCode = maybeStatusCode.right;
|
const statusCode = maybeStatusCode.right;
|
||||||
@ -338,7 +334,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (remove) {
|
if (shouldRemove) {
|
||||||
// If starting the container fialed, we remove it so that it doesn't litter
|
// If starting the container fialed, we remove it so that it doesn't litter
|
||||||
await container.remove({ v: true }).catch(_.noop);
|
await container.remove({ v: true }).catch(_.noop);
|
||||||
logger.logSystemEvent(LogTypes.startServiceError, {
|
logger.logSystemEvent(LogTypes.startServiceError, {
|
||||||
@ -366,17 +362,17 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
return container;
|
return container;
|
||||||
} finally {
|
} finally {
|
||||||
if (containerId != null) {
|
if (containerId != null) {
|
||||||
this.reportChange(containerId);
|
reportChange(containerId);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public listenToEvents() {
|
export function listenToEvents() {
|
||||||
if (this.listening) {
|
if (listening) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.listening = true;
|
listening = true;
|
||||||
|
|
||||||
const listen = async () => {
|
const listen = async () => {
|
||||||
const stream = await docker.getEvents({
|
const stream = await docker.getEvents({
|
||||||
@ -394,22 +390,19 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
try {
|
try {
|
||||||
let service: Service | null = null;
|
let service: Service | null = null;
|
||||||
try {
|
try {
|
||||||
service = await this.getByDockerContainerId(data.id);
|
service = await getByDockerContainerId(data.id);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (!NotFoundError(e)) {
|
if (!NotFoundError(e)) {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (service != null) {
|
if (service != null) {
|
||||||
this.emit('change');
|
events.emit('change');
|
||||||
if (status === 'die') {
|
if (status === 'die') {
|
||||||
logger.logSystemEvent(LogTypes.serviceExit, { service });
|
logger.logSystemEvent(LogTypes.serviceExit, { service });
|
||||||
this.containerHasDied[data.id] = true;
|
containerHasDied[data.id] = true;
|
||||||
} else if (
|
} else if (status === 'start' && containerHasDied[data.id]) {
|
||||||
status === 'start' &&
|
delete containerHasDied[data.id];
|
||||||
this.containerHasDied[data.id]
|
|
||||||
) {
|
|
||||||
delete this.containerHasDied[data.id];
|
|
||||||
logger.logSystemEvent(LogTypes.serviceRestart, {
|
logger.logSystemEvent(LogTypes.serviceRestart, {
|
||||||
service,
|
service,
|
||||||
});
|
});
|
||||||
@ -450,15 +443,15 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
log.error('Error listening to events:', e, e.stack);
|
log.error('Error listening to events:', e, e.stack);
|
||||||
})
|
})
|
||||||
.finally(() => {
|
.finally(() => {
|
||||||
this.listening = false;
|
listening = false;
|
||||||
setTimeout(() => this.listenToEvents(), 1000);
|
setTimeout(listenToEvents, 1000);
|
||||||
});
|
});
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async attachToRunning() {
|
export async function attachToRunning() {
|
||||||
const services = await this.getAll();
|
const services = await getAll();
|
||||||
for (const service of services) {
|
for (const service of services) {
|
||||||
if (service.status === 'Running') {
|
if (service.status === 'Running') {
|
||||||
const serviceId = service.serviceId;
|
const serviceId = service.serviceId;
|
||||||
@ -480,46 +473,47 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async getContainerIdMap(appId: number): Promise<Dictionary<string>> {
|
export async function getContainerIdMap(
|
||||||
return _(await this.getAllByAppId(appId))
|
appId: number,
|
||||||
|
): Promise<Dictionary<string>> {
|
||||||
|
return _(await getAllByAppId(appId))
|
||||||
.keyBy('serviceName')
|
.keyBy('serviceName')
|
||||||
.mapValues('containerId')
|
.mapValues('containerId')
|
||||||
.value() as Dictionary<string>;
|
.value() as Dictionary<string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
private reportChange(containerId?: string, status?: Partial<Service>) {
|
function reportChange(containerId?: string, status?: Partial<Service>) {
|
||||||
if (containerId != null) {
|
if (containerId != null) {
|
||||||
if (status != null) {
|
if (status != null) {
|
||||||
this.volatileState[containerId] = {};
|
volatileState[containerId] = { ...status };
|
||||||
_.merge(this.volatileState[containerId], status);
|
} else if (volatileState[containerId] != null) {
|
||||||
} else if (this.volatileState[containerId] != null) {
|
delete volatileState[containerId];
|
||||||
delete this.volatileState[containerId];
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.emit('change');
|
events.emit('change');
|
||||||
}
|
}
|
||||||
|
|
||||||
private reportNewStatus(
|
function reportNewStatus(
|
||||||
containerId: string,
|
containerId: string,
|
||||||
service: Partial<Service>,
|
service: Partial<Service>,
|
||||||
status: string,
|
status: string,
|
||||||
) {
|
) {
|
||||||
this.reportChange(
|
reportChange(
|
||||||
containerId,
|
containerId,
|
||||||
_.merge(
|
_.merge(
|
||||||
{ status },
|
{ status },
|
||||||
_.pick(service, ['imageId', 'appId', 'releaseId', 'commit']),
|
_.pick(service, ['imageId', 'appId', 'releaseId', 'commit']),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
private killContainer(
|
function killContainer(
|
||||||
containerId: string,
|
containerId: string,
|
||||||
service: Partial<Service> = {},
|
service: Partial<Service> = {},
|
||||||
{ removeContainer = true, wait = false }: KillOpts = {},
|
{ removeContainer = true, wait = false }: KillOpts = {},
|
||||||
): Bluebird<void> {
|
): Bluebird<void> {
|
||||||
// To maintain compatibility of the `wait` flag, this function is not
|
// To maintain compatibility of the `wait` flag, this function is not
|
||||||
// async, but it feels like whether or not the promise should be waited on
|
// async, but it feels like whether or not the promise should be waited on
|
||||||
// should performed by the caller
|
// should performed by the caller
|
||||||
@ -528,7 +522,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
return Bluebird.try(() => {
|
return Bluebird.try(() => {
|
||||||
logger.logSystemEvent(LogTypes.stopService, { service });
|
logger.logSystemEvent(LogTypes.stopService, { service });
|
||||||
if (service.imageId != null) {
|
if (service.imageId != null) {
|
||||||
this.reportNewStatus(containerId, service, 'Stopping');
|
reportNewStatus(containerId, service, 'Stopping');
|
||||||
}
|
}
|
||||||
|
|
||||||
const containerObj = docker.getContainer(containerId);
|
const containerObj = docker.getContainer(containerId);
|
||||||
@ -566,7 +560,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
.tap(() => {
|
.tap(() => {
|
||||||
delete this.containerHasDied[containerId];
|
delete containerHasDied[containerId];
|
||||||
logger.logSystemEvent(LogTypes.stopServiceSuccess, { service });
|
logger.logSystemEvent(LogTypes.stopServiceSuccess, { service });
|
||||||
})
|
})
|
||||||
.catch((e) => {
|
.catch((e) => {
|
||||||
@ -577,7 +571,7 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
})
|
})
|
||||||
.finally(() => {
|
.finally(() => {
|
||||||
if (service.imageId != null) {
|
if (service.imageId != null) {
|
||||||
this.reportChange(containerId);
|
reportChange(containerId);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -586,11 +580,11 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async listWithBothLabels(
|
async function listWithBothLabels(
|
||||||
labelList: string[],
|
labelList: string[],
|
||||||
): Promise<Dockerode.ContainerInfo[]> {
|
): Promise<Dockerode.ContainerInfo[]> {
|
||||||
const listWithPrefix = (prefix: string) =>
|
const listWithPrefix = (prefix: string) =>
|
||||||
docker.listContainers({
|
docker.listContainers({
|
||||||
all: true,
|
all: true,
|
||||||
@ -605,10 +599,10 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
return _.unionBy(legacy, current, 'Id');
|
return _.unionBy(legacy, current, 'Id');
|
||||||
}
|
}
|
||||||
|
|
||||||
private async prepareForHandover(service: Service) {
|
async function prepareForHandover(service: Service) {
|
||||||
const svc = await this.get(service);
|
const svc = await get(service);
|
||||||
if (svc.containerId == null) {
|
if (svc.containerId == null) {
|
||||||
throw new InternalInconsistencyError(
|
throw new InternalInconsistencyError(
|
||||||
`No containerId provided for service ${service.serviceName} in ServiceManager.prepareForHandover. Service: ${service}`,
|
`No containerId provided for service ${service.serviceName} in ServiceManager.prepareForHandover. Service: ${service}`,
|
||||||
@ -619,9 +613,9 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
return await container.rename({
|
return await container.rename({
|
||||||
name: `old_${service.serviceName}_${service.imageId}_${service.imageId}_${service.releaseId}`,
|
name: `old_${service.serviceName}_${service.imageId}_${service.imageId}_${service.releaseId}`,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private waitToKill(service: Service, timeout: number | string) {
|
function waitToKill(service: Service, timeout: number | string) {
|
||||||
const pollInterval = 100;
|
const pollInterval = 100;
|
||||||
timeout = checkInt(timeout, { positive: true }) || 60000;
|
timeout = checkInt(timeout, { positive: true }) || 60000;
|
||||||
const deadline = Date.now() + timeout;
|
const deadline = Date.now() + timeout;
|
||||||
@ -651,7 +645,4 @@ export class ServiceManager extends (EventEmitter as new () => ServiceManagerEve
|
|||||||
return wait().then(() => {
|
return wait().then(() => {
|
||||||
log.success(`Handover complete for service ${service.serviceName}`);
|
log.success(`Handover complete for service ${service.serviceName}`);
|
||||||
});
|
});
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export default ServiceManager;
|
|
||||||
|
@ -10,6 +10,7 @@ import * as db from '../db';
|
|||||||
import * as logger from '../logger';
|
import * as logger from '../logger';
|
||||||
import * as images from '../compose/images';
|
import * as images from '../compose/images';
|
||||||
import * as volumeManager from '../compose/volume-manager';
|
import * as volumeManager from '../compose/volume-manager';
|
||||||
|
import * as serviceManager from '../compose/service-manager';
|
||||||
import { spawnJournalctl } from '../lib/journald';
|
import { spawnJournalctl } from '../lib/journald';
|
||||||
import {
|
import {
|
||||||
appNotFoundMessage,
|
appNotFoundMessage,
|
||||||
@ -153,7 +154,7 @@ export function createV2Api(router: Router, applications: ApplicationManager) {
|
|||||||
// It's kinda hacky to access the services and db via the application manager
|
// It's kinda hacky to access the services and db via the application manager
|
||||||
// maybe refactor this code
|
// maybe refactor this code
|
||||||
Bluebird.join(
|
Bluebird.join(
|
||||||
applications.services.getStatus(),
|
serviceManager.getStatus(),
|
||||||
images.getStatus(),
|
images.getStatus(),
|
||||||
db.models('app').select(['appId', 'commit', 'name']),
|
db.models('app').select(['appId', 'commit', 'name']),
|
||||||
(
|
(
|
||||||
@ -359,7 +360,7 @@ export function createV2Api(router: Router, applications: ApplicationManager) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
router.get('/v2/containerId', async (req, res) => {
|
router.get('/v2/containerId', async (req, res) => {
|
||||||
const services = await applications.services.getAll();
|
const services = await serviceManager.getAll();
|
||||||
|
|
||||||
if (req.query.serviceName != null || req.query.service != null) {
|
if (req.query.serviceName != null || req.query.service != null) {
|
||||||
const serviceName = req.query.serviceName || req.query.service;
|
const serviceName = req.query.serviceName || req.query.service;
|
||||||
@ -393,7 +394,7 @@ export function createV2Api(router: Router, applications: ApplicationManager) {
|
|||||||
const currentRelease = await config.get('currentCommit');
|
const currentRelease = await config.get('currentCommit');
|
||||||
|
|
||||||
const pending = applications.deviceState.applyInProgress;
|
const pending = applications.deviceState.applyInProgress;
|
||||||
const containerStates = (await applications.services.getAll()).map((svc) =>
|
const containerStates = (await serviceManager.getAll()).map((svc) =>
|
||||||
_.pick(
|
_.pick(
|
||||||
svc,
|
svc,
|
||||||
'status',
|
'status',
|
||||||
|
@ -13,6 +13,7 @@ import { ApplicationManager } from '../application-manager';
|
|||||||
import * as config from '../config';
|
import * as config from '../config';
|
||||||
import * as db from '../db';
|
import * as db from '../db';
|
||||||
import * as volumeManager from '../compose/volume-manager';
|
import * as volumeManager from '../compose/volume-manager';
|
||||||
|
import * as serviceManager from '../compose/service-manager';
|
||||||
import DeviceState from '../device-state';
|
import DeviceState from '../device-state';
|
||||||
import * as constants from '../lib/constants';
|
import * as constants from '../lib/constants';
|
||||||
import { BackupError, DatabaseParseError, NotFoundError } from '../lib/errors';
|
import { BackupError, DatabaseParseError, NotFoundError } from '../lib/errors';
|
||||||
@ -244,7 +245,7 @@ export async function normaliseLegacyDatabase(
|
|||||||
}
|
}
|
||||||
|
|
||||||
log.debug('Killing legacy containers');
|
log.debug('Killing legacy containers');
|
||||||
await application.services.killAllLegacy();
|
await serviceManager.killAllLegacy();
|
||||||
log.debug('Migrating legacy app volumes');
|
log.debug('Migrating legacy app volumes');
|
||||||
|
|
||||||
const targetApps = await application.getTargetApps();
|
const targetApps = await application.getTargetApps();
|
||||||
|
@ -1,10 +1,9 @@
|
|||||||
import { Router } from 'express';
|
import { Router } from 'express';
|
||||||
import { fs } from 'mz';
|
import { fs } from 'mz';
|
||||||
import { stub } from 'sinon';
|
|
||||||
|
|
||||||
import { ApplicationManager } from '../../src/application-manager';
|
import { ApplicationManager } from '../../src/application-manager';
|
||||||
import * as networkManager from '../../src/compose/network-manager';
|
import * as networkManager from '../../src/compose/network-manager';
|
||||||
import { ServiceManager } from '../../src/compose/service-manager';
|
import * as serviceManager from '../../src/compose/service-manager';
|
||||||
import * as volumeManager from '../../src/compose/volume-manager';
|
import * as volumeManager from '../../src/compose/volume-manager';
|
||||||
import * as config from '../../src/config';
|
import * as config from '../../src/config';
|
||||||
import * as db from '../../src/db';
|
import * as db from '../../src/db';
|
||||||
@ -135,22 +134,23 @@ function buildRoutes(appManager: ApplicationManager): Router {
|
|||||||
|
|
||||||
const originalNetGetAll = networkManager.getAllByAppId;
|
const originalNetGetAll = networkManager.getAllByAppId;
|
||||||
const originalVolGetAll = volumeManager.getAllByAppId;
|
const originalVolGetAll = volumeManager.getAllByAppId;
|
||||||
|
const originalSvcGetStatus = serviceManager.getStatus;
|
||||||
function setupStubs() {
|
function setupStubs() {
|
||||||
stub(ServiceManager.prototype, 'getStatus').resolves(STUBBED_VALUES.services);
|
|
||||||
|
|
||||||
// @ts-expect-error Assigning to a RO property
|
// @ts-expect-error Assigning to a RO property
|
||||||
networkManager.getAllByAppId = () => Promise.resolve(STUBBED_VALUES.networks);
|
networkManager.getAllByAppId = async () => STUBBED_VALUES.networks;
|
||||||
// @ts-expect-error Assigning to a RO property
|
// @ts-expect-error Assigning to a RO property
|
||||||
volumeManager.getAllByAppId = () => Promise.resolve(STUBBED_VALUES.volumes);
|
volumeManager.getAllByAppId = async () => STUBBED_VALUES.volumes;
|
||||||
|
// @ts-expect-error Assigning to a RO property
|
||||||
|
serviceManager.getStatus = async () => STUBBED_VALUES.services;
|
||||||
}
|
}
|
||||||
|
|
||||||
function restoreStubs() {
|
function restoreStubs() {
|
||||||
(ServiceManager.prototype as any).getStatus.restore();
|
|
||||||
|
|
||||||
// @ts-expect-error Assigning to a RO property
|
// @ts-expect-error Assigning to a RO property
|
||||||
networkManager.getAllByAppId = originalNetGetAll;
|
networkManager.getAllByAppId = originalNetGetAll;
|
||||||
// @ts-expect-error Assigning to a RO property
|
// @ts-expect-error Assigning to a RO property
|
||||||
volumeManager.getAllByAppId = originalVolGetAll;
|
volumeManager.getAllByAppId = originalVolGetAll;
|
||||||
|
// @ts-expect-error Assigning to a RO property
|
||||||
|
serviceManager.getStatus = originalSvcGetStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface SupervisorAPIOpts {
|
interface SupervisorAPIOpts {
|
||||||
|
Loading…
x
Reference in New Issue
Block a user