Add tests for doRestart action and v1/v2 app restart routes

Signed-off-by: Christina Ying Wang <christina@balena.io>

docker-compose.test.yml

@@ -10,6 +10,7 @@ services:
       args:
         ARCH: ${ARCH:-amd64}
     command: [ '/wait-for-it.sh', '--', '/usr/src/app/entry.sh' ]
+    stop_grace_period: 3s
     # Use bridge networking for the tests
     network_mode: 'bridge'
     networks:
@@ -34,6 +35,7 @@ services:
 
   dbus:
     image: balenablocks/dbus
+    stop_grace_period: 3s
     environment:
       DBUS_CONFIG: session.conf
       DBUS_ADDRESS: unix:path=/run/dbus/system_bus_socket
@@ -44,6 +46,7 @@ services:
   # requests
   dbus-services:
     build: ./test/lib/dbus
+    stop_grace_period: 3s
     depends_on:
       - dbus
     volumes:
@@ -53,6 +56,7 @@ services:
 
   docker:
     image: docker:dind
+    stop_grace_period: 3s
     privileged: true
     environment:
       DOCKER_TLS_CERTDIR: ''
@@ -81,6 +85,7 @@ services:
       - docker
       - dbus
       - dbus-services
+    stop_grace_period: 3s
     volumes:
       - dbus:/run/dbus
     # Set required supervisor configuration variables here

src/device-api/actions.ts

@@ -1,8 +1,14 @@
+import * as Bluebird from 'bluebird';
+
 import { getGlobalApiKey, refreshKey } from '.';
 import * as eventTracker from '../event-tracker';
 import * as deviceState from '../device-state';
+import * as applicationManager from '../compose/application-manager';
+import * as serviceManager from '../compose/service-manager';
 import log from '../lib/supervisor-console';
 import blink = require('../lib/blink');
+import { lock } from '../lib/update-lock';
+import { InternalInconsistencyError } from '../lib/errors';
 
 /**
  * Run an array of healthchecks, outputting whether all passed or not
@@ -59,3 +65,33 @@ export const regenerateKey = async (oldKey: string) => {
 
 	return newKey;
 };
+
+/**
+ * Restarts an application by recreating containers.
+ * Used by:
+ * - POST /v1/restart
+ * - POST /v2/applications/:appId/restart
+ */
+export const doRestart = async (appId: number, force: boolean = false) => {
+	await deviceState.initialized();
+
+	return await lock(appId, { force }, async () => {
+		const currentState = await deviceState.getCurrentState();
+		if (currentState.local.apps?.[appId] == null) {
+			throw new InternalInconsistencyError(
+				`Application with ID ${appId} is not in the current state`,
+			);
+		}
+		const { services } = currentState.local.apps?.[appId];
+		applicationManager.clearTargetVolatileForServices(
+			services.map((svc) => svc.imageId),
+		);
+
+		return deviceState.pausingApply(async () => {
+			for (const service of services) {
+				await serviceManager.kill(service, { wait: true });
+				await serviceManager.start(service);
+			}
+		});
+	});
+};

src/device-api/common.ts

@@ -4,42 +4,13 @@ import * as _ from 'lodash';
 import * as logger from '../logger';
 import * as deviceState from '../device-state';
 import * as applicationManager from '../compose/application-manager';
-import * as serviceManager from '../compose/service-manager';
 import * as volumeManager from '../compose/volume-manager';
 import { App } from '../compose/app';
-import { InternalInconsistencyError } from '../lib/errors';
 import { lock } from '../lib/update-lock';
 import { appNotFoundMessage } from './messages';
 
 import type { InstancedDeviceState } from '../types';
 
-export async function doRestart(appId: number, force: boolean) {
-	await deviceState.initialized();
-	await applicationManager.initialized();
-
-	return lock(appId, { force }, () =>
-		deviceState.getCurrentState().then(function (currentState) {
-			if (currentState.local.apps?.[appId] == null) {
-				throw new InternalInconsistencyError(
-					`Application with ID ${appId} is not in the current state`,
-				);
-			}
-			const allApps = currentState.local.apps;
-
-			const app = allApps[appId];
-			const imageIds = _.map(app.services, 'imageId');
-			applicationManager.clearTargetVolatileForServices(imageIds);
-
-			return deviceState.pausingApply(async () => {
-				return Bluebird.each(app.services, async (service) => {
-					await serviceManager.kill(service, { wait: true });
-					await serviceManager.start(service);
-				});
-			});
-		}),
-	);
-}
-
 export async function doPurge(appId: number, force: boolean) {
 	await deviceState.initialized();
 	await applicationManager.initialized();

src/device-api/v1.ts

@@ -1,7 +1,8 @@
 import * as express from 'express';
 import * as _ from 'lodash';
 
-import { doRestart, doPurge } from './common';
+import * as actions from './actions';
+import { doPurge } from './common';
 import { AuthorizedRequest } from './api-keys';
 import * as eventTracker from '../event-tracker';
 import { isReadyForUpdates } from '../api-binder';
@@ -40,7 +41,8 @@ router.post('/v1/restart', (req: AuthorizedRequest, res, next) => {
 		return;
 	}
 
-	return doRestart(appId, force)
+	return actions
+		.doRestart(appId, force)
 		.then(() => res.status(200).send('OK'))
 		.catch(next);
 });

src/device-api/v2.ts

@@ -31,7 +31,8 @@ import log from '../lib/supervisor-console';
 import supervisorVersion = require('../lib/supervisor-version');
 import { checkInt, checkTruthy } from '../lib/validation';
 import { isVPNActive } from '../network';
-import { doPurge, doRestart, safeStateClone } from './common';
+import * as actions from './actions';
+import { doPurge, safeStateClone } from './common';
 import { AuthorizedRequest } from './api-keys';
 import { fromV2TargetState } from '../lib/legacy';
 
@@ -160,8 +161,8 @@ router.post(
 router.post(
 	'/v2/applications/:appId/restart',
 	(req: AuthorizedRequest, res: Response, next: NextFunction) => {
-		const { force } = req.body;
 		const appId = checkInt(req.params.appId);
+		const force = checkTruthy(req.body.force);
 		if (!appId) {
 			return res.status(400).json({
 				status: 'failed',
@@ -178,7 +179,8 @@ router.post(
 			return;
 		}
 
-		return doRestart(appId, force)
+		return actions
+			.doRestart(appId, force)
 			.then(() => {
 				res.status(200).send('OK');
 			})

test/integration/device-api/actions.spec.ts

@@ -1,21 +1,25 @@
 import { expect } from 'chai';
 import { stub, SinonStub } from 'sinon';
+import * as Docker from 'dockerode';
+import * as request from 'supertest';
+import { setTimeout } from 'timers/promises';
 
 import * as deviceState from '~/src/device-state';
-import { getGlobalApiKey, generateScopedKey } from '~/src/device-api';
+import * as deviceApi from '~/src/device-api';
 import * as actions from '~/src/device-api/actions';
+import { cleanupDocker } from '~/test-lib/docker-helper';
 
 describe('regenerates API keys', () => {
 	// Stub external dependency - current state report should be tested separately.
-	// api-key.ts methods are also tested separately.
+	// API key related methods are tested in api-keys.spec.ts.
 	beforeEach(() => stub(deviceState, 'reportCurrentState'));
 	afterEach(() => (deviceState.reportCurrentState as SinonStub).restore());
 
 	it("communicates new key to cloud if it's a global key", async () => {
-		const originalGlobalKey = await getGlobalApiKey();
+		const originalGlobalKey = await deviceApi.getGlobalApiKey();
 		const newKey = await actions.regenerateKey(originalGlobalKey);
 		expect(originalGlobalKey).to.not.equal(newKey);
-		expect(newKey).to.equal(await getGlobalApiKey());
+		expect(newKey).to.equal(await deviceApi.getGlobalApiKey());
 		expect(deviceState.reportCurrentState as SinonStub).to.have.been.calledOnce;
 		expect(
 			(deviceState.reportCurrentState as SinonStub).firstCall.args[0],
@@ -25,10 +29,279 @@ describe('regenerates API keys', () => {
 	});
 
 	it("doesn't communicate new key if it's a service key", async () => {
-		const originalScopedKey = await generateScopedKey(111, 'main');
+		const originalScopedKey = await deviceApi.generateScopedKey(111, 'main');
 		const newKey = await actions.regenerateKey(originalScopedKey);
 		expect(originalScopedKey).to.not.equal(newKey);
-		expect(newKey).to.not.equal(await getGlobalApiKey());
+		expect(newKey).to.not.equal(await deviceApi.getGlobalApiKey());
 		expect(deviceState.reportCurrentState as SinonStub).to.not.have.been.called;
 	});
 });
+
+// TODO: test all the container stop / start / recreate / purge related actions
+// together here to avoid repeated setup of containers and images.
+describe('manages application lifecycle', () => {
+	const BASE_IMAGE = 'alpine:latest';
+	const BALENA_SUPERVISOR_ADDRESS =
+		process.env.BALENA_SUPERVISOR_ADDRESS || 'http://balena-supervisor:48484';
+	const APP_ID = 1;
+	const docker = new Docker();
+
+	const getSupervisorTarget = async () =>
+		await request(BALENA_SUPERVISOR_ADDRESS)
+			.get('/v2/local/target-state')
+			.expect(200)
+			.then(({ body }) => body.state.local);
+
+	const setSupervisorTarget = async (
+		target: Awaited<ReturnType<typeof generateTarget>>,
+	) =>
+		await request(BALENA_SUPERVISOR_ADDRESS)
+			.post('/v2/local/target-state')
+			.set('Content-Type', 'application/json')
+			.send(JSON.stringify(target))
+			.expect(200);
+
+	const generateTargetApps = ({
+		serviceCount,
+		appId,
+		serviceNames,
+	}: {
+		serviceCount: number;
+		appId: number;
+		serviceNames: string[];
+	}) => {
+		// Populate app services
+		const services: Dictionary<any> = {};
+		for (let i = 1; i <= serviceCount; i++) {
+			services[i] = {
+				environment: {},
+				image: BASE_IMAGE,
+				imageId: `${i}`,
+				labels: {
+					'io.balena.testing': '1',
+				},
+				restart: 'unless-stopped',
+				running: true,
+				serviceName: serviceNames[i - 1],
+				serviceId: `${i}`,
+				volumes: ['data:/data'],
+				command: 'sleep infinity',
+				// Kill container immediately instead of waiting for 10s
+				stop_signal: 'SIGKILL',
+			};
+		}
+
+		return {
+			[appId]: {
+				name: 'localapp',
+				commit: 'localcommit',
+				releaseId: '1',
+				services,
+				volumes: {
+					data: {},
+				},
+			},
+		};
+	};
+
+	const generateTarget = async ({
+		serviceCount,
+		appId = APP_ID,
+		serviceNames = ['server', 'client'],
+	}: {
+		serviceCount: number;
+		appId?: number;
+		serviceNames?: string[];
+	}) => {
+		const { name, config } = await getSupervisorTarget();
+		return {
+			local: {
+				// We don't want to change name or config as this may result in
+				// unintended reboots. We just want to test state changes in containers.
+				name,
+				config,
+				apps:
+					serviceCount === 0
+						? {}
+						: generateTargetApps({
+								serviceCount,
+								appId,
+								serviceNames,
+						  }),
+			},
+		};
+	};
+
+	// Wait until containers are in a ready state prior to testing assertions
+	const waitForSetup = async (
+		targetState: Dictionary<any>,
+		isWaitComplete: (ctns: Docker.ContainerInspectInfo[]) => boolean = (ctns) =>
+			ctns.every((ctn) => ctn.State.Running),
+	) => {
+		// Get expected number of containers from target state
+		const expected = Object.keys(
+			targetState.local.apps[`${APP_ID}`].services,
+		).length;
+
+		// Wait for engine until number of containers are reached.
+		// This test suite will timeout if anything goes wrong, since
+		// we don't have any way of knowing whether Docker has finished
+		// setting up containers or not.
+		while (true) {
+			const containers = await docker.listContainers({ all: true });
+			const containerInspects = await Promise.all(
+				containers.map(({ Id }) => docker.getContainer(Id).inspect()),
+			);
+			if (expected === containers.length && isWaitComplete(containerInspects)) {
+				return containerInspects;
+			} else {
+				await setTimeout(500);
+			}
+		}
+	};
+
+	// Get NEW container inspects. This function should be passed to waitForSetup
+	// when checking a container has started or been recreated. This is necessary
+	// because waitForSetup may erroneously return the existing 2 containers
+	// in its while loop if stopping them takes some time.
+	const startTimesChanged = (startedAt: string[]) => {
+		return (ctns: Docker.ContainerInspectInfo[]) =>
+			ctns.every(({ State }) => !startedAt.includes(State.StartedAt));
+	};
+
+	before(async () => {
+		// Images are ignored in local mode so we need to pull the base image
+		await docker.pull(BASE_IMAGE);
+		// Wait for base image to finish pulling
+		while (true) {
+			const images = await docker.listImages();
+			if (images.length > 0) {
+				break;
+			}
+			await setTimeout(500);
+		}
+
+		// Make sure Supervisor doesn't have any apps running before assertions
+		await setSupervisorTarget(await generateTarget({ serviceCount: 0 }));
+	});
+
+	after(async () => {
+		// Reset Supervisor to state from before lifecycle tests
+		await setSupervisorTarget(await generateTarget({ serviceCount: 0 }));
+
+		// Remove any leftover engine artifacts
+		await cleanupDocker(docker);
+	});
+
+	describe('manages single container application lifecycle', () => {
+		const serviceCount = 1;
+		const serviceNames = ['server'];
+		let targetState: Awaited<ReturnType<typeof generateTarget>>;
+		let containers: Docker.ContainerInspectInfo[];
+
+		before(async () => {
+			targetState = await generateTarget({
+				serviceCount,
+				serviceNames,
+			});
+
+			// Create a single-container application in local mode
+			await setSupervisorTarget(targetState);
+		});
+
+		afterEach(async () => {
+			// Make sure target state has reset to single-container app between assertions
+			await setSupervisorTarget(targetState);
+		});
+
+		// Make sure the app is running and correct before testing more assertions
+		it('should setup a single container app (sanity check)', async () => {
+			containers = await waitForSetup(targetState);
+			// Containers should have correct metadata;
+			// Testing their names should be sufficient.
+			containers.forEach((ctn) => {
+				expect(serviceNames.some((name) => new RegExp(name).test(ctn.Name))).to
+					.be.true;
+			});
+		});
+
+		it('should restart an application by recreating containers', async () => {
+			containers = await waitForSetup(targetState);
+			const isRestartSuccessful = startTimesChanged(
+				containers.map((ctn) => ctn.State.StartedAt),
+			);
+
+			await actions.doRestart(APP_ID);
+
+			const restartedContainers = await waitForSetup(
+				targetState,
+				isRestartSuccessful,
+			);
+
+			// Technically the wait function above should already verify that the two
+			// containers have been restarted, but verify explcitly with an assertion
+			expect(isRestartSuccessful(restartedContainers)).to.be.true;
+
+			// Containers should have different Ids since they're recreated
+			expect(restartedContainers.map(({ Id }) => Id)).to.not.have.members(
+				containers.map((ctn) => ctn.Id),
+			);
+		});
+	});
+
+	describe('manages multi-container application lifecycle', () => {
+		const serviceCount = 2;
+		const serviceNames = ['server', 'client'];
+		let targetState: Awaited<ReturnType<typeof generateTarget>>;
+		let containers: Docker.ContainerInspectInfo[];
+
+		before(async () => {
+			targetState = await generateTarget({
+				serviceCount,
+				serviceNames,
+			});
+
+			// Create a single-container application in local mode
+			await setSupervisorTarget(targetState);
+		});
+
+		afterEach(async () => {
+			// Make sure target state has reset to single-container app between assertions
+			await setSupervisorTarget(targetState);
+		});
+
+		// Make sure the app is running and correct before testing more assertions
+		it('should setup a multi-container app (sanity check)', async () => {
+			containers = await waitForSetup(targetState);
+			// Containers should have correct metadata;
+			// Testing their names should be sufficient.
+			containers.forEach((ctn) => {
+				expect(serviceNames.some((name) => new RegExp(name).test(ctn.Name))).to
+					.be.true;
+			});
+		});
+
+		it('should restart an application by recreating containers', async () => {
+			containers = await waitForSetup(targetState);
+			const isRestartSuccessful = startTimesChanged(
+				containers.map((ctn) => ctn.State.StartedAt),
+			);
+
+			await actions.doRestart(APP_ID);
+
+			const restartedContainers = await waitForSetup(
+				targetState,
+				isRestartSuccessful,
+			);
+
+			// Technically the wait function above should already verify that the two
+			// containers have been restarted, but verify explcitly with an assertion
+			expect(isRestartSuccessful(restartedContainers)).to.be.true;
+
+			// Containers should have different Ids since they're recreated
+			expect(restartedContainers.map(({ Id }) => Id)).to.not.have.members(
+				containers.map((ctn) => ctn.Id),
+			);
+		});
+	});
+});

test/integration/device-api/v1.spec.ts

@@ -4,10 +4,14 @@ import { SinonStub, stub } from 'sinon';
 import * as request from 'supertest';
 
 import * as config from '~/src/config';
+import * as db from '~/src/db';
 import * as deviceApi from '~/src/device-api';
 import * as actions from '~/src/device-api/actions';
 import * as v1 from '~/src/device-api/v1';
+import { UpdatesLockedError } from '~/lib/errors';
 
+// All routes that require Authorization are integration tests due to
+// the api-key module relying on the database.
 describe('device-api/v1', () => {
 	let api: express.Application;
 
@@ -94,4 +98,87 @@ describe('device-api/v1', () => {
 				.expect(503);
 		});
 	});
+
+	describe('POST /v1/restart', () => {
+		// Actions are tested elsewhere so we can stub the dependency here
+		let doRestartStub: SinonStub;
+		beforeEach(() => {
+			doRestartStub = stub(actions, 'doRestart').resolves();
+		});
+		afterEach(async () => {
+			doRestartStub.restore();
+			// Remove all scoped API keys between tests
+			await db.models('apiSecret').whereNot({ appId: 0 }).del();
+		});
+
+		it('validates data from request body', async () => {
+			// Parses force: false
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 1234567, force: false })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+			expect(doRestartStub).to.have.been.calledWith(1234567, false);
+			doRestartStub.resetHistory();
+
+			// Parses force: true
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 7654321, force: true })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+			expect(doRestartStub).to.have.been.calledWith(7654321, true);
+			doRestartStub.resetHistory();
+
+			// Defaults to force: false
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 7654321 })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+			expect(doRestartStub).to.have.been.calledWith(7654321, false);
+		});
+
+		it('responds with 400 if appId is missing', async () => {
+			await request(api)
+				.post('/v1/restart')
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(400);
+		});
+
+		it("responds with 401 if caller's API key is not in scope of appId", async () => {
+			const scopedKey = await deviceApi.generateScopedKey(1234567, 'main');
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 7654321 })
+				.set('Authorization', `Bearer ${scopedKey}`)
+				.expect(401);
+		});
+
+		it('responds with 200 if restart succeeded', async () => {
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 1234567 })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+		});
+
+		it('responds with 423 if there are update locks', async () => {
+			doRestartStub.throws(new UpdatesLockedError());
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 1234567 })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(423);
+		});
+
+		it('responds with 503 for other errors that occur during restart', async () => {
+			doRestartStub.throws(new Error());
+			await request(api)
+				.post('/v1/restart')
+				.send({ appId: 1234567 })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(503);
+		});
+	});
 });

test/integration/device-api/v2.spec.ts

@@ -0,0 +1,108 @@
+import { expect } from 'chai';
+import * as express from 'express';
+import { SinonStub, stub } from 'sinon';
+import * as request from 'supertest';
+
+import * as config from '~/src/config';
+import * as db from '~/src/db';
+import * as deviceApi from '~/src/device-api';
+import * as actions from '~/src/device-api/actions';
+import * as v2 from '~/src/device-api/v2';
+import { UpdatesLockedError } from '~/lib/errors';
+
+// All routes that require Authorization are integration tests due to
+// the api-key module relying on the database.
+describe('device-api/v2', () => {
+	let api: express.Application;
+
+	before(async () => {
+		await config.initialized();
+
+		// `api` is a private property on SupervisorAPI but
+		// passing it directly to supertest is easier than
+		// setting up an API listen port & timeout
+		api = new deviceApi.SupervisorAPI({
+			routers: [v2.router],
+			healthchecks: [],
+			// @ts-expect-error
+		}).api;
+	});
+
+	describe('POST /v2/applications/:appId/restart', () => {
+		// Actions are tested elsewhere so we can stub the dependency here
+		let doRestartStub: SinonStub;
+		beforeEach(() => {
+			doRestartStub = stub(actions, 'doRestart').resolves();
+		});
+		afterEach(async () => {
+			doRestartStub.restore();
+			// Remove all scoped API keys between tests
+			await db.models('apiSecret').whereNot({ appId: 0 }).del();
+		});
+
+		it('validates data from request body', async () => {
+			// Parses force: false
+			await request(api)
+				.post('/v2/applications/1234567/restart')
+				.send({ force: false })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+			expect(doRestartStub).to.have.been.calledWith(1234567, false);
+			doRestartStub.resetHistory();
+
+			// Parses force: true
+			await request(api)
+				.post('/v2/applications/7654321/restart')
+				.send({ force: true })
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+			expect(doRestartStub).to.have.been.calledWith(7654321, true);
+			doRestartStub.resetHistory();
+
+			// Defaults to force: false
+			await request(api)
+				.post('/v2/applications/7654321/restart')
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+			expect(doRestartStub).to.have.been.calledWith(7654321, false);
+		});
+
+		it('responds with 400 if appId is missing', async () => {
+			await request(api)
+				.post('/v2/applications/badAppId/restart')
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(400);
+		});
+
+		it("responds with 401 if caller's API key is not in scope of appId", async () => {
+			const scopedKey = await deviceApi.generateScopedKey(1234567, 'main');
+			await request(api)
+				.post('/v2/applications/7654321/restart')
+				.set('Authorization', `Bearer ${scopedKey}`)
+				.expect(401);
+		});
+
+		it('responds with 200 if restart succeeded', async () => {
+			await request(api)
+				.post('/v2/applications/1234567/restart')
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(200);
+		});
+
+		it('responds with 423 if there are update locks', async () => {
+			doRestartStub.throws(new UpdatesLockedError());
+			await request(api)
+				.post('/v2/applications/1234567/restart')
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(423);
+		});
+
+		it('responds with 503 for other errors that occur during restart', async () => {
+			doRestartStub.throws(new Error());
+			await request(api)
+				.post('/v2/applications/7654321/restart')
+				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
+				.expect(503);
+		});
+	});
+});

test/integration/lib/docker-utils.spec.ts

@@ -2,7 +2,7 @@ import { expect } from 'chai';
 import { stub } from 'sinon';
 
 import * as dockerUtils from '~/lib/docker-utils';
-import { createDockerImage } from '~/test-lib/docker-helper';
+import { createDockerImage, cleanupDocker } from '~/test-lib/docker-helper';
 import * as Docker from 'dockerode';
 
 describe('lib/docker-utils', () => {
@@ -10,6 +10,8 @@ describe('lib/docker-utils', () => {
 
 	describe('getNetworkGateway', async () => {
 		before(async () => {
+			// Remove network if it already exists
+			await cleanupDocker(docker);
 			await docker.createNetwork({
 				Name: 'supervisor0',
 				Options: {
@@ -28,14 +30,7 @@ describe('lib/docker-utils', () => {
 		});
 
 		after(async () => {
-			const allNetworks = await docker.listNetworks();
+			await cleanupDocker(docker);
-
-			// Delete any remaining networks
-			await Promise.all(
-				allNetworks
-					.filter(({ Name }) => !['bridge', 'host', 'none'].includes(Name)) // exclude docker default network from the cleanup
-					.map(({ Name }) => docker.getNetwork(Name).remove()),
-			);
 		});
 
 		// test using existing data...

test/legacy/41-device-api-v1.spec.ts

@@ -1,4 +1,3 @@
-import * as _ from 'lodash';
 import { expect } from 'chai';
 import { stub, spy, SinonStub, SinonSpy } from 'sinon';
 import * as supertest from 'supertest';
@@ -110,57 +109,6 @@ describe('SupervisorAPI [V1 Endpoints]', () => {
 		loggerStub.restore();
 	});
 
-	describe('POST /v1/restart', () => {
-		it('restarts all containers in release', async () => {
-			// Perform the test with our mocked release
-			await mockedDockerode.testWithData({ containers, images }, async () => {
-				// Perform test
-				await request
-					.post('/v1/restart')
-					.send({ appId: 2 })
-					.set('Accept', 'application/json')
-					.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
-					.expect(sampleResponses.V1.POST['/restart'].statusCode)
-					.then((response) => {
-						expect(response.body).to.deep.equal(
-							sampleResponses.V1.POST['/restart'].body,
-						);
-						expect(response.text).to.deep.equal(
-							sampleResponses.V1.POST['/restart'].text,
-						);
-					});
-				// Check that mockedDockerode contains 3 stop and start actions
-				const removeSteps = _(mockedDockerode.actions)
-					.pickBy({ name: 'stop' })
-					.map()
-					.value();
-				expect(removeSteps).to.have.lengthOf(3);
-				const startSteps = _(mockedDockerode.actions)
-					.pickBy({ name: 'start' })
-					.map()
-					.value();
-				expect(startSteps).to.have.lengthOf(3);
-			});
-		});
-
-		it('validates request body parameters', async () => {
-			await request
-				.post('/v1/restart')
-				.send({ thing: '' })
-				.set('Accept', 'application/json')
-				.set('Authorization', `Bearer ${await deviceApi.getGlobalApiKey()}`)
-				.expect(sampleResponses.V1.POST['/restart [Invalid Body]'].statusCode)
-				.then((response) => {
-					expect(response.body).to.deep.equal(
-						sampleResponses.V1.POST['/restart [Invalid Body]'].body,
-					);
-					expect(response.text).to.deep.equal(
-						sampleResponses.V1.POST['/restart [Invalid Body]'].text,
-					);
-				});
-		});
-	});
-
 	describe('GET /v1/apps/:appId', () => {
 		it('does not return information for an application when there is more than 1 container', async () => {
 			await request

test/lib/docker-helper.ts

@@ -1,8 +1,9 @@
 import * as Docker from 'dockerode';
 import * as tar from 'tar-stream';
-
 import { strict as assert } from 'assert';
 
+import { isStatusError } from '~/lib/errors';
+
 // Creates an image from scratch with just some labels
 export async function createDockerImage(
 	name: string,
@@ -41,3 +42,38 @@ export async function createDockerImage(
 		});
 	});
 }
+
+// Clean up all Docker relics from tests
+export const cleanupDocker = async (docker = new Docker()) => {
+	// Remove all containers
+	// Some containers may still be running so a prune won't suffice
+	try {
+		const containers = await docker.listContainers({ all: true });
+		await Promise.all(
+			containers.map(({ Id }) =>
+				docker.getContainer(Id).remove({ force: true }),
+			),
+		);
+	} catch (e: unknown) {
+		// Sometimes a container is already in the process of being removed
+		// This is safe to ignore since we're removing them anyway.
+		if (isStatusError(e) && e.statusCode !== 409) {
+			throw e;
+		}
+	}
+
+	// Remove all networks except defaults
+	const networks = await docker.listNetworks();
+	await Promise.all(
+		networks
+			.filter(({ Name }) => !['bridge', 'host', 'none'].includes(Name)) // exclude docker default network from the cleanup
+			.map(({ Name }) => docker.getNetwork(Name).remove()),
+	);
+
+	// Remove all volumes
+	const { Volumes } = await docker.listVolumes();
+	await Promise.all(Volumes.map(({ Name }) => docker.getVolume(Name).remove()));
+
+	// Remove all images
+	await docker.pruneImages({ filters: { dangling: { false: true } } });
+};