Merge pull request #2170 from balena-os/handle-engine-host-resource-race-condition

Handle Engine-host race condition

src/compose/app.ts

@@ -1,10 +1,10 @@
 import * as _ from 'lodash';
 import { promises as fs } from 'fs';
+import { ImageInspectInfo } from 'dockerode';
 
 import Network from './network';
 import Volume from './volume';
 import Service from './service';
-
 import * as imageManager from './images';
 import type { Image } from './images';
 import {
@@ -12,8 +12,10 @@ import {
 	generateStep,
 	CompositionStepAction,
 } from './composition-steps';
+import { isOlderThan } from './utils';
+import { inspectByDockerContainerId } from './service-manager';
 import * as targetStateCache from '../device-state/target-state-cache';
-import * as dockerUtils from '../lib/docker-utils';
+import { getNetworkGateway } from '../lib/docker-utils';
 import * as constants from '../lib/constants';
 
 import { getStepsFromStrategy } from './update-strategies';
@@ -22,10 +24,11 @@ import { InternalInconsistencyError, isNotFoundError } from '../lib/errors';
 import * as config from '../config';
 import { checkTruthy, checkString } from '../lib/validation';
 import { ServiceComposeConfig, DeviceMetadata } from './types/service';
-import { ImageInspectInfo } from 'dockerode';
 import { pathExistsOnRoot } from '../lib/host-utils';
 import { isSupervisor } from '../lib/supervisor-metadata';
 
+const SECONDS_TO_WAIT_FOR_START = 60;
+
 export interface AppConstructOpts {
 	appId: number;
 	appUuid?: string;
@@ -99,10 +102,10 @@ export class App {
 		}
 	}
 
-	public nextStepsForAppUpdate(
+	public async nextStepsForAppUpdate(
 		state: UpdateState,
 		target: App,
-	): CompositionStep[] {
+	): Promise<CompositionStep[]> {
 		// Check to see if we need to polyfill in some "new" data for legacy services
 		this.migrateLegacy(target);
 
@@ -128,7 +131,8 @@ export class App {
 			}
 		}
 
-		const { removePairs, installPairs, updatePairs } = this.compareServices(
+		const { removePairs, installPairs, updatePairs } =
+			await this.compareServices(
 				this.services,
 				target.services,
 				state.containerIds,
@@ -145,9 +149,8 @@ export class App {
 
 		// For every service which needs to be updated, update via update strategy.
 		const servicePairs = updatePairs.concat(installPairs);
-		steps = steps.concat(
+		const serviceSteps = await Promise.all(
-			servicePairs
+			servicePairs.map((pair) =>
-				.map((pair) =>
 				this.generateStepsForService(pair, {
 					...state,
 					servicePairs,
@@ -155,8 +158,10 @@ export class App {
 					networkPairs: networkChanges,
 					volumePairs: volumeChanges,
 				}),
-				)
+			),
-				.filter((step) => step != null) as CompositionStep[],
+		);
+		steps = steps.concat(
+			serviceSteps.filter((step) => step != null) as CompositionStep[],
 		);
 
 		// Generate volume steps
@@ -184,7 +189,6 @@ export class App {
 				}),
 			);
 		}
-
 		return steps;
 	}
 
@@ -296,15 +300,15 @@ export class App {
 		return outputs;
 	}
 
-	private compareServices(
+	private async compareServices(
 		current: Service[],
 		target: Service[],
 		containerIds: Dictionary<string>,
-	): {
+	): Promise<{
 		installPairs: Array<ChangingPair<Service>>;
 		removePairs: Array<ChangingPair<Service>>;
 		updatePairs: Array<ChangingPair<Service>>;
-	} {
+	}> {
 		const currentByServiceName = _.keyBy(current, 'serviceName');
 		const targetByServiceName = _.keyBy(target, 'serviceName');
 
@@ -363,7 +367,7 @@ export class App {
 		 * @param serviceCurrent
 		 * @param serviceTarget
 		 */
-		const shouldBeStarted = (
+		const shouldBeStarted = async (
 			serviceCurrent: Service,
 			serviceTarget: Service,
 		) => {
@@ -377,9 +381,20 @@ export class App {
 
 			// Only start a Service if we have never started it before and the service matches target!
 			// This is so the engine can handle the restart policy configured for the container.
+			//
+			// However, there is a certain race condition where the container's compose depends on a host
+			// resource that may not be there when the Engine starts the container, such as a port binding
+			// of 192.168.88.1:3000:3000, where 192.168.88.1 is a user-defined interface configured in system-connections
+			// and created by the host. This interface creation may not occur before the container creation.
+			// In this case, the container is created and never started, and the Engine does not attempt to restart it
+			// regardless of restart policy.
 			return (
 				(serviceCurrent.status === 'Installing' ||
-					serviceCurrent.status === 'Installed') &&
+					serviceCurrent.status === 'Installed' ||
+					(await this.requirementsMetForSpecialStart(
+						serviceCurrent,
+						serviceTarget,
+					))) &&
 				isEqualExceptForRunningState(serviceCurrent, serviceTarget)
 			);
 		};
@@ -415,17 +430,23 @@ export class App {
 		/**
 		 * Filter all the services which should be updated due to run state change, or config mismatch.
 		 */
-		const toBeUpdated = maybeUpdate
+		const satisfiesRequirementsForUpdate = async (c: Service, t: Service) =>
-			.map((serviceName) => ({
+			!isEqualExceptForRunningState(c, t) ||
+			(await shouldBeStarted(c, t)) ||
+			shouldBeStopped(c, t) ||
+			shouldWaitForStop(c);
+
+		const maybeUpdatePairs = maybeUpdate.map((serviceName) => ({
 			current: currentByServiceName[serviceName],
 			target: targetByServiceName[serviceName],
-			}))
+		}));
-			.filter(
+		const maybeUpdatePairsFiltered = await Promise.all(
-				({ current: c, target: t }) =>
+			maybeUpdatePairs.map(({ current: c, target: t }) =>
-					!isEqualExceptForRunningState(c, t) ||
+				satisfiesRequirementsForUpdate(c, t),
-					shouldBeStarted(c, t) ||
+			),
-					shouldBeStopped(c, t) ||
+		);
-					shouldWaitForStop(c),
+		const toBeUpdated = maybeUpdatePairs.filter(
+			(__, idx) => maybeUpdatePairsFiltered[idx],
 		);
 
 		return {
@@ -492,7 +513,7 @@ export class App {
 		return steps;
 	}
 
-	private generateStepsForService(
+	private async generateStepsForService(
 		{ current, target }: ChangingPair<Service>,
 		context: {
 			availableImages: Image[];
@@ -503,7 +524,7 @@ export class App {
 			volumePairs: Array<ChangingPair<Volume>>;
 			servicePairs: Array<ChangingPair<Service>>;
 		},
-	): Nullable<CompositionStep> {
+	): Promise<Nullable<CompositionStep>> {
 		if (current?.status === 'Stopping') {
 			// Theres a kill step happening already, emit a noop to ensure we stay alive while
 			// this happens
@@ -557,7 +578,7 @@ export class App {
 				current.isEqualConfig(target, context.containerIds)
 			) {
 				// we're only starting/stopping a service
-				return this.generateContainerStep(current, target);
+				return await this.generateContainerStep(current, target);
 			}
 
 			let strategy =
@@ -631,11 +652,56 @@ export class App {
 		);
 	}
 
-	private generateContainerStep(current: Service, target: Service) {
+	// In the case where the Engine does not start the container despite the
+	// restart policy (this can happen in cases of Engine race conditions with
+	// host resources that are slower to be created but that a service relies on),
+	// we need to start the container after a delay.
+	private async requirementsMetForSpecialStart(
+		current: Service,
+		target: Service,
+	): Promise<boolean> {
+		// Shortcut the Engine inspect queries if status isn't exited
+		if (
+			current.status !== 'exited' ||
+			current.config.running !== false ||
+			target.config.running !== true
+		) {
+			return false;
+		}
+		let conditionsMetWithRestartPolicy = ['always', 'unless-stopped'].includes(
+			target.config.restart,
+		);
+
+		if (current.containerId != null && !conditionsMetWithRestartPolicy) {
+			const containerInspect = await inspectByDockerContainerId(
+				current.containerId,
+			);
+			// If the container has previously been started but exited unsuccessfully, it needs to be started.
+			// If the container has a restart policy of 'no' and has never been started, it
+			// should also be started, however, in that case, the status of the container will
+			// be 'Installed' and the state funnel will already be trying to start it until success,
+			// so we don't need to add any additional handling.
+			if (
+				target.config.restart === 'on-failure' &&
+				containerInspect.State.ExitCode !== 0
+			) {
+				conditionsMetWithRestartPolicy = true;
+			}
+		}
+		return conditionsMetWithRestartPolicy;
+	}
+
+	private async generateContainerStep(current: Service, target: Service) {
 		// if the services release doesn't match, then rename the container...
 		if (current.commit !== target.commit) {
 			return generateStep('updateMetadata', { current, target });
 		} else if (target.config.running !== current.config.running) {
+			if (
+				(await this.requirementsMetForSpecialStart(current, target)) &&
+				!isOlderThan(current.createdAt, SECONDS_TO_WAIT_FOR_START)
+			) {
+				return generateStep('noop', {});
+			}
 			if (target.config.running) {
 				return generateStep('start', { target });
 			} else {
@@ -772,9 +838,9 @@ export class App {
 		const [opts, supervisorApiHost, hostPathExists, hostname] =
 			await Promise.all([
 				config.get('extendedEnvOptions'),
-				dockerUtils
+				getNetworkGateway(constants.supervisorNetworkInterface).catch(
-					.getNetworkGateway(constants.supervisorNetworkInterface)
+					() => '127.0.0.1',
-					.catch(() => '127.0.0.1'),
+				),
 				(async () => ({
 					firmware: await pathExistsOnRoot('/lib/firmware'),
 					modules: await pathExistsOnRoot('/lib/modules'),

src/compose/application-manager.ts

@@ -207,7 +207,7 @@ export async function inferNextSteps(
 			// do to move to the target state
 			for (const id of targetAndCurrent) {
 				steps = steps.concat(
-					currentApps[id].nextStepsForAppUpdate(
+					await currentApps[id].nextStepsForAppUpdate(
 						{
 							availableImages,
 							containerIds: containerIdsByAppId[id],
@@ -243,7 +243,7 @@ export async function inferNextSteps(
 					false,
 				);
 				steps = steps.concat(
-					emptyCurrent.nextStepsForAppUpdate(
+					await emptyCurrent.nextStepsForAppUpdate(
 						{
 							availableImages,
 							containerIds: containerIdsByAppId[id] ?? {},

src/compose/service-manager.ts

@@ -135,7 +135,7 @@ export async function getState() {
 export async function getByDockerContainerId(
 	containerId: string,
 ): Promise<Service | null> {
-	const container = await docker.getContainer(containerId).inspect();
+	const container = await inspectByDockerContainerId(containerId);
 	if (
 		container.Config.Labels['io.balena.supervised'] == null &&
 		container.Config.Labels['io.resin.supervised'] == null
@@ -145,6 +145,12 @@ export async function getByDockerContainerId(
 	return Service.fromDockerContainer(container);
 }
 
+export async function inspectByDockerContainerId(
+	containerId: string,
+): Promise<Dockerode.ContainerInspectInfo> {
+	return await docker.getContainer(containerId).inspect();
+}
+
 export async function updateMetadata(service: Service, target: Service) {
 	const svc = await get(service);
 	if (svc.containerId == null) {

src/compose/utils.ts

@@ -693,3 +693,10 @@ export function dockerMountToServiceMount(
 
 	return mount as LongDefinition;
 }
+
+export function isOlderThan(currentTime: Date | null, seconds: number) {
+	if (currentTime == null) {
+		return false;
+	}
+	return new Date().getTime() - currentTime.getTime() > seconds * 1000;
+}

test/integration/compose/application-manager.spec.ts

@@ -1417,4 +1417,349 @@ describe('compose/application-manager', () => {
 			});
 		});
 	});
+
+	// In the case where a container requires a host resource such as a network interface that is not created by the time the Engine
+	// comes up, the Engine will not attempt to restart the container which seems to be Docker's implemented behavior (if not the correct behavior).
+	// An example of a host resource would be a port binding such as 192.168.88.1:3000:3000, where the IP is an interface delayed in creation by host.
+	// In this case, the Supervisor needs to wait a grace period for the Engine to start the container, and if this does not occur, the Supervisor
+	// deduces the existence of this race condition and generates another start step after a delay (1 minute by default).
+	describe('handling Engine restart policy inaction when host resource required by container is delayed in creation', () => {
+		// Time 61 seconds ago
+		const date61SecondsAgo = new Date();
+		date61SecondsAgo.setSeconds(date61SecondsAgo.getSeconds() - 61);
+
+		// Time 59 seconds ago
+		const date50SecondsAgo = new Date();
+		date50SecondsAgo.setSeconds(date50SecondsAgo.getSeconds() - 50);
+
+		// TODO: We need to be able to start a service with restart policy "no" if that service did not start at all due to
+		// the host resource race condition described above. However, this is harder to parse as the containers do not include
+		// the proper metadata for this. The last resort would be parsing the error message that caused the container to exit.
+		it('should not infer any steps for a service with a status of "exited" if restart policy is "no" or "on-failure"', async () => {
+			// Conditions:
+			// - restart: "no" || "on-failure"
+			// - status: "exited"
+			const targetApps = createApps(
+				{
+					services: [
+						await createService({
+							image: 'image-1',
+							serviceName: 'one',
+							composition: {
+								restart: 'no',
+							},
+						}),
+						await createService({
+							image: 'image-2',
+							serviceName: 'two',
+							composition: {
+								restart: 'no',
+							},
+						}),
+						await createService({
+							image: 'image-3',
+							serviceName: 'three',
+							composition: {
+								restart: 'on-failure',
+							},
+						}),
+						await createService({
+							image: 'image-4',
+							serviceName: 'four',
+							composition: {
+								restart: 'on-failure',
+							},
+						}),
+					],
+					networks: [DEFAULT_NETWORK],
+				},
+				true,
+			);
+
+			const { currentApps, availableImages, downloading, containerIdsByAppId } =
+				createCurrentState({
+					services: [
+						await createService(
+							{
+								image: 'image-1',
+								serviceName: 'one',
+								composition: {
+									restart: 'no',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									// Should not generate noop if exited within 1 minute
+									createdAt: date50SecondsAgo,
+								},
+							},
+						),
+						await createService(
+							{
+								image: 'image-2',
+								serviceName: 'two',
+								composition: {
+									restart: 'no',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									// Should not generate start if exited more than 1 minute ago
+									createdAt: date61SecondsAgo,
+								},
+							},
+						),
+						await createService(
+							{
+								image: 'image-3',
+								serviceName: 'three',
+								composition: {
+									restart: 'on-failure',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									// Should not generate noop if exited within 1 minute
+									createdAt: date50SecondsAgo,
+								},
+							},
+						),
+						await createService(
+							{
+								image: 'image-4',
+								serviceName: 'four',
+								composition: {
+									restart: 'on-failure',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									// Should not generate start if exited more than 1 minute ago
+									createdAt: date61SecondsAgo,
+								},
+							},
+						),
+					],
+					networks: [DEFAULT_NETWORK],
+					images: [
+						createImage({
+							name: 'image-1',
+							serviceName: 'one',
+						}),
+						createImage({
+							name: 'image-2',
+							serviceName: 'two',
+						}),
+						createImage({
+							name: 'image-3',
+							serviceName: 'three',
+						}),
+						createImage({
+							name: 'image-4',
+							serviceName: 'four',
+						}),
+					],
+				});
+
+			const [...steps] = await applicationManager.inferNextSteps(
+				currentApps,
+				targetApps,
+				{
+					downloading,
+					availableImages,
+					containerIdsByAppId,
+				},
+			);
+
+			expect(steps).to.have.lengthOf(0);
+		});
+
+		it('should infer a noop step for a service that was created <=1 min ago with status of "exited" if restart policy is "always" or "unless-stopped"', async () => {
+			// Conditions:
+			// - restart: "always" || "unless-stopped"
+			// - status: "exited"
+			// - createdAt: <= SECONDS_TO_WAIT_FOR_START ago
+			const targetApps = createApps(
+				{
+					services: [
+						await createService({
+							image: 'image-1',
+							serviceName: 'one',
+							composition: {
+								restart: 'always',
+							},
+						}),
+						await createService({
+							image: 'image-2',
+							serviceName: 'two',
+							composition: {
+								restart: 'unless-stopped',
+							},
+						}),
+					],
+					networks: [DEFAULT_NETWORK],
+				},
+				true,
+			);
+
+			const { currentApps, availableImages, downloading, containerIdsByAppId } =
+				createCurrentState({
+					services: [
+						await createService(
+							{
+								image: 'image-1',
+								serviceName: 'one',
+								running: false,
+								composition: {
+									restart: 'always',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									createdAt: date50SecondsAgo,
+								},
+							},
+						),
+						await createService(
+							{
+								image: 'image-2',
+								serviceName: 'two',
+								running: false,
+								composition: {
+									restart: 'unless-stopped',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									createdAt: date50SecondsAgo,
+								},
+							},
+						),
+					],
+					networks: [DEFAULT_NETWORK],
+					images: [
+						createImage({
+							name: 'image-1',
+							serviceName: 'one',
+						}),
+						createImage({
+							name: 'image-2',
+							serviceName: 'two',
+						}),
+					],
+				});
+
+			const [noopStep1, noopStep2, ...nextSteps] =
+				await applicationManager.inferNextSteps(currentApps, targetApps, {
+					downloading,
+					availableImages,
+					containerIdsByAppId,
+				});
+
+			expect(noopStep1).to.have.property('action').that.equals('noop');
+			expect(noopStep2).to.have.property('action').that.equals('noop');
+
+			expect(nextSteps).to.have.lengthOf(0);
+		});
+
+		it('should infer a start step for a service that was created >1 min ago with status of "exited" if restart policy is "always" or "unless-stopped"', async () => {
+			// Conditions:
+			// - restart: "always" || "unless-stopped"
+			// - status: "exited"
+			// - createdAt: <= SECONDS_TO_WAIT_FOR_START ago
+			const targetApps = createApps(
+				{
+					services: [
+						await createService({
+							image: 'image-1',
+							serviceName: 'one',
+							composition: {
+								restart: 'always',
+							},
+						}),
+						await createService({
+							image: 'image-2',
+							serviceName: 'two',
+							composition: {
+								restart: 'unless-stopped',
+							},
+						}),
+					],
+					networks: [DEFAULT_NETWORK],
+				},
+				true,
+			);
+
+			const { currentApps, availableImages, downloading, containerIdsByAppId } =
+				createCurrentState({
+					services: [
+						await createService(
+							{
+								image: 'image-1',
+								serviceName: 'one',
+								running: false,
+								composition: {
+									restart: 'always',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									createdAt: date61SecondsAgo,
+								},
+							},
+						),
+						await createService(
+							{
+								image: 'image-2',
+								serviceName: 'two',
+								running: false,
+								composition: {
+									restart: 'unless-stopped',
+								},
+							},
+							{
+								state: {
+									status: 'exited',
+									createdAt: date61SecondsAgo,
+								},
+							},
+						),
+					],
+					networks: [DEFAULT_NETWORK],
+					images: [
+						createImage({
+							name: 'image-1',
+							serviceName: 'one',
+						}),
+						createImage({
+							name: 'image-2',
+							serviceName: 'two',
+						}),
+					],
+				});
+
+			const [startStep1, startStep2, ...nextSteps] =
+				await applicationManager.inferNextSteps(currentApps, targetApps, {
+					downloading,
+					availableImages,
+					containerIdsByAppId,
+				});
+
+			[startStep1, startStep2].forEach((step) => {
+				expect(step).to.have.property('action').that.equals('start');
+				expect(step)
+					.to.have.property('target')
+					.that.has.property('serviceName')
+					.that.is.oneOf(['one', 'two']);
+			});
+			expect(nextSteps).to.have.lengthOf(0);
+		});
+	});
 });

test/integration/state-engine.spec.ts

@@ -49,6 +49,8 @@ const setTargetState = async (
 		while (true) {
 			const status = await getStatus();
 			if (status.appState === 'applied') {
+				// Wait a tiny bit more after applied for state to settle
+				await delay(1000);
 				clearTimeout(timer);
 				resolve(true);
 				break;

test/unit/compose/app.spec.ts

@@ -111,7 +111,7 @@ const defaultNetwork = Network.fromComposeObject('default', 1, 'appuuid', {});
 
 describe('compose/app', () => {
 	describe('volume state behavior', () => {
-		it('should correctly infer a volume create step', () => {
+		it('should correctly infer a volume create step', async () => {
 			// Setup current and target apps
 			const current = createApp();
 			const target = createApp({
@@ -120,7 +120,7 @@ describe('compose/app', () => {
 			});
 
 			// Calculate the steps
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			// Check that a createVolume step has been created
 			const [createVolumeStep] = expectSteps('createVolume', steps);
@@ -129,7 +129,7 @@ describe('compose/app', () => {
 				.that.deep.includes({ name: 'test-volume' });
 		});
 
-		it('should correctly infer more than one volume create step', () => {
+		it('should correctly infer more than one volume create step', async () => {
 			const current = createApp();
 			const target = createApp({
 				volumes: [
@@ -139,7 +139,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			// Check that 2 createVolume steps are found
 			const createVolumeSteps = expectSteps('createVolume', steps, 2);
@@ -160,7 +160,7 @@ describe('compose/app', () => {
 		});
 
 		// We don't remove volumes until the end
-		it('should not infer a volume remove step when the app is still referenced', () => {
+		it('should not infer a volume remove step when the app is still referenced', async () => {
 			const current = createApp({
 				volumes: [
 					Volume.fromComposeObject('test-volume', 1, 'deadbeef'),
@@ -172,11 +172,11 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			expectNoStep('removeVolume', steps);
 		});
 
-		it('should correctly infer volume recreation steps', () => {
+		it('should correctly infer volume recreation steps', async () => {
 			const current = createApp({
 				volumes: [Volume.fromComposeObject('test-volume', 1, 'deadbeef')],
 			});
@@ -190,7 +190,7 @@ describe('compose/app', () => {
 			});
 
 			// First step should create a volume removal step
-			const stepsForRemoval = current.nextStepsForAppUpdate(
+			const stepsForRemoval = await current.nextStepsForAppUpdate(
 				defaultContext,
 				target,
 			);
@@ -212,7 +212,7 @@ describe('compose/app', () => {
 			});
 
 			// This test is extra since we have already tested that the volume gets created
-			const stepsForCreation = intermediate.nextStepsForAppUpdate(
+			const stepsForCreation = await intermediate.nextStepsForAppUpdate(
 				defaultContext,
 				target,
 			);
@@ -254,7 +254,7 @@ describe('compose/app', () => {
 			});
 
 			// Calculate steps
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [killStep] = expectSteps('kill', steps);
 			expect(killStep)
@@ -294,7 +294,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			expectNoStep('kill', steps);
 		});
 
@@ -333,7 +333,7 @@ describe('compose/app', () => {
 			});
 
 			// Step 1: kill
-			const steps = current.nextStepsForAppUpdate(
+			const steps = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				intermediateTarget,
 			);
@@ -341,7 +341,7 @@ describe('compose/app', () => {
 
 			// Step 2: noop (service is stopping)
 			service.status = 'Stopping';
-			const secondStageSteps = current.nextStepsForAppUpdate(
+			const secondStageSteps = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				intermediateTarget,
 			);
@@ -355,7 +355,7 @@ describe('compose/app', () => {
 				volumes: [volume],
 			});
 			expect(
-				currentWithServiceRemoved.nextStepsForAppUpdate(
+				await currentWithServiceRemoved.nextStepsForAppUpdate(
 					contextWithImages,
 					intermediateTarget,
 				),
@@ -377,7 +377,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 			const recreateVolumeSteps =
-				currentWithVolumesRemoved.nextStepsForAppUpdate(
+				await currentWithVolumesRemoved.nextStepsForAppUpdate(
 					contextWithImages,
 					target,
 				);
@@ -393,7 +393,7 @@ describe('compose/app', () => {
 			});
 
 			const createServiceSteps =
-				currentWithVolumeRecreated.nextStepsForAppUpdate(
+				await currentWithVolumeRecreated.nextStepsForAppUpdate(
 					contextWithImages,
 					target,
 				);
@@ -402,14 +402,14 @@ describe('compose/app', () => {
 	});
 
 	describe('network state behavior', () => {
-		it('should correctly infer a network create step', () => {
+		it('should correctly infer a network create step', async () => {
 			const current = createApp({ networks: [] });
 			const target = createApp({
 				networks: [Network.fromComposeObject('default', 1, 'deadbeef', {})],
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [createNetworkStep] = expectSteps('createNetwork', steps);
 			expect(createNetworkStep).to.have.property('target').that.deep.includes({
@@ -417,7 +417,7 @@ describe('compose/app', () => {
 			});
 		});
 
-		it('should correctly infer a network remove step', () => {
+		it('should correctly infer a network remove step', async () => {
 			const current = createApp({
 				networks: [
 					Network.fromComposeObject('test-network', 1, 'deadbeef', {}),
@@ -425,7 +425,7 @@ describe('compose/app', () => {
 			});
 			const target = createApp({ networks: [], isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [removeNetworkStep] = expectSteps('removeNetwork', steps);
 
@@ -434,7 +434,7 @@ describe('compose/app', () => {
 			});
 		});
 
-		it('should correctly remove default duplicate networks', () => {
+		it('should correctly remove default duplicate networks', async () => {
 			const current = createApp({
 				networks: [defaultNetwork, defaultNetwork],
 			});
@@ -443,7 +443,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [removeNetworkStep] = expectSteps('removeNetwork', steps);
 
@@ -452,7 +452,7 @@ describe('compose/app', () => {
 			});
 		});
 
-		it('should correctly remove duplicate networks', () => {
+		it('should correctly remove duplicate networks', async () => {
 			const current = createApp({
 				networks: [
 					Network.fromComposeObject('test-network', 1, 'deadbeef', {}),
@@ -468,7 +468,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [removeNetworkStep] = expectSteps('removeNetwork', steps);
 
@@ -477,7 +477,7 @@ describe('compose/app', () => {
 			});
 		});
 
-		it('should ignore the duplicates if there are changes already', () => {
+		it('should ignore the duplicates if there are changes already', async () => {
 			const current = createApp({
 				networks: [
 					Network.fromComposeObject('test-network', 1, 'deadbeef', {}),
@@ -494,7 +494,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [removeNetworkStep] = expectSteps('removeNetwork', steps);
 
 			expect(removeNetworkStep).to.have.property('current').that.deep.includes({
@@ -534,7 +534,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [removeNetworkStep] = expectSteps('kill', steps);
 
@@ -543,7 +543,7 @@ describe('compose/app', () => {
 			});
 		});
 
-		it('should correctly infer more than one network removal step', () => {
+		it('should correctly infer more than one network removal step', async () => {
 			const current = createApp({
 				networks: [
 					Network.fromComposeObject('test-network', 1, 'deadbeef', {}),
@@ -553,7 +553,7 @@ describe('compose/app', () => {
 			});
 			const target = createApp({ networks: [], isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [first, second] = expectSteps('removeNetwork', steps, 2);
 
@@ -565,7 +565,7 @@ describe('compose/app', () => {
 			});
 		});
 
-		it('should correctly infer a network recreation step', () => {
+		it('should correctly infer a network recreation step', async () => {
 			const current = createApp({
 				networks: [
 					Network.fromComposeObject('test-network', 1, 'deadbeef', {}),
@@ -580,7 +580,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const stepsForRemoval = current.nextStepsForAppUpdate(
+			const stepsForRemoval = await current.nextStepsForAppUpdate(
 				defaultContext,
 				target,
 			);
@@ -595,7 +595,7 @@ describe('compose/app', () => {
 				networks: [],
 			});
 
-			const stepsForCreation = intermediate.nextStepsForAppUpdate(
+			const stepsForCreation = await intermediate.nextStepsForAppUpdate(
 				defaultContext,
 				target,
 			);
@@ -641,7 +641,7 @@ describe('compose/app', () => {
 
 			const availableImages = [createImage({ appUuid: 'deadbeef' })];
 
-			const steps = current.nextStepsForAppUpdate(
+			const steps = await current.nextStepsForAppUpdate(
 				{ ...defaultContext, availableImages },
 				target,
 			);
@@ -674,7 +674,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [killStep] = expectSteps('kill', steps);
 
 			expect(killStep)
@@ -730,7 +730,7 @@ describe('compose/app', () => {
 				createImage({ appId: 1, serviceName: 'two', name: 'alpine' }),
 			];
 
-			const steps = current.nextStepsForAppUpdate(
+			const steps = await current.nextStepsForAppUpdate(
 				{ ...defaultContext, availableImages },
 				target,
 			);
@@ -793,7 +793,7 @@ describe('compose/app', () => {
 				createImage({ appId: 1, serviceName: 'two', name: 'alpine' }),
 			];
 
-			const steps = current.nextStepsForAppUpdate(
+			const steps = await current.nextStepsForAppUpdate(
 				{ ...defaultContext, availableImages },
 				target,
 			);
@@ -808,11 +808,11 @@ describe('compose/app', () => {
 			expectNoStep('removeNetwork', steps);
 		});
 
-		it('should create the default network if it does not exist', () => {
+		it('should create the default network if it does not exist', async () => {
 			const current = createApp({ networks: [] });
 			const target = createApp({ networks: [], isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			// A default network should always be created
 			const [createNetworkStep] = expectSteps('createNetwork', steps);
@@ -821,13 +821,13 @@ describe('compose/app', () => {
 				.that.deep.includes({ name: 'default' });
 		});
 
-		it('should not create the default network if it already exists', () => {
+		it('should not create the default network if it already exists', async () => {
 			const current = createApp({
 				networks: [Network.fromComposeObject('default', 1, 'deadbeef', {})],
 			});
 			const target = createApp({ networks: [], isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			// The network should not be created again
 			expectNoStep('createNetwork', steps);
@@ -854,7 +854,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [createNetworkStep] = expectSteps('createNetwork', steps);
 			expect(createNetworkStep)
@@ -883,7 +883,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [createNetworkStep] = expectSteps('createNetwork', steps);
 			expect(createNetworkStep)
@@ -896,7 +896,7 @@ describe('compose/app', () => {
 			const current = createApp({});
 			const target = createApp({ isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [createNetworkStep] = expectSteps('createNetwork', steps);
 			expect(createNetworkStep)
@@ -921,7 +921,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [killStep] = expectSteps('kill', steps);
 			expect(killStep)
 				.to.have.property('current')
@@ -939,7 +939,7 @@ describe('compose/app', () => {
 			});
 			const target = createApp({ services: [], isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			expectSteps('noop', steps);
 
 			// Kill was already emitted for this service
@@ -959,7 +959,7 @@ describe('compose/app', () => {
 				services: [await createService({ serviceName: 'main', running: true })],
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			expectSteps('noop', steps);
 		});
 
@@ -977,7 +977,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [removeStep] = expectSteps('remove', steps);
 
 			expect(removeStep)
@@ -996,7 +996,7 @@ describe('compose/app', () => {
 			});
 			const target = createApp({ services: [], isTarget: true });
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [removeStep] = expectSteps('remove', steps);
 
 			expect(removeStep)
@@ -1013,7 +1013,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(
+			const steps = await current.nextStepsForAppUpdate(
 				{ ...defaultContext, ...{ downloading: ['main-image'] } },
 				target,
 			);
@@ -1034,7 +1034,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [updateMetadataStep] = expectSteps('updateMetadata', steps);
 
 			expect(updateMetadataStep)
@@ -1057,7 +1057,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [stopStep] = expectSteps('stop', steps);
 			expect(stopStep)
 				.to.have.property('current')
@@ -1086,7 +1086,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			expectNoStep('start', steps);
 		});
 
@@ -1118,7 +1118,7 @@ describe('compose/app', () => {
 			});
 
 			// should see a 'stop'
-			const stepsToIntermediate = current.nextStepsForAppUpdate(
+			const stepsToIntermediate = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1135,7 +1135,7 @@ describe('compose/app', () => {
 			});
 
 			// now should see a 'start'
-			const stepsToTarget = intermediate.nextStepsForAppUpdate(
+			const stepsToTarget = await intermediate.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1193,7 +1193,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const stepsToIntermediate = current.nextStepsForAppUpdate(
+			const stepsToIntermediate = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1216,7 +1216,7 @@ describe('compose/app', () => {
 			});
 
 			// we should now see a start for the 'main' service...
-			const stepsToTarget = intermediate.nextStepsForAppUpdate(
+			const stepsToTarget = await intermediate.nextStepsForAppUpdate(
 				{ ...contextWithImages, ...{ containerIds: { dep: 'dep-id' } } },
 				target,
 			);
@@ -1248,7 +1248,10 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(contextWithImages, target);
+			const steps = await current.nextStepsForAppUpdate(
+				contextWithImages,
+				target,
+			);
 
 			// There should be no steps since the engine manages restart policy for stopped containers
 			expect(steps.length).to.equal(0);
@@ -1292,7 +1295,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const stepsToIntermediate = current.nextStepsForAppUpdate(
+			const stepsToIntermediate = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1308,7 +1311,7 @@ describe('compose/app', () => {
 				networks: [defaultNetwork],
 			});
 
-			const stepsToTarget = intermediate.nextStepsForAppUpdate(
+			const stepsToTarget = await intermediate.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1379,7 +1382,10 @@ describe('compose/app', () => {
 			});
 
 			// No kill steps should be generated
-			const steps = current.nextStepsForAppUpdate(contextWithImages, target);
+			const steps = await current.nextStepsForAppUpdate(
+				contextWithImages,
+				target,
+			);
 			expectNoStep('kill', steps);
 		});
 
@@ -1421,7 +1427,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const stepsFirstTry = current.nextStepsForAppUpdate(
+			const stepsFirstTry = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1432,7 +1438,7 @@ describe('compose/app', () => {
 				.that.deep.includes({ serviceName: 'main' });
 
 			// if at first you don't succeed
-			const stepsSecondTry = current.nextStepsForAppUpdate(
+			const stepsSecondTry = await current.nextStepsForAppUpdate(
 				contextWithImages,
 				target,
 			);
@@ -1464,7 +1470,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [killStep] = expectSteps('kill', steps);
 			expect(killStep)
 				.to.have.property('current')
@@ -1487,7 +1493,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [createNetworkStep] = expectSteps('createNetwork', steps);
 			expect(createNetworkStep)
 				.to.have.property('target')
@@ -1528,7 +1534,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			expectSteps('kill', steps, 2);
 		});
 
@@ -1590,7 +1596,10 @@ describe('compose/app', () => {
 			});
 
 			// No kill steps should be generated
-			const steps = current.nextStepsForAppUpdate(contextWithImages, target);
+			const steps = await current.nextStepsForAppUpdate(
+				contextWithImages,
+				target,
+			);
 			expectNoStep('kill', steps);
 		});
 
@@ -1629,7 +1638,10 @@ describe('compose/app', () => {
 			});
 
 			// No kill steps should be generated
-			const steps = current.nextStepsForAppUpdate(contextWithImages, target);
+			const steps = await current.nextStepsForAppUpdate(
+				contextWithImages,
+				target,
+			);
 			expectNoStep('start', steps);
 		});
 
@@ -1673,7 +1685,10 @@ describe('compose/app', () => {
 			});
 
 			// No kill steps should be generated
-			const steps = current.nextStepsForAppUpdate(contextWithImages, target);
+			const steps = await current.nextStepsForAppUpdate(
+				contextWithImages,
+				target,
+			);
 			expectSteps('start', steps, 2);
 		});
 	});
@@ -1686,7 +1701,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 			const [fetchStep] = expectSteps('fetch', steps);
 			expect(fetchStep)
 				.to.have.property('image')
@@ -1708,7 +1723,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(
+			const steps = await current.nextStepsForAppUpdate(
 				contextWithDownloading,
 				target,
 			);
@@ -1724,7 +1739,7 @@ describe('compose/app', () => {
 				isTarget: true,
 			});
 
-			const steps = current.nextStepsForAppUpdate(defaultContext, target);
+			const steps = await current.nextStepsForAppUpdate(defaultContext, target);
 
 			const [fetchStep] = expectSteps('fetch', steps);
 			expect(fetchStep)