Handle Engine-host race condition for "always" and "unless-stopped" restart policy

There exists a race condition between Engine and a host resource that may not
be immediately created. In this race condition, if a container's compose config
depends on the existence of that host resource, such as a network interface, and the
Engine tries to create & start the container before the host resource is created, the
Engine will not reattempt to start the container, regardless of the restart policy.
This is undesireable behavior but seems to be the behavior as implemented by Docker.

To rectify this, the Supervisor state funnel noops for a grace period of 1 minute
after starting a container to see that the container's status has become 'running`.
If the container exits because of the race condition, the status becomes 'exited' and the
Supervisor will attempt to generate another start step. This noop-wait-start step loop
will repeat until the container is able to start.

If the container is never able to start, there was a problem in the host in the creation of the
host resource, and that should be fixed at the host level.

This commit does not handle the case of services with restart policies "no" or "on-failure"
which encounter this host race, as metadata from container inspects needs to be introduced
during step calculation in order to figure out whether services with those restart policies
need to be started. This will be fixed in a future PR.

Change-type: patch
Signed-off-by: Christina Ying Wang <christina@balena.io>

src/compose/app.ts

@@ -4,7 +4,6 @@ import { promises as fs } from 'fs';
 import Network from './network';
 import Volume from './volume';
 import Service from './service';
-
 import * as imageManager from './images';
 import type { Image } from './images';
 import {
@@ -12,6 +11,7 @@ import {
 	generateStep,
 	CompositionStepAction,
 } from './composition-steps';
+import { isOlderThan } from './utils';
 import * as targetStateCache from '../device-state/target-state-cache';
 import * as dockerUtils from '../lib/docker-utils';
 import * as constants from '../lib/constants';
@@ -26,6 +26,8 @@ import { ImageInspectInfo } from 'dockerode';
 import { pathExistsOnRoot } from '../lib/host-utils';
 import { isSupervisor } from '../lib/supervisor-metadata';
 
+const SECONDS_TO_WAIT_FOR_START = 60;
+
 export interface AppConstructOpts {
 	appId: number;
 	appUuid?: string;
@@ -184,7 +186,6 @@ export class App {
 				}),
 			);
 		}
-
 		return steps;
 	}
 
@@ -377,9 +378,22 @@ export class App {
 
 			// Only start a Service if we have never started it before and the service matches target!
 			// This is so the engine can handle the restart policy configured for the container.
+			//
+			// However, there is a certain race condition where the container's compose depends on a host
+			// resource that may not be there when the Engine starts the container, such as a port binding
+			// of 192.168.88.1:3000:3000, where 192.168.88.1 is a user-defined interface configured in system-connections
+			// and created by the host. This interface creation may not occur before the container creation.
+			// In this case, the container is created and never started, and the Engine does not attempt to restart it
+			// regardless of restart policy.
+			const requiresExplicitStart =
+				['always', 'unless-stopped'].includes(serviceTarget.config.restart) &&
+				serviceCurrent.status === 'exited' &&
+				serviceCurrent.createdAt != null &&
+				isOlderThan(serviceCurrent.createdAt, SECONDS_TO_WAIT_FOR_START);
 			return (
 				(serviceCurrent.status === 'Installing' ||
-					serviceCurrent.status === 'Installed') &&
+					serviceCurrent.status === 'Installed' ||
+					requiresExplicitStart) &&
 				isEqualExceptForRunningState(serviceCurrent, serviceTarget)
 			);
 		};
@@ -401,6 +415,26 @@ export class App {
 			);
 		};
 
+		/**
+		 * Under the race condition of a container depending on a host resource that may not be there when the Engine
+		 * starts up, resulting in the Engine not restarting the container regardless of restart policy, the Supervisor
+		 * state loop needs to stay alive so that the Supervisor may issue a start step after a wait.
+		 * @param serviceCurrent
+		 */
+		const shouldWaitForStart = (
+			serviceCurrent: Service,
+			serviceTarget: Service,
+		) => {
+			return (
+				['always', 'unless-stopped'].includes(serviceTarget.config.restart) &&
+				serviceCurrent.config.running === false &&
+				serviceTarget.config.running === true &&
+				serviceCurrent.status === 'exited' &&
+				serviceCurrent.createdAt != null &&
+				!isOlderThan(serviceCurrent.createdAt, SECONDS_TO_WAIT_FOR_START)
+			);
+		};
+
 		/**
 		 * Checks if Supervisor should keep the state loop alive while waiting on a service to stop
 		 * @param serviceCurrent
@@ -425,7 +459,8 @@ export class App {
 					!isEqualExceptForRunningState(c, t) ||
 					shouldBeStarted(c, t) ||
 					shouldBeStopped(c, t) ||
-					shouldWaitForStop(c),
+					shouldWaitForStop(c) ||
+					shouldWaitForStart(c, t),
 			);
 
 		return {
@@ -636,6 +671,26 @@ export class App {
 		if (current.commit !== target.commit) {
 			return generateStep('updateMetadata', { current, target });
 		} else if (target.config.running !== current.config.running) {
+			// In the case where the Engine does not start the container despite the
+			// restart policy (this can happen in cases of Engine race conditions with
+			// host resources that are slower to be created but that a service relies on),
+			// we need to start the container after a delay.
+			// FIXME: It makes absolutely zero sense that an empty string is the value
+			// for config.restart if the restart policy is 'no'.
+			// TODO: There is no way to tell from a Service instance whether a service
+			// exited with a non-zero exit code, therefore this method cannot determine
+			// whether to start a service with a restart policy of 'on-failure'. There
+			// is no way to get the exit message from a Service instance either, so we
+			// can't parse the error message to determine whether to start a service with
+			// a restart policy of 'no' (in case a oneshot suffered from this race condition).
+			if (
+				['always', 'unless-stopped'].includes(target.config.restart) &&
+				current.status === 'exited' &&
+				current.createdAt != null &&
+				!isOlderThan(current.createdAt, SECONDS_TO_WAIT_FOR_START)
+			) {
+				return generateStep('noop', {});
+			}
 			if (target.config.running) {
 				return generateStep('start', { target });
 			} else {

src/compose/utils.ts

@@ -693,3 +693,7 @@ export function dockerMountToServiceMount(
 
 	return mount as LongDefinition;
 }
+
+export function isOlderThan(currentTime: Date, seconds: number) {
+	return new Date().getTime() - currentTime.getTime() > seconds * 1000;
+}