Run the built supervisor as part of docker-compose tests

This allows to test that the supervisor build actually runs and opens up the
possibility of running more exhaustive API tests against a working supervisor.

Change-type: patch

docker-compose.test.yml

@@ -5,9 +5,32 @@ services:
   # be run through the `sut` service
   balena-supervisor:
     build:
-      dockerfile: Dockerfile
       context: ./
-    command: sleep infinity
+      dockerfile: Dockerfile.template
+      args:
+        ARCH: ${ARCH:-amd64}
+    command: ['/wait-for-it.sh', '--', '/usr/src/app/entry.sh']
+    # Use bridge networking for the tests
+    network_mode: 'bridge'
+    networks:
+      - default
+    environment:
+      DOCKER_HOST: tcp://docker:2375
+      DBUS_SYSTEM_BUS_ADDRESS: unix:path=/run/dbus/system_bus_socket
+      # Required by migrations
+      CONFIG_MOUNT_POINT: /mnt/root/mnt/boot/config.json
+      # Read by constants to setup `bootMountpoint`
+      BOOT_MOUNTPOINT: /mnt/boot
+    depends_on:
+      - docker
+      - dbus
+      - dbus-services
+    volumes:
+      - dbus:/run/dbus
+      - ./test/data/root:/mnt/root
+      - ./test/lib/wait-for-it.sh:/wait-for-it.sh
+    tmpfs:
+      - /data
 
   dbus:
     image: balenablocks/dbus
@@ -33,7 +56,7 @@ services:
     privileged: true
     environment:
       DOCKER_TLS_CERTDIR: ''
-    command: --tls=false --debug
+    command: --tls=false # --debug
 
   sut:
     # Build the supervisor code for development and testing
@@ -44,6 +67,15 @@ services:
       args:
         # Change this if testing in another architecture
         ARCH: ${ARCH:-amd64}
+    command:
+      [
+        './test/lib/wait-for-it.sh',
+        '--supervisor',
+        '--',
+        'npm',
+        'run',
+        'test:integration',
+      ]
     depends_on:
       - balena-supervisor
       - docker
@@ -55,6 +87,7 @@ services:
     environment:
       DOCKER_HOST: tcp://docker:2375
       DBUS_SYSTEM_BUS_ADDRESS: unix:path=/run/dbus/system_bus_socket
+      BALENA_SUPERVISOR_ADDRESS: http://balena-supervisor:48484
       # Required by migrations
       CONFIG_MOUNT_POINT: /mnt/root/mnt/boot/config.json
       # Read by constants to setup `bootMountpoint`
@@ -73,3 +106,6 @@ volumes:
       # Use tmpfs to avoid files remaining between runs
       type: tmpfs
       device: tmpfs
+
+networks:
+  default:

entry.sh

@@ -12,7 +12,7 @@ set -o errexit
 [ -d /mnt/root/tmp/balena-supervisor ] ||
 	mkdir -p /mnt/root/tmp/balena-supervisor
 
-export DBUS_SYSTEM_BUS_ADDRESS="unix:path=/mnt/root/run/dbus/system_bus_socket"
+export DBUS_SYSTEM_BUS_ADDRESS="${DBUS_SYSTEM_BUS_ADDRESS:-unix:path=/mnt/root/run/dbus/system_bus_socket}"
 
 # Include self-signed CAs, should they exist
 if [ -n "${BALENA_ROOT_CA}" ]; then

src/logging/monitor.ts

@@ -29,6 +29,7 @@ const DB_FLUSH_INTERVAL = 10 * 60 * 1000;
 
 // Wait 5s when journalctl failed before trying to read the logs again
 const JOURNALCTL_ERROR_RETRY_DELAY = 5000;
+const JOURNALCTL_ERROR_RETRY_DELAY_MAX = 15 * 60 * 1000;
 
 function messageFieldToString(entry: JournalRow['MESSAGE']): string | null {
 	if (Array.isArray(entry)) {
@@ -55,6 +56,7 @@ class LogMonitor {
 			writeRequired: boolean;
 		};
 	} = {};
+	private setupAttempts = 0;
 
 	public constructor() {
 		setInterval(() => this.flushDb(), DB_FLUSH_INTERVAL);
@@ -70,6 +72,7 @@ class LogMonitor {
 			},
 			(row: JournalRow) => {
 				if (row.CONTAINER_ID_FULL && this.containers[row.CONTAINER_ID_FULL]) {
+					this.setupAttempts = 0;
 					this.handleRow(row);
 				}
 			},
@@ -82,8 +85,16 @@ class LogMonitor {
 			async () => {
 				log.debug('balena.service journalctl process exit.');
 				// On exit of process try to create another
-				await delay(JOURNALCTL_ERROR_RETRY_DELAY);
-				log.debug('Spawning another process to watch balena.service logs.');
+				const wait = Math.min(
+					2 ** this.setupAttempts++ * JOURNALCTL_ERROR_RETRY_DELAY,
+					JOURNALCTL_ERROR_RETRY_DELAY_MAX,
+				);
+				log.debug(
+					`Spawning another process to watch balena.service logs in ${
+						wait / 1000
+					}s`,
+				);
+				await delay(wait);
 				return this.start();
 			},
 		);

test/data/root/etc/hostname

@@ -0,0 +1 @@
+balena

test/data/root/etc/os-release

@@ -0,0 +1,10 @@
+ID="balena-os"
+NAME="balenaOS"
+VERSION="2.103.1+rev1"
+VERSION_ID="2.103.1+rev1"
+PRETTY_NAME="balenaOS 2.103.1+rev1"
+MACHINE="raspberrypi4-64"
+META_BALENA_VERSION="2.103.1"
+BALENA_BOARD_REV=""
+META_BALENA_REV=""
+SLUG="raspberrypi4-64"

test/data/root/mnt/boot/config.json

@@ -0,0 +1,12 @@
+{
+	"appUpdatePollInterval": 900000,
+	"deltaEndpoint": "https://delta.balena-cloud.com",
+	"developmentMode": "true",
+	"deviceType": "raspberrypi4-64",
+	"listenPort": "48484",
+	"registryEndpoint": "registry2.balena-cloud.com",
+	"vpnEndpoint": "cloudlink.balena-cloud.com",
+	"vpnPort": "443",
+	"uuid": "151e8026dd444b028cbf40d76c244b93",
+	"deviceApiKey": "8d6d8933c14242249da4979a5f48f333"
+}

test/data/root/mnt/boot/config.txt

@@ -0,0 +1,11 @@
+hdmi_force_hotplug1=1
+hdmi_force_hotplug_1=1
+hdmi_force_hotplug__1=1
+avoid_warnings=1
+disable_splash=1
+dtoverlay=vc4-kms-v3d
+dtoverlay=dwc2
+dtparam=i2c_arm=on
+dtparam=spi=on
+dtparam=audio=on
+gpu_mem=16

test/data/root/mnt/boot/device-type.json

@@ -0,0 +1,7 @@
+{
+	"slug": "raspberrypi4-64",
+	"version": 1,
+	"aliases": ["raspberrypi4-64"],
+	"name": "Raspberry Pi 4",
+	"arch": "aarch64"
+}

test/integration/config.spec.ts

@@ -153,7 +153,7 @@ describe('config', () => {
 	});
 
 	it('reads and exposes MAC addresses', async () => {
-		// FIXME: this variable defaults to `/mnt/root/sys/class/net`. The supervisor runs with network_mode: false
+		// FIXME: this variable defaults to `/mnt/root/sys/class/net`. The supervisor runs with network_mode: host
 		// which means that it can just use the container `/sys/class/net` and the result should be the same
 		constants.macAddressPath = '/sys/class/net';
 		const macAddress = await conf.get('macAddress');

test/integration/supervisor.spec.ts

@@ -0,0 +1,10 @@
+import * as request from 'supertest';
+
+const BALENA_SUPERVISOR_ADDRESS =
+	process.env.BALENA_SUPERVISOR_ADDRESS || 'http://balena-supervisor:48484';
+
+describe('supervisor app', () => {
+	it('the supervisor app runs and the API responds with a healthy status', async () => {
+		await request(BALENA_SUPERVISOR_ADDRESS).get('/v1/healthy').expect(200);
+	});
+});

test/legacy/18-startup.spec.ts

@@ -1,48 +0,0 @@
-import { SinonStub, stub } from 'sinon';
-import { expect } from 'chai';
-import * as _ from 'lodash';
-
-import * as apiBinder from '~/src/api-binder';
-import * as applicationManager from '~/src/compose/application-manager';
-import * as deviceState from '~/src/device-state';
-import * as constants from '~/lib/constants';
-import { docker } from '~/lib/docker-utils';
-import { Supervisor } from '~/src/supervisor';
-
-// TODO: remove this when we can test proper supervisor startup during
-// integration tests
-describe('Startup', () => {
-	let startStub: SinonStub;
-	let vpnStatusPathStub: SinonStub;
-	let deviceStateStub: SinonStub;
-	let dockerStub: SinonStub;
-
-	before(async () => {
-		startStub = stub(apiBinder as any, 'start').resolves();
-		deviceStateStub = stub(deviceState, 'applyTarget').resolves();
-		// @ts-expect-error
-		applicationManager.initialized = () => Promise.resolve();
-		vpnStatusPathStub = stub(constants, 'vpnStatusPath').returns('');
-		dockerStub = stub(docker, 'listContainers').returns(Promise.resolve([]));
-	});
-
-	after(() => {
-		startStub.restore();
-		vpnStatusPathStub.restore();
-		deviceStateStub.restore();
-		dockerStub.restore();
-	});
-
-	it('should startup correctly', async () => {
-		const supervisor = new Supervisor();
-		await supervisor.init();
-
-		// Cast as any to access private properties
-		const anySupervisor = supervisor as any;
-		expect(anySupervisor.db).to.not.be.null;
-		expect(anySupervisor.config).to.not.be.null;
-		expect(anySupervisor.logger).to.not.be.null;
-		expect(anySupervisor.deviceState).to.not.be.null;
-		expect(anySupervisor.apiBinder).to.not.be.null;
-	});
-});

test/lib/wait-for-it.sh

@@ -0,0 +1,113 @@
+#!/bin/sh
+
+set -e
+
+timeout=30
+while :; do
+	case $1 in
+	-s | --supervisor)
+		with_supervisor=1
+		shift
+		;;
+	-t | --timeout) # Takes an option argument, ensuring it has been specified.
+		if [ -n "$2" ]; then
+			timeout=$2
+			shift
+		else
+			printf 'ERROR: "--timeout" requires a non-empty option argument.\n' >&2
+		fi
+		break
+		;;
+	--) # End of all options.
+		shift
+		break
+		;;
+	-?*)
+		printf 'WARN: Unknown option (ignored): %s\n' "$1" >&2
+		;;
+	*) # Default case: If no more options then break out of the loop.
+		break ;;
+	esac
+
+	shift
+done
+
+cmd="$*"
+
+# Use a local socket for docker by default
+DOCKER_HOST="${DOCKER_HOST:-'unix:///var/run/docker.sock'}"
+
+path=${DOCKER_HOST#*//}
+host=${path%%/*}
+proto=${DOCKER_HOST%:*}
+
+# Install curl
+apk add --update curl
+
+docker_healthy() {
+	if [ "${proto}" = "unix" ]; then
+		curl -s -S --unix-socket "${path}" "http://localhost/_ping"
+	else
+		curl -s -S "http://${host}/_ping"
+	fi
+}
+
+dbus_healthy() {
+	# The dbus service creates a fake openvpn unit, if the query below
+	# succeeds, then the service is ready
+	dbus-send --system \
+		--print-reply \
+		--type=method_call \
+		--dest=org.freedesktop.systemd1 \
+		/org/freedesktop/systemd1 \
+		org.freedesktop.systemd1.Manager.GetUnit string:openvpn.service
+}
+
+set_abort_timer() {
+	sleep "$1"
+	# Send a USR2 signal to the given pid after the timeout happens
+	kill -USR2 "$2"
+}
+
+# Flag indicating whether the required services are ready
+ready=0
+abort_if_not_ready() {
+	# If the timeout is reached and the required services are not ready, it probably
+	# means something went wrong so we terminate the program with an error
+	if [ "${ready}" = "0" ]; then
+		echo "Something happened, failed to start in ${timeout}s" >&2
+		exit 1
+	fi
+}
+
+# Trap the signal and start the timer if user timeout is greater than 0
+if [ "$timeout" -gt 0 ]; then
+	trap 'abort_if_not_ready' USR2
+	set_abort_timer "$timeout" $$ &
+fi
+
+# Wait for docker
+until docker_healthy; do
+	echo "Waiting for docker at ${DOCKER_HOST}"
+	sleep 1
+done
+
+# Wait for dbus
+until dbus_healthy >/dev/null; do
+	echo "Waiting for dbus"
+	sleep 1
+done
+
+# Wait for supervisor if user requested
+BALENA_SUPERVISOR_ADDRESS="${BALENA_SUPERVISOR_ADDRESS:-'http://balena-supervisor:48484'}"
+if [ "${with_supervisor}" = "1" ]; then
+	until curl -s -f "${BALENA_SUPERVISOR_ADDRESS}/v1/healthy" >/dev/null; do
+		echo "Waiting for supervisor at ${BALENA_SUPERVISOR_ADDRESS}"
+		sleep 1
+	done
+fi
+
+# Ignore signal if received
+ready=1
+
+exec ${cmd}