⚡ Update synchronisation scripts for supervisor development

We move the old sync.js script to tools/, and delete the now-broken
sync-debug.js.

We add a command `npm run sync`, which starts a livepush process
with the supervisor on a device.

Change-type: minor
Signed-off-by: Cameron Diver <cameron@balena.io>

Dockerfile

@@ -45,6 +45,13 @@ COPY package*.json ./
 
 RUN npm ci
 
+# TODO: Once we support live copies and live runs, convert
+# these
+# issue: https://github.com/balena-io-modules/livepush/issues/73
+RUN apk add --no-cache ip6tables iptables
+COPY entry.sh .
+#dev-cmd-live=LIVEPUSH=1 ./entry.sh
+
 COPY webpack.config.js fix-jsonstream.js hardcode-migrations.js tsconfig.json tsconfig.release.json ./
 COPY src ./src
 COPY test ./test

entry.sh

@@ -53,4 +53,9 @@ fi
 # not a problem.
 modprobe ip6_tables || true
 
-exec node /usr/src/app/dist/app.js
+if [ "${LIVEPUSH}" -eq "1" ]; then
+	exec npx nodemon --watch src --watch typings --ignore tests \
+		 --exec node -r ts-node/register/transpile-only -r coffeescript/register src/app.ts
+else
+	exec node /usr/src/app/dist/app.js
+fi

package.json

@@ -26,7 +26,8 @@
     "packagejson:copy": "cp package.json build/",
     "testitems:copy": "cp -r test/data build/test/",
     "lint:coffee": "balena-lint src/ test/",
-    "lint:typescript": "balena-lint -e ts -e js --typescript src/ test/ typings/ && tsc --noEmit && tsc --noEmit --project tsconfig.js.json"
+    "lint:typescript": "balena-lint -e ts -e js --typescript src/ test/ typings/ && tsc --noEmit && tsc --noEmit --project tsconfig.js.json",
+    "sync": "ts-node sync/sync.ts"
   },
   "private": true,
   "dependencies": {
@@ -62,6 +63,7 @@
     "@types/sinon": "^7.5.2",
     "@types/sinon-chai": "^3.2.3",
     "@types/tmp": "^0.1.0",
+    "@types/yargs": "^15.0.4",
     "blinking": "~0.0.3",
     "bluebird": "^3.7.2",
     "body-parser": "^1.19.0",
@@ -90,7 +92,7 @@
     "json-mask": "^0.3.9",
     "knex": "^0.15.2",
     "lint-staged": "^10.0.8",
-    "livepush": "^3.0.3",
+    "livepush": "^3.2.2",
     "lockfile": "^1.0.4",
     "lodash": "^4.17.15",
     "log-timestamp": "^0.1.2",
@@ -101,10 +103,12 @@
     "morgan": "^1.10.0",
     "mz": "^2.7.0",
     "network-checker": "^0.1.1",
+    "nodemon": "^2.0.2",
     "pinejs-client-request": "^5.2.0",
     "pretty-ms": "^5.1.0",
     "request": "^2.88.2",
     "resin-cli-visuals": "^1.5.2",
+    "resin-docker-build": "^1.1.4",
     "resin-register-device": "^3.0.0",
     "resumable-request": "^2.0.0",
     "rimraf": "^2.7.1",
@@ -113,6 +117,7 @@
     "sinon": "^7.5.0",
     "sinon-chai": "^3.5.0",
     "strict-event-emitter-types": "^2.0.0",
+    "tar-stream": "^2.1.2",
     "terser": "^3.17.0",
     "tmp": "^0.1.0",
     "ts-loader": "^5.4.5",

sync/device.ts

@@ -0,0 +1,178 @@
+import * as Docker from 'dockerode';
+import { Dockerfile } from 'livepush';
+import * as _ from 'lodash';
+import { Builder } from 'resin-docker-build';
+
+import { promises as fs } from 'fs';
+import { child_process } from 'mz';
+import * as Path from 'path';
+import { Duplex, Readable } from 'stream';
+import * as tar from 'tar-stream';
+
+export function getDocker(deviceAddress: string): Docker {
+	return new Docker({
+		host: deviceAddress,
+		// TODO: Make this configurable
+		port: 2375,
+	});
+}
+
+export async function getSupervisorContainer(
+	docker: Docker,
+	requireRunning: boolean = false,
+): Promise<Docker.ContainerInfo> {
+	// First get the supervisor container id
+	const containers = await docker.listContainers({
+		filters: { name: ['resin_supervisor'] },
+		all: !requireRunning,
+	});
+
+	if (containers.length !== 1) {
+		throw new Error('supervisor container not found');
+	}
+	return containers[0];
+}
+
+export async function getDeviceArch(docker: Docker): Promise<string> {
+	try {
+		const supervisorContainer = await getSupervisorContainer(docker);
+		const arch = supervisorContainer.Labels?.['io.balena.architecture'];
+		if (arch == null) {
+			// We can try to inspect the image for the
+			// architecture if this fails
+			const match = /(amd64|i386|aarch64|armv7hf|rpi)/.exec(
+				supervisorContainer.Image,
+			);
+			if (match != null) {
+				return match[1];
+			}
+
+			throw new Error('supervisor container does not have architecture label');
+		}
+
+		return arch.trim();
+	} catch (e) {
+		throw new Error(
+			`Unable to get device architecture: ${e.message}.\nTry specifying the architecture with -a.`,
+		);
+	}
+}
+
+export async function getCacheFrom(docker: Docker): Promise<string[]> {
+	const container = await getSupervisorContainer(docker);
+	return [container.Image];
+}
+
+// perform the build and return the image id
+export async function performBuild(
+	docker: Docker,
+	dockerfile: Dockerfile,
+	dockerOpts: { [key: string]: any },
+): Promise<void> {
+	const builder = Builder.fromDockerode(docker);
+
+	// tar the directory, but replace the dockerfile with the
+	// livepush generated one
+	const tarStream = await tarDirectory(Path.join(__dirname, '..'), dockerfile);
+
+	return new Promise((resolve, reject) => {
+		builder.createBuildStream(dockerOpts, {
+			buildSuccess: () => {
+				resolve();
+			},
+			buildFailure: reject,
+			buildStream: (stream: Duplex) => {
+				stream.pipe(process.stdout);
+				tarStream.pipe(stream);
+			},
+		});
+	});
+}
+
+async function tarDirectory(
+	dir: string,
+	dockerfile: Dockerfile,
+): Promise<Readable> {
+	const pack = tar.pack();
+
+	const add = async (path: string) => {
+		const entries = await fs.readdir(path);
+		for (const entry of entries) {
+			const newPath = Path.resolve(path, entry);
+			const stat = await fs.stat(newPath);
+			if (stat.isDirectory()) {
+				await add(newPath);
+			} else {
+				// Here we filter the things we don't want
+				if (
+					newPath.includes('node_modules/') ||
+					newPath.includes('.git/') ||
+					newPath.includes('build/') ||
+					newPath.includes('coverage/')
+				) {
+					continue;
+				}
+
+				if (newPath.endsWith('Dockerfile')) {
+					pack.entry(
+						{ name: 'Dockerfile', mode: stat.mode, size: stat.size },
+						dockerfile.generateLiveDockerfile(),
+					);
+					continue;
+				}
+
+				pack.entry(
+					{
+						name: Path.relative(dir, newPath),
+						mode: stat.mode,
+						size: stat.size,
+					},
+					await fs.readFile(newPath),
+				);
+			}
+		}
+	};
+
+	await add(dir);
+	pack.finalize();
+	return pack;
+}
+
+// Absolutely no escaping in this function, just be careful
+async function runSshCommand(address: string, command: string) {
+	// TODO: Make the port configurable
+	const [stdout] = await child_process.exec(
+		'ssh -p 22222 -o LogLevel=ERROR ' +
+			'-o StrictHostKeyChecking=no ' +
+			'-o UserKnownHostsFile=/dev/null ' +
+			`root@${address} ` +
+			`"${command}"`,
+	);
+	return stdout;
+}
+
+export function stopSupervisor(address: string) {
+	return runSshCommand(address, 'systemctl stop resin-supervisor');
+}
+
+export function startSupervisor(address: string) {
+	return runSshCommand(address, 'systemctl start resin-supervisor');
+}
+
+export async function replaceSupervisorImage(
+	address: string,
+	imageName: string,
+	imageTag: string,
+) {
+	// TODO: Maybe don't overwrite the LED file?
+	const fileStr = `#This file was edited by livepush
+SUPERVISOR_IMAGE=${imageName}
+SUPERVISOR_TAG=${imageTag}
+LED_FILE=/dev/null
+`;
+
+	return runSshCommand(
+		address,
+		`echo '${fileStr}' > /tmp/update-supervisor.conf`,
+	);
+}

sync/init.ts

@@ -0,0 +1,53 @@
+import * as Bluebird from 'bluebird';
+import * as Docker from 'dockerode';
+import { Dockerfile } from 'livepush';
+
+import * as device from './device';
+
+interface Opts {
+	address: string;
+	imageName: string;
+	imageTag: string;
+	docker: Docker;
+	dockerfile: Dockerfile;
+	nocache: boolean;
+	arch?: string;
+}
+
+export async function initDevice(opts: Opts) {
+	const arch = opts.arch ?? (await device.getDeviceArch(opts.docker));
+	const image = `${opts.imageName}:${opts.imageTag}`;
+
+	await device.performBuild(opts.docker, opts.dockerfile, {
+		buildargs: { ARCH: arch },
+		t: image,
+		labels: { 'io.balena.livepush-image': '1', 'io.balena.architecture': arch },
+		cachefrom: (await device.getCacheFrom(opts.docker)).concat(image),
+		nocache: opts.nocache,
+	});
+
+	// Now that we have our new image on the device, we need
+	// to stop the supervisor, update
+	// /tmp/update-supervisor.conf with our version, and
+	// restart the supervisor
+	await device.stopSupervisor(opts.address);
+	await device.replaceSupervisorImage(
+		opts.address,
+		opts.imageName,
+		opts.imageTag,
+	);
+	await device.startSupervisor(opts.address);
+
+	let supervisorContainer: undefined | Docker.ContainerInfo;
+	while (supervisorContainer == null) {
+		try {
+			supervisorContainer = await device.getSupervisorContainer(
+				opts.docker,
+				true,
+			);
+		} catch (e) {
+			await Bluebird.delay(500);
+		}
+	}
+	return supervisorContainer.Id;
+}

sync/livepush.ts

@@ -0,0 +1,63 @@
+import * as chokidar from 'chokidar';
+import * as Docker from 'dockerode';
+import * as _ from 'lodash';
+import * as Path from 'path';
+
+import { Dockerfile, Livepush } from 'livepush';
+
+// TODO: Pass build args to the livepush process
+export async function startLivepush(opts: {
+	dockerfile: Dockerfile;
+	containerId: string;
+	docker: Docker;
+	noinit: boolean;
+}) {
+	const livepush = await Livepush.init({
+		...opts,
+		context: Path.join(__dirname, '..'),
+		stageImages: [],
+	});
+
+	livepush.addListener('commandExecute', ({ command }) => {
+		console.log(`Executing command: ${command} `);
+	});
+	livepush.addListener('commandReturn', ({ returnCode }) => {
+		if (returnCode !== 0) {
+			console.log(`  Command executed with code ${returnCode}`);
+		}
+	});
+	livepush.addListener('commandOutput', ({ output }) => {
+		console.log(output.data.toString());
+	});
+	livepush.addListener('containerRestart', () => {
+		console.log('Restarting container');
+	});
+
+	const livepushExecutor = getExecutor(livepush);
+
+	chokidar
+		.watch('.', {
+			ignored: /((^|[\/\\])\..|node_modules.*|sync\/.*)/,
+			ignoreInitial: opts.noinit,
+		})
+		.on('add', path => livepushExecutor(path))
+		.on('change', path => livepushExecutor(path))
+		.on('unlink', path => livepushExecutor(undefined, path));
+}
+
+const getExecutor = (livepush: Livepush) => {
+	const changedFiles: string[] = [];
+	const deletedFiles: string[] = [];
+	const actualExecutor = _.debounce(async () => {
+		await livepush.performLivepush(changedFiles, deletedFiles);
+	});
+	return (changed?: string, deleted?: string) => {
+		if (changed) {
+			changedFiles.push(changed);
+		}
+		if (deleted) {
+			deletedFiles.push(deleted);
+		}
+		actualExecutor();
+	};
+};

sync/logs.ts

@@ -0,0 +1,46 @@
+import * as Docker from 'dockerode';
+import * as _ from 'lodash';
+
+export async function setupLogs(
+	docker: Docker,
+	containerId: string,
+	lastReadTimestamp = 0,
+) {
+	const container = docker.getContainer(containerId);
+
+	const stream = await container.logs({
+		stdout: true,
+		stderr: true,
+		follow: true,
+		timestamps: true,
+		since: lastReadTimestamp,
+	});
+
+	stream.on('data', chunk => {
+		const { message, timestamp } = extractMessage(chunk);
+		// Add one here, other we can end up constantly reading
+		// the same log line
+		lastReadTimestamp = Math.floor(timestamp.getTime() / 1000) + 1;
+		process.stdout.write(message);
+	});
+
+	// This happens when a container is restarted
+	stream.on('end', () => {
+		setupLogs(docker, containerId, lastReadTimestamp);
+	});
+}
+
+function extractMessage(msgBuf: Buffer) {
+	// Non-tty message format from:
+	// https://docs.docker.com/engine/api/v1.30/#operation/ContainerAttach
+	if ([0, 1, 2].includes(msgBuf[0])) {
+		// Take the header from this message, and parse it as normal
+		msgBuf = msgBuf.slice(8);
+	}
+	const str = msgBuf.toString();
+	const space = str.indexOf(' ');
+	return {
+		timestamp: new Date(str.slice(0, space)),
+		message: str.slice(space + 1),
+	};
+}

sync/sync.ts

@@ -0,0 +1,88 @@
+import * as packageJson from '../package.json';
+
+import * as livepush from 'livepush';
+
+import { fs } from 'mz';
+import * as yargs from 'yargs';
+
+import * as device from './device';
+import * as init from './init';
+import { startLivepush } from './livepush';
+import { setupLogs } from './logs';
+
+const helpText = `Sync changes code to a running supervisor on a device on the local network
+
+Usage:
+  npm run sync <device IP>
+`;
+
+const argv = yargs
+	.command(
+		'$0 <device-address>',
+		'Sync changes in code to a running debug mode supervisor on a local device',
+		y =>
+			y.positional('device-address', {
+				type: 'string',
+				describe: 'The address of a local device',
+			}),
+	)
+	.option('device-arch', {
+		alias: 'a',
+		type: 'string',
+		description:
+			'Specify the device architecture (use this when the automatic detection fails)',
+		choices: ['amd64', 'i386', 'aarch64', 'armv7hf', 'rpi'],
+	})
+	.options('image-name', {
+		alias: 'i',
+		type: 'string',
+		description: 'Specify the name to use for the supervisor image on device',
+		default: 'livepush-supervisor',
+	})
+	.options('image-tag', {
+		alias: 't',
+		type: 'string',
+		description: 'Specify the tag to use for the supervisor image on device',
+		default: packageJson.version,
+	})
+	.options('nocache', {
+		description: 'Run the intial build without cache',
+		type: 'boolean',
+		default: false,
+	})
+	.usage(helpText)
+	.version(false)
+	.scriptName('npm run sync --')
+	.alias('h', 'help').argv;
+
+(async () => {
+	const address = argv['device-address']!;
+	const dockerfile = new livepush.Dockerfile(await fs.readFile('Dockerfile'));
+
+	try {
+		const docker = device.getDocker(address);
+		const containerId = await init.initDevice({
+			address,
+			docker,
+			dockerfile,
+			imageName: argv['image-name'],
+			imageTag: argv['image-tag'],
+			arch: argv['device-arch'],
+			nocache: argv['nocache'],
+		});
+
+		console.log('==================================================');
+		console.log('Supervisor container id: ', containerId);
+		console.log('==================================================');
+		await setupLogs(docker, containerId);
+		await startLivepush({
+			dockerfile,
+			containerId,
+			docker,
+			noinit: true,
+		});
+	} catch (e) {
+		console.error('Error:');
+		console.error(e.message);
+	}
+})();