Convert more code to Typescript (compose.js)

Change-type: patch

lib/commands/build.ts

@@ -22,8 +22,8 @@ import * as compose from '../utils/compose';
 import type { Application, ApplicationType, BalenaSDK } from 'balena-sdk';
 import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import type { ComposeCliFlags, ComposeOpts } from '../utils/compose-types';
-import { composeCliFlags } from '../utils/compose_ts';
+import { buildProject, composeCliFlags } from '../utils/compose_ts';
-import type { DockerCliFlags } from '../utils/docker';
+import type { BuildOpts, DockerCliFlags } from '../utils/docker';
 import { dockerCliFlags } from '../utils/docker';
 
 interface FlagsDef extends ComposeCliFlags, DockerCliFlags {
@@ -219,7 +219,7 @@ ${dockerignoreHelp}
 			arch: string;
 			deviceType: string;
 			buildEmulated: boolean;
-			buildOpts: any;
+			buildOpts: BuildOpts;
 		},
 	) {
 		const { loadProject } = await import('../utils/compose_ts');
@@ -238,21 +238,21 @@ ${dockerignoreHelp}
 			);
 		}
 
-		await compose.buildProject(
+		await buildProject({
 			docker,
 			logger,
-			project.path,
+			projectPath: project.path,
-			project.name,
+			projectName: project.name,
-			project.composition,
+			composition: project.composition,
-			opts.arch,
+			arch: opts.arch,
-			opts.deviceType,
+			deviceType: opts.deviceType,
-			opts.buildEmulated,
+			emulated: opts.buildEmulated,
-			opts.buildOpts,
+			buildOpts: opts.buildOpts,
-			composeOpts.inlineLogs,
+			inlineLogs: composeOpts.inlineLogs,
-			composeOpts.convertEol,
+			convertEol: composeOpts.convertEol,
-			composeOpts.dockerfilePath,
+			dockerfilePath: composeOpts.dockerfilePath,
-			composeOpts.nogitignore,
+			nogitignore: composeOpts.nogitignore,
-			composeOpts.multiDockerignore,
+			multiDockerignore: composeOpts.multiDockerignore,
-		);
+		});
 	}
 }

lib/commands/deploy.ts

@@ -16,14 +16,24 @@
  */
 
 import { flags } from '@oclif/command';
+import type { ImageDescriptor } from 'resin-compose-parse';
+
 import Command from '../command';
 import { ExpectedError } from '../errors';
 import { getBalenaSdk, getChalk } from '../utils/lazy';
 import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import * as compose from '../utils/compose';
-import type { ComposeCliFlags, ComposeOpts } from '../utils/compose-types';
+import type {
+	BuiltImage,
+	ComposeCliFlags,
+	ComposeOpts,
+} from '../utils/compose-types';
 import type { DockerCliFlags } from '../utils/docker';
-import { composeCliFlags } from '../utils/compose_ts';
+import {
+	buildProject,
+	composeCliFlags,
+	isBuildConfig,
+} from '../utils/compose_ts';
 import { dockerCliFlags } from '../utils/docker';
 import type { Application, ApplicationType, DeviceType } from 'balena-sdk';
 
@@ -214,22 +224,21 @@ ${dockerignoreHelp}
 			}
 
 			// find which services use images that already exist locally
-			let servicesToSkip = await Promise.all(
+			let servicesToSkip: string[] = await Promise.all(
-				project.descriptors.map(async function (d: any) {
+				project.descriptors.map(async function (d: ImageDescriptor) {
 					// unconditionally build (or pull) if explicitly requested
 					if (opts.shouldPerformBuild) {
-						return d;
+						return '';
 					}
 					try {
 						await docker
-							.getImage(
+							.getImage((isBuildConfig(d.image) ? d.image.tag : d.image) || '')
-								(typeof d.image === 'string' ? d.image : d.image.tag) || '',
-							)
 							.inspect();
 
 						return d.serviceName;
 					} catch {
 						// Ignore
+						return '';
 					}
 				}),
 			);
@@ -243,35 +252,35 @@ ${dockerignoreHelp}
 				compositionToBuild.services,
 				servicesToSkip,
 			);
-			let builtImagesByService: Dictionary<any> = {};
+			let builtImagesByService: Dictionary<BuiltImage> = {};
 			if (_.size(compositionToBuild.services) === 0) {
 				logger.logInfo(
 					'Everything is up to date (use --build to force a rebuild)',
 				);
 			} else {
-				const builtImages = await compose.buildProject(
+				const builtImages = await buildProject({
 					docker,
 					logger,
-					project.path,
+					projectPath: project.path,
-					project.name,
+					projectName: project.name,
-					compositionToBuild,
+					composition: compositionToBuild,
-					opts.app.arch,
+					arch: opts.app.arch,
-					(opts.app?.is_for__device_type as DeviceType[])?.[0].slug,
+					deviceType: (opts.app?.is_for__device_type as DeviceType[])?.[0].slug,
-					opts.buildEmulated,
+					emulated: opts.buildEmulated,
-					opts.buildOpts,
+					buildOpts: opts.buildOpts,
-					composeOpts.inlineLogs,
+					inlineLogs: composeOpts.inlineLogs,
-					composeOpts.convertEol,
+					convertEol: composeOpts.convertEol,
-					composeOpts.dockerfilePath,
+					dockerfilePath: composeOpts.dockerfilePath,
-					composeOpts.nogitignore,
+					nogitignore: composeOpts.nogitignore,
-					composeOpts.multiDockerignore,
+					multiDockerignore: composeOpts.multiDockerignore,
-				);
+				});
 				builtImagesByService = _.keyBy(builtImages, 'serviceName');
 			}
-			const images = project.descriptors.map(
+			const images: BuiltImage[] = project.descriptors.map(
 				(d) =>
 					builtImagesByService[d.serviceName] ?? {
 						serviceName: d.serviceName,
-						name: typeof d.image === 'string' ? d.image : d.image.tag,
+						name: (isBuildConfig(d.image) ? d.image.tag : d.image) || '',
 						logs: 'Build skipped; image for service already exists.',
 						props: {},
 					},

lib/utils/compose-types.d.ts

@@ -30,6 +30,8 @@ export interface BuiltImage {
 		dockerfile?: string;
 		projectType?: string;
 		size?: number;
+		startTime?: Date;
+		endTime?: Date;
 	};
 	serviceName: string;
 }
@@ -64,7 +66,7 @@ export interface ComposeCliFlags {
 	'multi-dockerignore': boolean;
 	nogitignore: boolean;
 	'noparent-check': boolean;
-	'registry-secrets'?: string | RegistrySecrets;
+	'registry-secrets'?: RegistrySecrets;
 	'convert-eol': boolean;
 	'noconvert-eol': boolean;
 	projectName?: string;

lib/utils/compose.js

@@ -85,23 +85,6 @@ export function createProject(composePath, composeStr, projectName = null) {
 	};
 }
 
-/**
- * Create a tar stream out of the local filesystem at the given directory,
- * while optionally applying file filters such as '.dockerignore' and
- * optionally converting text file line endings (CRLF to LF).
- * @param {string} dir Source directory
- * @param {import('./compose-types').TarDirectoryOptions} param
- * @returns {Promise<import('stream').Readable>}
- */
-export function tarDirectory(dir, param) {
-	let { nogitignore = false } = param;
-	if (nogitignore) {
-		return require('./compose_ts').tarDirectory(dir, param);
-	} else {
-		return originalTarDirectory(dir, param);
-	}
-}
-
 /**
  * This is the CLI v10 / v11 "original" tarDirectory function. It is still
  * around for the benefit of the `--gitignore` option, but is expected to be
@@ -110,7 +93,7 @@ export function tarDirectory(dir, param) {
  * @param {import('./compose-types').TarDirectoryOptions} param
  * @returns {Promise<import('stream').Readable>}
  */
-function originalTarDirectory(dir, param) {
+export async function originalTarDirectory(dir, param) {
 	let {
 		preFinalizeCallback = null,
 		convertEol = false,
@@ -185,265 +168,6 @@ function originalTarDirectory(dir, param) {
 		});
 }
 
-/**
- * @param {string} str
- * @param {number} len
- * @returns {string}
- */
-const truncateString = function (str, len) {
-	if (str.length < len) {
-		return str;
-	}
-	str = str.slice(0, len);
-	// return everything up to the last line. this is a cheeky way to avoid
-	// having to deal with splitting the string midway through some special
-	// character sequence.
-	return str.slice(0, str.lastIndexOf('\n'));
-};
-
-const LOG_LENGTH_MAX = 512 * 1024; // 512KB
-
-export function buildProject(
-	docker,
-	logger,
-	projectPath,
-	projectName,
-	composition,
-	arch,
-	deviceType,
-	emulated,
-	buildOpts,
-	inlineLogs,
-	convertEol,
-	dockerfilePath,
-	nogitignore,
-	multiDockerignore,
-) {
-	const Bluebird = require('bluebird');
-	const _ = require('lodash');
-	const humanize = require('humanize');
-	const compose = require('resin-compose-parse');
-	const builder = require('resin-multibuild');
-	const transpose = require('docker-qemu-transpose');
-	const { BALENA_ENGINE_TMP_PATH } = require('../config');
-	const {
-		checkBuildSecretsRequirements,
-		makeBuildTasks,
-	} = require('./compose_ts');
-	const qemu = require('./qemu');
-	const { toPosixPath } = builder.PathUtils;
-
-	logger.logInfo(`Building for ${arch}/${deviceType}`);
-
-	const imageDescriptors = compose.parse(composition);
-	const imageDescriptorsByServiceName = _.keyBy(
-		imageDescriptors,
-		'serviceName',
-	);
-
-	let renderer;
-	if (inlineLogs) {
-		renderer = new BuildProgressInline(
-			logger.streams['build'],
-			imageDescriptors,
-		);
-	} else {
-		const tty = require('./tty')(process.stdout);
-		renderer = new BuildProgressUI(tty, imageDescriptors);
-	}
-	renderer.start();
-
-	return Bluebird.resolve(checkBuildSecretsRequirements(docker, projectPath))
-		.then(() => qemu.installQemuIfNeeded(emulated, logger, arch, docker))
-		.tap(function (needsQemu) {
-			if (!needsQemu) {
-				return;
-			}
-			logger.logInfo('Emulation is enabled');
-			// Copy qemu into all build contexts
-			return Promise.all(
-				imageDescriptors.map(function (d) {
-					if (typeof d.image === 'string' || d.image.context == null) {
-						return;
-					}
-					// external image
-					return qemu.copyQemu(path.join(projectPath, d.image.context), arch);
-				}),
-			);
-		})
-		.then((
-			needsQemu, // Tar up the directory, ready for the build stream
-		) =>
-			Bluebird.resolve(
-				tarDirectory(projectPath, {
-					composition,
-					convertEol,
-					multiDockerignore,
-					nogitignore,
-				}),
-			)
-				.then((tarStream) =>
-					makeBuildTasks(
-						composition,
-						tarStream,
-						{ arch, deviceType },
-						logger,
-						projectName,
-					),
-				)
-				.map(function (/** @type {any} */ task) {
-					const d = imageDescriptorsByServiceName[task.serviceName];
-
-					// multibuild parses the composition internally so any tags we've
-					// set before are lost; re-assign them here
-					task.tag ??= [projectName, task.serviceName].join('_').toLowerCase();
-
-					if (typeof d.image !== 'string' && d.image.context != null) {
-						d.image.tag = task.tag;
-					}
-
-					// configure build opts appropriately
-					task.dockerOpts ??= {};
-
-					_.merge(task.dockerOpts, buildOpts, { t: task.tag });
-					if (typeof d.image !== 'string') {
-						/** @type {any} */
-						const context = d.image.context;
-						if (context?.args != null) {
-							task.dockerOpts.buildargs ??= {};
-							_.merge(task.dockerOpts.buildargs, context.args);
-						}
-					}
-
-					// Get the service-specific log stream
-					// Caveat: `multibuild.BuildTask` defines no `logStream` property
-					// but it's convenient to store it there; it's JS ultimately.
-					task.logStream = renderer.streams[task.serviceName];
-					task.logBuffer = [];
-
-					// Setup emulation if needed
-					if (task.external || !needsQemu) {
-						return [task, null];
-					}
-					const binPath = qemu.qemuPathInContext(
-						path.join(projectPath, task.context ?? ''),
-					);
-					if (task.buildStream == null) {
-						throw new Error(`No buildStream for task '${task.tag}'`);
-					}
-					return transpose
-						.transposeTarStream(
-							task.buildStream,
-							{
-								hostQemuPath: toPosixPath(binPath),
-								containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`,
-								qemuFileMode: 0o555,
-							},
-							dockerfilePath || undefined,
-						)
-						.then((/** @type {any} */ stream) => {
-							task.buildStream = stream;
-						})
-						.return([task, binPath]);
-				}),
-		)
-		.map(function ([task, qemuPath]) {
-			const captureStream = buildLogCapture(task.external, task.logBuffer);
-
-			if (task.external) {
-				// External image -- there's no build to be performed,
-				// just follow pull progress.
-				captureStream.pipe(task.logStream);
-				task.progressHook = pullProgressAdapter(captureStream);
-			} else {
-				task.streamHook = function (stream) {
-					let rawStream;
-					stream = createLogStream(stream);
-					if (qemuPath != null) {
-						const buildThroughStream = transpose.getBuildThroughStream({
-							hostQemuPath: toPosixPath(qemuPath),
-							containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`,
-						});
-						rawStream = stream.pipe(buildThroughStream);
-					} else {
-						rawStream = stream;
-					}
-					// `stream` sends out raw strings in contrast to `task.progressHook`
-					// where we're given objects. capture these strings as they come
-					// before we parse them.
-					return rawStream
-						.pipe(dropEmptyLinesStream())
-						.pipe(captureStream)
-						.pipe(buildProgressAdapter(inlineLogs))
-						.pipe(task.logStream);
-				};
-			}
-			return task;
-		})
-		.then(function (tasks) {
-			logger.logDebug('Prepared tasks; building...');
-			return builder
-				.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH)
-				.then(function (builtImages) {
-					return Promise.all(
-						builtImages.map(function (builtImage) {
-							if (!builtImage.successful) {
-								/** @type {Error & {serviceName?: string}} */
-								const error = builtImage.error ?? new Error();
-								error.serviceName = builtImage.serviceName;
-								throw error;
-							}
-
-							const d = imageDescriptorsByServiceName[builtImage.serviceName];
-							const task = _.find(tasks, {
-								serviceName: builtImage.serviceName,
-							});
-
-							const image = {
-								serviceName: d.serviceName,
-								name: typeof d.image === 'string' ? d.image : d.image.tag,
-								logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX),
-								props: {
-									dockerfile: builtImage.dockerfile,
-									projectType: builtImage.projectType,
-								},
-							};
-
-							// Times here are timestamps, so test whether they're null
-							// before creating a date out of them, as `new Date(null)`
-							// creates a date representing UNIX time 0.
-							if (builtImage.startTime) {
-								image.props.startTime = new Date(builtImage.startTime);
-							}
-							if (builtImage.endTime) {
-								image.props.endTime = new Date(builtImage.endTime);
-							}
-							return docker
-								.getImage(image.name)
-								.inspect()
-								.get('Size')
-								.then((size) => {
-									image.props.size = size;
-								})
-								.return(image);
-						}),
-					);
-				})
-				.then(function (images) {
-					const summary = _(images)
-						.map(({ serviceName, props }) => [
-							serviceName,
-							`Image size: ${humanize.filesize(props.size)}`,
-						])
-						.fromPairs()
-						.value();
-					renderer.end(summary);
-					return images;
-				});
-		})
-		.finally(renderer.end);
-}
-
 /**
  * @param {string} apiEndpoint
  * @param {string} auth
@@ -708,102 +432,7 @@ var pushProgressRenderer = function (tty, prefix) {
 	return fn;
 };
 
-var createLogStream = function (input) {
+export class BuildProgressUI {
-	const split = require('split');
-	const stripAnsi = require('strip-ansi-stream');
-	return input.pipe(stripAnsi()).pipe(split());
-};
-
-var dropEmptyLinesStream = function () {
-	const through = require('through2');
-	return through(function (data, _enc, cb) {
-		const str = data.toString('utf-8');
-		if (str.trim()) {
-			this.push(str);
-		}
-		return cb();
-	});
-};
-
-var buildLogCapture = function (objectMode, buffer) {
-	const through = require('through2');
-
-	return through({ objectMode }, function (data, _enc, cb) {
-		// data from pull stream
-		if (data.error) {
-			buffer.push(`${data.error}`);
-		} else if (data.progress && data.status) {
-			buffer.push(`${data.progress}% ${data.status}`);
-		} else if (data.status) {
-			buffer.push(`${data.status}`);
-
-			// data from build stream
-		} else {
-			buffer.push(data);
-		}
-
-		return cb(null, data);
-	});
-};
-
-var buildProgressAdapter = function (inline) {
-	const through = require('through2');
-
-	const stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/;
-
-	let step = null;
-	let numSteps = null;
-	let progress;
-
-	return through({ objectMode: true }, function (str, _enc, cb) {
-		if (str == null) {
-			return cb(null, str);
-		}
-
-		if (inline) {
-			return cb(null, { status: str });
-		}
-
-		if (/^Successfully tagged /.test(str)) {
-			progress = undefined;
-		} else {
-			const match = stepRegex.exec(str);
-			if (match) {
-				step = match[1];
-				numSteps ??= match[2];
-				str = match[3];
-			}
-			if (step != null) {
-				str = `Step ${step}/${numSteps}: ${str}`;
-				progress = Math.floor(
-					(parseInt(step, 10) * 100) / parseInt(numSteps, 10),
-				);
-			}
-		}
-
-		return cb(null, { status: str, progress });
-	});
-};
-
-var pullProgressAdapter = (outStream) =>
-	function ({ status, id, percentage, error, errorDetail }) {
-		if (status != null) {
-			status = status.replace(/^Status: /, '');
-		}
-		if (id != null) {
-			status = `${id}: ${status}`;
-		}
-		if (percentage === 100) {
-			percentage = undefined;
-		}
-		return outStream.write({
-			status,
-			progress: percentage,
-			error: errorDetail?.message ?? error,
-		});
-	};
-
-class BuildProgressUI {
 	constructor(tty, descriptors) {
 		this._handleEvent = this._handleEvent.bind(this);
 		this._handleInterrupt = this._handleInterrupt.bind(this);
@@ -978,7 +607,7 @@ class BuildProgressUI {
 	}
 }
 
-class BuildProgressInline {
+export class BuildProgressInline {
 	constructor(outStream, descriptors) {
 		this.start = this.start.bind(this);
 		this.end = this.end.bind(this);

lib/utils/compose_ts.ts

@@ -14,17 +14,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { flags } from '@oclif/command';
 import { BalenaSDK } from 'balena-sdk';
+import type { TransposeOptions } from 'docker-qemu-transpose';
 import type * as Dockerode from 'dockerode';
 import * as _ from 'lodash';
 import { promises as fs } from 'fs';
 import * as path from 'path';
-import type { Composition } from 'resin-compose-parse';
+import type {
+	BuildConfig,
+	Composition,
+	ImageDescriptor,
+} from 'resin-compose-parse';
 import type * as MultiBuild from 'resin-multibuild';
-import type { Readable } from 'stream';
+import type { Duplex, Readable } from 'stream';
 import type { Pack } from 'tar-stream';
+
 import { ExpectedError } from '../errors';
-import { getBalenaSdk, getChalk, stripIndent } from './lazy';
 import {
 	BuiltImage,
 	ComposeCliFlags,
@@ -34,16 +40,9 @@ import {
 	TaggedImage,
 	TarDirectoryOptions,
 } from './compose-types';
-import { DeviceInfo } from './device/api';
+import type { DeviceInfo } from './device/api';
+import { getBalenaSdk, getChalk, stripIndent } from './lazy';
 import Logger = require('./logger');
-import { flags } from '@oclif/command';
-
-export interface RegistrySecrets {
-	[registryAddress: string]: {
-		username: string;
-		password: string;
-	};
-}
 
 const exists = async (filename: string) => {
 	try {
@@ -54,8 +53,8 @@ const exists = async (filename: string) => {
 	}
 };
 
+const LOG_LENGTH_MAX = 512 * 1024; // 512KB
 const compositionFileNames = ['docker-compose.yml', 'docker-compose.yaml'];
-
 const hr =
 	'----------------------------------------------------------------------';
 
@@ -131,6 +130,363 @@ async function resolveProject(
 	return [composeFileName, composeFileContents];
 }
 
+interface BuildTaskPlus extends MultiBuild.BuildTask {
+	logBuffer?: string[];
+}
+
+interface Renderer {
+	start: () => void;
+	end: (buildSummaryByService?: Dictionary<string>) => void;
+	streams: Dictionary<NodeJS.ReadWriteStream>;
+}
+
+export async function buildProject(opts: {
+	docker: Dockerode;
+	logger: Logger;
+	projectPath: string;
+	projectName: string;
+	composition: Composition;
+	arch: string;
+	deviceType: string;
+	emulated: boolean;
+	buildOpts: import('./docker').BuildOpts;
+	inlineLogs?: boolean;
+	convertEol: boolean;
+	dockerfilePath?: string;
+	nogitignore: boolean;
+	multiDockerignore: boolean;
+}): Promise<BuiltImage[]> {
+	const { logger, projectName } = opts;
+	logger.logInfo(`Building for ${opts.arch}/${opts.deviceType}`);
+
+	let buildSummaryByService: Dictionary<string> | undefined;
+	const compose = await import('resin-compose-parse');
+	const imageDescriptors = compose.parse(opts.composition);
+	const imageDescriptorsByServiceName = _.keyBy(
+		imageDescriptors,
+		'serviceName',
+	);
+	const renderer = await startRenderer({ imageDescriptors, ...opts });
+	try {
+		await checkBuildSecretsRequirements(opts.docker, opts.projectPath);
+
+		const needsQemu = await installQemuIfNeeded({ ...opts, imageDescriptors });
+
+		const tarStream = await tarDirectory(opts.projectPath, opts);
+
+		const tasks: BuildTaskPlus[] = await makeBuildTasks(
+			opts.composition,
+			tarStream,
+			opts,
+			logger,
+			projectName,
+		);
+
+		setTaskAttributes({ tasks, imageDescriptorsByServiceName, ...opts });
+
+		const transposeOptArray: Array<
+			TransposeOptions | undefined
+		> = await Promise.all(
+			tasks.map((task) => {
+				// Setup emulation if needed
+				if (needsQemu && !task.external) {
+					return qemuTransposeBuildStream({ task, ...opts });
+				}
+			}),
+		);
+
+		await Promise.all(
+			// transposeOptions may be undefined. That's OK.
+			transposeOptArray.map((transposeOptions, index) =>
+				setTaskProgressHooks({
+					task: tasks[index],
+					renderer,
+					transposeOptions,
+					...opts,
+				}),
+			),
+		);
+
+		logger.logDebug('Prepared tasks; building...');
+
+		const { BALENA_ENGINE_TMP_PATH } = await import('../config');
+		const builder = await import('resin-multibuild');
+
+		const builtImages = await builder.performBuilds(
+			tasks,
+			opts.docker,
+			BALENA_ENGINE_TMP_PATH,
+		);
+
+		const [images, summaryMsgByService] = await inspectBuiltImages({
+			builtImages,
+			imageDescriptorsByServiceName,
+			tasks,
+			...opts,
+		});
+		buildSummaryByService = summaryMsgByService;
+
+		return images;
+	} finally {
+		renderer.end(buildSummaryByService);
+	}
+}
+
+async function startRenderer({
+	imageDescriptors,
+	inlineLogs,
+	logger,
+}: {
+	imageDescriptors: ImageDescriptor[];
+	inlineLogs?: boolean;
+	logger: Logger;
+}): Promise<Renderer> {
+	let renderer: Renderer;
+	if (inlineLogs) {
+		renderer = new (await import('./compose')).BuildProgressInline(
+			logger.streams['build'],
+			imageDescriptors,
+		);
+	} else {
+		const tty = (await import('./tty'))(process.stdout);
+		renderer = new (await import('./compose')).BuildProgressUI(
+			tty,
+			imageDescriptors,
+		);
+	}
+	renderer.start();
+	return renderer;
+}
+
+async function installQemuIfNeeded({
+	arch,
+	docker,
+	emulated,
+	imageDescriptors,
+	logger,
+	projectPath,
+}: {
+	arch: string;
+	docker: Dockerode;
+	emulated: boolean;
+	imageDescriptors: ImageDescriptor[];
+	logger: Logger;
+	projectPath: string;
+}): Promise<boolean> {
+	const qemu = await import('./qemu');
+	const needsQemu = await qemu.installQemuIfNeeded(
+		emulated,
+		logger,
+		arch,
+		docker,
+	);
+	if (needsQemu) {
+		logger.logInfo('Emulation is enabled');
+		// Copy qemu into all build contexts
+		await Promise.all(
+			imageDescriptors.map(function (d) {
+				if (isBuildConfig(d.image)) {
+					return qemu.copyQemu(
+						path.join(projectPath, d.image.context || '.'),
+						arch,
+					);
+				}
+			}),
+		);
+	}
+	return needsQemu;
+}
+
+function setTaskAttributes({
+	tasks,
+	buildOpts,
+	imageDescriptorsByServiceName,
+	projectName,
+}: {
+	tasks: BuildTaskPlus[];
+	buildOpts: import('./docker').BuildOpts;
+	imageDescriptorsByServiceName: Dictionary<ImageDescriptor>;
+	projectName: string;
+}) {
+	for (const task of tasks) {
+		const d = imageDescriptorsByServiceName[task.serviceName];
+		// multibuild (splitBuildStream) parses the composition internally so
+		// any tags we've set before are lost; re-assign them here
+		task.tag ??= [projectName, task.serviceName].join('_').toLowerCase();
+
+		task.dockerOpts ??= {};
+		if (isBuildConfig(d.image)) {
+			d.image.tag = task.tag;
+			if (d.image.args) {
+				task.dockerOpts.buildargs ??= {};
+				_.merge(task.dockerOpts.buildargs, d.image.args);
+			}
+		}
+		_.merge(task.dockerOpts, buildOpts, { t: task.tag });
+	}
+}
+
+async function qemuTransposeBuildStream({
+	task,
+	dockerfilePath,
+	projectPath,
+}: {
+	task: BuildTaskPlus;
+	dockerfilePath?: string;
+	projectPath: string;
+}): Promise<TransposeOptions> {
+	const qemu = await import('./qemu');
+	const binPath = qemu.qemuPathInContext(
+		path.join(projectPath, task.context ?? ''),
+	);
+	if (task.buildStream == null) {
+		throw new Error(`No buildStream for task '${task.tag}'`);
+	}
+
+	const transpose = await import('docker-qemu-transpose');
+	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
+
+	const transposeOptions: TransposeOptions = {
+		hostQemuPath: toPosixPath(binPath),
+		containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`,
+		qemuFileMode: 0o555,
+	};
+
+	task.buildStream = (await transpose.transposeTarStream(
+		task.buildStream,
+		transposeOptions,
+		dockerfilePath || undefined,
+	)) as Pack;
+
+	return transposeOptions;
+}
+
+async function setTaskProgressHooks({
+	inlineLogs,
+	renderer,
+	task,
+	transposeOptions,
+}: {
+	inlineLogs?: boolean;
+	renderer: Renderer;
+	task: BuildTaskPlus;
+	transposeOptions?: import('docker-qemu-transpose').TransposeOptions;
+}) {
+	const transpose = await import('docker-qemu-transpose');
+	// Get the service-specific log stream
+	const logStream = renderer.streams[task.serviceName];
+	task.logBuffer = [];
+	const captureStream = buildLogCapture(task.external, task.logBuffer);
+
+	if (task.external) {
+		// External image -- there's no build to be performed,
+		// just follow pull progress.
+		captureStream.pipe(logStream);
+		task.progressHook = pullProgressAdapter(captureStream);
+	} else {
+		task.streamHook = function (stream) {
+			let rawStream;
+			stream = createLogStream(stream);
+			if (transposeOptions) {
+				const buildThroughStream = transpose.getBuildThroughStream(
+					transposeOptions,
+				);
+				rawStream = stream.pipe(buildThroughStream);
+			} else {
+				rawStream = stream;
+			}
+			// `stream` sends out raw strings in contrast to `task.progressHook`
+			// where we're given objects. capture these strings as they come
+			// before we parse them.
+			return rawStream
+				.pipe(dropEmptyLinesStream())
+				.pipe(captureStream)
+				.pipe(buildProgressAdapter(!!inlineLogs))
+				.pipe(logStream);
+		};
+	}
+}
+
+async function inspectBuiltImages({
+	builtImages,
+	docker,
+	imageDescriptorsByServiceName,
+	tasks,
+}: {
+	builtImages: MultiBuild.LocalImage[];
+	docker: Dockerode;
+	imageDescriptorsByServiceName: Dictionary<ImageDescriptor>;
+	tasks: BuildTaskPlus[];
+}): Promise<[BuiltImage[], Dictionary<string>]> {
+	const images: BuiltImage[] = await Promise.all(
+		builtImages.map((builtImage: MultiBuild.LocalImage) =>
+			inspectBuiltImage({
+				builtImage,
+				docker,
+				imageDescriptorsByServiceName,
+				tasks,
+			}),
+		),
+	);
+
+	const humanize = require('humanize');
+	const summaryMsgByService: { [serviceName: string]: string } = {};
+	for (const image of images) {
+		summaryMsgByService[image.serviceName] = `Image size: ${humanize.filesize(
+			image.props.size,
+		)}`;
+	}
+
+	return [images, summaryMsgByService];
+}
+
+async function inspectBuiltImage({
+	builtImage,
+	docker,
+	imageDescriptorsByServiceName,
+	tasks,
+}: {
+	builtImage: MultiBuild.LocalImage;
+	docker: Dockerode;
+	imageDescriptorsByServiceName: Dictionary<ImageDescriptor>;
+	tasks: BuildTaskPlus[];
+}): Promise<BuiltImage> {
+	if (!builtImage.successful) {
+		const error: Error & { serviceName?: string } =
+			builtImage.error ?? new Error();
+		error.serviceName = builtImage.serviceName;
+		throw error;
+	}
+
+	const d = imageDescriptorsByServiceName[builtImage.serviceName];
+	const task = _.find(tasks, {
+		serviceName: builtImage.serviceName,
+	});
+
+	const image: BuiltImage = {
+		serviceName: d.serviceName,
+		name: (isBuildConfig(d.image) ? d.image.tag : d.image) || '',
+		logs: truncateString(task?.logBuffer?.join('\n') || '', LOG_LENGTH_MAX),
+		props: {
+			dockerfile: builtImage.dockerfile,
+			projectType: builtImage.projectType,
+		},
+	};
+
+	// Times here are timestamps, so test whether they're null
+	// before creating a date out of them, as `new Date(null)`
+	// creates a date representing UNIX time 0.
+	if (builtImage.startTime) {
+		image.props.startTime = new Date(builtImage.startTime);
+	}
+	if (builtImage.endTime) {
+		image.props.endTime = new Date(builtImage.endTime);
+	}
+	image.props.size = (await docker.getImage(image.name).inspect()).Size;
+
+	return image;
+}
+
 /**
  * Load the ".balena/balena.yml" file (or resin.yml, or yaml or json),
  * which contains "build metadata" for features like "build secrets" and
@@ -207,9 +563,9 @@ async function getServiceDirsFromComposition(
 		const relPrefix = '.' + path.sep;
 		for (const [serviceName, service] of Object.entries(composition.services)) {
 			let dir =
-				typeof service.build === 'string'
+				(typeof service.build === 'string'
 					? service.build
-					: service.build?.context || '.';
+					: service.build?.context) || '.';
 			// Convert forward slashes to backslashes on Windows
 			dir = path.normalize(dir);
 			// Make sure the path is relative to the project directory
@@ -230,15 +586,58 @@ async function getServiceDirsFromComposition(
 	return serviceDirs;
 }
 
+/**
+ * Return true if `image` is actually a docker-compose.yml `services.service.build`
+ * configuration object, rather than an "external image" (`services.service.image`).
+ *
+ * The `image` argument may therefore refere to either a `build` or `image` property
+ * of a service in a docker-compose.yml file, which is a bit confusing but it matches
+ * the `ImageDescriptor.image` property as defined by `resin-compose-parse`.
+ *
+ * Note that `resin-compose-parse` "normalizes" the docker-compose.yml file such
+ * that, if `services.service.build` is a string, it is converted to a BuildConfig
+ * object with the string value assigned to `services.service.build.context`:
+ * https://github.com/balena-io-modules/resin-compose-parse/blob/v2.1.3/src/compose.ts#L166-L167
+ * This is why this implementation works when `services.service.build` is defined
+ * as a string in the docker-compose.yml file.
+ *
+ * @param image The `ImageDescriptor.image` attribute parsed with `resin-compose-parse`
+ */
+export function isBuildConfig(
+	image: string | BuildConfig,
+): image is BuildConfig {
+	return image != null && typeof image !== 'string';
+}
+
 /**
  * Create a tar stream out of the local filesystem at the given directory,
  * while optionally applying file filters such as '.dockerignore' and
  * optionally converting text file line endings (CRLF to LF).
  * @param dir Source directory
  * @param param Options
- * @returns {Promise<import('stream').Readable>}
+ * @returns Readable stream
  */
 export async function tarDirectory(
+	dir: string,
+	param: TarDirectoryOptions,
+): Promise<import('stream').Readable> {
+	const { nogitignore = false } = param;
+	if (nogitignore) {
+		return newTarDirectory(dir, param);
+	} else {
+		return (await import('./compose')).originalTarDirectory(dir, param);
+	}
+}
+
+/**
+ * Create a tar stream out of the local filesystem at the given directory,
+ * while optionally applying file filters such as '.dockerignore' and
+ * optionally converting text file line endings (CRLF to LF).
+ * @param dir Source directory
+ * @param param Options
+ * @returns Readable stream
+ */
+async function newTarDirectory(
 	dir: string,
 	{
 		composition,
@@ -441,7 +840,7 @@ export async function checkBuildSecretsRequirements(
 export async function getRegistrySecrets(
 	sdk: BalenaSDK,
 	inputFilename?: string,
-): Promise<RegistrySecrets> {
+): Promise<MultiBuild.RegistrySecrets> {
 	if (inputFilename != null) {
 		return await parseRegistrySecrets(inputFilename);
 	}
@@ -464,7 +863,7 @@ export async function getRegistrySecrets(
 
 async function parseRegistrySecrets(
 	secretsFilename: string,
-): Promise<RegistrySecrets> {
+): Promise<MultiBuild.RegistrySecrets> {
 	try {
 		let isYaml = false;
 		if (/.+\.ya?ml$/i.test(secretsFilename)) {
@@ -661,7 +1060,7 @@ async function validateSpecifiedDockerfile(
 
 export interface ProjectValidationResult {
 	dockerfilePath: string;
-	registrySecrets: RegistrySecrets;
+	registrySecrets: MultiBuild.RegistrySecrets;
 }
 
 /**
@@ -797,7 +1196,7 @@ async function pushServiceImages(
 export async function deployProject(
 	docker: import('docker-toolbelt'),
 	logger: Logger,
-	composition: import('resin-compose-parse').Composition,
+	composition: Composition,
 	images: BuiltImage[],
 	appId: number,
 	userId: number,
@@ -907,6 +1306,123 @@ export function createRunLoop(tick: (...args: any[]) => void) {
 	return runloop;
 }
 
+function createLogStream(input: Readable) {
+	const split = require('split') as typeof import('split');
+	const stripAnsi = require('strip-ansi-stream');
+	return input.pipe<Duplex>(stripAnsi()).pipe(split());
+}
+
+function dropEmptyLinesStream() {
+	const through = require('through2') as typeof import('through2');
+	return through(function (data, _enc, cb) {
+		const str = data.toString('utf-8');
+		if (str.trim()) {
+			this.push(str);
+		}
+		return cb();
+	});
+}
+
+function buildLogCapture(objectMode: boolean, buffer: string[]) {
+	const through = require('through2') as typeof import('through2');
+
+	return through({ objectMode }, function (data, _enc, cb) {
+		// data from pull stream
+		if (data.error) {
+			buffer.push(`${data.error}`);
+		} else if (data.progress && data.status) {
+			buffer.push(`${data.progress}% ${data.status}`);
+		} else if (data.status) {
+			buffer.push(`${data.status}`);
+
+			// data from build stream
+		} else {
+			buffer.push(data);
+		}
+
+		return cb(null, data);
+	});
+}
+
+function buildProgressAdapter(inline: boolean) {
+	const through = require('through2') as typeof import('through2');
+
+	const stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/;
+
+	let step = '';
+	let numSteps = '';
+	let progress: number | undefined;
+
+	return through({ objectMode: true }, function (str, _enc, cb) {
+		if (str == null) {
+			return cb(null, str);
+		}
+
+		if (inline) {
+			return cb(null, { status: str });
+		}
+
+		if (!/^Successfully tagged /.test(str)) {
+			const match = stepRegex.exec(str);
+			if (match) {
+				step = match[1];
+				numSteps ??= match[2];
+				str = match[3];
+			}
+			if (step) {
+				str = `Step ${step}/${numSteps}: ${str}`;
+				progress = Math.floor(
+					(parseInt(step, 10) * 100) / parseInt(numSteps, 10),
+				);
+			}
+		}
+
+		return cb(null, { status: str, progress });
+	});
+}
+
+function pullProgressAdapter(outStream: Duplex) {
+	return function ({
+		status,
+		id,
+		percentage,
+		error,
+		errorDetail,
+	}: {
+		status: string;
+		id: string;
+		percentage: number | undefined;
+		error: Error;
+		errorDetail: Error;
+	}) {
+		if (status != null) {
+			status = status.replace(/^Status: /, '');
+		}
+		if (id != null) {
+			status = `${id}: ${status}`;
+		}
+		if (percentage === 100) {
+			percentage = undefined;
+		}
+		return outStream.write({
+			status,
+			progress: percentage,
+			error: errorDetail?.message ?? error,
+		});
+	};
+}
+
+function truncateString(str: string, len: number): string {
+	if (str.length < len) {
+		return str;
+	}
+	str = str.slice(0, len);
+	// return everything up to the last line. this is a cheeky way to avoid
+	// having to deal with splitting the string midway through some special
+	// character sequence.
+	return str.slice(0, str.lastIndexOf('\n'));
+}
+
 export const composeCliFlags: flags.Input<ComposeCliFlags> = {
 	emulated: flags.boolean({
 		description:

lib/utils/device/deploy.ts

@@ -32,6 +32,7 @@ import {
 	checkBuildSecretsRequirements,
 	loadProject,
 	makeBuildTasks,
+	tarDirectory,
 } from '../compose_ts';
 import Logger = require('../logger');
 import { DeviceAPI, DeviceInfo } from './api';
@@ -121,7 +122,6 @@ async function environmentFromInput(
 }
 
 export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
-	const { tarDirectory } = await import('../compose');
 	const { exitWithExpectedError } = await import('../../errors');
 	const { displayDeviceLogs } = await import('./logs');
 
@@ -400,7 +400,6 @@ export async function rebuildSingleTask(
 	// this should provide the following callback
 	containerIdCb?: (id: string) => void,
 ): Promise<string> {
-	const { tarDirectory } = await import('../compose');
 	const multibuild = await import('resin-multibuild');
 	// First we run the build task, to get the new image id
 	let buildLogs = '';

lib/utils/docker-js.js

@@ -91,48 +91,6 @@ const generateConnectOpts = async function (opts) {
 	return connectOpts;
 };
 
-const parseBuildArgs = function (args) {
-	if (!Array.isArray(args)) {
-		args = [args];
-	}
-	const buildArgs = {};
-	args.forEach(function (arg) {
-		// note: [^] matches any character, including line breaks
-		const pair = /^([^\s]+?)=([^]*)$/.exec(arg);
-		if (pair != null) {
-			buildArgs[pair[1]] = pair[2] ?? '';
-		} else {
-			throw new ExpectedError(`Could not parse build argument: '${arg}'`);
-		}
-	});
-	return buildArgs;
-};
-
-export function generateBuildOpts(options) {
-	const opts = {};
-	if (options.tag != null) {
-		opts.t = options.tag;
-	}
-	if (options.nocache != null) {
-		opts.nocache = true;
-	}
-	if (options['cache-from']?.trim()) {
-		opts.cachefrom = options['cache-from'].split(',').filter((i) => !!i.trim());
-	}
-	if (options.pull != null) {
-		opts.pull = true;
-	}
-	if (options.squash != null) {
-		opts.squash = true;
-	}
-	if (options.buildArg != null) {
-		opts.buildargs = parseBuildArgs(options.buildArg);
-	}
-	if (!_.isEmpty(options['registry-secrets'])) {
-		opts.registryconfig = options['registry-secrets'];
-	}
-	return opts;
-}
 /**
  * @param {{
  * 	ca?: string; // path to ca (Certificate Authority) file (TLS)

lib/utils/docker.ts

@@ -17,6 +17,8 @@
 
 import type * as dockerode from 'dockerode';
 import { flags } from '@oclif/command';
+
+import { ExpectedError } from '../errors';
 import { parseAsInteger } from './validation';
 
 export * from './docker-js';
@@ -98,6 +100,70 @@ Implements the same feature as the "docker build --cache-from" option.`,
 	...dockerConnectionCliFlags,
 };
 
+export interface BuildOpts {
+	buildargs?: Dictionary<string>;
+	cachefrom?: string[];
+	nocache?: boolean;
+	pull?: boolean;
+	registryconfig?: import('resin-multibuild').RegistrySecrets;
+	squash?: boolean;
+	t?: string;
+}
+
+function parseBuildArgs(args: string[]): Dictionary<string> {
+	if (!Array.isArray(args)) {
+		args = [args];
+	}
+	const buildArgs: Dictionary<string> = {};
+	args.forEach(function (arg) {
+		// note: [^] matches any character, including line breaks
+		const pair = /^([^\s]+?)=([^]*)$/.exec(arg);
+		if (pair != null) {
+			buildArgs[pair[1]] = pair[2] ?? '';
+		} else {
+			throw new ExpectedError(`Could not parse build argument: '${arg}'`);
+		}
+	});
+	return buildArgs;
+}
+
+export function generateBuildOpts(options: {
+	buildArg?: string[];
+	'cache-from'?: string;
+	nocache: boolean;
+	pull?: boolean;
+	'registry-secrets'?: import('resin-multibuild').RegistrySecrets;
+	squash: boolean;
+	tag?: string;
+}): BuildOpts {
+	const opts: BuildOpts = {};
+	if (options.buildArg != null) {
+		opts.buildargs = parseBuildArgs(options.buildArg);
+	}
+	if (options['cache-from']?.trim()) {
+		opts.cachefrom = options['cache-from'].split(',').filter((i) => !!i.trim());
+	}
+	if (options.nocache != null) {
+		opts.nocache = true;
+	}
+	if (options.pull != null) {
+		opts.pull = true;
+	}
+	if (
+		options['registry-secrets'] &&
+		Object.keys(options['registry-secrets']).length
+	) {
+		opts.registryconfig = options['registry-secrets'];
+	}
+	if (options.squash != null) {
+		opts.squash = true;
+	}
+	if (options.tag != null) {
+		opts.t = options.tag;
+	}
+	return opts;
+}
+
 export async function isBalenaEngine(docker: dockerode): Promise<boolean> {
 	// dockerVersion.Engine should equal 'balena-engine' for the current/latest
 	// version of balenaEngine, but it was at one point (mis)spelt 'balaena':

lib/utils/remote-build.ts

@@ -24,7 +24,7 @@ import type { Pack } from 'tar-stream';
 
 import { ExpectedError } from '../errors';
 import { exitWithExpectedError } from '../errors';
-import { tarDirectory } from './compose';
+import { tarDirectory } from './compose_ts';
 import { getVisuals, stripIndent } from './lazy';
 import Logger = require('./logger');
 

tests/utils/tarDirectory.spec.ts

@@ -20,7 +20,7 @@ import * as _ from 'lodash';
 import * as path from 'path';
 import * as tar from 'tar-stream';
 
-import { tarDirectory } from '../../build/utils/compose';
+import { tarDirectory } from '../../build/utils/compose_ts';
 import { setupDockerignoreTestData } from '../projects';
 
 const repoPath = path.normalize(path.join(__dirname, '..', '..'));