Integrate new resin-multibuild major version (private docker registry

authentication support for the docker-compose 'image' instruction).

Resolves: #1114
Change-type: minor
Signed-off-by: Paulo Castro <paulo@balena.io>

lib/actions/push.ts

@@ -109,7 +109,10 @@ async function parseRegistrySecrets(
 	secretsFilename: string,
 ): Promise<RegistrySecrets> {
 	const { fs } = await require('mz');
-	const { RegistrySecretValidator } = await require('resin-multibuild');
+	const {
+		addCanonicalDockerHubEntry,
+		RegistrySecretValidator,
+	} = await require('resin-multibuild');
 	try {
 		let isYaml = false;
 		if (/.+\.ya?ml$/i.test(secretsFilename)) {
@@ -118,9 +121,11 @@ async function parseRegistrySecrets(
 			throw new Error('Filename must end with .json, .yml or .yaml');
 		}
 		const raw = (await fs.readFile(secretsFilename)).toString();
-		return new RegistrySecretValidator().validateRegistrySecrets(
+		const registrySecrets = new RegistrySecretValidator().validateRegistrySecrets(
 			isYaml ? (await require('js-yaml')).safeLoad(raw) : JSON.parse(raw),
 		);
+		addCanonicalDockerHubEntry(registrySecrets);
+		return registrySecrets;
 	} catch (error) {
 		error.message =
 			`Error validating registry secrets file "${secretsFilename}":\n` +

lib/utils/compose.coffee

@@ -197,16 +197,8 @@ exports.buildProject = (
 		# Tar up the directory, ready for the build stream
 		tarDirectory(projectPath)
 		.then (tarStream) ->
-			builder.splitBuildStream(composition, tarStream)
-		.tap (tasks) ->
-			# Updates each task as a side-effect
-			builder.performResolution(tasks, arch, deviceType)
-			.map (task) ->
-				if not task.external and not task.resolved
-					throw new Error(
-						"Project type for service '#{task.serviceName}' could not be determined. " +
-						'Please add a Dockerfile'
-					)
+			{ makeBuildTasks } = require('./compose_ts')
+			Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger))
 		.map (task) ->
 			d = imageDescriptorsByServiceName[task.serviceName]
 

lib/utils/compose_ts.ts

@@ -14,28 +14,98 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import * as _ from 'lodash';
+import * as Bluebird from 'bluebird';
+import * as tar from 'tar-stream';
+import { Readable } from 'stream';
 
-import { RegistrySecrets } from 'resin-multibuild';
-import { Pack } from 'tar-stream';
+import * as MultiBuild from 'resin-multibuild';
+import { Composition } from 'resin-compose-parse';
+
+import { DeviceInfo } from './device/api';
+import Logger = require('./logger');
 
 /**
- * Return a callback function that takes a tar-stream Pack object as argument
- * and uses it to add the '.balena/registry-secrets.json' metadata file that
- * contains usernames and passwords for private docker registries. The builder
- * will remove the file from the tar stream and use the secrets to pull base
- * images from users' private registries.
- * @param registrySecrets JS object containing registry usernames and passwords
- * @returns A callback function, or undefined if registrySecrets is empty
+ * Create a BuildTask array of "resolved build tasks" by calling multibuild
+ * .splitBuildStream() and performResolution(), and add build stream error
+ * handlers and debug logging.
+ * Both `balena build` and `balena deploy` call this function.
  */
-export function getTarStreamCallbackForRegistrySecrets(
-	registrySecrets: RegistrySecrets,
-): ((pack: Pack) => void) | undefined {
-	if (Object.keys(registrySecrets).length > 0) {
-		return (pack: Pack) => {
-			pack.entry(
-				{ name: '.balena/registry-secrets.json' },
-				JSON.stringify(registrySecrets),
-			);
-		};
-	}
+export async function makeBuildTasks(
+	composition: Composition,
+	tarStream: Readable,
+	deviceInfo: DeviceInfo,
+	logger: Logger,
+): Promise<MultiBuild.BuildTask[]> {
+	const buildTasks = await MultiBuild.splitBuildStream(composition, tarStream);
+
+	logger.logDebug('Found build tasks:');
+	_.each(buildTasks, task => {
+		let infoStr: string;
+		if (task.external) {
+			infoStr = `image pull [${task.imageName}]`;
+		} else {
+			infoStr = `build [${task.context}]`;
+		}
+		logger.logDebug(`    ${task.serviceName}: ${infoStr}`);
+	});
+
+	logger.logDebug(
+		`Resolving services with [${deviceInfo.deviceType}|${deviceInfo.arch}]`,
+	);
+
+	await performResolution(buildTasks, deviceInfo);
+
+	logger.logDebug('Found project types:');
+	_.each(buildTasks, task => {
+		if (task.external) {
+			logger.logDebug(`    ${task.serviceName}: External image`);
+		} else {
+			logger.logDebug(`    ${task.serviceName}: ${task.projectType}`);
+		}
+	});
+
+	return buildTasks;
+}
+
+async function performResolution(
+	tasks: MultiBuild.BuildTask[],
+	deviceInfo: DeviceInfo,
+): Promise<MultiBuild.BuildTask[]> {
+	const { cloneTarStream } = require('tar-utils');
+
+	return await new Promise<MultiBuild.BuildTask[]>((resolve, reject) => {
+		const buildTasks = MultiBuild.performResolution(
+			tasks,
+			deviceInfo.arch,
+			deviceInfo.deviceType,
+			{ error: [reject] },
+		);
+		// Do one task at a time (Bluebird.each instead of Bluebird.all)
+		// in order to reduce peak memory usage. Resolves to buildTasks.
+		Bluebird.each(buildTasks, buildTask => {
+			// buildStream is falsy for "external" tasks (image pull)
+			if (!buildTask.buildStream) {
+				return buildTask;
+			}
+			// Consume each task.buildStream in order to trigger the
+			// resolution events that define fields like:
+			//     task.dockerfile, task.dockerfilePath,
+			//     task.projectType, task.resolved
+			// This mimics what is currently done in `resin-builder`.
+			return cloneTarStream(buildTask.buildStream).then(
+				(clonedStream: tar.Pack) => {
+					buildTask.buildStream = clonedStream;
+					if (!buildTask.external && !buildTask.resolved) {
+						throw new Error(
+							`Project type for service "${
+								buildTask.serviceName
+							}" could not be determined. Missing a Dockerfile?`,
+						);
+					}
+					return buildTask;
+				},
+			);
+		}).then(resolve, reject);
+	});
 }

lib/utils/device/deploy.ts

@@ -19,13 +19,18 @@ import * as Bluebird from 'bluebird';
 import * as Docker from 'dockerode';
 import * as _ from 'lodash';
 import { Composition } from 'resin-compose-parse';
-import { BuildTask, LocalImage, RegistrySecrets } from 'resin-multibuild';
+import {
+	BuildTask,
+	getAuthConfigObj,
+	LocalImage,
+	RegistrySecrets,
+} from 'resin-multibuild';
 import * as semver from 'resin-semver';
 import { Readable } from 'stream';
 
 import Logger = require('../logger');
 import { displayBuildLog } from './logs';
-
+import { makeBuildTasks } from '../compose_ts';
 import { DeviceInfo } from './api';
 import * as LocalPushErrors from './errors';
 
@@ -148,36 +153,12 @@ export async function performBuilds(
 ): Promise<void> {
 	const multibuild = await import('resin-multibuild');
 
-	const buildTasks = await multibuild.splitBuildStream(composition, tarStream);
-
-	logger.logDebug('Found build tasks:');
-	_.each(buildTasks, task => {
-		let infoStr: string;
-		if (task.external) {
-			infoStr = `image pull [${task.imageName}]`;
-		} else {
-			infoStr = `build [${task.context}]`;
-		}
-		logger.logDebug(`    ${task.serviceName}: ${infoStr}`);
-	});
-
-	logger.logDebug(
-		`Resolving services with [${deviceInfo.deviceType}|${deviceInfo.arch}]`,
+	const buildTasks = await makeBuildTasks(
+		composition,
+		tarStream,
+		deviceInfo,
+		logger,
 	);
-	await multibuild.performResolution(
-		buildTasks,
-		deviceInfo.arch,
-		deviceInfo.deviceType,
-	);
-
-	logger.logDebug('Found project types:');
-	_.each(buildTasks, task => {
-		if (!task.external) {
-			logger.logDebug(`    ${task.serviceName}: ${task.projectType}`);
-		} else {
-			logger.logDebug(`    ${task.serviceName}: External image`);
-		}
-	});
 
 	logger.logDebug('Probing remote daemon for cache images');
 	await assignDockerBuildOpts(docker, buildTasks, opts);
@@ -247,16 +228,23 @@ async function assignDockerBuildOpts(
 
 	logger.logDebug(`Using ${images.length} on-device images for cache...`);
 
-	_.each(buildTasks, (task: BuildTask) => {
+	await Bluebird.map(buildTasks, async (task: BuildTask) => {
 		task.dockerOpts = {
 			cachefrom: images,
 			labels: {
 				'io.resin.local.image': '1',
 				'io.resin.local.service': task.serviceName,
 			},
-			registryconfig: opts.registrySecrets,
 			t: generateImageName(task.serviceName),
 		};
+		if (task.external) {
+			task.dockerOpts.authconfig = await getAuthConfigObj(
+				task.imageName!,
+				opts.registrySecrets,
+			);
+		} else {
+			task.dockerOpts.registryconfig = opts.registrySecrets;
+		}
 	});
 }
 

lib/utils/patterns.ts

@@ -288,6 +288,15 @@ export function printErrorMessage(message: string) {
 	console.error(chalk.red(`\n${messages.getHelp}\n`));
 }
 
+/**
+ * Print a friendly error message and exit the CLI with an error code, BYPASSING
+ * error reporting through Sentry.io's platform (raven.Raven.captureException).
+ * Note that lib/errors.ts provides top-level error handling code to catch any
+ * otherwise uncaught errors, AND to report them through Sentry.io. But many
+ * "expected" errors (say, a JSON parsing error in a file provided by the user)
+ * don't warrant reporting through Sentry.io.  For such mundane errors, catch
+ * them and call this function.
+ */
 export function exitWithExpectedError(message: string | Error): never {
 	if (message instanceof Error) {
 		({ message } = message);

package.json

@@ -155,13 +155,12 @@
     "raven": "^2.5.0",
     "reconfix": "^0.1.0",
     "request": "^2.81.0",
-    "resin-bundle-resolve": "^0.6.0",
     "resin-cli-form": "^2.0.1",
     "resin-cli-visuals": "^1.4.0",
     "resin-compose-parse": "^2.0.0",
     "resin-doodles": "0.0.1",
     "resin-image-fs": "^5.0.2",
-    "resin-multibuild": "^0.10.0",
+    "resin-multibuild": "^2.1.0",
     "resin-release": "^1.2.0",
     "resin-semver": "^1.4.0",
     "resin-stream-logger": "^0.1.2",
@@ -171,7 +170,8 @@
     "split": "^1.0.1",
     "string-width": "^2.1.1",
     "strip-ansi-stream": "^1.0.0",
-    "tar-stream": "^1.5.5",
+    "tar-stream": "^1.6.2",
+    "tar-utils": "^1.1.0",
     "through2": "^2.0.3",
     "tmp": "0.0.31",
     "typed-error": "^3.0.0",