Merge pull request #1041 from balena-io/1015-build-secrets-command-line-options

Add --registry-secrets option for balena push (private registry auth)

doc/cli.markdown

@@ -1360,16 +1360,28 @@ Docker host TLS key file
 
 ## push <applicationOrDevice>
 
-This command can be used to start a build on the remote
-balena cloud builders, or a local mode balena device.
+This command can be used to start a build on the remote balena cloud builders,
+or a local mode balena device.
 
 When building on the balena cloud the given source directory will be sent to the
 balena builder, and the build will proceed. This can be used as a drop-in
 replacement for git push to deploy.
 
-When building on a local mode device, the given source directory will be built on
-device, and the resulting containers will be run on the device. Logs will be
-streamed back from the device as part of the same invocation.
+When building on a local mode device, the given source directory will be built
+on the device, and the resulting containers will be run on the device. Logs will
+be streamed back from the device as part of the same invocation.
+
+The --registry-secrets option specifies a JSON or YAML file containing private
+Docker registry usernames and passwords to be used when pulling base images.
+Sample registry-secrets YAML file:
+
+	'https://idx.docker.io/v1/':
+		username: mike
+		password: cze14
+	'myregistry.com:25000':
+		username: ann
+		password: hunter2
+
 
 Examples:
 
@@ -1395,6 +1407,10 @@ Force an emulated build to occur on the remote builder
 
 Don't use cache when building this project
 
+#### --registry-secrets, -R <secrets.yml|.json>
+
+Path to a local YAML or JSON file containing Docker registry passwords used to pull base images
+
 # Settings
 
 ## settings

lib/actions/config.coffee

@@ -1,5 +1,5 @@
 ###
-Copyright 2016-2017 Balena
+Copyright 2016-2018 Balena Ltd.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -306,7 +306,7 @@ exports.generate =
 			'''
 
 		if !options.application and options.deviceType
-			patterns.exitWithExpectedError '''
+			exitWithExpectedError '''
 				Specifying a different device type is only supported when
 				generating a config for an application:
 

lib/actions/push.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2016-2017 Balena
+Copyright 2016-2018 Balena Ltd.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -98,6 +98,37 @@ async function getAppOwner(sdk: BalenaSDK, appName: string) {
 	return selected.extra;
 }
 
+interface RegistrySecrets {
+	[registryAddress: string]: {
+		username: string;
+		password: string;
+	};
+}
+
+async function parseRegistrySecrets(
+	secretsFilename: string,
+): Promise<RegistrySecrets> {
+	const { fs } = await require('mz');
+	const { RegistrySecretValidator } = await require('resin-multibuild');
+	try {
+		let isYaml = false;
+		if (/.+\.ya?ml$/i.test(secretsFilename)) {
+			isYaml = true;
+		} else if (!/.+\.json$/i.test(secretsFilename)) {
+			throw new Error('Filename must end with .json, .yml or .yaml');
+		}
+		const raw = (await fs.readFile(secretsFilename)).toString();
+		return new RegistrySecretValidator().validateRegistrySecrets(
+			isYaml ? (await require('js-yaml')).safeLoad(raw) : JSON.parse(raw),
+		);
+	} catch (error) {
+		error.message =
+			`Error validating registry secrets file "${secretsFilename}":\n` +
+			error.message;
+		throw error;
+	}
+}
+
 export const push: CommandDefinition<
 	{
 		applicationOrDevice: string;
@@ -106,22 +137,35 @@ export const push: CommandDefinition<
 		source: string;
 		emulated: boolean;
 		nocache: boolean;
+		'registry-secrets': string;
 	}
 > = {
 	signature: 'push <applicationOrDevice>',
 	description:
 		'Start a remote build on the balena cloud build servers or a local mode device',
 	help: stripIndent`
-		This command can be used to start a build on the remote
-		balena cloud builders, or a local mode balena device.
+		This command can be used to start a build on the remote balena cloud builders,
+		or a local mode balena device.
 
 		When building on the balena cloud the given source directory will be sent to the
 		balena builder, and the build will proceed. This can be used as a drop-in
 		replacement for git push to deploy.
 
-		When building on a local mode device, the given source directory will be built on
-		device, and the resulting containers will be run on the device. Logs will be
-		streamed back from the device as part of the same invocation.
+		When building on a local mode device, the given source directory will be built
+		on the device, and the resulting containers will be run on the device. Logs will
+		be streamed back from the device as part of the same invocation.
+
+		The --registry-secrets option specifies a JSON or YAML file containing private
+		Docker registry usernames and passwords to be used when pulling base images.
+		Sample registry-secrets YAML file:
+
+			'https://idx.docker.io/v1/':
+				username: mike
+				password: cze14
+			'myregistry.com:25000':
+				username: ann
+				password: hunter2
+
 
 		Examples:
 
@@ -154,6 +198,13 @@ export const push: CommandDefinition<
 			description: "Don't use cache when building this project",
 			boolean: true,
 		},
+		{
+			signature: 'registry-secrets',
+			alias: 'R',
+			parameter: 'secrets.yml|.json',
+			description: stripIndent`
+				Path to a local YAML or JSON file containing Docker registry passwords used to pull base images`,
+		},
 	],
 	async action(params, options, done) {
 		const sdk = (await import('balena-sdk')).fromSharedOptions();
@@ -172,6 +223,10 @@ export const push: CommandDefinition<
 			console.log(`[debug] Using ${source} as build source`);
 		}
 
+		const registrySecrets = options['registry-secrets']
+			? await parseRegistrySecrets(options['registry-secrets'])
+			: {};
+
 		const buildTarget = getBuildTarget(appOrDevice);
 		switch (buildTarget) {
 			case BuildTarget.Cloud:
@@ -184,6 +239,7 @@ export const push: CommandDefinition<
 						const opts = {
 							emulated: options.emulated,
 							nocache: options.nocache,
+							registrySecrets,
 						};
 						const args = {
 							app,
@@ -206,6 +262,7 @@ export const push: CommandDefinition<
 					deviceDeploy.deployToDevice({
 						source,
 						deviceHost: device,
+						registrySecrets,
 					}),
 				)
 					.catch(BuildError, e => {

lib/utils/compose.coffee

@@ -1,3 +1,20 @@
+###*
+# @license
+# Copyright 2018 Balena Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+
 Promise = require('bluebird')
 path = require('path')
 
@@ -100,7 +117,7 @@ toPosixPath = (systemPath) ->
 	path = require('path')
 	systemPath.replace(new RegExp('\\' + path.sep, 'g'), '/')
 
-exports.tarDirectory = tarDirectory = (dir) ->
+exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) ->
 	tar = require('tar-stream')
 	klaw = require('klaw')
 	path = require('path')
@@ -126,6 +143,8 @@ exports.tarDirectory = tarDirectory = (dir) ->
 		Promise.join relPath, fs.stat(file), fs.readFile(file),
 			(filename, stats, data) ->
 				pack.entry({ name: toPosixPath(filename), size: stats.size, mode: stats.mode }, data)
+	.then ->
+		preFinalizeCallback?(pack)
 	.then ->
 		pack.finalize()
 		return pack

lib/utils/compose.d.ts

@@ -1,6 +1,24 @@
+/**
+ * @license
+ * Copyright 2018 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import * as Bluebird from 'bluebird';
 import * as Stream from 'stream';
 import { Composition } from 'resin-compose-parse';
+import { Pack } from 'tar-stream';
 import Logger = require('./logger');
 
 interface Image {
@@ -29,4 +47,7 @@ export function loadProject(
 	image?: string,
 ): Bluebird<ComposeProject>;
 
-export function tarDirectory(source: string): Promise<Stream.Readable>;
+export function tarDirectory(
+	source: string,
+	preFinalizeCallback?: (pack: Pack) => void,
+): Promise<Stream.Readable>;

lib/utils/compose_ts.ts

@@ -0,0 +1,41 @@
+/**
+ * @license
+ * Copyright 2018 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RegistrySecrets } from 'resin-multibuild';
+import { Pack } from 'tar-stream';
+
+/**
+ * Return a callback function that takes a tar-stream Pack object as argument
+ * and uses it to add the '.balena/registry-secrets.json' metadata file that
+ * contains usernames and passwords for private docker registries. The builder
+ * will remove the file from the tar stream and use the secrets to pull base
+ * images from users' private registries.
+ * @param registrySecrets JS object containing registry usernames and passwords
+ * @returns A callback function, or undefined if registrySecrets is empty
+ */
+export function getTarStreamCallbackForRegistrySecrets(
+	registrySecrets: RegistrySecrets,
+): ((pack: Pack) => void) | undefined {
+	if (Object.keys(registrySecrets).length > 0) {
+		return (pack: Pack) => {
+			pack.entry(
+				{ name: '.balena/registry-secrets.json' },
+				JSON.stringify(registrySecrets),
+			);
+		};
+	}
+}

lib/utils/device/deploy.ts

@@ -1,8 +1,25 @@
+/**
+ * @license
+ * Copyright 2018 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import * as Bluebird from 'bluebird';
 import * as Docker from 'dockerode';
 import * as _ from 'lodash';
 import { Composition } from 'resin-compose-parse';
-import { BuildTask, LocalImage } from 'resin-multibuild';
+import { BuildTask, LocalImage, RegistrySecrets } from 'resin-multibuild';
 import * as semver from 'resin-semver';
 import { Readable } from 'stream';
 
@@ -20,6 +37,7 @@ export interface DeviceDeployOptions {
 	source: string;
 	deviceHost: string;
 	devicePort?: number;
+	registrySecrets: RegistrySecrets;
 }
 
 async function checkSource(source: string): Promise<boolean> {
@@ -86,6 +104,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 		docker,
 		deviceInfo,
 		logger,
+		opts,
 	);
 
 	logger.logDebug('Setting device state...');
@@ -125,6 +144,7 @@ export async function performBuilds(
 	docker: Docker,
 	deviceInfo: DeviceInfo,
 	logger: Logger,
+	opts: DeviceDeployOptions,
 ): Promise<void> {
 	const multibuild = await import('resin-multibuild');
 
@@ -160,7 +180,7 @@ export async function performBuilds(
 	});
 
 	logger.logDebug('Probing remote daemon for cache images');
-	await assignDockerBuildOpts(docker, buildTasks);
+	await assignDockerBuildOpts(docker, buildTasks, opts);
 
 	logger.logDebug('Starting builds...');
 	await assignOutputHandlers(buildTasks, logger);
@@ -219,6 +239,7 @@ async function getDeviceDockerImages(docker: Docker): Promise<string[]> {
 async function assignDockerBuildOpts(
 	docker: Docker,
 	buildTasks: BuildTask[],
+	opts: DeviceDeployOptions,
 ): Promise<void> {
 	// Get all of the images on the remote docker daemon, so
 	// that we can use all of them for cache
@@ -233,6 +254,7 @@ async function assignDockerBuildOpts(
 				'io.resin.local.image': '1',
 				'io.resin.local.service': task.serviceName,
 			},
+			registryconfig: opts.registrySecrets,
 			t: generateImageName(task.serviceName),
 		};
 	});
@@ -301,6 +323,6 @@ async function inspectBuildResults(images: LocalImage[]): Promise<void> {
 	});
 
 	if (failures.length > 0) {
-		exitWithExpectedError(new LocalPushErrors.BuildError(failures));
+		exitWithExpectedError(new LocalPushErrors.BuildError(failures).toString());
 	}
 }

lib/utils/remote-build.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2016-2017 Balena
+Copyright 2016-2018 Balena Ltd.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,8 +18,10 @@ import * as JSONStream from 'JSONStream';
 import * as request from 'request';
 import { BalenaSDK } from 'balena-sdk';
 import * as Stream from 'stream';
+import { Pack } from 'tar-stream';
 import { TypedError } from 'typed-error';
 
+import { RegistrySecrets } from 'resin-multibuild';
 import { tarDirectory } from './compose';
 
 const DEBUG_MODE = !!process.env.DEBUG;
@@ -30,6 +32,7 @@ const TRIM_REGEX = /\n+$/;
 export interface BuildOpts {
 	emulated: boolean;
 	nocache: boolean;
+	registrySecrets: RegistrySecrets;
 }
 
 export interface RemoteBuild {
@@ -76,6 +79,8 @@ async function getBuilderEndpoint(
 		emulated: opts.emulated,
 		nocache: opts.nocache,
 	});
+	// Note that using https (rather than http) is a requirement when using the
+	// --registry-secrets feature, as the secrets are not otherwise encrypted.
 	return `https://builder.${baseUrl}/v3/build?${args}`;
 }
 
@@ -210,6 +215,28 @@ async function cancelBuildIfNecessary(build: RemoteBuild): Promise<void> {
 	}
 }
 
+/**
+ * Return a callback function that takes a tar-stream Pack object as argument
+ * and uses it to add the '.balena/registry-secrets.json' metadata file that
+ * contains usernames and passwords to private docker registries. The builder
+ * will remove the file from the tar stream and use the secrets to pull base
+ * images from users' private registries.
+ * @param registrySecrets JS object containing registry usernames and passwords
+ * @returns A callback function, or undefined if registrySecrets is empty
+ */
+function getTarStreamCallbackForRegistrySecrets(
+	registrySecrets: RegistrySecrets,
+): ((pack: Pack) => void) | undefined {
+	if (Object.keys(registrySecrets).length > 0) {
+		return (pack: Pack) => {
+			pack.entry(
+				{ name: '.balena/registry-secrets.json' },
+				JSON.stringify(registrySecrets),
+			);
+		};
+	}
+}
+
 async function getRequestStream(build: RemoteBuild): Promise<Stream.Duplex> {
 	const path = await import('path');
 	const visuals = await import('resin-cli-visuals');
@@ -218,19 +245,24 @@ async function getRequestStream(build: RemoteBuild): Promise<Stream.Duplex> {
 	const tarSpinner = new visuals.Spinner('Packaging the project source...');
 	tarSpinner.start();
 	// Tar the directory so that we can send it to the builder
-	const tarStream = await tarDirectory(path.resolve(build.source));
+	const tarStream = await tarDirectory(
+		path.resolve(build.source),
+		getTarStreamCallbackForRegistrySecrets(build.opts.registrySecrets),
+	);
 	tarSpinner.stop();
 
+	const url = await getBuilderEndpoint(
+		build.baseUrl,
+		build.owner,
+		build.app,
+		build.opts,
+	);
+
 	if (DEBUG_MODE) {
-		console.log('[debug] Opening builder connection');
+		console.log(`[debug] Connecting to builder at ${url}`);
 	}
 	const post = request.post({
-		url: await getBuilderEndpoint(
-			build.baseUrl,
-			build.owner,
-			build.app,
-			build.opts,
-		),
+		url,
 		auth: {
 			bearer: build.auth,
 		},

package.json

@@ -88,13 +88,14 @@
     "prettier": "^1.14.2",
     "publish-release": "^1.3.3",
     "require-npm4-to-publish": "^1.0.0",
-    "resin-lint": "^2.0.0",
+    "resin-lint": "^2.0.1",
     "rewire": "^3.0.2",
     "ts-node": "^4.0.1",
     "typescript": "2.8.1"
   },
   "dependencies": {
     "@resin.io/valid-email": "^0.1.0",
+    "@types/dockerode": "2.5.5",
     "@types/stream-to-promise": "2.2.0",
     "@types/through2": "^2.0.33",
     "@zeit/dockerignore": "0.0.3",
@@ -112,7 +113,7 @@
     "bash": "0.0.1",
     "bluebird": "^3.3.3",
     "body-parser": "^1.14.1",
-    "capitano": "^1.7.0",
+    "capitano": "^1.8.2",
     "chalk": "^2.3.0",
     "cli-truncate": "^1.1.0",
     "coffeescript": "^1.12.6",
@@ -158,7 +159,7 @@
     "resin-compose-parse": "^2.0.0",
     "resin-doodles": "0.0.1",
     "resin-image-fs": "^5.0.2",
-    "resin-multibuild": "^0.9.0",
+    "resin-multibuild": "^0.10.0",
     "resin-release": "^1.2.0",
     "resin-semver": "^1.4.0",
     "resin-stream-logger": "^0.1.2",