Merge pull request #1118 from balena-io/1116-private-reg-build-deploy

Extend private registry support to balena build and deploy commands

doc/cli.markdown

@@ -1351,15 +1351,15 @@ Pin the preloaded device to the preloaded release on provision
 
 #### --docker, -P <docker>
 
-Path to a local docker socket
+Path to a local docker socket (e.g. /var/run/docker.sock)
 
 #### --dockerHost, -h <dockerHost>
 
-The address of the host containing the docker daemon
+Docker daemon hostname or IP address (dev machine or balena device) 
 
 #### --dockerPort, -p <dockerPort>
 
-The port on which the host docker daemon is listening
+Docker daemon TCP port number (hint: 2375 for balena devices)
 
 #### --ca <ca>
 
@@ -1691,7 +1691,9 @@ name of container to stop
 ## build [source]
 
 Use this command to build an image or a complete multicontainer project
-with the provided docker daemon.
+with the provided docker daemon in your development machine or balena
+device. (See also the `balena push` command for the option of building
+images in balena's cloud builders.)
 
 You must provide either an application or a device-type/architecture
 pair to use the balena Dockerfile pre-processor
@@ -1738,17 +1740,21 @@ Run an emulated build using Qemu
 
 Display full log output
 
+#### --registry-secrets, -R <secrets.yml|.json>
+
+Path to a YAML or JSON file with passwords for a private Docker registry
+
 #### --docker, -P <docker>
 
-Path to a local docker socket
+Path to a local docker socket (e.g. /var/run/docker.sock)
 
 #### --dockerHost, -h <dockerHost>
 
-The address of the host containing the docker daemon
+Docker daemon hostname or IP address (dev machine or balena device) 
 
 #### --dockerPort, -p <dockerPort>
 
-The port on which the host docker daemon is listening
+Docker daemon TCP port number (hint: 2375 for balena devices)
 
 #### --ca <ca>
 
@@ -1780,11 +1786,14 @@ Squash newly built layers into a single new layer
 
 ## deploy <appName> [image]
 
-Use this command to deploy an image or a complete multicontainer project
-to an application, optionally building it first.
-
 Usage: `deploy <appName> ([image] | --build [--source build-dir])`
 
+Use this command to deploy an image or a complete multicontainer project to an
+application, optionally building it first. The source images are searched for
+(and optionally built) using the docker daemon in your development machine or
+balena device. (See also the `balena push` command for the option of building
+the image in balena's cloud builders.)
+
 Unless an image is specified, this command will look into the current directory
 (or the one specified by --source) for a compose file. If one is found, this
 command will deploy each service defined in the compose file, building it first
@@ -1830,17 +1839,21 @@ Run an emulated build using Qemu
 
 Display full log output
 
+#### --registry-secrets, -R &#60;secrets.yml|.json&#62;
+
+Path to a YAML or JSON file with passwords for a private Docker registry
+
 #### --docker, -P &#60;docker&#62;
 
-Path to a local docker socket
+Path to a local docker socket (e.g. /var/run/docker.sock)
 
 #### --dockerHost, -h &#60;dockerHost&#62;
 
-The address of the host containing the docker daemon
+Docker daemon hostname or IP address (dev machine or balena device) 
 
 #### --dockerPort, -p &#60;dockerPort&#62;
 
-The port on which the host docker daemon is listening
+Docker daemon TCP port number (hint: 2375 for balena devices)
 
 #### --ca &#60;ca&#62;
 

lib/actions/build.coffee

@@ -50,7 +50,9 @@ module.exports =
 	primary: true
 	help: '''
 		Use this command to build an image or a complete multicontainer project
-		with the provided docker daemon.
+		with the provided docker daemon in your development machine or balena
+		device. (See also the `balena push` command for the option of building
+		images in balena's cloud builders.)
 
 		You must provide either an application or a device-type/architecture
 		pair to use the balena Dockerfile pre-processor
@@ -94,7 +96,7 @@ module.exports =
 	action: (params, options, done) ->
 		# compositions with many services trigger misleading warnings
 		require('events').defaultMaxListeners = 1000
-
+		{ validateComposeOptions } = require('../utils/compose_ts')
 		{ exitWithExpectedError } = require('../utils/patterns')
 		helpers = require('../utils/helpers')
 		Logger = require('../utils/logger')
@@ -109,6 +111,8 @@ module.exports =
 			options.source ?= params.source
 			delete params.source
 
+			validateComposeOptions(options)
+
 			{ application, arch, deviceType } = options
 
 			if (not (arch? and deviceType?) and not application?) or (application? and (arch? or deviceType?))

lib/actions/deploy.coffee

@@ -122,11 +122,14 @@ module.exports =
 	signature: 'deploy <appName> [image]'
 	description: 'Deploy a single image or a multicontainer project to a balena application'
 	help: '''
-		Use this command to deploy an image or a complete multicontainer project
-		to an application, optionally building it first.
-
 		Usage: `deploy <appName> ([image] | --build [--source build-dir])`
 
+		Use this command to deploy an image or a complete multicontainer project to an
+		application, optionally building it first. The source images are searched for
+		(and optionally built) using the docker daemon in your development machine or
+		balena device. (See also the `balena push` command for the option of building
+		the image in balena's cloud builders.)
+
 		Unless an image is specified, this command will look into the current directory
 		(or the one specified by --source) for a compose file. If one is found, this
 		command will deploy each service defined in the compose file, building it first
@@ -170,7 +173,7 @@ module.exports =
 	action: (params, options, done) ->
 		# compositions with many services trigger misleading warnings
 		require('events').defaultMaxListeners = 1000
-
+		{ validateComposeOptions } = require('../utils/compose_ts')
 		helpers = require('../utils/helpers')
 		Logger = require('../utils/logger')
 
@@ -185,6 +188,8 @@ module.exports =
 			appName = options.application if not appName?
 			delete options.application
 
+			validateComposeOptions(options)
+
 			if not appName?
 				throw new Error('Please specify the name of the application to deploy')
 

lib/actions/push.ts

@@ -18,8 +18,6 @@ import { CommandDefinition } from 'capitano';
 import { stripIndent } from 'common-tags';
 import { BalenaSDK } from 'balena-sdk';
 
-import { BuildError } from '../utils/device/errors';
-
 // An regex to detect an IP address, from https://www.regular-expressions.info/ip.html
 const IP_REGEX = new RegExp(
 	/\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/,
@@ -98,42 +96,6 @@ async function getAppOwner(sdk: BalenaSDK, appName: string) {
 	return selected.extra;
 }
 
-interface RegistrySecrets {
-	[registryAddress: string]: {
-		username: string;
-		password: string;
-	};
-}
-
-async function parseRegistrySecrets(
-	secretsFilename: string,
-): Promise<RegistrySecrets> {
-	const { fs } = await require('mz');
-	const {
-		addCanonicalDockerHubEntry,
-		RegistrySecretValidator,
-	} = await require('resin-multibuild');
-	try {
-		let isYaml = false;
-		if (/.+\.ya?ml$/i.test(secretsFilename)) {
-			isYaml = true;
-		} else if (!/.+\.json$/i.test(secretsFilename)) {
-			throw new Error('Filename must end with .json, .yml or .yaml');
-		}
-		const raw = (await fs.readFile(secretsFilename)).toString();
-		const registrySecrets = new RegistrySecretValidator().validateRegistrySecrets(
-			isYaml ? (await require('js-yaml')).safeLoad(raw) : JSON.parse(raw),
-		);
-		addCanonicalDockerHubEntry(registrySecrets);
-		return registrySecrets;
-	} catch (error) {
-		error.message =
-			`Error validating registry secrets file "${secretsFilename}":\n` +
-			error.message;
-		throw error;
-	}
-}
-
 export const push: CommandDefinition<
 	{
 		applicationOrDevice: string;
@@ -217,6 +179,8 @@ export const push: CommandDefinition<
 		const remote = await import('../utils/remote-build');
 		const deviceDeploy = await import('../utils/device/deploy');
 		const { exitWithExpectedError } = await import('../utils/patterns');
+		const { parseRegistrySecrets } = await import('../utils/compose_ts');
+		const { BuildError } = await import('../utils/device/errors');
 
 		const appOrDevice: string | null = params.applicationOrDevice;
 		if (appOrDevice == null) {

lib/utils/compose.coffee

@@ -41,6 +41,12 @@ exports.appendOptions = (opts) ->
 			description: 'Display full log output'
 			boolean: true
 		},
+		{
+			signature: 'registry-secrets',
+			alias: 'R',
+			parameter: 'secrets.yml|.json',
+			description: 'Path to a YAML or JSON file with passwords for a private Docker registry',
+		},
 	]
 
 exports.generateOpts = (options) ->

lib/utils/compose_ts.ts

@@ -25,6 +25,51 @@ import { Composition } from 'resin-compose-parse';
 import { DeviceInfo } from './device/api';
 import Logger = require('./logger');
 
+export interface RegistrySecrets {
+	[registryAddress: string]: {
+		username: string;
+		password: string;
+	};
+}
+
+export async function parseRegistrySecrets(
+	secretsFilename: string,
+): Promise<RegistrySecrets> {
+	const { fs } = require('mz');
+	try {
+		let isYaml = false;
+		if (/.+\.ya?ml$/i.test(secretsFilename)) {
+			isYaml = true;
+		} else if (!/.+\.json$/i.test(secretsFilename)) {
+			throw new Error('Filename must end with .json, .yml or .yaml');
+		}
+		const raw = (await fs.readFile(secretsFilename)).toString();
+		const registrySecrets = new MultiBuild.RegistrySecretValidator().validateRegistrySecrets(
+			isYaml ? require('js-yaml').safeLoad(raw) : JSON.parse(raw),
+		);
+		MultiBuild.addCanonicalDockerHubEntry(registrySecrets);
+		return registrySecrets;
+	} catch (error) {
+		error.message =
+			`Error validating registry secrets file "${secretsFilename}":\n` +
+			error.message;
+		throw error;
+	}
+}
+
+/**
+ * Validate the compose-specific command-line options defined in compose.coffee.
+ * This function is meant to be called very early on to validate users' input,
+ * before any project loading / building / deploying.
+ */
+export async function validateComposeOptions(options: { [opt: string]: any }) {
+	if (options['registry-secrets']) {
+		options['registry-secrets'] = await parseRegistrySecrets(
+			options['registry-secrets'],
+		);
+	}
+}
+
 /**
  * Create a BuildTask array of "resolved build tasks" by calling multibuild
  * .splitBuildStream() and performResolution(), and add build stream error

lib/utils/docker.coffee

@@ -14,19 +14,19 @@ exports.appendConnectionOptions = appendConnectionOptions = (opts) ->
 		{
 			signature: 'docker'
 			parameter: 'docker'
-			description: 'Path to a local docker socket'
+			description: 'Path to a local docker socket (e.g. /var/run/docker.sock)'
 			alias: 'P'
 		},
 		{
 			signature: 'dockerHost'
 			parameter: 'dockerHost'
-			description: 'The address of the host containing the docker daemon'
+			description: 'Docker daemon hostname or IP address (dev machine or balena device) '
 			alias: 'h'
 		},
 		{
 			signature: 'dockerPort'
 			parameter: 'dockerPort'
-			description: 'The port on which the host docker daemon is listening'
+			description: 'Docker daemon TCP port number (hint: 2375 for balena devices)'
 			alias: 'p'
 		},
 		{
@@ -147,6 +147,8 @@ exports.generateBuildOpts = (options) ->
 		opts.squash = true
 	if options.buildArg?
 		opts.buildargs = parseBuildArgs(options.buildArg)
+	if not _.isEmpty(options['registry-secrets'])
+		opts.registryconfig = options['registry-secrets']
 	return opts
 
 exports.getDocker = (options) ->