From 211fb824a1e009e8158415716050f76c0b4e6e75 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Tue, 26 Feb 2019 13:32:27 +0000 Subject: [PATCH] Extend private registry support to balena build and deploy commands Resolves: #1116 Change-type: minor Signed-off-by: Paulo Castro --- doc/cli.markdown | 39 ++++++++++++++++++++++----------- lib/actions/build.coffee | 8 +++++-- lib/actions/deploy.coffee | 13 +++++++---- lib/actions/push.ts | 40 ++-------------------------------- lib/utils/compose.coffee | 6 ++++++ lib/utils/compose_ts.ts | 45 +++++++++++++++++++++++++++++++++++++++ lib/utils/docker.coffee | 8 ++++--- 7 files changed, 99 insertions(+), 60 deletions(-) diff --git a/doc/cli.markdown b/doc/cli.markdown index f20bc56a..cf085fca 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -1351,15 +1351,15 @@ Pin the preloaded device to the preloaded release on provision #### --docker, -P <docker> -Path to a local docker socket +Path to a local docker socket (e.g. /var/run/docker.sock) #### --dockerHost, -h <dockerHost> -The address of the host containing the docker daemon +Docker daemon hostname or IP address (dev machine or balena device) #### --dockerPort, -p <dockerPort> -The port on which the host docker daemon is listening +Docker daemon TCP port number (hint: 2375 for balena devices) #### --ca <ca> @@ -1691,7 +1691,9 @@ name of container to stop ## build [source] Use this command to build an image or a complete multicontainer project -with the provided docker daemon. +with the provided docker daemon in your development machine or balena +device. (See also the `balena push` command for the option of building +images in balena's cloud builders.) You must provide either an application or a device-type/architecture pair to use the balena Dockerfile pre-processor @@ -1738,17 +1740,21 @@ Run an emulated build using Qemu Display full log output +#### --registry-secrets, -R <secrets.yml|.json> + +Path to a YAML or JSON file with passwords for a private Docker registry + #### --docker, -P <docker> -Path to a local docker socket +Path to a local docker socket (e.g. /var/run/docker.sock) #### --dockerHost, -h <dockerHost> -The address of the host containing the docker daemon +Docker daemon hostname or IP address (dev machine or balena device) #### --dockerPort, -p <dockerPort> -The port on which the host docker daemon is listening +Docker daemon TCP port number (hint: 2375 for balena devices) #### --ca <ca> @@ -1780,11 +1786,14 @@ Squash newly built layers into a single new layer ## deploy <appName> [image] -Use this command to deploy an image or a complete multicontainer project -to an application, optionally building it first. - Usage: `deploy ([image] | --build [--source build-dir])` +Use this command to deploy an image or a complete multicontainer project to an +application, optionally building it first. The source images are searched for +(and optionally built) using the docker daemon in your development machine or +balena device. (See also the `balena push` command for the option of building +the image in balena's cloud builders.) + Unless an image is specified, this command will look into the current directory (or the one specified by --source) for a compose file. If one is found, this command will deploy each service defined in the compose file, building it first @@ -1830,17 +1839,21 @@ Run an emulated build using Qemu Display full log output +#### --registry-secrets, -R <secrets.yml|.json> + +Path to a YAML or JSON file with passwords for a private Docker registry + #### --docker, -P <docker> -Path to a local docker socket +Path to a local docker socket (e.g. /var/run/docker.sock) #### --dockerHost, -h <dockerHost> -The address of the host containing the docker daemon +Docker daemon hostname or IP address (dev machine or balena device) #### --dockerPort, -p <dockerPort> -The port on which the host docker daemon is listening +Docker daemon TCP port number (hint: 2375 for balena devices) #### --ca <ca> diff --git a/lib/actions/build.coffee b/lib/actions/build.coffee index f3a6dba6..08c6a080 100644 --- a/lib/actions/build.coffee +++ b/lib/actions/build.coffee @@ -50,7 +50,9 @@ module.exports = primary: true help: ''' Use this command to build an image or a complete multicontainer project - with the provided docker daemon. + with the provided docker daemon in your development machine or balena + device. (See also the `balena push` command for the option of building + images in balena's cloud builders.) You must provide either an application or a device-type/architecture pair to use the balena Dockerfile pre-processor @@ -94,7 +96,7 @@ module.exports = action: (params, options, done) -> # compositions with many services trigger misleading warnings require('events').defaultMaxListeners = 1000 - + { validateComposeOptions } = require('../utils/compose_ts') { exitWithExpectedError } = require('../utils/patterns') helpers = require('../utils/helpers') Logger = require('../utils/logger') @@ -109,6 +111,8 @@ module.exports = options.source ?= params.source delete params.source + validateComposeOptions(options) + { application, arch, deviceType } = options if (not (arch? and deviceType?) and not application?) or (application? and (arch? or deviceType?)) diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 00c7161f..ccdae53b 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -122,11 +122,14 @@ module.exports = signature: 'deploy [image]' description: 'Deploy a single image or a multicontainer project to a balena application' help: ''' - Use this command to deploy an image or a complete multicontainer project - to an application, optionally building it first. - Usage: `deploy ([image] | --build [--source build-dir])` + Use this command to deploy an image or a complete multicontainer project to an + application, optionally building it first. The source images are searched for + (and optionally built) using the docker daemon in your development machine or + balena device. (See also the `balena push` command for the option of building + the image in balena's cloud builders.) + Unless an image is specified, this command will look into the current directory (or the one specified by --source) for a compose file. If one is found, this command will deploy each service defined in the compose file, building it first @@ -170,7 +173,7 @@ module.exports = action: (params, options, done) -> # compositions with many services trigger misleading warnings require('events').defaultMaxListeners = 1000 - + { validateComposeOptions } = require('../utils/compose_ts') helpers = require('../utils/helpers') Logger = require('../utils/logger') @@ -185,6 +188,8 @@ module.exports = appName = options.application if not appName? delete options.application + validateComposeOptions(options) + if not appName? throw new Error('Please specify the name of the application to deploy') diff --git a/lib/actions/push.ts b/lib/actions/push.ts index 728c46f8..e5726018 100644 --- a/lib/actions/push.ts +++ b/lib/actions/push.ts @@ -18,8 +18,6 @@ import { CommandDefinition } from 'capitano'; import { stripIndent } from 'common-tags'; import { BalenaSDK } from 'balena-sdk'; -import { BuildError } from '../utils/device/errors'; - // An regex to detect an IP address, from https://www.regular-expressions.info/ip.html const IP_REGEX = new RegExp( /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/, @@ -98,42 +96,6 @@ async function getAppOwner(sdk: BalenaSDK, appName: string) { return selected.extra; } -interface RegistrySecrets { - [registryAddress: string]: { - username: string; - password: string; - }; -} - -async function parseRegistrySecrets( - secretsFilename: string, -): Promise { - const { fs } = await require('mz'); - const { - addCanonicalDockerHubEntry, - RegistrySecretValidator, - } = await require('resin-multibuild'); - try { - let isYaml = false; - if (/.+\.ya?ml$/i.test(secretsFilename)) { - isYaml = true; - } else if (!/.+\.json$/i.test(secretsFilename)) { - throw new Error('Filename must end with .json, .yml or .yaml'); - } - const raw = (await fs.readFile(secretsFilename)).toString(); - const registrySecrets = new RegistrySecretValidator().validateRegistrySecrets( - isYaml ? (await require('js-yaml')).safeLoad(raw) : JSON.parse(raw), - ); - addCanonicalDockerHubEntry(registrySecrets); - return registrySecrets; - } catch (error) { - error.message = - `Error validating registry secrets file "${secretsFilename}":\n` + - error.message; - throw error; - } -} - export const push: CommandDefinition< { applicationOrDevice: string; @@ -217,6 +179,8 @@ export const push: CommandDefinition< const remote = await import('../utils/remote-build'); const deviceDeploy = await import('../utils/device/deploy'); const { exitWithExpectedError } = await import('../utils/patterns'); + const { parseRegistrySecrets } = await import('../utils/compose_ts'); + const { BuildError } = await import('../utils/device/errors'); const appOrDevice: string | null = params.applicationOrDevice; if (appOrDevice == null) { diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index 6b79a39e..58127acb 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -41,6 +41,12 @@ exports.appendOptions = (opts) -> description: 'Display full log output' boolean: true }, + { + signature: 'registry-secrets', + alias: 'R', + parameter: 'secrets.yml|.json', + description: 'Path to a YAML or JSON file with passwords for a private Docker registry', + }, ] exports.generateOpts = (options) -> diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index a09fb3f1..2519938e 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -25,6 +25,51 @@ import { Composition } from 'resin-compose-parse'; import { DeviceInfo } from './device/api'; import Logger = require('./logger'); +export interface RegistrySecrets { + [registryAddress: string]: { + username: string; + password: string; + }; +} + +export async function parseRegistrySecrets( + secretsFilename: string, +): Promise { + const { fs } = require('mz'); + try { + let isYaml = false; + if (/.+\.ya?ml$/i.test(secretsFilename)) { + isYaml = true; + } else if (!/.+\.json$/i.test(secretsFilename)) { + throw new Error('Filename must end with .json, .yml or .yaml'); + } + const raw = (await fs.readFile(secretsFilename)).toString(); + const registrySecrets = new MultiBuild.RegistrySecretValidator().validateRegistrySecrets( + isYaml ? require('js-yaml').safeLoad(raw) : JSON.parse(raw), + ); + MultiBuild.addCanonicalDockerHubEntry(registrySecrets); + return registrySecrets; + } catch (error) { + error.message = + `Error validating registry secrets file "${secretsFilename}":\n` + + error.message; + throw error; + } +} + +/** + * Validate the compose-specific command-line options defined in compose.coffee. + * This function is meant to be called very early on to validate users' input, + * before any project loading / building / deploying. + */ +export async function validateComposeOptions(options: { [opt: string]: any }) { + if (options['registry-secrets']) { + options['registry-secrets'] = await parseRegistrySecrets( + options['registry-secrets'], + ); + } +} + /** * Create a BuildTask array of "resolved build tasks" by calling multibuild * .splitBuildStream() and performResolution(), and add build stream error diff --git a/lib/utils/docker.coffee b/lib/utils/docker.coffee index 418197b2..b9456917 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker.coffee @@ -14,19 +14,19 @@ exports.appendConnectionOptions = appendConnectionOptions = (opts) -> { signature: 'docker' parameter: 'docker' - description: 'Path to a local docker socket' + description: 'Path to a local docker socket (e.g. /var/run/docker.sock)' alias: 'P' }, { signature: 'dockerHost' parameter: 'dockerHost' - description: 'The address of the host containing the docker daemon' + description: 'Docker daemon hostname or IP address (dev machine or balena device) ' alias: 'h' }, { signature: 'dockerPort' parameter: 'dockerPort' - description: 'The port on which the host docker daemon is listening' + description: 'Docker daemon TCP port number (hint: 2375 for balena devices)' alias: 'p' }, { @@ -147,6 +147,8 @@ exports.generateBuildOpts = (options) -> opts.squash = true if options.buildArg? opts.buildargs = parseBuildArgs(options.buildArg) + if not _.isEmpty(options['registry-secrets']) + opts.registryconfig = options['registry-secrets'] return opts exports.getDocker = (options) ->