From 0f302d30eccfaa9ea72f1c114c615794b3ba26dc Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Wed, 7 Nov 2018 18:15:05 +0000 Subject: [PATCH] Add push --registry-secrets option for private docker registry authentication Change-type: minor Signed-off-by: Paulo Castro --- doc/cli.markdown | 26 +++++++++++--- lib/actions/config.coffee | 4 +-- lib/actions/push.ts | 69 ++++++++++++++++++++++++++++++++++---- lib/utils/compose.coffee | 21 +++++++++++- lib/utils/compose.d.ts | 23 ++++++++++++- lib/utils/compose_ts.ts | 41 ++++++++++++++++++++++ lib/utils/device/deploy.ts | 28 ++++++++++++++-- lib/utils/remote-build.ts | 50 ++++++++++++++++++++++----- package.json | 7 ++-- 9 files changed, 239 insertions(+), 30 deletions(-) create mode 100644 lib/utils/compose_ts.ts diff --git a/doc/cli.markdown b/doc/cli.markdown index 17177dbd..8dc2ced1 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -1360,16 +1360,28 @@ Docker host TLS key file ## push <applicationOrDevice> -This command can be used to start a build on the remote -balena cloud builders, or a local mode balena device. +This command can be used to start a build on the remote balena cloud builders, +or a local mode balena device. When building on the balena cloud the given source directory will be sent to the balena builder, and the build will proceed. This can be used as a drop-in replacement for git push to deploy. -When building on a local mode device, the given source directory will be built on -device, and the resulting containers will be run on the device. Logs will be -streamed back from the device as part of the same invocation. +When building on a local mode device, the given source directory will be built +on the device, and the resulting containers will be run on the device. Logs will +be streamed back from the device as part of the same invocation. + +The --registry-secrets option specifies a JSON or YAML file containing private +Docker registry usernames and passwords to be used when pulling base images. +Sample registry-secrets YAML file: + + 'https://idx.docker.io/v1/': + username: mike + password: cze14 + 'myregistry.com:25000': + username: ann + password: hunter2 + Examples: @@ -1395,6 +1407,10 @@ Force an emulated build to occur on the remote builder Don't use cache when building this project +#### --registry-secrets, -R <secrets.yml|.json> + +Path to a local YAML or JSON file containing Docker registry passwords used to pull base images + # Settings ## settings diff --git a/lib/actions/config.coffee b/lib/actions/config.coffee index bc6cc975..935f774c 100644 --- a/lib/actions/config.coffee +++ b/lib/actions/config.coffee @@ -1,5 +1,5 @@ ### -Copyright 2016-2017 Balena +Copyright 2016-2018 Balena Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -306,7 +306,7 @@ exports.generate = ''' if !options.application and options.deviceType - patterns.exitWithExpectedError ''' + exitWithExpectedError ''' Specifying a different device type is only supported when generating a config for an application: diff --git a/lib/actions/push.ts b/lib/actions/push.ts index 37dae6ce..08d15ecc 100644 --- a/lib/actions/push.ts +++ b/lib/actions/push.ts @@ -1,5 +1,5 @@ /* -Copyright 2016-2017 Balena +Copyright 2016-2018 Balena Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -98,6 +98,37 @@ async function getAppOwner(sdk: BalenaSDK, appName: string) { return selected.extra; } +interface RegistrySecrets { + [registryAddress: string]: { + username: string; + password: string; + }; +} + +async function parseRegistrySecrets( + secretsFilename: string, +): Promise { + const { fs } = await require('mz'); + const { RegistrySecretValidator } = await require('resin-multibuild'); + try { + let isYaml = false; + if (/.+\.ya?ml$/i.test(secretsFilename)) { + isYaml = true; + } else if (!/.+\.json$/i.test(secretsFilename)) { + throw new Error('Filename must end with .json, .yml or .yaml'); + } + const raw = (await fs.readFile(secretsFilename)).toString(); + return new RegistrySecretValidator().validateRegistrySecrets( + isYaml ? (await require('js-yaml')).safeLoad(raw) : JSON.parse(raw), + ); + } catch (error) { + error.message = + `Error validating registry secrets file "${secretsFilename}":\n` + + error.message; + throw error; + } +} + export const push: CommandDefinition< { applicationOrDevice: string; @@ -106,22 +137,35 @@ export const push: CommandDefinition< source: string; emulated: boolean; nocache: boolean; + 'registry-secrets': string; } > = { signature: 'push ', description: 'Start a remote build on the balena cloud build servers or a local mode device', help: stripIndent` - This command can be used to start a build on the remote - balena cloud builders, or a local mode balena device. + This command can be used to start a build on the remote balena cloud builders, + or a local mode balena device. When building on the balena cloud the given source directory will be sent to the balena builder, and the build will proceed. This can be used as a drop-in replacement for git push to deploy. - When building on a local mode device, the given source directory will be built on - device, and the resulting containers will be run on the device. Logs will be - streamed back from the device as part of the same invocation. + When building on a local mode device, the given source directory will be built + on the device, and the resulting containers will be run on the device. Logs will + be streamed back from the device as part of the same invocation. + + The --registry-secrets option specifies a JSON or YAML file containing private + Docker registry usernames and passwords to be used when pulling base images. + Sample registry-secrets YAML file: + + 'https://idx.docker.io/v1/': + username: mike + password: cze14 + 'myregistry.com:25000': + username: ann + password: hunter2 + Examples: @@ -154,6 +198,13 @@ export const push: CommandDefinition< description: "Don't use cache when building this project", boolean: true, }, + { + signature: 'registry-secrets', + alias: 'R', + parameter: 'secrets.yml|.json', + description: stripIndent` + Path to a local YAML or JSON file containing Docker registry passwords used to pull base images`, + }, ], async action(params, options, done) { const sdk = (await import('balena-sdk')).fromSharedOptions(); @@ -172,6 +223,10 @@ export const push: CommandDefinition< console.log(`[debug] Using ${source} as build source`); } + const registrySecrets = options['registry-secrets'] + ? await parseRegistrySecrets(options['registry-secrets']) + : {}; + const buildTarget = getBuildTarget(appOrDevice); switch (buildTarget) { case BuildTarget.Cloud: @@ -184,6 +239,7 @@ export const push: CommandDefinition< const opts = { emulated: options.emulated, nocache: options.nocache, + registrySecrets, }; const args = { app, @@ -206,6 +262,7 @@ export const push: CommandDefinition< deviceDeploy.deployToDevice({ source, deviceHost: device, + registrySecrets, }), ) .catch(BuildError, e => { diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index a594c6b1..34f7765b 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -1,3 +1,20 @@ +###* +# @license +# Copyright 2018 Balena Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + Promise = require('bluebird') path = require('path') @@ -100,7 +117,7 @@ toPosixPath = (systemPath) -> path = require('path') systemPath.replace(new RegExp('\\' + path.sep, 'g'), '/') -exports.tarDirectory = tarDirectory = (dir) -> +exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) -> tar = require('tar-stream') klaw = require('klaw') path = require('path') @@ -126,6 +143,8 @@ exports.tarDirectory = tarDirectory = (dir) -> Promise.join relPath, fs.stat(file), fs.readFile(file), (filename, stats, data) -> pack.entry({ name: toPosixPath(filename), size: stats.size, mode: stats.mode }, data) + .then -> + preFinalizeCallback?(pack) .then -> pack.finalize() return pack diff --git a/lib/utils/compose.d.ts b/lib/utils/compose.d.ts index 39cb23d1..53c753c0 100644 --- a/lib/utils/compose.d.ts +++ b/lib/utils/compose.d.ts @@ -1,6 +1,24 @@ +/** + * @license + * Copyright 2018 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import * as Bluebird from 'bluebird'; import * as Stream from 'stream'; import { Composition } from 'resin-compose-parse'; +import { Pack } from 'tar-stream'; import Logger = require('./logger'); interface Image { @@ -29,4 +47,7 @@ export function loadProject( image?: string, ): Bluebird; -export function tarDirectory(source: string): Promise; +export function tarDirectory( + source: string, + preFinalizeCallback?: (pack: Pack) => void, +): Promise; diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts new file mode 100644 index 00000000..c6bb930c --- /dev/null +++ b/lib/utils/compose_ts.ts @@ -0,0 +1,41 @@ +/** + * @license + * Copyright 2018 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { RegistrySecrets } from 'resin-multibuild'; +import { Pack } from 'tar-stream'; + +/** + * Return a callback function that takes a tar-stream Pack object as argument + * and uses it to add the '.balena/registry-secrets.json' metadata file that + * contains usernames and passwords for private docker registries. The builder + * will remove the file from the tar stream and use the secrets to pull base + * images from users' private registries. + * @param registrySecrets JS object containing registry usernames and passwords + * @returns A callback function, or undefined if registrySecrets is empty + */ +export function getTarStreamCallbackForRegistrySecrets( + registrySecrets: RegistrySecrets, +): ((pack: Pack) => void) | undefined { + if (Object.keys(registrySecrets).length > 0) { + return (pack: Pack) => { + pack.entry( + { name: '.balena/registry-secrets.json' }, + JSON.stringify(registrySecrets), + ); + }; + } +} diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts index e830c360..3a2cd59d 100644 --- a/lib/utils/device/deploy.ts +++ b/lib/utils/device/deploy.ts @@ -1,8 +1,25 @@ +/** + * @license + * Copyright 2018 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import * as Bluebird from 'bluebird'; import * as Docker from 'dockerode'; import * as _ from 'lodash'; import { Composition } from 'resin-compose-parse'; -import { BuildTask, LocalImage } from 'resin-multibuild'; +import { BuildTask, LocalImage, RegistrySecrets } from 'resin-multibuild'; import * as semver from 'resin-semver'; import { Readable } from 'stream'; @@ -20,6 +37,7 @@ export interface DeviceDeployOptions { source: string; deviceHost: string; devicePort?: number; + registrySecrets: RegistrySecrets; } async function checkSource(source: string): Promise { @@ -86,6 +104,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise { docker, deviceInfo, logger, + opts, ); logger.logDebug('Setting device state...'); @@ -125,6 +144,7 @@ export async function performBuilds( docker: Docker, deviceInfo: DeviceInfo, logger: Logger, + opts: DeviceDeployOptions, ): Promise { const multibuild = await import('resin-multibuild'); @@ -160,7 +180,7 @@ export async function performBuilds( }); logger.logDebug('Probing remote daemon for cache images'); - await assignDockerBuildOpts(docker, buildTasks); + await assignDockerBuildOpts(docker, buildTasks, opts); logger.logDebug('Starting builds...'); await assignOutputHandlers(buildTasks, logger); @@ -219,6 +239,7 @@ async function getDeviceDockerImages(docker: Docker): Promise { async function assignDockerBuildOpts( docker: Docker, buildTasks: BuildTask[], + opts: DeviceDeployOptions, ): Promise { // Get all of the images on the remote docker daemon, so // that we can use all of them for cache @@ -233,6 +254,7 @@ async function assignDockerBuildOpts( 'io.resin.local.image': '1', 'io.resin.local.service': task.serviceName, }, + registryconfig: opts.registrySecrets, t: generateImageName(task.serviceName), }; }); @@ -301,6 +323,6 @@ async function inspectBuildResults(images: LocalImage[]): Promise { }); if (failures.length > 0) { - exitWithExpectedError(new LocalPushErrors.BuildError(failures)); + exitWithExpectedError(new LocalPushErrors.BuildError(failures).toString()); } } diff --git a/lib/utils/remote-build.ts b/lib/utils/remote-build.ts index 42a7184f..e80fc318 100644 --- a/lib/utils/remote-build.ts +++ b/lib/utils/remote-build.ts @@ -1,5 +1,5 @@ /* -Copyright 2016-2017 Balena +Copyright 2016-2018 Balena Ltd. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,8 +18,10 @@ import * as JSONStream from 'JSONStream'; import * as request from 'request'; import { BalenaSDK } from 'balena-sdk'; import * as Stream from 'stream'; +import { Pack } from 'tar-stream'; import { TypedError } from 'typed-error'; +import { RegistrySecrets } from 'resin-multibuild'; import { tarDirectory } from './compose'; const DEBUG_MODE = !!process.env.DEBUG; @@ -30,6 +32,7 @@ const TRIM_REGEX = /\n+$/; export interface BuildOpts { emulated: boolean; nocache: boolean; + registrySecrets: RegistrySecrets; } export interface RemoteBuild { @@ -76,6 +79,8 @@ async function getBuilderEndpoint( emulated: opts.emulated, nocache: opts.nocache, }); + // Note that using https (rather than http) is a requirement when using the + // --registry-secrets feature, as the secrets are not otherwise encrypted. return `https://builder.${baseUrl}/v3/build?${args}`; } @@ -210,6 +215,28 @@ async function cancelBuildIfNecessary(build: RemoteBuild): Promise { } } +/** + * Return a callback function that takes a tar-stream Pack object as argument + * and uses it to add the '.balena/registry-secrets.json' metadata file that + * contains usernames and passwords to private docker registries. The builder + * will remove the file from the tar stream and use the secrets to pull base + * images from users' private registries. + * @param registrySecrets JS object containing registry usernames and passwords + * @returns A callback function, or undefined if registrySecrets is empty + */ +function getTarStreamCallbackForRegistrySecrets( + registrySecrets: RegistrySecrets, +): ((pack: Pack) => void) | undefined { + if (Object.keys(registrySecrets).length > 0) { + return (pack: Pack) => { + pack.entry( + { name: '.balena/registry-secrets.json' }, + JSON.stringify(registrySecrets), + ); + }; + } +} + async function getRequestStream(build: RemoteBuild): Promise { const path = await import('path'); const visuals = await import('resin-cli-visuals'); @@ -218,19 +245,24 @@ async function getRequestStream(build: RemoteBuild): Promise { const tarSpinner = new visuals.Spinner('Packaging the project source...'); tarSpinner.start(); // Tar the directory so that we can send it to the builder - const tarStream = await tarDirectory(path.resolve(build.source)); + const tarStream = await tarDirectory( + path.resolve(build.source), + getTarStreamCallbackForRegistrySecrets(build.opts.registrySecrets), + ); tarSpinner.stop(); + const url = await getBuilderEndpoint( + build.baseUrl, + build.owner, + build.app, + build.opts, + ); + if (DEBUG_MODE) { - console.log('[debug] Opening builder connection'); + console.log(`[debug] Connecting to builder at ${url}`); } const post = request.post({ - url: await getBuilderEndpoint( - build.baseUrl, - build.owner, - build.app, - build.opts, - ), + url, auth: { bearer: build.auth, }, diff --git a/package.json b/package.json index 5c273572..26198599 100644 --- a/package.json +++ b/package.json @@ -88,13 +88,14 @@ "prettier": "^1.14.2", "publish-release": "^1.3.3", "require-npm4-to-publish": "^1.0.0", - "resin-lint": "^2.0.0", + "resin-lint": "^2.0.1", "rewire": "^3.0.2", "ts-node": "^4.0.1", "typescript": "2.8.1" }, "dependencies": { "@resin.io/valid-email": "^0.1.0", + "@types/dockerode": "2.5.5", "@types/stream-to-promise": "2.2.0", "@types/through2": "^2.0.33", "@zeit/dockerignore": "0.0.3", @@ -112,7 +113,7 @@ "bash": "0.0.1", "bluebird": "^3.3.3", "body-parser": "^1.14.1", - "capitano": "^1.7.0", + "capitano": "^1.8.2", "chalk": "^2.3.0", "cli-truncate": "^1.1.0", "coffeescript": "^1.12.6", @@ -158,7 +159,7 @@ "resin-compose-parse": "^2.0.0", "resin-doodles": "0.0.1", "resin-image-fs": "^5.0.2", - "resin-multibuild": "^0.9.0", + "resin-multibuild": "^0.10.0", "resin-release": "^1.2.0", "resin-semver": "^1.4.0", "resin-stream-logger": "^0.1.2",