From c1e94e661fa47af1c3b754d09c322a768e10c77e Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Thu, 7 Feb 2019 15:10:16 +0000 Subject: [PATCH] Integrate new resin-multibuild major version (private docker registry authentication support for the docker-compose 'image' instruction). Resolves: #1114 Change-type: minor Signed-off-by: Paulo Castro --- lib/actions/push.ts | 9 ++- lib/utils/compose.coffee | 12 +--- lib/utils/compose_ts.ts | 110 ++++++++++++++++++++++++++++++------- lib/utils/device/deploy.ts | 54 +++++++----------- lib/utils/patterns.ts | 9 +++ package.json | 6 +- 6 files changed, 132 insertions(+), 68 deletions(-) diff --git a/lib/actions/push.ts b/lib/actions/push.ts index 08d15ecc..728c46f8 100644 --- a/lib/actions/push.ts +++ b/lib/actions/push.ts @@ -109,7 +109,10 @@ async function parseRegistrySecrets( secretsFilename: string, ): Promise { const { fs } = await require('mz'); - const { RegistrySecretValidator } = await require('resin-multibuild'); + const { + addCanonicalDockerHubEntry, + RegistrySecretValidator, + } = await require('resin-multibuild'); try { let isYaml = false; if (/.+\.ya?ml$/i.test(secretsFilename)) { @@ -118,9 +121,11 @@ async function parseRegistrySecrets( throw new Error('Filename must end with .json, .yml or .yaml'); } const raw = (await fs.readFile(secretsFilename)).toString(); - return new RegistrySecretValidator().validateRegistrySecrets( + const registrySecrets = new RegistrySecretValidator().validateRegistrySecrets( isYaml ? (await require('js-yaml')).safeLoad(raw) : JSON.parse(raw), ); + addCanonicalDockerHubEntry(registrySecrets); + return registrySecrets; } catch (error) { error.message = `Error validating registry secrets file "${secretsFilename}":\n` + diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index 50bffad9..6b79a39e 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -197,16 +197,8 @@ exports.buildProject = ( # Tar up the directory, ready for the build stream tarDirectory(projectPath) .then (tarStream) -> - builder.splitBuildStream(composition, tarStream) - .tap (tasks) -> - # Updates each task as a side-effect - builder.performResolution(tasks, arch, deviceType) - .map (task) -> - if not task.external and not task.resolved - throw new Error( - "Project type for service '#{task.serviceName}' could not be determined. " + - 'Please add a Dockerfile' - ) + { makeBuildTasks } = require('./compose_ts') + Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger)) .map (task) -> d = imageDescriptorsByServiceName[task.serviceName] diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index c6bb930c..a09fb3f1 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -14,28 +14,98 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import * as _ from 'lodash'; +import * as Bluebird from 'bluebird'; +import * as tar from 'tar-stream'; +import { Readable } from 'stream'; -import { RegistrySecrets } from 'resin-multibuild'; -import { Pack } from 'tar-stream'; +import * as MultiBuild from 'resin-multibuild'; +import { Composition } from 'resin-compose-parse'; + +import { DeviceInfo } from './device/api'; +import Logger = require('./logger'); /** - * Return a callback function that takes a tar-stream Pack object as argument - * and uses it to add the '.balena/registry-secrets.json' metadata file that - * contains usernames and passwords for private docker registries. The builder - * will remove the file from the tar stream and use the secrets to pull base - * images from users' private registries. - * @param registrySecrets JS object containing registry usernames and passwords - * @returns A callback function, or undefined if registrySecrets is empty + * Create a BuildTask array of "resolved build tasks" by calling multibuild + * .splitBuildStream() and performResolution(), and add build stream error + * handlers and debug logging. + * Both `balena build` and `balena deploy` call this function. */ -export function getTarStreamCallbackForRegistrySecrets( - registrySecrets: RegistrySecrets, -): ((pack: Pack) => void) | undefined { - if (Object.keys(registrySecrets).length > 0) { - return (pack: Pack) => { - pack.entry( - { name: '.balena/registry-secrets.json' }, - JSON.stringify(registrySecrets), - ); - }; - } +export async function makeBuildTasks( + composition: Composition, + tarStream: Readable, + deviceInfo: DeviceInfo, + logger: Logger, +): Promise { + const buildTasks = await MultiBuild.splitBuildStream(composition, tarStream); + + logger.logDebug('Found build tasks:'); + _.each(buildTasks, task => { + let infoStr: string; + if (task.external) { + infoStr = `image pull [${task.imageName}]`; + } else { + infoStr = `build [${task.context}]`; + } + logger.logDebug(` ${task.serviceName}: ${infoStr}`); + }); + + logger.logDebug( + `Resolving services with [${deviceInfo.deviceType}|${deviceInfo.arch}]`, + ); + + await performResolution(buildTasks, deviceInfo); + + logger.logDebug('Found project types:'); + _.each(buildTasks, task => { + if (task.external) { + logger.logDebug(` ${task.serviceName}: External image`); + } else { + logger.logDebug(` ${task.serviceName}: ${task.projectType}`); + } + }); + + return buildTasks; +} + +async function performResolution( + tasks: MultiBuild.BuildTask[], + deviceInfo: DeviceInfo, +): Promise { + const { cloneTarStream } = require('tar-utils'); + + return await new Promise((resolve, reject) => { + const buildTasks = MultiBuild.performResolution( + tasks, + deviceInfo.arch, + deviceInfo.deviceType, + { error: [reject] }, + ); + // Do one task at a time (Bluebird.each instead of Bluebird.all) + // in order to reduce peak memory usage. Resolves to buildTasks. + Bluebird.each(buildTasks, buildTask => { + // buildStream is falsy for "external" tasks (image pull) + if (!buildTask.buildStream) { + return buildTask; + } + // Consume each task.buildStream in order to trigger the + // resolution events that define fields like: + // task.dockerfile, task.dockerfilePath, + // task.projectType, task.resolved + // This mimics what is currently done in `resin-builder`. + return cloneTarStream(buildTask.buildStream).then( + (clonedStream: tar.Pack) => { + buildTask.buildStream = clonedStream; + if (!buildTask.external && !buildTask.resolved) { + throw new Error( + `Project type for service "${ + buildTask.serviceName + }" could not be determined. Missing a Dockerfile?`, + ); + } + return buildTask; + }, + ); + }).then(resolve, reject); + }); } diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts index ae79fc51..5646076d 100644 --- a/lib/utils/device/deploy.ts +++ b/lib/utils/device/deploy.ts @@ -19,13 +19,18 @@ import * as Bluebird from 'bluebird'; import * as Docker from 'dockerode'; import * as _ from 'lodash'; import { Composition } from 'resin-compose-parse'; -import { BuildTask, LocalImage, RegistrySecrets } from 'resin-multibuild'; +import { + BuildTask, + getAuthConfigObj, + LocalImage, + RegistrySecrets, +} from 'resin-multibuild'; import * as semver from 'resin-semver'; import { Readable } from 'stream'; import Logger = require('../logger'); import { displayBuildLog } from './logs'; - +import { makeBuildTasks } from '../compose_ts'; import { DeviceInfo } from './api'; import * as LocalPushErrors from './errors'; @@ -148,36 +153,12 @@ export async function performBuilds( ): Promise { const multibuild = await import('resin-multibuild'); - const buildTasks = await multibuild.splitBuildStream(composition, tarStream); - - logger.logDebug('Found build tasks:'); - _.each(buildTasks, task => { - let infoStr: string; - if (task.external) { - infoStr = `image pull [${task.imageName}]`; - } else { - infoStr = `build [${task.context}]`; - } - logger.logDebug(` ${task.serviceName}: ${infoStr}`); - }); - - logger.logDebug( - `Resolving services with [${deviceInfo.deviceType}|${deviceInfo.arch}]`, + const buildTasks = await makeBuildTasks( + composition, + tarStream, + deviceInfo, + logger, ); - await multibuild.performResolution( - buildTasks, - deviceInfo.arch, - deviceInfo.deviceType, - ); - - logger.logDebug('Found project types:'); - _.each(buildTasks, task => { - if (!task.external) { - logger.logDebug(` ${task.serviceName}: ${task.projectType}`); - } else { - logger.logDebug(` ${task.serviceName}: External image`); - } - }); logger.logDebug('Probing remote daemon for cache images'); await assignDockerBuildOpts(docker, buildTasks, opts); @@ -247,16 +228,23 @@ async function assignDockerBuildOpts( logger.logDebug(`Using ${images.length} on-device images for cache...`); - _.each(buildTasks, (task: BuildTask) => { + await Bluebird.map(buildTasks, async (task: BuildTask) => { task.dockerOpts = { cachefrom: images, labels: { 'io.resin.local.image': '1', 'io.resin.local.service': task.serviceName, }, - registryconfig: opts.registrySecrets, t: generateImageName(task.serviceName), }; + if (task.external) { + task.dockerOpts.authconfig = await getAuthConfigObj( + task.imageName!, + opts.registrySecrets, + ); + } else { + task.dockerOpts.registryconfig = opts.registrySecrets; + } }); } diff --git a/lib/utils/patterns.ts b/lib/utils/patterns.ts index e3a3b29e..0789ffb3 100644 --- a/lib/utils/patterns.ts +++ b/lib/utils/patterns.ts @@ -288,6 +288,15 @@ export function printErrorMessage(message: string) { console.error(chalk.red(`\n${messages.getHelp}\n`)); } +/** + * Print a friendly error message and exit the CLI with an error code, BYPASSING + * error reporting through Sentry.io's platform (raven.Raven.captureException). + * Note that lib/errors.ts provides top-level error handling code to catch any + * otherwise uncaught errors, AND to report them through Sentry.io. But many + * "expected" errors (say, a JSON parsing error in a file provided by the user) + * don't warrant reporting through Sentry.io. For such mundane errors, catch + * them and call this function. + */ export function exitWithExpectedError(message: string | Error): never { if (message instanceof Error) { ({ message } = message); diff --git a/package.json b/package.json index a425fdc3..c7bff768 100644 --- a/package.json +++ b/package.json @@ -155,13 +155,12 @@ "raven": "^2.5.0", "reconfix": "^0.1.0", "request": "^2.81.0", - "resin-bundle-resolve": "^0.6.0", "resin-cli-form": "^2.0.1", "resin-cli-visuals": "^1.4.0", "resin-compose-parse": "^2.0.0", "resin-doodles": "0.0.1", "resin-image-fs": "^5.0.2", - "resin-multibuild": "^0.10.0", + "resin-multibuild": "^2.1.0", "resin-release": "^1.2.0", "resin-semver": "^1.4.0", "resin-stream-logger": "^0.1.2", @@ -171,7 +170,8 @@ "split": "^1.0.1", "string-width": "^2.1.1", "strip-ansi-stream": "^1.0.0", - "tar-stream": "^1.5.5", + "tar-stream": "^1.6.2", + "tar-utils": "^1.1.0", "through2": "^2.0.3", "tmp": "0.0.31", "typed-error": "^3.0.0",