From 23b07f8a41c82c0b23a38bac55cbd3874300dd58 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Fri, 12 Nov 2021 01:37:48 +0000 Subject: [PATCH] deploy: Ensure the release fails if an image's digest (hash) is missing Change-type: patch --- .mocharc.js | 3 ++ lib/utils/compose.js | 72 +------------------------------- lib/utils/compose_ts.ts | 92 +++++++++++++++++++++++++++++++++++++++-- lib/utils/tty.ts | 25 +++++++---- 4 files changed, 110 insertions(+), 82 deletions(-) diff --git a/.mocharc.js b/.mocharc.js index b4209e59..b8181136 100644 --- a/.mocharc.js +++ b/.mocharc.js @@ -3,5 +3,8 @@ module.exports = { require: 'ts-node/register/transpile-only', file: './tests/config-tests', timeout: 12000, + // To test only, say, 'push.spec.ts', do it as follows so that + // requests are authenticated: + // spec: ['tests/auth/*.spec.ts', 'tests/**/deploy.spec.ts'], spec: 'tests/**/*.spec.ts', }; diff --git a/lib/utils/compose.js b/lib/utils/compose.js index 78ae1308..e2e30dbb 100644 --- a/lib/utils/compose.js +++ b/lib/utils/compose.js @@ -354,76 +354,6 @@ export const authorizePush = function ( .catch(() => ''); }; -/** - * @param {import('dockerode')} docker - * @param {string} token - * @param {Array} images - * @param {(serviceImage: import('balena-release/build/models').ImageModel, props: object) => void} afterEach - */ -export const pushAndUpdateServiceImages = function ( - docker, - token, - images, - afterEach, -) { - const { DockerProgress } = require('docker-progress'); - const { retry } = require('./helpers'); - const tty = require('./tty')(process.stdout); - const Bluebird = require('bluebird'); - - const opts = { authconfig: { registrytoken: token } }; - - const progress = new DockerProgress({ docker }); - const renderer = pushProgressRenderer( - tty, - getChalk().blue('[Push]') + ' ', - ); - const reporters = progress.aggregateProgress(images.length, renderer); - - return Bluebird.using(tty.cursorHidden(), () => - Promise.all( - images.map(({ serviceImage, localImage, props, logs }, index) => - Promise.all([ - localImage.inspect().then((img) => img.Size), - retry({ - // @ts-ignore - func: () => progress.push(localImage.name, reporters[index], opts), - maxAttempts: 3, // try calling func 3 times (max) - // @ts-ignore - label: localImage.name, // label for retry log messages - initialDelayMs: 2000, // wait 2 seconds before the 1st retry - backoffScaler: 1.4, // wait multiplier for each retry - }).finally(renderer.end), - ]) - .then( - /** @type {([number, string]) => void} */ - function ([size, digest]) { - serviceImage.image_size = size; - serviceImage.content_hash = digest; - serviceImage.build_log = logs; - serviceImage.dockerfile = props.dockerfile; - serviceImage.project_type = props.projectType; - if (props.startTime) { - serviceImage.start_timestamp = props.startTime; - } - if (props.endTime) { - serviceImage.end_timestamp = props.endTime; - } - serviceImage.push_timestamp = new Date(); - serviceImage.status = 'success'; - }, - ) - .catch(function (e) { - serviceImage.error_message = '' + e; - serviceImage.status = 'failed'; - throw e; - }) - .finally(() => afterEach?.(serviceImage, props)), - ), - ), - ); -}; - // utilities const renderProgressBar = function (percentage, stepCount) { @@ -435,7 +365,7 @@ const renderProgressBar = function (percentage, stepCount) { return `${bar} ${_.padStart(percentage, 3)}%`; }; -var pushProgressRenderer = function (tty, prefix) { +export const pushProgressRenderer = function (tty, prefix) { const fn = function (e) { const { error, percentage } = e; if (error != null) { diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index afa74006..7dfe60fa 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -1306,15 +1306,101 @@ async function getTokenForPreviousRepos( return token; } +async function pushAndUpdateServiceImages( + docker: Dockerode, + token: string, + images: TaggedImage[], + afterEach: ( + serviceImage: import('balena-release/build/models').ImageModel, + props: object, + ) => void, +) { + const { DockerProgress } = await import('docker-progress'); + const { retry } = await import('./helpers'); + const { pushProgressRenderer } = await import('./compose'); + const tty = (await import('./tty'))(process.stdout); + const opts = { authconfig: { registrytoken: token } }; + const progress = new DockerProgress({ docker }); + const renderer = pushProgressRenderer( + tty, + getChalk().blue('[Push]') + ' ', + ); + const reporters = progress.aggregateProgress(images.length, renderer); + + const pushImage = async ( + localImage: Dockerode.Image, + index: number, + ): Promise => { + try { + // TODO 'localImage as any': find out exactly why tsc warns about + // 'name' that exists as a matter of fact, with a value similar to: + // "name": "registry2.balena-cloud.com/v2/aa27790dff571ec7d2b4fbcf3d4648d5:latest" + const imgName: string = (localImage as any).name || ''; + const imageDigest: string = await retry({ + func: () => progress.push(imgName, reporters[index], opts), + maxAttempts: 3, // try calling func 3 times (max) + label: imgName, // label for retry log messages + initialDelayMs: 2000, // wait 2 seconds before the 1st retry + backoffScaler: 1.4, // wait multiplier for each retry + }); + if (!imageDigest) { + throw new ExpectedError(stripIndent`\ + Unable to extract image digest (content hash) from image upload progress stream for image: + ${imgName}`); + } + return imageDigest; + } finally { + renderer.end(); + } + }; + + const inspectAndPushImage = async ( + { serviceImage, localImage, props, logs }: TaggedImage, + index: number, + ) => { + try { + const [imgInfo, imgDigest] = await Promise.all([ + localImage.inspect(), + pushImage(localImage, index), + ]); + serviceImage.image_size = imgInfo.Size; + serviceImage.content_hash = imgDigest; + serviceImage.build_log = logs; + serviceImage.dockerfile = props.dockerfile; + serviceImage.project_type = props.projectType; + if (props.startTime) { + serviceImage.start_timestamp = props.startTime; + } + if (props.endTime) { + serviceImage.end_timestamp = props.endTime; + } + serviceImage.push_timestamp = new Date(); + serviceImage.status = 'success'; + } catch (error) { + serviceImage.error_message = '' + error; + serviceImage.status = 'failed'; + throw error; + } finally { + await afterEach(serviceImage, props); + } + }; + + tty.hideCursor(); + try { + await Promise.all(images.map(inspectAndPushImage)); + } finally { + tty.showCursor(); + } +} + async function pushServiceImages( - docker: import('dockerode'), + docker: Dockerode, logger: Logger, pineClient: ReturnType, taggedImages: TaggedImage[], token: string, skipLogUpload: boolean, ): Promise { - const { pushAndUpdateServiceImages } = await import('./compose'); const releaseMod = await import('balena-release'); logger.logInfo('Pushing images to registry...'); await pushAndUpdateServiceImages( @@ -1337,7 +1423,7 @@ async function pushServiceImages( const PLAIN_SEMVER_REGEX = /^([0-9]+)\.([0-9]+)\.([0-9]+)$/; export async function deployProject( - docker: import('dockerode'), + docker: Dockerode, logger: Logger, composition: Composition, images: BuiltImage[], diff --git a/lib/utils/tty.ts b/lib/utils/tty.ts index a3d85614..7c786956 100644 --- a/lib/utils/tty.ts +++ b/lib/utils/tty.ts @@ -1,3 +1,20 @@ +/** + * @license + * Copyright 2018-2021 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + const windowSize: { width?: number; height?: number } = {}; const updateWindowSize = () => { @@ -29,13 +46,6 @@ export = (stream: NodeJS.WriteStream = process.stdout) => { const cursorDown = (rows: number = 0) => stream.write(`\u001B[${rows}B`); - const cursorHidden = () => { - const Bluebird = require('bluebird') as typeof import('bluebird'); - return Bluebird.try(hideCursor).disposer(() => { - showCursor(); - }); - }; - const write = (str: string) => stream.write(str); const writeLine = (str: string) => stream.write(`${str}\n`); @@ -54,7 +64,6 @@ export = (stream: NodeJS.WriteStream = process.stdout) => { currentWindowSize, hideCursor, showCursor, - cursorHidden, cursorUp, cursorDown, write,