/** * @license * Copyright 2017-2021 Balena Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import type { Renderer } from './compose_ts'; import type * as SDK from 'balena-sdk'; import type Dockerode = require('dockerode'); import * as path from 'path'; import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse'; import type { RetryParametersObj } from 'pinejs-client-core'; import type { BuiltImage, ComposeOpts, ComposeProject, Release, TaggedImage, } from './compose-types'; import { getChalk } from './lazy'; import Logger = require('./logger'); import { ProgressCallback } from 'docker-progress'; export function generateOpts(options: { source?: string; projectName?: string; nologs: boolean; 'noconvert-eol': boolean; dockerfile?: string; 'multi-dockerignore': boolean; 'noparent-check': boolean; }): Promise { const { promises: fs } = require('fs') as typeof import('fs'); return fs.realpath(options.source || '.').then((projectPath) => ({ projectName: options.projectName, projectPath, inlineLogs: !options.nologs, convertEol: !options['noconvert-eol'], dockerfilePath: options.dockerfile, multiDockerignore: !!options['multi-dockerignore'], noParentCheck: options['noparent-check'], })); } /** Parse the given composition and return a structure with info. Input is: * - composePath: the *absolute* path to the directory containing the compose file * - composeStr: the contents of the compose file, as a string */ export function createProject( composePath: string, composeStr: string, projectName = '', imageTag = '', ): ComposeProject { const yml = require('js-yaml') as typeof import('js-yaml'); const compose = require('@balena/compose/dist/parse') as typeof import('@balena/compose/dist/parse'); // both methods below may throw. const rawComposition = yml.load(composeStr); const composition = compose.normalize(rawComposition); projectName ||= path.basename(composePath); const descriptors = compose.parse(composition).map(function (descr) { // generate an image name based on the project and service names // if one is not given and the service requires a build if ( typeof descr.image !== 'string' && descr.image.context != null && descr.image.tag == null ) { const { makeImageName } = require('./compose_ts') as typeof import('./compose_ts'); descr.image.tag = makeImageName(projectName, descr.serviceName, imageTag); } return descr; }); return { path: composePath, name: projectName, composition, descriptors, }; } const getRequestRetryParameters = (): RetryParametersObj => { if ( process.env.BALENA_CLI_TEST_TYPE != null && process.env.BALENA_CLI_TEST_TYPE !== '' ) { // We only read the test env vars when in test mode. const { intVar } = require('@balena/env-parsing') as typeof import('@balena/env-parsing'); // We use the BALENARCTEST namespace and only parse the env vars while in test mode // since we plan to switch all pinejs clients with the one of the SDK and might not // want to have to support these env vars. return { minDelayMs: intVar('BALENARCTEST_API_RETRY_MIN_DELAY_MS'), maxDelayMs: intVar('BALENARCTEST_API_RETRY_MAX_DELAY_MS'), maxAttempts: intVar('BALENARCTEST_API_RETRY_MAX_ATTEMPTS'), }; } return { minDelayMs: 1000, maxDelayMs: 60000, maxAttempts: 7, }; }; export const createRelease = async function ( logger: Logger, apiEndpoint: string, auth: string, userId: number, appId: number, composition: Composition, draft: boolean, semver: string | undefined, contract: string | undefined, ): Promise { const _ = require('lodash') as typeof import('lodash'); const crypto = require('crypto') as typeof import('crypto'); const releaseMod = require('@balena/compose/dist/release') as typeof import('@balena/compose/dist/release'); const client = releaseMod.createClient({ apiEndpoint, auth, retry: { ...getRequestRetryParameters(), onRetry: (err, delayMs, attempt, maxAttempts) => { const code = err?.statusCode ?? 0; logger.logDebug( `API call failed with code ${code}. Attempting retry ${attempt} of ${maxAttempts} in ${ delayMs / 1000 } seconds`, ); }, }, }); const { release, serviceImages } = await releaseMod.create({ client, user: userId, application: appId, composition, source: 'local', commit: crypto.pseudoRandomBytes(16).toString('hex').toLowerCase(), semver, is_final: !draft, contract, }); return { client, release: _.pick(release, [ 'id', 'status', 'commit', 'composition', 'source', 'is_final', 'contract', 'semver', 'start_timestamp', 'end_timestamp', ]), serviceImages: _.mapValues( serviceImages, (serviceImage) => _.omit(serviceImage, [ 'created_at', 'is_a_build_of__service', '__metadata', ]) as Omit< typeof serviceImage, 'created_at' | 'is_a_build_of__service' | '__metadata' >, ), }; }; export const tagServiceImages = ( docker: Dockerode, images: BuiltImage[], serviceImages: Release['serviceImages'], ): Promise => Promise.all( images.map(function (d) { const serviceImage = serviceImages[d.serviceName]; const imageName = serviceImage.is_stored_at__image_location; const match = /(.*?)\/(.*?)(?::([^/]*))?$/.exec(imageName); if (match == null) { throw new Error(`Could not parse imageName: '${imageName}'`); } const [, registry, repo, tag = 'latest'] = match; const name = `${registry}/${repo}`; return docker .getImage(d.name) .tag({ repo: name, tag, force: true }) .then(() => docker.getImage(`${name}:${tag}`)) .then((localImage) => ({ serviceName: d.serviceName, serviceImage, localImage, registry, repo, logs: d.logs, props: d.props, })); }), ); export const getPreviousRepos = ( sdk: SDK.BalenaSDK, logger: Logger, appID: number, ): Promise => sdk.pine .get({ resource: 'release', options: { $select: 'id', $filter: { belongs_to__application: appID, status: 'success', }, $expand: { contains__image: { $select: 'image', $expand: { image: { $select: 'is_stored_at__image_location' } }, }, }, $orderby: 'id desc', $top: 1, }, }) .then(function (release) { // grab all images from the latest release, return all image locations in the registry if (release.length > 0) { const images = release[0].contains__image as Array<{ image: [SDK.Image]; }>; const { getRegistryAndName } = require('@balena/compose/dist/multibuild') as typeof import('@balena/compose/dist/multibuild'); return Promise.all( images.map(function (d) { const imageName = d.image[0].is_stored_at__image_location || ''; const registry = getRegistryAndName(imageName); logger.logDebug( `Requesting access to previously pushed image repo (${registry.imageName})`, ); return registry.imageName; }), ); } else { return []; } }) .catch((e) => { logger.logDebug(`Failed to access previously pushed image repo: ${e}`); return []; }); export const authorizePush = function ( sdk: SDK.BalenaSDK, tokenAuthEndpoint: string, registry: string, images: string[], previousRepos: string[], ): Promise { if (!Array.isArray(images)) { images = [images]; } images.push(...previousRepos); return sdk.request .send({ baseUrl: tokenAuthEndpoint, url: '/auth/v1/token', qs: { service: registry, scope: images.map((repo) => `repository:${repo}:pull,push`), }, }) .then(({ body }) => body.token) .catch(() => ''); }; // utilities const renderProgressBar = function (percentage: number, stepCount: number) { const _ = require('lodash') as typeof import('lodash'); percentage = _.clamp(percentage, 0, 100); const barCount = Math.floor((stepCount * percentage) / 100); const spaceCount = stepCount - barCount; const bar = `[${_.repeat('=', barCount)}>${_.repeat(' ', spaceCount)}]`; return `${bar} ${_.padStart(`${percentage}`, 3)}%`; }; export const pushProgressRenderer = function ( tty: ReturnType, prefix: string, ): ProgressCallback & { end: () => void } { const fn: ProgressCallback & { end: () => void } = function (e) { const { error, percentage } = e; if (error != null) { throw new Error(error); } const bar = renderProgressBar(percentage, 40); return tty.replaceLine(`${prefix}${bar}\r`); }; fn.end = () => { tty.clearLine(); }; return fn; }; export class BuildProgressUI implements Renderer { public streams; private _prefix; private _prefixWidth; private _tty; private _services; private _startTime: undefined | number; private _ended; private _serviceToDataMap: Dictionary<{ status?: string; progress?: number; error?: Error; }> = {}; private _cancelled; private _spinner; private _runloop: | undefined | ReturnType; // these are to handle window wrapping private _maxLineWidth: undefined | number; private _lineWidths: number[] = []; constructor( tty: ReturnType, descriptors: ImageDescriptor[], ) { this._handleEvent = this._handleEvent.bind(this); this.start = this.start.bind(this); this.end = this.end.bind(this); this._display = this._display.bind(this); const _ = require('lodash') as typeof import('lodash'); const through = require('through2') as typeof import('through2'); const eventHandler = this._handleEvent; const services = _.map(descriptors, 'serviceName'); const streams = _(services) .map(function (service) { const stream = through.obj(function (event, _enc, cb) { eventHandler(service, event); return cb(); }); stream.pipe(tty.stream, { end: false }); return [service, stream]; }) .fromPairs() .value(); this._tty = tty; this._services = services; // Logger magically prefixes the log line with [Build] etc., but it doesn't // work well with the spinner we're also showing. Manually build the prefix // here and bypass the logger. const prefix = getChalk().blue('[Build]') + ' '; const offset = 10; // account for escape sequences inserted for colouring this._prefixWidth = offset + prefix.length + _.max(_.map(services, (s) => s.length))!; this._prefix = prefix; this._ended = false; this._cancelled = false; this._spinner = ( require('./compose_ts') as typeof import('./compose_ts') ).createSpinner(); this.streams = streams; } _handleEvent( service: string, event: { status?: string; progress?: number; error?: Error }, ) { this._serviceToDataMap[service] = event; } start() { this._tty.hideCursor(); this._services.forEach((service) => { this.streams[service].write({ status: 'Preparing...' }); }); this._runloop = ( require('./compose_ts') as typeof import('./compose_ts') ).createRunLoop(this._display); this._startTime = Date.now(); } end(summary?: Dictionary) { if (this._ended) { return; } this._ended = true; this._runloop?.end(); this._runloop = undefined; this._clear(); this._renderStatus(true); this._renderSummary(summary ?? this._getServiceSummary()); this._tty.showCursor(); } _display() { this._clear(); this._renderStatus(); this._renderSummary(this._getServiceSummary()); this._tty.cursorUp(this._services.length + 1); // for status line } _clear() { this._tty.deleteToEnd(); this._maxLineWidth = this._tty.currentWindowSize().width; } _getServiceSummary() { const _ = require('lodash') as typeof import('lodash'); const services = this._services; const serviceToDataMap = this._serviceToDataMap; return _(services) .map(function (service) { const { status, progress, error } = serviceToDataMap[service] ?? {}; if (error) { return `${error}`; } else if (progress) { const bar = renderProgressBar(progress, 20); if (status) { return `${bar} ${status}`; } return `${bar}`; } else if (status) { return `${status}`; } else { return 'Waiting...'; } }) .map((data, index) => [services[index], data]) .fromPairs() .value(); } _renderStatus(end = false) { const moment = require('moment') as typeof import('moment'); ( require('moment-duration-format') as typeof import('moment-duration-format') )(moment); this._tty.clearLine(); this._tty.write(this._prefix); if (end && this._cancelled) { this._tty.writeLine('Build cancelled'); } else if (end) { const serviceCount = this._services.length; const serviceStr = serviceCount === 1 ? '1 service' : `${serviceCount} services`; const durationStr = this._startTime == null ? 'unknown time' : moment .duration( Math.floor((Date.now() - this._startTime) / 1000), 'seconds', ) .format(); this._tty.writeLine(`Built ${serviceStr} in ${durationStr}`); } else { this._tty.writeLine(`Building services... ${this._spinner()}`); } } _renderSummary(serviceToStrMap: Dictionary) { const _ = require('lodash') as typeof import('lodash'); const chalk = getChalk(); const truncate = require('cli-truncate') as typeof import('cli-truncate'); const strlen = require('string-width') as typeof import('string-width'); this._services.forEach((service, index) => { let str = _.padEnd(this._prefix + chalk.bold(service), this._prefixWidth); str += serviceToStrMap[service]; if (this._maxLineWidth != null) { str = truncate(str, this._maxLineWidth); } this._lineWidths[index] = strlen(str); this._tty.clearLine(); this._tty.writeLine(str); }); } } export class BuildProgressInline implements Renderer { public streams; private _prefixWidth; private _outStream; private _services; private _startTime: number | undefined; private _ended; constructor( outStream: NodeJS.ReadWriteStream, descriptors: Array<{ serviceName: string }>, ) { this.start = this.start.bind(this); this.end = this.end.bind(this); this._renderEvent = this._renderEvent.bind(this); const _ = require('lodash') as typeof import('lodash'); const through = require('through2') as typeof import('through2'); const services = _.map(descriptors, 'serviceName'); const eventHandler = this._renderEvent; const streams = _(services) .map(function (service) { const stream = through.obj(function (event, _enc, cb) { eventHandler(service, event); return cb(); }); stream.pipe(outStream, { end: false }); return [service, stream]; }) .fromPairs() .value(); const offset = 10; // account for escape sequences inserted for colouring this._prefixWidth = offset + _.max(_.map(services, (s) => s.length))!; this._outStream = outStream; this._services = services; this._ended = false; this.streams = streams; } start() { this._outStream.write('Building services...\n'); this._services.forEach((service) => { this.streams[service].write({ status: 'Preparing...' }); }); this._startTime = Date.now(); } end(summary?: Dictionary) { const moment = require('moment') as typeof import('moment'); ( require('moment-duration-format') as typeof import('moment-duration-format') )(moment); if (this._ended) { return; } this._ended = true; if (summary != null) { this._services.forEach((service) => { this._renderEvent(service, { status: summary[service] }); }); } const serviceCount = this._services.length; const serviceStr = serviceCount === 1 ? '1 service' : `${serviceCount} services`; const durationStr = this._startTime == null ? 'unknown time' : moment .duration( Math.floor((Date.now() - this._startTime) / 1000), 'seconds', ) .format(); this._outStream.write(`Built ${serviceStr} in ${durationStr}\n`); } _renderEvent(service: string, event: { status?: string; error?: Error }) { const _ = require('lodash') as typeof import('lodash'); const str = (function () { const { status, error } = event; if (error) { return `${error}`; } else if (status) { return `${status}`; } else { return 'Waiting...'; } })(); const prefix = _.padEnd(getChalk().bold(service), this._prefixWidth); this._outStream.write(prefix); this._outStream.write(str); this._outStream.write('\n'); } }