From 560b0abbe7a6966a6ba9094527ff3223cf3514aa Mon Sep 17 00:00:00 2001 From: Pagan Gazzard Date: Thu, 23 Apr 2020 13:46:19 +0100 Subject: [PATCH] Convert lib/utils/compose.coffee to javascript Change-type: patch --- .../{compose.d.ts => compose-types.d.ts} | 22 +- lib/utils/compose.coffee | 796 ------------ lib/utils/compose.js | 1123 +++++++++++++++++ lib/utils/compose_ts.ts | 4 +- npm-shrinkwrap.json | 85 +- package.json | 2 +- 6 files changed, 1198 insertions(+), 834 deletions(-) rename lib/utils/{compose.d.ts => compose-types.d.ts} (70%) delete mode 100644 lib/utils/compose.coffee create mode 100644 lib/utils/compose.js diff --git a/lib/utils/compose.d.ts b/lib/utils/compose-types.d.ts similarity index 70% rename from lib/utils/compose.d.ts rename to lib/utils/compose-types.d.ts index 8dcc8bfa..59ef838e 100644 --- a/lib/utils/compose.d.ts +++ b/lib/utils/compose-types.d.ts @@ -15,9 +15,7 @@ * limitations under the License. */ -import * as Bluebird from 'bluebird'; -import { Composition } from 'resin-compose-parse'; -import * as Stream from 'stream'; +import { Composition, ImageDescriptor } from 'resin-compose-parse'; import { Pack } from 'tar-stream'; import Logger = require('./logger'); @@ -27,11 +25,6 @@ interface Image { tag: string; } -interface Descriptor { - image: Image | string; - serviceName: string; -} - export interface ComposeOpts { dockerfilePath?: string; inlineLogs?: boolean; @@ -44,21 +37,10 @@ export interface ComposeProject { path: string; name: string; composition: Composition; - descriptors: Descriptor[]; + descriptors: ImageDescriptor[]; } -export function createProject( - composePath: string, - composeStr: string, - projectName: string | null = null, -): ComposeProject; - interface TarDirectoryOptions { preFinalizeCallback?: (pack: Pack) => void; convertEol?: boolean; } - -export function tarDirectory( - source: string, - options?: TarDirectoryOptions, -): Promise; diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee deleted file mode 100644 index c602cea2..00000000 --- a/lib/utils/compose.coffee +++ /dev/null @@ -1,796 +0,0 @@ -###* -# @license -# Copyright 2017-2020 Balena Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -### - -Promise = require('bluebird') -path = require('path') -{ getBalenaSdk, getChalk } = require('./lazy') - -exports.appendProjectOptions = appendProjectOptions = (opts) -> - opts.concat [ - { - signature: 'projectName' - parameter: 'projectName' - description: 'Specify an alternate project name; default is the directory name' - alias: 'n' - }, - ] - -exports.appendOptions = (opts) -> - appendProjectOptions(opts).concat [ - { - signature: 'emulated' - description: 'Run an emulated build using Qemu' - boolean: true - alias: 'e' - }, - { - signature: 'dockerfile' - parameter: 'Dockerfile' - description: 'Alternative Dockerfile name/path, relative to the source folder' - }, - { - signature: 'logs' - description: 'Display full log output' - boolean: true - }, - { - signature: 'noparent-check' - description: 'Disable project validation check of \'docker-compose.yml\' file in parent folder' - boolean: true - }, - { - signature: 'registry-secrets' - alias: 'R' - parameter: 'secrets.yml|.json' - description: 'Path to a YAML or JSON file with passwords for a private Docker registry' - }, - { - signature: 'convert-eol' - description: ' - On Windows only, convert line endings from CRLF (Windows format) to LF (Unix format). - Source files are not modified.' - boolean: true - alias: 'l' - } - ] - -exports.generateOpts = (options) -> - fs = require('mz/fs') - fs.realpath(options.source || '.').then (projectPath) -> - projectName: options.projectName - projectPath: projectPath - inlineLogs: !!options.logs - dockerfilePath: options.dockerfile - noParentCheck: options['noparent-check'] - -# Parse the given composition and return a structure with info. Input is: -# - composePath: the *absolute* path to the directory containing the compose file -# - composeStr: the contents of the compose file, as a string -exports.createProject = (composePath, composeStr, projectName = null) -> - yml = require('js-yaml') - compose = require('resin-compose-parse') - - # both methods below may throw. - composition = yml.safeLoad(composeStr, schema: yml.FAILSAFE_SCHEMA) - composition = compose.normalize(composition) - - projectName ?= path.basename(composePath) - descriptors = compose.parse(composition).map (descr) -> - # generate an image name based on the project and service names - # if one is not given and the service requires a build - if descr.image.context? and not descr.image.tag? - descr.image.tag = [ projectName, descr.serviceName ].join('_').toLowerCase() - return descr - return { - path: composePath, - name: projectName, - composition, - descriptors - } - -exports.tarDirectory = tarDirectory = (dir, { preFinalizeCallback, convertEol } = {}) -> - preFinalizeCallback ?= null - convertEol ?= false - - tar = require('tar-stream') - klaw = require('klaw') - path = require('path') - fs = require('mz/fs') - streamToPromise = require('stream-to-promise') - { FileIgnorer } = require('./ignore') - { toPosixPath } = require('resin-multibuild').PathUtils - if process.platform == 'win32' - { readFileWithEolConversion } = require('./eol-conversion') - readFile = (file) -> readFileWithEolConversion(file, convertEol) - else - readFile = fs.readFile - - getFiles = -> - streamToPromise(klaw(dir)) - .filter((item) -> not item.stats.isDirectory()) - .map((item) -> item.path) - - ignore = new FileIgnorer(dir) - pack = tar.pack() - getFiles(dir) - .each (file) -> - type = ignore.getIgnoreFileType(path.relative(dir, file)) - if type? - ignore.addIgnoreFile(file, type) - .filter(ignore.filter) - .map (file) -> - relPath = path.relative(path.resolve(dir), file) - Promise.join relPath, fs.stat(file), readFile(file), - (filename, stats, data) -> - pack.entry({ name: toPosixPath(filename), size: stats.size, mode: stats.mode }, data) - .then -> - preFinalizeCallback?(pack) - .then -> - pack.finalize() - return pack - -truncateString = (str, len) -> - return str if str.length < len - str = str.slice(0, len) - # return everything up to the last line. this is a cheeky way to avoid - # having to deal with splitting the string midway through some special - # character sequence. - return str.slice(0, str.lastIndexOf('\n')) - -LOG_LENGTH_MAX = 512 * 1024 # 512KB - -exports.buildProject = ( - docker, logger, - projectPath, projectName, composition, - arch, deviceType, - emulated, buildOpts, - inlineLogs, - convertEol, - dockerfilePath, -) -> - _ = require('lodash') - humanize = require('humanize') - compose = require('resin-compose-parse') - builder = require('resin-multibuild') - transpose = require('docker-qemu-transpose') - { BALENA_ENGINE_TMP_PATH } = require('../config') - { checkBuildSecretsRequirements, makeBuildTasks } = require('./compose_ts') - qemu = require('./qemu-ts') - { toPosixPath } = builder.PathUtils - - logger.logInfo("Building for #{arch}/#{deviceType}") - - imageDescriptors = compose.parse(composition) - imageDescriptorsByServiceName = _.keyBy(imageDescriptors, 'serviceName') - - if inlineLogs - renderer = new BuildProgressInline(logger.streams['build'], imageDescriptors) - else - tty = require('./tty')(process.stdout) - renderer = new BuildProgressUI(tty, imageDescriptors) - renderer.start() - - Promise.resolve(checkBuildSecretsRequirements(docker, projectPath)) - .then -> qemu.installQemuIfNeeded(emulated, logger, arch, docker) - .tap (needsQemu) -> - return if not needsQemu - logger.logInfo('Emulation is enabled') - # Copy qemu into all build contexts - Promise.map imageDescriptors, (d) -> - return if not d.image.context? # external image - return qemu.copyQemu(path.join(projectPath, d.image.context), arch) - .then (needsQemu) -> - # Tar up the directory, ready for the build stream - tarDirectory(projectPath, { convertEol }) - .then (tarStream) -> - Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger, projectName)) - .map (task) -> - d = imageDescriptorsByServiceName[task.serviceName] - - # multibuild parses the composition internally so any tags we've - # set before are lost; re-assign them here - task.tag ?= [ projectName, task.serviceName ].join('_').toLowerCase() - if d.image.context? - d.image.tag = task.tag - - # configure build opts appropriately - task.dockerOpts ?= {} - _.merge(task.dockerOpts, buildOpts, { t: task.tag }) - if d.image.context?.args? - task.dockerOpts.buildargs ?= {} - _.merge(task.dockerOpts.buildargs, d.image.context.args) - - # Get the service-specific log stream - # Caveat: `multibuild.BuildTask` defines no `logStream` property - # but it's convenient to store it there; it's JS ultimately. - task.logStream = renderer.streams[task.serviceName] - task.logBuffer = [] - - # Setup emulation if needed - return [ task, null ] if task.external or not needsQemu - binPath = qemu.qemuPathInContext(path.join(projectPath, task.context)) - transpose.transposeTarStream( - task.buildStream, - { - hostQemuPath: toPosixPath(binPath), - containerQemuPath: "/tmp/#{qemu.QEMU_BIN_NAME}", - qemuFileMode: 0o555, - }, - dockerfilePath or undefined, - ).then (stream) -> - task.buildStream = stream - .return([ task, binPath ]) - .map ([ task, qemuPath ]) -> - Promise.resolve(task).tap (task) -> - captureStream = buildLogCapture(task.external, task.logBuffer) - - if task.external - # External image -- there's no build to be performed, - # just follow pull progress. - captureStream.pipe(task.logStream) - task.progressHook = pullProgressAdapter(captureStream) - else - task.streamHook = (stream) -> - stream = createLogStream(stream) - if qemuPath? - buildThroughStream = transpose.getBuildThroughStream - hostQemuPath: toPosixPath(qemuPath) - containerQemuPath: "/tmp/#{qemu.QEMU_BIN_NAME}" - rawStream = stream.pipe(buildThroughStream) - else - rawStream = stream - # `stream` sends out raw strings in contrast to `task.progressHook` - # where we're given objects. capture these strings as they come - # before we parse them. - rawStream - .pipe(dropEmptyLinesStream()) - .pipe(captureStream) - .pipe(buildProgressAdapter(inlineLogs)) - .pipe(task.logStream) - .then (tasks) -> - logger.logDebug 'Prepared tasks; building...' - Promise.resolve(builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH)) - .map (builtImage) -> - if not builtImage.successful - builtImage.error.serviceName = builtImage.serviceName - throw builtImage.error - - d = imageDescriptorsByServiceName[builtImage.serviceName] - task = _.find(tasks, serviceName: builtImage.serviceName) - - image = - serviceName: d.serviceName - name: d.image.tag ? d.image - logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX) - props: - dockerfile: builtImage.dockerfile - projectType: builtImage.projectType - - # Times here are timestamps, so test whether they're null - # before creating a date out of them, as `new Date(null)` - # creates a date representing UNIX time 0. - if (startTime = builtImage.startTime) - image.props.startTime = new Date(startTime) - if (endTime = builtImage.endTime) - image.props.endTime = new Date(endTime) - docker.getImage(image.name).inspect().get('Size').then (size) -> - image.props.size = size - .return(image) - .tap (images) -> - summary = _(images).map ({ serviceName, props }) -> - [ serviceName, "Image size: #{humanize.filesize(props.size)}" ] - .fromPairs() - .value() - renderer.end(summary) - .finally(renderer.end) - -createRelease = (apiEndpoint, auth, userId, appId, composition) -> - _ = require('lodash') - crypto = require('crypto') - releaseMod = require('balena-release') - - client = releaseMod.createClient({ apiEndpoint, auth }) - - releaseMod.create - client: client - user: userId - application: appId - composition: composition - source: 'local' - commit: crypto.pseudoRandomBytes(16).toString('hex').toLowerCase() - .then ({ release, serviceImages }) -> - release = _.omit(release, [ - 'created_at' - 'belongs_to__application' - 'is_created_by__user' - '__metadata' - ]) - serviceImages = _.mapValues serviceImages, (serviceImage) -> - _.omit(serviceImage, [ - 'created_at' - 'is_a_build_of__service' - '__metadata' - ]) - return { client, release, serviceImages } - -tagServiceImages = (docker, images, serviceImages) -> - Promise.map images, (d) -> - serviceImage = serviceImages[d.serviceName] - imageName = serviceImage.is_stored_at__image_location - # coffeelint: disable-next-line=check_scope ("Variable is assigned to but never read") - [ _match, registry, repo, tag = 'latest' ] = /(.*?)\/(.*?)(?::([^/]*))?$/.exec(imageName) - name = "#{registry}/#{repo}" - docker.getImage(d.name).tag({ repo: name, tag, force: true }) - .then -> - docker.getImage("#{name}:#{tag}") - .then (localImage) -> - serviceName: d.serviceName - serviceImage: serviceImage - localImage: localImage - registry: registry - repo: repo - logs: d.logs - props: d.props - - -getPreviousRepos = (sdk, docker, logger, appID) -> - sdk.pine.get( - resource: 'release' - options: - $filter: - belongs_to__application: appID - status: 'success' - $select: - [ 'id' ] - $expand: - contains__image: - $expand: 'image' - $orderby: 'id desc' - $top: 1 - ) - .then (release) -> - # grab all images from the latest release, return all image locations in the registry - if release?.length > 0 - images = release[0].contains__image - Promise.map images, (d) -> - imageName = d.image[0].is_stored_at__image_location - docker.getRegistryAndName(imageName) - .then ( registry ) -> - logger.logDebug("Requesting access to previously pushed image repo (#{registry.imageName})") - return registry.imageName - .catch (e) -> - logger.logDebug("Failed to access previously pushed image repo: #{e}") - -authorizePush = (sdk, logger, tokenAuthEndpoint, registry, images, previousRepos) -> - if not Array.isArray(images) - images = [ images ] - - images.push previousRepos... - sdk.request.send - baseUrl: tokenAuthEndpoint - url: '/auth/v1/token' - qs: - service: registry - scope: images.map (repo) -> - "repository:#{repo}:pull,push" - .get('body') - .get('token') - .catchReturn({}) - -pushAndUpdateServiceImages = (docker, token, images, afterEach) -> - { DockerProgress } = require('docker-progress') - { retry } = require('./helpers') - tty = require('./tty')(process.stdout) - - opts = { authconfig: registrytoken: token } - - progress = new DockerProgress(dockerToolbelt: docker) - renderer = pushProgressRenderer(tty, getChalk().blue('[Push]') + ' ') - reporters = progress.aggregateProgress(images.length, renderer) - - Promise.using tty.cursorHidden(), -> - Promise.map images, ({ serviceImage, localImage, props, logs }, index) -> - Promise.join( - localImage.inspect().get('Size') - retry( - -> progress.push(localImage.name, reporters[index], opts) - 3 # `times` - retry 3 times - localImage.name # `label` included in retry log messages - 2000 # `delayMs` - wait 2 seconds before the 1st retry - 1.4 # `backoffScaler` - wait multiplier for each retry - ).finally(renderer.end) - (size, digest) -> - serviceImage.image_size = size - serviceImage.content_hash = digest - serviceImage.build_log = logs - serviceImage.dockerfile = props.dockerfile - serviceImage.project_type = props.projectType - serviceImage.start_timestamp = props.startTime if props.startTime - serviceImage.end_timestamp = props.endTime if props.endTime - serviceImage.push_timestamp = new Date() - serviceImage.status = 'success' - ) - .tapCatch (e) -> - serviceImage.error_message = '' + e - serviceImage.status = 'failed' - .finally -> - afterEach?(serviceImage, props) - -exports.deployProject = ( - docker, logger, - composition, images, - appId, userId, auth, - apiEndpoint, - skipLogUpload -) -> - _ = require('lodash') - releaseMod = require('balena-release') - tty = require('./tty')(process.stdout) - - prefix = getChalk().cyan('[Info]') + ' ' - spinner = createSpinner() - runloop = runSpinner(tty, spinner, "#{prefix}Creating release...") - - createRelease(apiEndpoint, auth, userId, appId, composition) - .finally(runloop.end) - .then ({ client, release, serviceImages }) -> - logger.logDebug('Tagging images...') - tagServiceImages(docker, images, serviceImages) - .tap (images) -> - logger.logDebug('Authorizing push...') - sdk = getBalenaSdk() - getPreviousRepos(sdk, docker, logger, appId) - .then (previousRepos) -> - authorizePush(sdk, logger, apiEndpoint, images[0].registry, _.map(images, 'repo'), previousRepos) - .then (token) -> - logger.logInfo('Pushing images to registry...') - pushAndUpdateServiceImages docker, token, images, (serviceImage) -> - logger.logDebug("Saving image #{serviceImage.is_stored_at__image_location}") - if skipLogUpload - delete serviceImage.build_log - releaseMod.updateImage(client, serviceImage.id, serviceImage) - .finally -> - logger.logDebug('Untagging images...') - Promise.map images, ({ localImage }) -> - localImage.remove() - .then -> - release.status = 'success' - .tapCatch -> - release.status = 'failed' - .finally -> - runloop = runSpinner(tty, spinner, "#{prefix}Saving release...") - release.end_timestamp = new Date() - releaseMod.updateRelease(client, release.id, release) - .finally(runloop.end) - .return(release) - -# utilities - -renderProgressBar = (percentage, stepCount) -> - _ = require('lodash') - percentage = _.clamp(percentage, 0, 100) - barCount = stepCount * percentage // 100 - spaceCount = stepCount - barCount - bar = "[#{_.repeat('=', barCount)}>#{_.repeat(' ', spaceCount)}]" - return "#{bar} #{_.padStart(percentage, 3)}%" - -pushProgressRenderer = (tty, prefix) -> - fn = (e) -> - { error, percentage } = e - throw new Error(error) if error? - bar = renderProgressBar(percentage, 40) - tty.replaceLine("#{prefix}#{bar}\r") - fn.end = -> - tty.clearLine() - return fn - -createLogStream = (input) -> - split = require('split') - stripAnsi = require('strip-ansi-stream') - return input.pipe(stripAnsi()).pipe(split()) - -dropEmptyLinesStream = -> - through = require('through2') - through (data, enc, cb) -> - str = data.toString('utf-8') - @push(str) if str.trim() - cb() - -buildLogCapture = (objectMode, buffer) -> - through = require('through2') - - through { objectMode }, (data, enc, cb) -> - # data from pull stream - if data.error - buffer.push("#{data.error}") - else if data.progress and data.status - buffer.push("#{data.progress}% #{data.status}") - else if data.status - buffer.push("#{data.status}") - - # data from build stream - else - buffer.push(data) - - cb(null, data) - -buildProgressAdapter = (inline) -> - through = require('through2') - - stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/ - - [ step, numSteps, progress ] = [ null, null, undefined ] - - through { objectMode: true }, (str, enc, cb) -> - return cb(null, str) if not str? - - if inline - return cb(null, { status: str }) - - if /^Successfully tagged /.test(str) - progress = undefined - else - if (match = stepRegex.exec(str)) - step = match[1] - numSteps ?= match[2] - str = match[3] - if step? - str = "Step #{step}/#{numSteps}: #{str}" - progress = parseInt(step, 10) * 100 // parseInt(numSteps, 10) - - cb(null, { status: str, progress }) - -pullProgressAdapter = (outStream) -> - return ({ status, id, percentage, error, errorDetail }) -> - if status? - status = status.replace(/^Status: /, '') - if id? - status = "#{id}: #{status}" - if percentage is 100 - percentage = undefined - outStream.write - status: status - progress: percentage - error: errorDetail?.message ? error - -createSpinner = -> - chars = '|/-\\' - index = 0 - -> chars[(index++) % chars.length] - -runSpinner = (tty, spinner, msg) -> - runloop = createRunLoop -> - tty.clearLine() - tty.writeLine("#{msg} #{spinner()}") - tty.cursorUp() - runloop.onEnd = -> - tty.clearLine() - tty.writeLine(msg) - return runloop - -createRunLoop = (tick) -> - timerId = setInterval(tick, 1000 / 10) - runloop = { - onEnd: -> - end: -> - clearInterval(timerId) - runloop.onEnd() - } - return runloop - -class BuildProgressUI - constructor: (tty, descriptors) -> - _ = require('lodash') - through = require('through2') - - eventHandler = @_handleEvent - services = _.map(descriptors, 'serviceName') - - streams = _(services).map (service) -> - stream = through.obj (event, _enc, cb) -> - eventHandler(service, event) - cb() - stream.pipe(tty.stream, end: false) - [ service, stream ] - .fromPairs() - .value() - - @_tty = tty - @_serviceToDataMap = {} - @_services = services - - # Logger magically prefixes the log line with [Build] etc., but it doesn't - # work well with the spinner we're also showing. Manually build the prefix - # here and bypass the logger. - prefix = getChalk().blue('[Build]') + ' ' - - offset = 10 # account for escape sequences inserted for colouring - @_prefixWidth = offset + prefix.length + _.max(_.map(services, 'length')) - @_prefix = prefix - - # these are to handle window wrapping - @_maxLineWidth = null - @_lineWidths = [] - - @_startTime = null - @_ended = false - @_cancelled = false - @_spinner = createSpinner() - - @streams = streams - - _handleEvent: (service, event) => - @_serviceToDataMap[service] = event - - _handleInterrupt: => - @_cancelled = true - @end() - process.exit(130) # 128 + SIGINT - - start: => - process.on('SIGINT', @_handleInterrupt) - @_tty.hideCursor() - @_services.forEach (service) => - @streams[service].write({ status: 'Preparing...' }) - @_runloop = createRunLoop(@_display) - @_startTime = Date.now() - - end: (summary = null) => - return if @_ended - @_ended = true - process.removeListener('SIGINT', @_handleInterrupt) - @_runloop.end() - @_runloop = null - - @_clear() - @_renderStatus(true) - @_renderSummary(summary ? @_getServiceSummary()) - @_tty.showCursor() - - _display: => - @_clear() - @_renderStatus() - @_renderSummary(@_getServiceSummary()) - @_tty.cursorUp(@_services.length + 1) # for status line - - _clear: -> - @_tty.deleteToEnd() - @_maxLineWidth = @_tty.currentWindowSize().width - - _getServiceSummary: -> - _ = require('lodash') - - services = @_services - serviceToDataMap = @_serviceToDataMap - - _(services).map (service) -> - { status, progress, error } = serviceToDataMap[service] ? {} - if error - return "#{error}" - else if progress - bar = renderProgressBar(progress, 20) - return "#{bar} #{status}" if status - return "#{bar}" - else if status - return "#{status}" - else - return 'Waiting...' - .map (data, index) -> - [ services[index], data ] - .fromPairs() - .value() - - _renderStatus: (end = false) -> - moment = require('moment') - require('moment-duration-format')(moment) - - @_tty.clearLine() - @_tty.write(@_prefix) - if end and @_cancelled - @_tty.writeLine('Build cancelled') - else if end - serviceCount = @_services.length - serviceStr = if serviceCount is 1 then '1 service' else "#{serviceCount} services" - runTime = Date.now() - @_startTime - durationStr = moment.duration(runTime // 1000, 'seconds').format() - @_tty.writeLine("Built #{serviceStr} in #{durationStr}") - else - @_tty.writeLine("Building services... #{@_spinner()}") - - _renderSummary: (serviceToStrMap) -> - _ = require('lodash') - chalk = getChalk() - truncate = require('cli-truncate') - strlen = require('string-width') - - @_services.forEach (service, index) => - str = _.padEnd(@_prefix + chalk.bold(service), @_prefixWidth) - str += serviceToStrMap[service] - if @_maxLineWidth? - str = truncate(str, @_maxLineWidth) - @_lineWidths[index] = strlen(str) - - @_tty.clearLine() - @_tty.writeLine(str) - -class BuildProgressInline - constructor: (outStream, descriptors) -> - _ = require('lodash') - through = require('through2') - - services = _.map(descriptors, 'serviceName') - eventHandler = @_renderEvent - streams = _(services).map (service) -> - stream = through.obj (event, _enc, cb) -> - eventHandler(service, event) - cb() - stream.pipe(outStream, end: false) - [ service, stream ] - .fromPairs() - .value() - - offset = 10 # account for escape sequences inserted for colouring - @_prefixWidth = offset + _.max(_.map(services, 'length')) - @_outStream = outStream - @_services = services - @_startTime = null - @_ended = false - - @streams = streams - - start: => - @_outStream.write('Building services...\n') - @_services.forEach (service) => - @streams[service].write({ status: 'Preparing...' }) - @_startTime = Date.now() - - end: (summary = null) => - moment = require('moment') - require('moment-duration-format')(moment) - - return if @_ended - @_ended = true - - if summary? - @_services.forEach (service) => - @_renderEvent(service, summary[service]) - - if @_cancelled - @_outStream.write('Build cancelled\n') - else - serviceCount = @_services.length - serviceStr = if serviceCount is 1 then '1 service' else "#{serviceCount} services" - runTime = Date.now() - @_startTime - durationStr = moment.duration(runTime // 1000, 'seconds').format() - @_outStream.write("Built #{serviceStr} in #{durationStr}\n") - - _renderEvent: (service, event) => - _ = require('lodash') - - str = do -> - { status, error } = event - if error - return "#{error}" - else if status - return "#{status}" - else - return 'Waiting...' - - prefix = _.padEnd(getChalk().bold(service), @_prefixWidth) - @_outStream.write(prefix) - @_outStream.write(str) - @_outStream.write('\n') diff --git a/lib/utils/compose.js b/lib/utils/compose.js new file mode 100644 index 00000000..80b69166 --- /dev/null +++ b/lib/utils/compose.js @@ -0,0 +1,1123 @@ +/** + * @license + * Copyright 2017-2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as Promise from 'bluebird'; +import * as path from 'path'; +import { getBalenaSdk, getChalk } from './lazy'; + +export const appendProjectOptions = opts => + opts.concat([ + { + signature: 'projectName', + parameter: 'projectName', + description: + 'Specify an alternate project name; default is the directory name', + alias: 'n', + }, + ]); + +export function appendOptions(opts) { + return appendProjectOptions(opts).concat([ + { + signature: 'emulated', + description: 'Run an emulated build using Qemu', + boolean: true, + alias: 'e', + }, + { + signature: 'dockerfile', + parameter: 'Dockerfile', + description: + 'Alternative Dockerfile name/path, relative to the source folder', + }, + { + signature: 'logs', + description: 'Display full log output', + boolean: true, + }, + { + signature: 'noparent-check', + description: + "Disable project validation check of 'docker-compose.yml' file in parent folder", + boolean: true, + }, + { + signature: 'registry-secrets', + alias: 'R', + parameter: 'secrets.yml|.json', + description: + 'Path to a YAML or JSON file with passwords for a private Docker registry', + }, + { + signature: 'convert-eol', + description: `\ +On Windows only, convert line endings from CRLF (Windows format) to LF (Unix format). \ +Source files are not modified.`, + boolean: true, + alias: 'l', + }, + ]); +} + +/** + * @returns Promise<{import('./compose-types').ComposeOpts}> + */ +export function generateOpts(options) { + const fs = require('mz/fs'); + return fs.realpath(options.source || '.').then(projectPath => ({ + projectName: options.projectName, + projectPath, + inlineLogs: !!options.logs, + dockerfilePath: options.dockerfile, + noParentCheck: options['noparent-check'], + })); +} + +// Parse the given composition and return a structure with info. Input is: +// - composePath: the *absolute* path to the directory containing the compose file +// - composeStr: the contents of the compose file, as a string +/** + * @param {string} composePath + * @param {string} composeStr + * @param {string | null} projectName + * @returns {import('./compose-types').ComposeProject} + */ +export function createProject(composePath, composeStr, projectName = null) { + const yml = require('js-yaml'); + const compose = require('resin-compose-parse'); + + // both methods below may throw. + const rawComposition = yml.safeLoad(composeStr, { + schema: yml.FAILSAFE_SCHEMA, + }); + const composition = compose.normalize(rawComposition); + + if (projectName == null) { + projectName = path.basename(composePath); + } + const descriptors = compose.parse(composition).map(function(descr) { + // generate an image name based on the project and service names + // if one is not given and the service requires a build + if ( + typeof descr.image !== 'string' && + descr.image.context != null && + descr.image.tag == null + ) { + descr.image.tag = [projectName, descr.serviceName] + .join('_') + .toLowerCase(); + } + return descr; + }); + return { + path: composePath, + name: projectName, + composition, + descriptors, + }; +} + +/** + * @param {string} dir + * @param {import('./compose-types').TarDirectoryOptions} [param] + * @returns {Promise} + */ +export const tarDirectory = function(dir, param) { + if (param == null) { + param = {}; + } + let { preFinalizeCallback = null, convertEol = false } = param; + if (convertEol == null) { + convertEol = false; + } + + const tar = require('tar-stream'); + const klaw = require('klaw'); + const fs = require('mz/fs'); + const streamToPromise = require('stream-to-promise'); + const { FileIgnorer } = require('./ignore'); + const { toPosixPath } = require('resin-multibuild').PathUtils; + let readFile; + if (process.platform === 'win32') { + const { readFileWithEolConversion } = require('./eol-conversion'); + readFile = file => readFileWithEolConversion(file, convertEol); + } else { + ({ readFile } = fs); + } + + const getFiles = () => + // @ts-ignore `klaw` returns a `Walker` which is close enough to a stream to work but ts complains + streamToPromise(klaw(dir)) + .filter(item => !item.stats.isDirectory()) + .map(item => item.path); + + const ignore = new FileIgnorer(dir); + const pack = tar.pack(); + return getFiles() + .each(function(file) { + const type = ignore.getIgnoreFileType(path.relative(dir, file)); + if (type != null) { + return ignore.addIgnoreFile(file, type); + } + }) + .filter(ignore.filter) + .map(function(file) { + const relPath = path.relative(path.resolve(dir), file); + return Promise.join( + relPath, + fs.stat(file), + readFile(file), + (filename, stats, data) => + pack.entry( + { name: toPosixPath(filename), size: stats.size, mode: stats.mode }, + data, + ), + ); + }) + .then(() => preFinalizeCallback?.(pack)) + .then(function() { + pack.finalize(); + return pack; + }); +}; + +const truncateString = function(str, len) { + if (str.length < len) { + return str; + } + str = str.slice(0, len); + // return everything up to the last line. this is a cheeky way to avoid + // having to deal with splitting the string midway through some special + // character sequence. + return str.slice(0, str.lastIndexOf('\n')); +}; + +const LOG_LENGTH_MAX = 512 * 1024; // 512KB + +export function buildProject( + docker, + logger, + projectPath, + projectName, + composition, + arch, + deviceType, + emulated, + buildOpts, + inlineLogs, + convertEol, + dockerfilePath, +) { + const _ = require('lodash'); + const humanize = require('humanize'); + const compose = require('resin-compose-parse'); + const builder = require('resin-multibuild'); + const transpose = require('docker-qemu-transpose'); + const { BALENA_ENGINE_TMP_PATH } = require('../config'); + const { + checkBuildSecretsRequirements, + makeBuildTasks, + } = require('./compose_ts'); + const qemu = require('./qemu-ts'); + const { toPosixPath } = builder.PathUtils; + + logger.logInfo(`Building for ${arch}/${deviceType}`); + + const imageDescriptors = compose.parse(composition); + const imageDescriptorsByServiceName = _.keyBy( + imageDescriptors, + 'serviceName', + ); + + let renderer; + if (inlineLogs) { + renderer = new BuildProgressInline( + logger.streams['build'], + imageDescriptors, + ); + } else { + const tty = require('./tty')(process.stdout); + renderer = new BuildProgressUI(tty, imageDescriptors); + } + renderer.start(); + + return Promise.resolve(checkBuildSecretsRequirements(docker, projectPath)) + .then(() => qemu.installQemuIfNeeded(emulated, logger, arch, docker)) + .tap(function(needsQemu) { + if (!needsQemu) { + return; + } + logger.logInfo('Emulation is enabled'); + // Copy qemu into all build contexts + return Promise.map(imageDescriptors, function(d) { + if (typeof d.image === 'string' || d.image.context == null) { + return; + } + // external image + return qemu.copyQemu(path.join(projectPath, d.image.context), arch); + }); + }) + .then(( + needsQemu, // Tar up the directory, ready for the build stream + ) => + tarDirectory(projectPath, { convertEol }) + .then(tarStream => + makeBuildTasks( + composition, + tarStream, + { arch, deviceType }, + logger, + projectName, + ), + ) + .map(function(/** @type {any} */ task) { + const d = imageDescriptorsByServiceName[task.serviceName]; + + // multibuild parses the composition internally so any tags we've + // set before are lost; re-assign them here + if (task.tag == null) { + task.tag = [projectName, task.serviceName].join('_').toLowerCase(); + } + if (typeof d.image !== 'string' && d.image.context != null) { + d.image.tag = task.tag; + } + + // configure build opts appropriately + if (task.dockerOpts == null) { + task.dockerOpts = {}; + } + _.merge(task.dockerOpts, buildOpts, { t: task.tag }); + if (typeof d.image !== 'string') { + /** @type {any} */ + const context = d.image.context; + if (context?.args != null) { + if (task.dockerOpts.buildargs == null) { + task.dockerOpts.buildargs = {}; + } + _.merge(task.dockerOpts.buildargs, context.args); + } + } + + // Get the service-specific log stream + // Caveat: `multibuild.BuildTask` defines no `logStream` property + // but it's convenient to store it there; it's JS ultimately. + task.logStream = renderer.streams[task.serviceName]; + task.logBuffer = []; + + // Setup emulation if needed + if (task.external || !needsQemu) { + return [task, null]; + } + const binPath = qemu.qemuPathInContext( + path.join(projectPath, task.context ?? ''), + ); + if (task.buildStream == null) { + throw new Error(`No buildStream for task '${task.tag}'`); + } + return transpose + .transposeTarStream( + task.buildStream, + { + hostQemuPath: toPosixPath(binPath), + containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`, + qemuFileMode: 0o555, + }, + dockerfilePath || undefined, + ) + .then((/** @type {any} */ stream) => { + task.buildStream = stream; + }) + .return([task, binPath]); + }), + ) + .map(function([task, qemuPath]) { + const captureStream = buildLogCapture(task.external, task.logBuffer); + + if (task.external) { + // External image -- there's no build to be performed, + // just follow pull progress. + captureStream.pipe(task.logStream); + task.progressHook = pullProgressAdapter(captureStream); + } else { + task.streamHook = function(stream) { + let rawStream; + stream = createLogStream(stream); + if (qemuPath != null) { + const buildThroughStream = transpose.getBuildThroughStream({ + hostQemuPath: toPosixPath(qemuPath), + containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`, + }); + rawStream = stream.pipe(buildThroughStream); + } else { + rawStream = stream; + } + // `stream` sends out raw strings in contrast to `task.progressHook` + // where we're given objects. capture these strings as they come + // before we parse them. + return rawStream + .pipe(dropEmptyLinesStream()) + .pipe(captureStream) + .pipe(buildProgressAdapter(inlineLogs)) + .pipe(task.logStream); + }; + } + return task; + }) + .then(function(tasks) { + logger.logDebug('Prepared tasks; building...'); + return Promise.map( + builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH), + function(builtImage) { + if (!builtImage.successful) { + /** @type {Error & {serviceName?: string}} */ + const error = builtImage.error ?? new Error(); + error.serviceName = builtImage.serviceName; + throw error; + } + + const d = imageDescriptorsByServiceName[builtImage.serviceName]; + const task = _.find(tasks, { serviceName: builtImage.serviceName }); + + const image = { + serviceName: d.serviceName, + name: typeof d.image === 'string' ? d.image : d.image.tag, + logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX), + props: { + dockerfile: builtImage.dockerfile, + projectType: builtImage.projectType, + }, + }; + + // Times here are timestamps, so test whether they're null + // before creating a date out of them, as `new Date(null)` + // creates a date representing UNIX time 0. + if (builtImage.startTime) { + image.props.startTime = new Date(builtImage.startTime); + } + if (builtImage.endTime) { + image.props.endTime = new Date(builtImage.endTime); + } + return docker + .getImage(image.name) + .inspect() + .get('Size') + .then(size => { + image.props.size = size; + }) + .return(image); + }, + ).tap(function(images) { + const summary = _(images) + .map(({ serviceName, props }) => [ + serviceName, + `Image size: ${humanize.filesize(props.size)}`, + ]) + .fromPairs() + .value(); + renderer.end(summary); + }); + }) + .finally(renderer.end); +} + +const createRelease = function(apiEndpoint, auth, userId, appId, composition) { + const _ = require('lodash'); + const crypto = require('crypto'); + const releaseMod = require('balena-release'); + + const client = releaseMod.createClient({ apiEndpoint, auth }); + + return releaseMod + .create({ + client, + user: userId, + application: appId, + composition, + source: 'local', + commit: crypto + .pseudoRandomBytes(16) + .toString('hex') + .toLowerCase(), + }) + .then(function({ release, serviceImages }) { + return { + client, + release: _.omit(release, [ + 'created_at', + 'belongs_to__application', + 'is_created_by__user', + '__metadata', + ]), + serviceImages: _.mapValues(serviceImages, serviceImage => + _.omit(serviceImage, [ + 'created_at', + 'is_a_build_of__service', + '__metadata', + ]), + ), + }; + }); +}; + +const tagServiceImages = (docker, images, serviceImages) => + Promise.map(images, function(d) { + const serviceImage = serviceImages[d.serviceName]; + const imageName = serviceImage.is_stored_at__image_location; + const match = /(.*?)\/(.*?)(?::([^/]*))?$/.exec(imageName); + if (match == null) { + throw new Error(`Could not parse imageName: '${imageName}'`); + } + const [, registry, repo, tag = 'latest'] = match; + const name = `${registry}/${repo}`; + return docker + .getImage(d.name) + .tag({ repo: name, tag, force: true }) + .then(() => docker.getImage(`${name}:${tag}`)) + .then(localImage => ({ + serviceName: d.serviceName, + serviceImage, + localImage, + registry, + repo, + logs: d.logs, + props: d.props, + })); + }); + +const getPreviousRepos = (sdk, docker, logger, appID) => + sdk.pine + .get({ + resource: 'release', + options: { + $filter: { + belongs_to__application: appID, + status: 'success', + }, + $select: ['id'], + $expand: { + contains__image: { + $expand: 'image', + }, + }, + $orderby: 'id desc', + $top: 1, + }, + }) + .then(function(release) { + // grab all images from the latest release, return all image locations in the registry + if (release.length > 0) { + const images = release[0].contains__image; + return Promise.map(images, function(d) { + const imageName = d.image[0].is_stored_at__image_location; + return docker.getRegistryAndName(imageName).then(function(registry) { + logger.logDebug( + `Requesting access to previously pushed image repo (${registry.imageName})`, + ); + return registry.imageName; + }); + }); + } + }) + .catch(e => { + logger.logDebug(`Failed to access previously pushed image repo: ${e}`); + }); + +const authorizePush = function( + sdk, + tokenAuthEndpoint, + registry, + images, + previousRepos, +) { + if (!Array.isArray(images)) { + images = [images]; + } + + images.push(...previousRepos); + return sdk.request + .send({ + baseUrl: tokenAuthEndpoint, + url: '/auth/v1/token', + qs: { + service: registry, + scope: images.map(repo => `repository:${repo}:pull,push`), + }, + }) + .get('body') + .get('token') + .catchReturn({}); +}; + +const pushAndUpdateServiceImages = function(docker, token, images, afterEach) { + const { DockerProgress } = require('docker-progress'); + const { retry } = require('./helpers'); + const tty = require('./tty')(process.stdout); + + const opts = { authconfig: { registrytoken: token } }; + + const progress = new DockerProgress({ dockerToolbelt: docker }); + const renderer = pushProgressRenderer( + tty, + getChalk().blue('[Push]') + ' ', + ); + const reporters = progress.aggregateProgress(images.length, renderer); + + return Promise.using(tty.cursorHidden(), () => + Promise.map(images, ({ serviceImage, localImage, props, logs }, index) => + Promise.join( + localImage.inspect().get('Size'), + retry( + () => { + progress.push(localImage.name, reporters[index], opts); + }, + 3, // `times` - retry 3 times + localImage.name, // `label` included in retry log messages + 2000, // `delayMs` - wait 2 seconds before the 1st retry + 1.4, // `backoffScaler` - wait multiplier for each retry + ).finally(renderer.end), + function(size, digest) { + serviceImage.image_size = size; + serviceImage.content_hash = digest; + serviceImage.build_log = logs; + serviceImage.dockerfile = props.dockerfile; + serviceImage.project_type = props.projectType; + if (props.startTime) { + serviceImage.start_timestamp = props.startTime; + } + if (props.endTime) { + serviceImage.end_timestamp = props.endTime; + } + serviceImage.push_timestamp = new Date(); + serviceImage.status = 'success'; + }, + ) + .tapCatch(function(e) { + serviceImage.error_message = '' + e; + serviceImage.status = 'failed'; + }) + .finally(() => afterEach?.(serviceImage, props)), + ), + ); +}; + +export function deployProject( + docker, + logger, + composition, + images, + appId, + userId, + auth, + apiEndpoint, + skipLogUpload, +) { + const _ = require('lodash'); + const releaseMod = require('balena-release'); + const tty = require('./tty')(process.stdout); + + const prefix = getChalk().cyan('[Info]') + ' '; + const spinner = createSpinner(); + let runloop = runSpinner(tty, spinner, `${prefix}Creating release...`); + + return createRelease(apiEndpoint, auth, userId, appId, composition) + .finally(runloop.end) + .then(function({ client, release, serviceImages }) { + logger.logDebug('Tagging images...'); + return tagServiceImages(docker, images, serviceImages) + .tap(function(taggedImages) { + logger.logDebug('Authorizing push...'); + const sdk = getBalenaSdk(); + return getPreviousRepos(sdk, docker, logger, appId) + .then(previousRepos => + authorizePush( + sdk, + apiEndpoint, + taggedImages[0].registry, + _.map(taggedImages, 'repo'), + previousRepos, + ), + ) + .then(function(token) { + logger.logInfo('Pushing images to registry...'); + return pushAndUpdateServiceImages( + docker, + token, + taggedImages, + function(serviceImage) { + logger.logDebug( + `Saving image ${serviceImage.is_stored_at__image_location}`, + ); + if (skipLogUpload) { + delete serviceImage.build_log; + } + return releaseMod.updateImage( + client, + serviceImage.id, + serviceImage, + ); + }, + ); + }) + .finally(function() { + logger.logDebug('Untagging images...'); + return Promise.map(taggedImages, ({ localImage }) => + localImage.remove(), + ); + }); + }) + .then(() => { + release.status = 'success'; + }) + .tapCatch(() => { + release.status = 'failed'; + }) + .finally(function() { + runloop = runSpinner(tty, spinner, `${prefix}Saving release...`); + release.end_timestamp = new Date(); + if (release.id == null) { + return; + } + return releaseMod + .updateRelease(client, release.id, release) + .finally(runloop.end); + }) + .return(release); + }); +} + +// utilities + +const renderProgressBar = function(percentage, stepCount) { + const _ = require('lodash'); + percentage = _.clamp(percentage, 0, 100); + const barCount = Math.floor((stepCount * percentage) / 100); + const spaceCount = stepCount - barCount; + const bar = `[${_.repeat('=', barCount)}>${_.repeat(' ', spaceCount)}]`; + return `${bar} ${_.padStart(percentage, 3)}%`; +}; + +var pushProgressRenderer = function(tty, prefix) { + const fn = function(e) { + const { error, percentage } = e; + if (error != null) { + throw new Error(error); + } + const bar = renderProgressBar(percentage, 40); + return tty.replaceLine(`${prefix}${bar}\r`); + }; + fn.end = () => { + tty.clearLine(); + }; + return fn; +}; + +var createLogStream = function(input) { + const split = require('split'); + const stripAnsi = require('strip-ansi-stream'); + return input.pipe(stripAnsi()).pipe(split()); +}; + +var dropEmptyLinesStream = function() { + const through = require('through2'); + return through(function(data, _enc, cb) { + const str = data.toString('utf-8'); + if (str.trim()) { + this.push(str); + } + return cb(); + }); +}; + +var buildLogCapture = function(objectMode, buffer) { + const through = require('through2'); + + return through({ objectMode }, function(data, _enc, cb) { + // data from pull stream + if (data.error) { + buffer.push(`${data.error}`); + } else if (data.progress && data.status) { + buffer.push(`${data.progress}% ${data.status}`); + } else if (data.status) { + buffer.push(`${data.status}`); + + // data from build stream + } else { + buffer.push(data); + } + + return cb(null, data); + }); +}; + +var buildProgressAdapter = function(inline) { + const through = require('through2'); + + const stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/; + + let step = null; + let numSteps = null; + let progress; + + return through({ objectMode: true }, function(str, _enc, cb) { + if (str == null) { + return cb(null, str); + } + + if (inline) { + return cb(null, { status: str }); + } + + if (/^Successfully tagged /.test(str)) { + progress = undefined; + } else { + const match = stepRegex.exec(str); + if (match) { + step = match[1]; + if (numSteps == null) { + numSteps = match[2]; + } + str = match[3]; + } + if (step != null) { + str = `Step ${step}/${numSteps}: ${str}`; + progress = Math.floor( + (parseInt(step, 10) * 100) / parseInt(numSteps, 10), + ); + } + } + + return cb(null, { status: str, progress }); + }); +}; + +var pullProgressAdapter = outStream => + function({ status, id, percentage, error, errorDetail }) { + if (status != null) { + status = status.replace(/^Status: /, ''); + } + if (id != null) { + status = `${id}: ${status}`; + } + if (percentage === 100) { + percentage = undefined; + } + return outStream.write({ + status, + progress: percentage, + error: errorDetail?.message ?? error, + }); + }; + +var createSpinner = function() { + const chars = '|/-\\'; + let index = 0; + return () => chars[index++ % chars.length]; +}; + +var runSpinner = function(tty, spinner, msg) { + const runloop = createRunLoop(function() { + tty.clearLine(); + tty.writeLine(`${msg} ${spinner()}`); + return tty.cursorUp(); + }); + runloop.onEnd = function() { + tty.clearLine(); + return tty.writeLine(msg); + }; + return runloop; +}; + +var createRunLoop = function(tick) { + const timerId = setInterval(tick, 1000 / 10); + var runloop = { + onEnd() { + // noop + }, + end() { + clearInterval(timerId); + return runloop.onEnd(); + }, + }; + return runloop; +}; + +class BuildProgressUI { + constructor(tty, descriptors) { + this._handleEvent = this._handleEvent.bind(this); + this._handleInterrupt = this._handleInterrupt.bind(this); + this.start = this.start.bind(this); + this.end = this.end.bind(this); + this._display = this._display.bind(this); + const _ = require('lodash'); + const through = require('through2'); + + const eventHandler = this._handleEvent; + const services = _.map(descriptors, 'serviceName'); + + const streams = _(services) + .map(function(service) { + const stream = through.obj(function(event, _enc, cb) { + eventHandler(service, event); + return cb(); + }); + stream.pipe(tty.stream, { end: false }); + return [service, stream]; + }) + .fromPairs() + .value(); + + this._tty = tty; + this._serviceToDataMap = {}; + this._services = services; + + // Logger magically prefixes the log line with [Build] etc., but it doesn't + // work well with the spinner we're also showing. Manually build the prefix + // here and bypass the logger. + const prefix = getChalk().blue('[Build]') + ' '; + + const offset = 10; // account for escape sequences inserted for colouring + this._prefixWidth = + offset + prefix.length + _.max(_.map(services, 'length')); + this._prefix = prefix; + + // these are to handle window wrapping + this._maxLineWidth = null; + this._lineWidths = []; + + this._startTime = null; + this._ended = false; + this._cancelled = false; + this._spinner = createSpinner(); + + this.streams = streams; + } + + _handleEvent(service, event) { + this._serviceToDataMap[service] = event; + } + + _handleInterrupt() { + this._cancelled = true; + this.end(); + return process.exit(130); // 128 + SIGINT + } + + start() { + process.on('SIGINT', this._handleInterrupt); + this._tty.hideCursor(); + this._services.forEach(service => { + this.streams[service].write({ status: 'Preparing...' }); + }); + this._runloop = createRunLoop(this._display); + this._startTime = Date.now(); + } + + end(summary = null) { + if (this._ended) { + return; + } + this._ended = true; + process.removeListener('SIGINT', this._handleInterrupt); + this._runloop?.end(); + this._runloop = null; + + this._clear(); + this._renderStatus(true); + this._renderSummary(summary ?? this._getServiceSummary()); + this._tty.showCursor(); + } + + _display() { + this._clear(); + this._renderStatus(); + this._renderSummary(this._getServiceSummary()); + this._tty.cursorUp(this._services.length + 1); // for status line + } + + _clear() { + this._tty.deleteToEnd(); + this._maxLineWidth = this._tty.currentWindowSize().width; + } + + _getServiceSummary() { + const _ = require('lodash'); + + const services = this._services; + const serviceToDataMap = this._serviceToDataMap; + + return _(services) + .map(function(service) { + const { status, progress, error } = serviceToDataMap[service] ?? {}; + if (error) { + return `${error}`; + } else if (progress) { + const bar = renderProgressBar(progress, 20); + if (status) { + return `${bar} ${status}`; + } + return `${bar}`; + } else if (status) { + return `${status}`; + } else { + return 'Waiting...'; + } + }) + .map((data, index) => [services[index], data]) + .fromPairs() + .value(); + } + + _renderStatus(end) { + if (end == null) { + end = false; + } + const moment = require('moment'); + require('moment-duration-format')(moment); + + this._tty.clearLine(); + this._tty.write(this._prefix); + if (end && this._cancelled) { + this._tty.writeLine('Build cancelled'); + } else if (end) { + const serviceCount = this._services.length; + const serviceStr = + serviceCount === 1 ? '1 service' : `${serviceCount} services`; + const durationStr = + this._startTime == null + ? 'unknown time' + : moment + .duration( + Math.floor((Date.now() - this._startTime) / 1000), + 'seconds', + ) + .format(); + this._tty.writeLine(`Built ${serviceStr} in ${durationStr}`); + } else { + this._tty.writeLine(`Building services... ${this._spinner()}`); + } + } + + _renderSummary(serviceToStrMap) { + const _ = require('lodash'); + const chalk = getChalk(); + const truncate = require('cli-truncate'); + const strlen = require('string-width'); + + this._services.forEach((service, index) => { + let str = _.padEnd(this._prefix + chalk.bold(service), this._prefixWidth); + str += serviceToStrMap[service]; + if (this._maxLineWidth != null) { + str = truncate(str, this._maxLineWidth); + } + this._lineWidths[index] = strlen(str); + + this._tty.clearLine(); + this._tty.writeLine(str); + }); + } +} + +class BuildProgressInline { + constructor(outStream, descriptors) { + this.start = this.start.bind(this); + this.end = this.end.bind(this); + this._renderEvent = this._renderEvent.bind(this); + const _ = require('lodash'); + const through = require('through2'); + + const services = _.map(descriptors, 'serviceName'); + const eventHandler = this._renderEvent; + const streams = _(services) + .map(function(service) { + const stream = through.obj(function(event, _enc, cb) { + eventHandler(service, event); + return cb(); + }); + stream.pipe(outStream, { end: false }); + return [service, stream]; + }) + .fromPairs() + .value(); + + const offset = 10; // account for escape sequences inserted for colouring + this._prefixWidth = offset + _.max(_.map(services, 'length')); + this._outStream = outStream; + this._services = services; + this._startTime = null; + this._ended = false; + + this.streams = streams; + } + + start() { + this._outStream.write('Building services...\n'); + this._services.forEach(service => { + this.streams[service].write({ status: 'Preparing...' }); + }); + this._startTime = Date.now(); + } + + end(summary = null) { + const moment = require('moment'); + require('moment-duration-format')(moment); + + if (this._ended) { + return; + } + this._ended = true; + + if (summary != null) { + this._services.forEach(service => { + this._renderEvent(service, summary[service]); + }); + } + + const serviceCount = this._services.length; + const serviceStr = + serviceCount === 1 ? '1 service' : `${serviceCount} services`; + const durationStr = + this._startTime == null + ? 'unknown time' + : moment + .duration( + Math.floor((Date.now() - this._startTime) / 1000), + 'seconds', + ) + .format(); + this._outStream.write(`Built ${serviceStr} in ${durationStr}\n`); + } + + _renderEvent(service, event) { + const _ = require('lodash'); + + const str = (function() { + const { status, error } = event; + if (error) { + return `${error}`; + } else if (status) { + return `${status}`; + } else { + return 'Waiting...'; + } + })(); + + const prefix = _.padEnd(getChalk().bold(service), this._prefixWidth); + this._outStream.write(prefix); + this._outStream.write(str); + this._outStream.write('\n'); + } +} diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index eef2a959..4d9a9453 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -47,9 +47,9 @@ const compositionFileNames = ['docker-compose.yml', 'docker-compose.yaml']; */ export async function loadProject( logger: Logger, - opts: import('./compose').ComposeOpts, + opts: import('./compose-types').ComposeOpts, image?: string, -): Promise { +): Promise { const compose = await import('resin-compose-parse'); const { createProject } = await import('./compose'); let composeName: string; diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 00d1d6d2..f3a5f4c5 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1061,9 +1061,9 @@ "integrity": "sha512-M5qhhfuTt4fwHGqqANNQilp3Htb5cHwBxlMHDUw/TYRVkEp3s3IIFSH3Fe9HIAeEtnO4p3SSowLmCVavdRYfpw==" }, "@types/jsesc": { - "version": "0.4.29", - "resolved": "https://registry.npmjs.org/@types/jsesc/-/jsesc-0.4.29.tgz", - "integrity": "sha1-2Ntltyd2mWwDhwMGpbBIwmmAz44=" + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@types/jsesc/-/jsesc-2.5.0.tgz", + "integrity": "sha512-rhKX1CpHA4NH3H/tXpuHAOLCIrJOV6Dm2rtv5J4jQt9tJ801+IIz9yhj2lHl/m5l7NCS94pODN7kLeAcqEaq/g==" }, "@types/jsonstream": { "version": "0.8.30", @@ -4554,30 +4554,85 @@ } }, "docker-qemu-transpose": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/docker-qemu-transpose/-/docker-qemu-transpose-1.0.4.tgz", - "integrity": "sha512-sSg13dXH3cRZ/svJc1zigMcgPuKm85Cjx+Eg6bKYumg1mUTIeiIsacK79sHxoxk8r4keVDfvQ+/8507HUJac3w==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/docker-qemu-transpose/-/docker-qemu-transpose-1.1.1.tgz", + "integrity": "sha512-GI9NvYFSYsxz1NL2nAJBY7LJxNhJ7X0QfrxHA1wSKyrnTB+lliIIs+KZA9kc8zEMqMorAhS2pC4CQJx5apsEcw==", "requires": { "@types/bluebird": "^3.5.30", "@types/event-stream": "^3.3.34", - "@types/jsesc": "^0.4.29", - "@types/lodash": "^4.14.149", - "@types/node": "^7.10.9", + "@types/jsesc": "^2.5.0", + "@types/lodash": "^4.14.150", + "@types/node": "^8.10.60", "bluebird": "^3.7.2", "common-tags": "^1.8.0", "docker-file-parser": "^1.0.4", - "event-stream": "^3.3.4", + "event-stream": "^3.3.5", "jsesc": "^2.5.2", "lodash": "^4.17.15", "stream-to-promise": "^2.2.0", - "tar-stream": "^2.1.0", - "tar-utils": "^2.0.0" + "tar-stream": "^2.1.2", + "tar-utils": "^2.1.0" }, "dependencies": { "@types/node": { - "version": "7.10.9", - "resolved": "https://registry.npmjs.org/@types/node/-/node-7.10.9.tgz", - "integrity": "sha512-usSpgoUsRtO5xNV5YEPU8PPnHisFx8u0rokj1BPVn/hDF7zwUDzVLiuKZM38B7z8V2111Fj6kd4rGtQFUZpNOw==" + "version": "8.10.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.60.tgz", + "integrity": "sha512-YjPbypHFuiOV0bTgeF07HpEEqhmHaZqYNSdCKeBJa+yFoQ/7BC+FpJcwmi34xUIIRVFktnUyP1dPU8U0612GOg==" + }, + "bl": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", + "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "event-stream": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.5.tgz", + "integrity": "sha512-vyibDcu5JL20Me1fP734QBH/kenBGLZap2n0+XXM7mvuUPzJ20Ydqj1aKcIeMdri1p+PU+4yAKugjN8KCVst+g==", + "requires": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "tar-stream": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.2.tgz", + "integrity": "sha512-UaF6FoJ32WqALZGOIAApXx+OdxhekNMChu6axLJR85zMMjXKWFGjbIRe+J6P4UnRGg9rAwWvbTT0oI7hD/Un7Q==", + "requires": { + "bl": "^4.0.1", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } } } }, diff --git a/package.json b/package.json index 976ded9e..be1b8135 100644 --- a/package.json +++ b/package.json @@ -191,7 +191,7 @@ "denymount": "^2.3.0", "docker-modem": "^2.1.1", "docker-progress": "^4.0.1", - "docker-qemu-transpose": "^1.0.4", + "docker-qemu-transpose": "^1.1.1", "docker-toolbelt": "^3.3.8", "dockerode": "^2.5.8", "ejs": "^3.0.1",