2018-11-07 18:15:05 +00:00
|
|
|
###*
|
|
|
|
# @license
|
2020-01-29 11:10:59 +00:00
|
|
|
# Copyright 2017-2020 Balena Ltd.
|
2018-11-07 18:15:05 +00:00
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
###
|
|
|
|
|
2017-12-11 22:37:56 +00:00
|
|
|
Promise = require('bluebird')
|
|
|
|
path = require('path')
|
2020-02-28 15:25:12 +00:00
|
|
|
{ getBalenaSdk, getChalk } = require('./lazy')
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
exports.appendProjectOptions = appendProjectOptions = (opts) ->
|
|
|
|
opts.concat [
|
|
|
|
{
|
|
|
|
signature: 'projectName'
|
|
|
|
parameter: 'projectName'
|
|
|
|
description: 'Specify an alternate project name; default is the directory name'
|
|
|
|
alias: 'n'
|
|
|
|
},
|
|
|
|
]
|
|
|
|
|
|
|
|
exports.appendOptions = (opts) ->
|
|
|
|
appendProjectOptions(opts).concat [
|
|
|
|
{
|
|
|
|
signature: 'emulated'
|
|
|
|
description: 'Run an emulated build using Qemu'
|
|
|
|
boolean: true
|
|
|
|
alias: 'e'
|
|
|
|
},
|
2019-04-11 11:49:19 +00:00
|
|
|
{
|
|
|
|
signature: 'dockerfile'
|
|
|
|
parameter: 'Dockerfile'
|
|
|
|
description: 'Alternative Dockerfile name/path, relative to the source folder'
|
|
|
|
},
|
2017-12-11 22:37:56 +00:00
|
|
|
{
|
|
|
|
signature: 'logs'
|
|
|
|
description: 'Display full log output'
|
|
|
|
boolean: true
|
|
|
|
},
|
2020-02-15 20:20:56 +00:00
|
|
|
{
|
|
|
|
signature: 'noparent-check'
|
|
|
|
description: 'Disable project validation check of \'docker-compose.yml\' file in parent folder'
|
|
|
|
boolean: true
|
|
|
|
},
|
2019-02-26 13:32:27 +00:00
|
|
|
{
|
2019-04-11 11:49:19 +00:00
|
|
|
signature: 'registry-secrets'
|
|
|
|
alias: 'R'
|
|
|
|
parameter: 'secrets.yml|.json'
|
|
|
|
description: 'Path to a YAML or JSON file with passwords for a private Docker registry'
|
2019-02-26 13:32:27 +00:00
|
|
|
},
|
2020-01-29 11:10:59 +00:00
|
|
|
{
|
|
|
|
signature: 'convert-eol'
|
2020-02-07 13:22:20 +00:00
|
|
|
description: '
|
|
|
|
On Windows only, convert line endings from CRLF (Windows format) to LF (Unix format).
|
|
|
|
Source files are not modified.'
|
2020-01-29 11:10:59 +00:00
|
|
|
boolean: true
|
|
|
|
alias: 'l'
|
|
|
|
}
|
2017-12-11 22:37:56 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
exports.generateOpts = (options) ->
|
|
|
|
fs = require('mz/fs')
|
|
|
|
fs.realpath(options.source || '.').then (projectPath) ->
|
|
|
|
projectName: options.projectName
|
|
|
|
projectPath: projectPath
|
|
|
|
inlineLogs: !!options.logs
|
2019-04-11 11:49:19 +00:00
|
|
|
dockerfilePath: options.dockerfile
|
2020-02-15 20:20:56 +00:00
|
|
|
noParentCheck: options['noparent-check']
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
# Parse the given composition and return a structure with info. Input is:
|
|
|
|
# - composePath: the *absolute* path to the directory containing the compose file
|
|
|
|
# - composeStr: the contents of the compose file, as a string
|
2020-02-15 20:20:56 +00:00
|
|
|
exports.createProject = (composePath, composeStr, projectName = null) ->
|
2017-12-11 22:37:56 +00:00
|
|
|
yml = require('js-yaml')
|
|
|
|
compose = require('resin-compose-parse')
|
|
|
|
|
|
|
|
# both methods below may throw.
|
|
|
|
composition = yml.safeLoad(composeStr, schema: yml.FAILSAFE_SCHEMA)
|
|
|
|
composition = compose.normalize(composition)
|
|
|
|
|
|
|
|
projectName ?= path.basename(composePath)
|
|
|
|
descriptors = compose.parse(composition).map (descr) ->
|
|
|
|
# generate an image name based on the project and service names
|
|
|
|
# if one is not given and the service requires a build
|
|
|
|
if descr.image.context? and not descr.image.tag?
|
2018-03-15 13:24:12 +00:00
|
|
|
descr.image.tag = [ projectName, descr.serviceName ].join('_').toLowerCase()
|
2017-12-11 22:37:56 +00:00
|
|
|
return descr
|
|
|
|
return {
|
|
|
|
path: composePath,
|
|
|
|
name: projectName,
|
|
|
|
composition,
|
|
|
|
descriptors
|
|
|
|
}
|
|
|
|
|
2020-01-29 11:10:59 +00:00
|
|
|
exports.tarDirectory = tarDirectory = (dir, { preFinalizeCallback, convertEol } = {}) ->
|
|
|
|
preFinalizeCallback ?= null
|
|
|
|
convertEol ?= false
|
|
|
|
|
2017-12-11 22:37:56 +00:00
|
|
|
tar = require('tar-stream')
|
|
|
|
klaw = require('klaw')
|
|
|
|
path = require('path')
|
|
|
|
fs = require('mz/fs')
|
|
|
|
streamToPromise = require('stream-to-promise')
|
2018-08-15 12:13:06 +00:00
|
|
|
{ FileIgnorer } = require('./ignore')
|
2019-04-11 11:49:19 +00:00
|
|
|
{ toPosixPath } = require('resin-multibuild').PathUtils
|
2020-02-07 13:22:20 +00:00
|
|
|
if process.platform == 'win32'
|
|
|
|
{ readFileWithEolConversion } = require('./eol-conversion')
|
|
|
|
readFile = (file) -> readFileWithEolConversion(file, convertEol)
|
|
|
|
else
|
|
|
|
readFile = fs.readFile
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
getFiles = ->
|
2018-08-01 14:43:06 +00:00
|
|
|
streamToPromise(klaw(dir))
|
2017-12-11 22:37:56 +00:00
|
|
|
.filter((item) -> not item.stats.isDirectory())
|
|
|
|
.map((item) -> item.path)
|
|
|
|
|
2018-08-15 12:13:06 +00:00
|
|
|
ignore = new FileIgnorer(dir)
|
2017-12-11 22:37:56 +00:00
|
|
|
pack = tar.pack()
|
|
|
|
getFiles(dir)
|
2018-08-15 12:13:06 +00:00
|
|
|
.each (file) ->
|
|
|
|
type = ignore.getIgnoreFileType(path.relative(dir, file))
|
|
|
|
if type?
|
|
|
|
ignore.addIgnoreFile(file, type)
|
|
|
|
.filter(ignore.filter)
|
2017-12-11 22:37:56 +00:00
|
|
|
.map (file) ->
|
|
|
|
relPath = path.relative(path.resolve(dir), file)
|
2020-02-07 13:22:20 +00:00
|
|
|
Promise.join relPath, fs.stat(file), readFile(file),
|
2017-12-11 22:37:56 +00:00
|
|
|
(filename, stats, data) ->
|
|
|
|
pack.entry({ name: toPosixPath(filename), size: stats.size, mode: stats.mode }, data)
|
2018-11-07 18:15:05 +00:00
|
|
|
.then ->
|
|
|
|
preFinalizeCallback?(pack)
|
2017-12-11 22:37:56 +00:00
|
|
|
.then ->
|
|
|
|
pack.finalize()
|
|
|
|
return pack
|
|
|
|
|
|
|
|
truncateString = (str, len) ->
|
|
|
|
return str if str.length < len
|
|
|
|
str = str.slice(0, len)
|
|
|
|
# return everything up to the last line. this is a cheeky way to avoid
|
|
|
|
# having to deal with splitting the string midway through some special
|
|
|
|
# character sequence.
|
|
|
|
return str.slice(0, str.lastIndexOf('\n'))
|
|
|
|
|
|
|
|
LOG_LENGTH_MAX = 512 * 1024 # 512KB
|
|
|
|
|
|
|
|
exports.buildProject = (
|
|
|
|
docker, logger,
|
|
|
|
projectPath, projectName, composition,
|
|
|
|
arch, deviceType,
|
|
|
|
emulated, buildOpts,
|
2020-01-29 11:10:59 +00:00
|
|
|
inlineLogs,
|
2020-02-25 01:18:22 +00:00
|
|
|
convertEol,
|
|
|
|
dockerfilePath,
|
2017-12-11 22:37:56 +00:00
|
|
|
) ->
|
|
|
|
_ = require('lodash')
|
|
|
|
humanize = require('humanize')
|
|
|
|
compose = require('resin-compose-parse')
|
|
|
|
builder = require('resin-multibuild')
|
|
|
|
transpose = require('docker-qemu-transpose')
|
2019-07-09 14:43:36 +00:00
|
|
|
{ BALENA_ENGINE_TMP_PATH } = require('../config')
|
|
|
|
{ checkBuildSecretsRequirements, makeBuildTasks } = require('./compose_ts')
|
2019-08-21 19:45:07 +00:00
|
|
|
qemu = require('./qemu-ts')
|
2019-04-11 11:49:19 +00:00
|
|
|
{ toPosixPath } = builder.PathUtils
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
logger.logInfo("Building for #{arch}/#{deviceType}")
|
|
|
|
|
|
|
|
imageDescriptors = compose.parse(composition)
|
|
|
|
imageDescriptorsByServiceName = _.keyBy(imageDescriptors, 'serviceName')
|
|
|
|
|
|
|
|
if inlineLogs
|
|
|
|
renderer = new BuildProgressInline(logger.streams['build'], imageDescriptors)
|
|
|
|
else
|
|
|
|
tty = require('./tty')(process.stdout)
|
|
|
|
renderer = new BuildProgressUI(tty, imageDescriptors)
|
|
|
|
renderer.start()
|
|
|
|
|
2019-07-09 14:43:36 +00:00
|
|
|
Promise.resolve(checkBuildSecretsRequirements(docker, projectPath))
|
2019-08-21 19:45:07 +00:00
|
|
|
.then -> qemu.installQemuIfNeeded(emulated, logger, arch, docker)
|
2017-12-11 22:37:56 +00:00
|
|
|
.tap (needsQemu) ->
|
|
|
|
return if not needsQemu
|
|
|
|
logger.logInfo('Emulation is enabled')
|
|
|
|
# Copy qemu into all build contexts
|
|
|
|
Promise.map imageDescriptors, (d) ->
|
|
|
|
return if not d.image.context? # external image
|
2018-11-03 03:55:24 +00:00
|
|
|
return qemu.copyQemu(path.join(projectPath, d.image.context), arch)
|
2017-12-11 22:37:56 +00:00
|
|
|
.then (needsQemu) ->
|
|
|
|
# Tar up the directory, ready for the build stream
|
2020-01-29 11:10:59 +00:00
|
|
|
tarDirectory(projectPath, { convertEol })
|
2017-12-11 22:37:56 +00:00
|
|
|
.then (tarStream) ->
|
2020-02-18 03:21:12 +00:00
|
|
|
Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger, projectName))
|
2017-12-11 22:37:56 +00:00
|
|
|
.map (task) ->
|
|
|
|
d = imageDescriptorsByServiceName[task.serviceName]
|
|
|
|
|
|
|
|
# multibuild parses the composition internally so any tags we've
|
|
|
|
# set before are lost; re-assign them here
|
2018-03-15 13:24:12 +00:00
|
|
|
task.tag ?= [ projectName, task.serviceName ].join('_').toLowerCase()
|
2017-12-11 22:37:56 +00:00
|
|
|
if d.image.context?
|
|
|
|
d.image.tag = task.tag
|
|
|
|
|
|
|
|
# configure build opts appropriately
|
|
|
|
task.dockerOpts ?= {}
|
|
|
|
_.merge(task.dockerOpts, buildOpts, { t: task.tag })
|
|
|
|
if d.image.context?.args?
|
|
|
|
task.dockerOpts.buildargs ?= {}
|
|
|
|
_.merge(task.dockerOpts.buildargs, d.image.context.args)
|
|
|
|
|
|
|
|
# Get the service-specific log stream
|
|
|
|
# Caveat: `multibuild.BuildTask` defines no `logStream` property
|
|
|
|
# but it's convenient to store it there; it's JS ultimately.
|
|
|
|
task.logStream = renderer.streams[task.serviceName]
|
|
|
|
task.logBuffer = []
|
|
|
|
|
|
|
|
# Setup emulation if needed
|
|
|
|
return [ task, null ] if task.external or not needsQemu
|
|
|
|
binPath = qemu.qemuPathInContext(path.join(projectPath, task.context))
|
2020-02-25 01:18:22 +00:00
|
|
|
transpose.transposeTarStream(
|
|
|
|
task.buildStream,
|
|
|
|
{
|
|
|
|
hostQemuPath: toPosixPath(binPath),
|
|
|
|
containerQemuPath: "/tmp/#{qemu.QEMU_BIN_NAME}",
|
|
|
|
qemuFileMode: 0o555,
|
|
|
|
},
|
2020-03-06 01:48:13 +00:00
|
|
|
dockerfilePath or undefined,
|
2020-02-25 01:18:22 +00:00
|
|
|
).then (stream) ->
|
2017-12-11 22:37:56 +00:00
|
|
|
task.buildStream = stream
|
|
|
|
.return([ task, binPath ])
|
|
|
|
.map ([ task, qemuPath ]) ->
|
|
|
|
Promise.resolve(task).tap (task) ->
|
|
|
|
captureStream = buildLogCapture(task.external, task.logBuffer)
|
|
|
|
|
|
|
|
if task.external
|
|
|
|
# External image -- there's no build to be performed,
|
|
|
|
# just follow pull progress.
|
|
|
|
captureStream.pipe(task.logStream)
|
|
|
|
task.progressHook = pullProgressAdapter(captureStream)
|
|
|
|
else
|
|
|
|
task.streamHook = (stream) ->
|
2018-03-15 13:16:33 +00:00
|
|
|
stream = createLogStream(stream)
|
2017-12-11 22:37:56 +00:00
|
|
|
if qemuPath?
|
|
|
|
buildThroughStream = transpose.getBuildThroughStream
|
|
|
|
hostQemuPath: toPosixPath(qemuPath)
|
|
|
|
containerQemuPath: "/tmp/#{qemu.QEMU_BIN_NAME}"
|
|
|
|
rawStream = stream.pipe(buildThroughStream)
|
|
|
|
else
|
|
|
|
rawStream = stream
|
|
|
|
# `stream` sends out raw strings in contrast to `task.progressHook`
|
|
|
|
# where we're given objects. capture these strings as they come
|
|
|
|
# before we parse them.
|
|
|
|
rawStream
|
2018-03-15 13:16:33 +00:00
|
|
|
.pipe(dropEmptyLinesStream())
|
2017-12-11 22:37:56 +00:00
|
|
|
.pipe(captureStream)
|
|
|
|
.pipe(buildProgressAdapter(inlineLogs))
|
|
|
|
.pipe(task.logStream)
|
|
|
|
.then (tasks) ->
|
|
|
|
logger.logDebug 'Prepared tasks; building...'
|
2019-07-09 14:43:36 +00:00
|
|
|
Promise.resolve(builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH))
|
2017-12-11 22:37:56 +00:00
|
|
|
.map (builtImage) ->
|
|
|
|
if not builtImage.successful
|
|
|
|
builtImage.error.serviceName = builtImage.serviceName
|
|
|
|
throw builtImage.error
|
|
|
|
|
|
|
|
d = imageDescriptorsByServiceName[builtImage.serviceName]
|
|
|
|
task = _.find(tasks, serviceName: builtImage.serviceName)
|
|
|
|
|
|
|
|
image =
|
|
|
|
serviceName: d.serviceName
|
|
|
|
name: d.image.tag ? d.image
|
|
|
|
logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX)
|
|
|
|
props:
|
|
|
|
dockerfile: builtImage.dockerfile
|
|
|
|
projectType: builtImage.projectType
|
|
|
|
|
|
|
|
# Times here are timestamps, so test whether they're null
|
|
|
|
# before creating a date out of them, as `new Date(null)`
|
|
|
|
# creates a date representing UNIX time 0.
|
|
|
|
if (startTime = builtImage.startTime)
|
|
|
|
image.props.startTime = new Date(startTime)
|
|
|
|
if (endTime = builtImage.endTime)
|
|
|
|
image.props.endTime = new Date(endTime)
|
|
|
|
docker.getImage(image.name).inspect().get('Size').then (size) ->
|
|
|
|
image.props.size = size
|
|
|
|
.return(image)
|
|
|
|
.tap (images) ->
|
|
|
|
summary = _(images).map ({ serviceName, props }) ->
|
|
|
|
[ serviceName, "Image size: #{humanize.filesize(props.size)}" ]
|
|
|
|
.fromPairs()
|
|
|
|
.value()
|
|
|
|
renderer.end(summary)
|
|
|
|
.finally(renderer.end)
|
|
|
|
|
|
|
|
createRelease = (apiEndpoint, auth, userId, appId, composition) ->
|
|
|
|
_ = require('lodash')
|
|
|
|
crypto = require('crypto')
|
2020-04-21 16:39:25 +00:00
|
|
|
releaseMod = require('balena-release')
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
client = releaseMod.createClient({ apiEndpoint, auth })
|
|
|
|
|
|
|
|
releaseMod.create
|
|
|
|
client: client
|
|
|
|
user: userId
|
|
|
|
application: appId
|
|
|
|
composition: composition
|
|
|
|
source: 'local'
|
|
|
|
commit: crypto.pseudoRandomBytes(16).toString('hex').toLowerCase()
|
|
|
|
.then ({ release, serviceImages }) ->
|
|
|
|
release = _.omit(release, [
|
|
|
|
'created_at'
|
|
|
|
'belongs_to__application'
|
|
|
|
'is_created_by__user'
|
|
|
|
'__metadata'
|
|
|
|
])
|
2018-10-20 20:57:36 +00:00
|
|
|
serviceImages = _.mapValues serviceImages, (serviceImage) ->
|
|
|
|
_.omit(serviceImage, [
|
2017-12-11 22:37:56 +00:00
|
|
|
'created_at'
|
|
|
|
'is_a_build_of__service'
|
|
|
|
'__metadata'
|
|
|
|
])
|
|
|
|
return { client, release, serviceImages }
|
|
|
|
|
|
|
|
tagServiceImages = (docker, images, serviceImages) ->
|
|
|
|
Promise.map images, (d) ->
|
|
|
|
serviceImage = serviceImages[d.serviceName]
|
|
|
|
imageName = serviceImage.is_stored_at__image_location
|
2019-03-12 22:07:57 +00:00
|
|
|
# coffeelint: disable-next-line=check_scope ("Variable is assigned to but never read")
|
2017-12-11 22:37:56 +00:00
|
|
|
[ _match, registry, repo, tag = 'latest' ] = /(.*?)\/(.*?)(?::([^/]*))?$/.exec(imageName)
|
|
|
|
name = "#{registry}/#{repo}"
|
|
|
|
docker.getImage(d.name).tag({ repo: name, tag, force: true })
|
|
|
|
.then ->
|
|
|
|
docker.getImage("#{name}:#{tag}")
|
|
|
|
.then (localImage) ->
|
|
|
|
serviceName: d.serviceName
|
|
|
|
serviceImage: serviceImage
|
|
|
|
localImage: localImage
|
|
|
|
registry: registry
|
|
|
|
repo: repo
|
|
|
|
logs: d.logs
|
|
|
|
props: d.props
|
|
|
|
|
2018-12-28 15:47:38 +00:00
|
|
|
|
|
|
|
getPreviousRepos = (sdk, docker, logger, appID) ->
|
|
|
|
sdk.pine.get(
|
|
|
|
resource: 'release'
|
|
|
|
options:
|
|
|
|
$filter:
|
|
|
|
belongs_to__application: appID
|
|
|
|
status: 'success'
|
|
|
|
$select:
|
|
|
|
[ 'id' ]
|
|
|
|
$expand:
|
|
|
|
contains__image:
|
|
|
|
$expand: 'image'
|
|
|
|
$orderby: 'id desc'
|
|
|
|
$top: 1
|
|
|
|
)
|
|
|
|
.then (release) ->
|
|
|
|
# grab all images from the latest release, return all image locations in the registry
|
|
|
|
if release?.length > 0
|
|
|
|
images = release[0].contains__image
|
|
|
|
Promise.map images, (d) ->
|
|
|
|
imageName = d.image[0].is_stored_at__image_location
|
|
|
|
docker.getRegistryAndName(imageName)
|
|
|
|
.then ( registry ) ->
|
|
|
|
logger.logDebug("Requesting access to previously pushed image repo (#{registry.imageName})")
|
|
|
|
return registry.imageName
|
|
|
|
.catch (e) ->
|
|
|
|
logger.logDebug("Failed to access previously pushed image repo: #{e}")
|
|
|
|
|
|
|
|
authorizePush = (sdk, logger, tokenAuthEndpoint, registry, images, previousRepos) ->
|
2020-03-13 16:09:43 +00:00
|
|
|
if not Array.isArray(images)
|
2017-12-11 22:37:56 +00:00
|
|
|
images = [ images ]
|
|
|
|
|
2018-12-28 15:47:38 +00:00
|
|
|
images.push previousRepos...
|
2017-12-11 22:37:56 +00:00
|
|
|
sdk.request.send
|
|
|
|
baseUrl: tokenAuthEndpoint
|
|
|
|
url: '/auth/v1/token'
|
|
|
|
qs:
|
|
|
|
service: registry
|
|
|
|
scope: images.map (repo) ->
|
|
|
|
"repository:#{repo}:pull,push"
|
|
|
|
.get('body')
|
|
|
|
.get('token')
|
|
|
|
.catchReturn({})
|
|
|
|
|
|
|
|
pushAndUpdateServiceImages = (docker, token, images, afterEach) ->
|
|
|
|
{ DockerProgress } = require('docker-progress')
|
2019-03-04 15:08:58 +00:00
|
|
|
{ retry } = require('./helpers')
|
2017-12-11 22:37:56 +00:00
|
|
|
tty = require('./tty')(process.stdout)
|
|
|
|
|
|
|
|
opts = { authconfig: registrytoken: token }
|
|
|
|
|
|
|
|
progress = new DockerProgress(dockerToolbelt: docker)
|
2020-02-28 15:25:12 +00:00
|
|
|
renderer = pushProgressRenderer(tty, getChalk().blue('[Push]') + ' ')
|
2017-12-11 22:37:56 +00:00
|
|
|
reporters = progress.aggregateProgress(images.length, renderer)
|
|
|
|
|
|
|
|
Promise.using tty.cursorHidden(), ->
|
|
|
|
Promise.map images, ({ serviceImage, localImage, props, logs }, index) ->
|
|
|
|
Promise.join(
|
|
|
|
localImage.inspect().get('Size')
|
2019-03-04 15:08:58 +00:00
|
|
|
retry(
|
|
|
|
-> progress.push(localImage.name, reporters[index], opts)
|
|
|
|
3 # `times` - retry 3 times
|
|
|
|
localImage.name # `label` included in retry log messages
|
|
|
|
2000 # `delayMs` - wait 2 seconds before the 1st retry
|
|
|
|
1.4 # `backoffScaler` - wait multiplier for each retry
|
|
|
|
).finally(renderer.end)
|
2017-12-11 22:37:56 +00:00
|
|
|
(size, digest) ->
|
|
|
|
serviceImage.image_size = size
|
|
|
|
serviceImage.content_hash = digest
|
|
|
|
serviceImage.build_log = logs
|
|
|
|
serviceImage.dockerfile = props.dockerfile
|
|
|
|
serviceImage.project_type = props.projectType
|
|
|
|
serviceImage.start_timestamp = props.startTime if props.startTime
|
|
|
|
serviceImage.end_timestamp = props.endTime if props.endTime
|
|
|
|
serviceImage.push_timestamp = new Date()
|
|
|
|
serviceImage.status = 'success'
|
|
|
|
)
|
|
|
|
.tapCatch (e) ->
|
|
|
|
serviceImage.error_message = '' + e
|
|
|
|
serviceImage.status = 'failed'
|
|
|
|
.finally ->
|
|
|
|
afterEach?(serviceImage, props)
|
|
|
|
|
|
|
|
exports.deployProject = (
|
|
|
|
docker, logger,
|
|
|
|
composition, images,
|
|
|
|
appId, userId, auth,
|
|
|
|
apiEndpoint,
|
|
|
|
skipLogUpload
|
|
|
|
) ->
|
|
|
|
_ = require('lodash')
|
2020-04-21 16:39:25 +00:00
|
|
|
releaseMod = require('balena-release')
|
2017-12-11 22:37:56 +00:00
|
|
|
tty = require('./tty')(process.stdout)
|
|
|
|
|
2020-02-28 15:25:12 +00:00
|
|
|
prefix = getChalk().cyan('[Info]') + ' '
|
2017-12-11 22:37:56 +00:00
|
|
|
spinner = createSpinner()
|
|
|
|
runloop = runSpinner(tty, spinner, "#{prefix}Creating release...")
|
|
|
|
|
|
|
|
createRelease(apiEndpoint, auth, userId, appId, composition)
|
|
|
|
.finally(runloop.end)
|
|
|
|
.then ({ client, release, serviceImages }) ->
|
|
|
|
logger.logDebug('Tagging images...')
|
|
|
|
tagServiceImages(docker, images, serviceImages)
|
|
|
|
.tap (images) ->
|
|
|
|
logger.logDebug('Authorizing push...')
|
2020-02-27 14:55:30 +00:00
|
|
|
sdk = getBalenaSdk()
|
2018-12-28 15:47:38 +00:00
|
|
|
getPreviousRepos(sdk, docker, logger, appId)
|
|
|
|
.then (previousRepos) ->
|
|
|
|
authorizePush(sdk, logger, apiEndpoint, images[0].registry, _.map(images, 'repo'), previousRepos)
|
2017-12-11 22:37:56 +00:00
|
|
|
.then (token) ->
|
|
|
|
logger.logInfo('Pushing images to registry...')
|
|
|
|
pushAndUpdateServiceImages docker, token, images, (serviceImage) ->
|
|
|
|
logger.logDebug("Saving image #{serviceImage.is_stored_at__image_location}")
|
|
|
|
if skipLogUpload
|
|
|
|
delete serviceImage.build_log
|
|
|
|
releaseMod.updateImage(client, serviceImage.id, serviceImage)
|
|
|
|
.finally ->
|
|
|
|
logger.logDebug('Untagging images...')
|
|
|
|
Promise.map images, ({ localImage }) ->
|
|
|
|
localImage.remove()
|
|
|
|
.then ->
|
|
|
|
release.status = 'success'
|
2018-10-20 20:57:36 +00:00
|
|
|
.tapCatch ->
|
2017-12-11 22:37:56 +00:00
|
|
|
release.status = 'failed'
|
|
|
|
.finally ->
|
|
|
|
runloop = runSpinner(tty, spinner, "#{prefix}Saving release...")
|
|
|
|
release.end_timestamp = new Date()
|
|
|
|
releaseMod.updateRelease(client, release.id, release)
|
|
|
|
.finally(runloop.end)
|
|
|
|
.return(release)
|
|
|
|
|
|
|
|
# utilities
|
|
|
|
|
|
|
|
renderProgressBar = (percentage, stepCount) ->
|
|
|
|
_ = require('lodash')
|
|
|
|
percentage = _.clamp(percentage, 0, 100)
|
|
|
|
barCount = stepCount * percentage // 100
|
|
|
|
spaceCount = stepCount - barCount
|
|
|
|
bar = "[#{_.repeat('=', barCount)}>#{_.repeat(' ', spaceCount)}]"
|
|
|
|
return "#{bar} #{_.padStart(percentage, 3)}%"
|
|
|
|
|
|
|
|
pushProgressRenderer = (tty, prefix) ->
|
|
|
|
fn = (e) ->
|
|
|
|
{ error, percentage } = e
|
|
|
|
throw new Error(error) if error?
|
|
|
|
bar = renderProgressBar(percentage, 40)
|
|
|
|
tty.replaceLine("#{prefix}#{bar}\r")
|
|
|
|
fn.end = ->
|
|
|
|
tty.clearLine()
|
|
|
|
return fn
|
|
|
|
|
2018-03-15 13:16:33 +00:00
|
|
|
createLogStream = (input) ->
|
|
|
|
split = require('split')
|
|
|
|
stripAnsi = require('strip-ansi-stream')
|
|
|
|
return input.pipe(stripAnsi()).pipe(split())
|
|
|
|
|
|
|
|
dropEmptyLinesStream = ->
|
|
|
|
through = require('through2')
|
|
|
|
through (data, enc, cb) ->
|
|
|
|
str = data.toString('utf-8')
|
|
|
|
@push(str) if str.trim()
|
|
|
|
cb()
|
|
|
|
|
2017-12-11 22:37:56 +00:00
|
|
|
buildLogCapture = (objectMode, buffer) ->
|
|
|
|
through = require('through2')
|
|
|
|
|
|
|
|
through { objectMode }, (data, enc, cb) ->
|
|
|
|
# data from pull stream
|
|
|
|
if data.error
|
|
|
|
buffer.push("#{data.error}")
|
|
|
|
else if data.progress and data.status
|
|
|
|
buffer.push("#{data.progress}% #{data.status}")
|
|
|
|
else if data.status
|
|
|
|
buffer.push("#{data.status}")
|
|
|
|
|
|
|
|
# data from build stream
|
|
|
|
else
|
2018-03-15 13:16:33 +00:00
|
|
|
buffer.push(data)
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
cb(null, data)
|
|
|
|
|
|
|
|
buildProgressAdapter = (inline) ->
|
|
|
|
through = require('through2')
|
|
|
|
|
2018-03-15 13:16:33 +00:00
|
|
|
stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
[ step, numSteps, progress ] = [ null, null, undefined ]
|
|
|
|
|
|
|
|
through { objectMode: true }, (str, enc, cb) ->
|
|
|
|
return cb(null, str) if not str?
|
|
|
|
|
|
|
|
if inline
|
|
|
|
return cb(null, { status: str })
|
|
|
|
|
|
|
|
if /^Successfully tagged /.test(str)
|
|
|
|
progress = undefined
|
|
|
|
else
|
|
|
|
if (match = stepRegex.exec(str))
|
|
|
|
step = match[1]
|
2018-03-15 13:16:33 +00:00
|
|
|
numSteps ?= match[2]
|
2017-12-11 22:37:56 +00:00
|
|
|
str = match[3]
|
|
|
|
if step?
|
|
|
|
str = "Step #{step}/#{numSteps}: #{str}"
|
|
|
|
progress = parseInt(step, 10) * 100 // parseInt(numSteps, 10)
|
|
|
|
|
|
|
|
cb(null, { status: str, progress })
|
|
|
|
|
|
|
|
pullProgressAdapter = (outStream) ->
|
|
|
|
return ({ status, id, percentage, error, errorDetail }) ->
|
|
|
|
if status?
|
|
|
|
status = status.replace(/^Status: /, '')
|
|
|
|
if id?
|
|
|
|
status = "#{id}: #{status}"
|
|
|
|
if percentage is 100
|
|
|
|
percentage = undefined
|
|
|
|
outStream.write
|
|
|
|
status: status
|
|
|
|
progress: percentage
|
|
|
|
error: errorDetail?.message ? error
|
|
|
|
|
|
|
|
createSpinner = ->
|
|
|
|
chars = '|/-\\'
|
|
|
|
index = 0
|
|
|
|
-> chars[(index++) % chars.length]
|
|
|
|
|
|
|
|
runSpinner = (tty, spinner, msg) ->
|
|
|
|
runloop = createRunLoop ->
|
|
|
|
tty.clearLine()
|
|
|
|
tty.writeLine("#{msg} #{spinner()}")
|
|
|
|
tty.cursorUp()
|
|
|
|
runloop.onEnd = ->
|
|
|
|
tty.clearLine()
|
|
|
|
tty.writeLine(msg)
|
|
|
|
return runloop
|
|
|
|
|
|
|
|
createRunLoop = (tick) ->
|
|
|
|
timerId = setInterval(tick, 1000 / 10)
|
|
|
|
runloop = {
|
|
|
|
onEnd: ->
|
|
|
|
end: ->
|
|
|
|
clearInterval(timerId)
|
|
|
|
runloop.onEnd()
|
|
|
|
}
|
|
|
|
return runloop
|
|
|
|
|
|
|
|
class BuildProgressUI
|
|
|
|
constructor: (tty, descriptors) ->
|
|
|
|
_ = require('lodash')
|
|
|
|
through = require('through2')
|
|
|
|
|
|
|
|
eventHandler = @_handleEvent
|
|
|
|
services = _.map(descriptors, 'serviceName')
|
|
|
|
|
|
|
|
streams = _(services).map (service) ->
|
|
|
|
stream = through.obj (event, _enc, cb) ->
|
|
|
|
eventHandler(service, event)
|
|
|
|
cb()
|
2019-07-12 15:18:28 +00:00
|
|
|
stream.pipe(tty.stream, end: false)
|
2017-12-11 22:37:56 +00:00
|
|
|
[ service, stream ]
|
|
|
|
.fromPairs()
|
|
|
|
.value()
|
|
|
|
|
|
|
|
@_tty = tty
|
|
|
|
@_serviceToDataMap = {}
|
|
|
|
@_services = services
|
|
|
|
|
|
|
|
# Logger magically prefixes the log line with [Build] etc., but it doesn't
|
|
|
|
# work well with the spinner we're also showing. Manually build the prefix
|
|
|
|
# here and bypass the logger.
|
2020-02-28 15:25:12 +00:00
|
|
|
prefix = getChalk().blue('[Build]') + ' '
|
2017-12-11 22:37:56 +00:00
|
|
|
|
|
|
|
offset = 10 # account for escape sequences inserted for colouring
|
|
|
|
@_prefixWidth = offset + prefix.length + _.max(_.map(services, 'length'))
|
|
|
|
@_prefix = prefix
|
|
|
|
|
|
|
|
# these are to handle window wrapping
|
|
|
|
@_maxLineWidth = null
|
|
|
|
@_lineWidths = []
|
|
|
|
|
|
|
|
@_startTime = null
|
|
|
|
@_ended = false
|
|
|
|
@_cancelled = false
|
|
|
|
@_spinner = createSpinner()
|
|
|
|
|
|
|
|
@streams = streams
|
|
|
|
|
|
|
|
_handleEvent: (service, event) =>
|
|
|
|
@_serviceToDataMap[service] = event
|
|
|
|
|
|
|
|
_handleInterrupt: =>
|
|
|
|
@_cancelled = true
|
|
|
|
@end()
|
|
|
|
process.exit(130) # 128 + SIGINT
|
|
|
|
|
|
|
|
start: =>
|
|
|
|
process.on('SIGINT', @_handleInterrupt)
|
|
|
|
@_tty.hideCursor()
|
|
|
|
@_services.forEach (service) =>
|
|
|
|
@streams[service].write({ status: 'Preparing...' })
|
|
|
|
@_runloop = createRunLoop(@_display)
|
|
|
|
@_startTime = Date.now()
|
|
|
|
|
|
|
|
end: (summary = null) =>
|
|
|
|
return if @_ended
|
|
|
|
@_ended = true
|
|
|
|
process.removeListener('SIGINT', @_handleInterrupt)
|
|
|
|
@_runloop.end()
|
|
|
|
@_runloop = null
|
|
|
|
|
|
|
|
@_clear()
|
|
|
|
@_renderStatus(true)
|
|
|
|
@_renderSummary(summary ? @_getServiceSummary())
|
|
|
|
@_tty.showCursor()
|
|
|
|
|
|
|
|
_display: =>
|
|
|
|
@_clear()
|
|
|
|
@_renderStatus()
|
|
|
|
@_renderSummary(@_getServiceSummary())
|
|
|
|
@_tty.cursorUp(@_services.length + 1) # for status line
|
|
|
|
|
|
|
|
_clear: ->
|
|
|
|
@_tty.deleteToEnd()
|
|
|
|
@_maxLineWidth = @_tty.currentWindowSize().width
|
|
|
|
|
|
|
|
_getServiceSummary: ->
|
|
|
|
_ = require('lodash')
|
|
|
|
|
|
|
|
services = @_services
|
|
|
|
serviceToDataMap = @_serviceToDataMap
|
|
|
|
|
|
|
|
_(services).map (service) ->
|
|
|
|
{ status, progress, error } = serviceToDataMap[service] ? {}
|
|
|
|
if error
|
|
|
|
return "#{error}"
|
|
|
|
else if progress
|
|
|
|
bar = renderProgressBar(progress, 20)
|
|
|
|
return "#{bar} #{status}" if status
|
|
|
|
return "#{bar}"
|
|
|
|
else if status
|
|
|
|
return "#{status}"
|
|
|
|
else
|
|
|
|
return 'Waiting...'
|
|
|
|
.map (data, index) ->
|
|
|
|
[ services[index], data ]
|
|
|
|
.fromPairs()
|
|
|
|
.value()
|
|
|
|
|
|
|
|
_renderStatus: (end = false) ->
|
|
|
|
moment = require('moment')
|
|
|
|
require('moment-duration-format')(moment)
|
|
|
|
|
|
|
|
@_tty.clearLine()
|
|
|
|
@_tty.write(@_prefix)
|
|
|
|
if end and @_cancelled
|
|
|
|
@_tty.writeLine('Build cancelled')
|
|
|
|
else if end
|
|
|
|
serviceCount = @_services.length
|
|
|
|
serviceStr = if serviceCount is 1 then '1 service' else "#{serviceCount} services"
|
|
|
|
runTime = Date.now() - @_startTime
|
|
|
|
durationStr = moment.duration(runTime // 1000, 'seconds').format()
|
|
|
|
@_tty.writeLine("Built #{serviceStr} in #{durationStr}")
|
|
|
|
else
|
|
|
|
@_tty.writeLine("Building services... #{@_spinner()}")
|
|
|
|
|
|
|
|
_renderSummary: (serviceToStrMap) ->
|
|
|
|
_ = require('lodash')
|
2020-02-28 15:25:12 +00:00
|
|
|
chalk = getChalk()
|
2017-12-11 22:37:56 +00:00
|
|
|
truncate = require('cli-truncate')
|
|
|
|
strlen = require('string-width')
|
|
|
|
|
|
|
|
@_services.forEach (service, index) =>
|
|
|
|
str = _.padEnd(@_prefix + chalk.bold(service), @_prefixWidth)
|
|
|
|
str += serviceToStrMap[service]
|
|
|
|
if @_maxLineWidth?
|
|
|
|
str = truncate(str, @_maxLineWidth)
|
|
|
|
@_lineWidths[index] = strlen(str)
|
|
|
|
|
|
|
|
@_tty.clearLine()
|
|
|
|
@_tty.writeLine(str)
|
|
|
|
|
|
|
|
class BuildProgressInline
|
|
|
|
constructor: (outStream, descriptors) ->
|
|
|
|
_ = require('lodash')
|
|
|
|
through = require('through2')
|
|
|
|
|
|
|
|
services = _.map(descriptors, 'serviceName')
|
|
|
|
eventHandler = @_renderEvent
|
|
|
|
streams = _(services).map (service) ->
|
|
|
|
stream = through.obj (event, _enc, cb) ->
|
|
|
|
eventHandler(service, event)
|
|
|
|
cb()
|
2019-07-12 15:18:28 +00:00
|
|
|
stream.pipe(outStream, end: false)
|
2017-12-11 22:37:56 +00:00
|
|
|
[ service, stream ]
|
|
|
|
.fromPairs()
|
|
|
|
.value()
|
|
|
|
|
|
|
|
offset = 10 # account for escape sequences inserted for colouring
|
|
|
|
@_prefixWidth = offset + _.max(_.map(services, 'length'))
|
|
|
|
@_outStream = outStream
|
|
|
|
@_services = services
|
|
|
|
@_startTime = null
|
|
|
|
@_ended = false
|
|
|
|
|
|
|
|
@streams = streams
|
|
|
|
|
|
|
|
start: =>
|
|
|
|
@_outStream.write('Building services...\n')
|
|
|
|
@_services.forEach (service) =>
|
|
|
|
@streams[service].write({ status: 'Preparing...' })
|
|
|
|
@_startTime = Date.now()
|
|
|
|
|
|
|
|
end: (summary = null) =>
|
|
|
|
moment = require('moment')
|
|
|
|
require('moment-duration-format')(moment)
|
|
|
|
|
|
|
|
return if @_ended
|
|
|
|
@_ended = true
|
|
|
|
|
|
|
|
if summary?
|
|
|
|
@_services.forEach (service) =>
|
|
|
|
@_renderEvent(service, summary[service])
|
|
|
|
|
|
|
|
if @_cancelled
|
|
|
|
@_outStream.write('Build cancelled\n')
|
|
|
|
else
|
|
|
|
serviceCount = @_services.length
|
|
|
|
serviceStr = if serviceCount is 1 then '1 service' else "#{serviceCount} services"
|
|
|
|
runTime = Date.now() - @_startTime
|
|
|
|
durationStr = moment.duration(runTime // 1000, 'seconds').format()
|
|
|
|
@_outStream.write("Built #{serviceStr} in #{durationStr}\n")
|
|
|
|
|
|
|
|
_renderEvent: (service, event) =>
|
|
|
|
_ = require('lodash')
|
|
|
|
|
|
|
|
str = do ->
|
|
|
|
{ status, error } = event
|
|
|
|
if error
|
|
|
|
return "#{error}"
|
|
|
|
else if status
|
|
|
|
return "#{status}"
|
|
|
|
else
|
|
|
|
return 'Waiting...'
|
|
|
|
|
2020-02-28 15:25:12 +00:00
|
|
|
prefix = _.padEnd(getChalk().bold(service), @_prefixWidth)
|
2017-12-11 22:37:56 +00:00
|
|
|
@_outStream.write(prefix)
|
|
|
|
@_outStream.write(str)
|
|
|
|
@_outStream.write('\n')
|