diff --git a/CHANGELOG.md b/CHANGELOG.md index 1488a5d1..76601ce4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +### Added + +- Add uploading of build logs when present with resin deploy + ## [5.9.1] - 2017-05-01 ### Fixed diff --git a/build/actions/deploy.js b/build/actions/deploy.js index fe7e8653..40bdddfb 100644 --- a/build/actions/deploy.js +++ b/build/actions/deploy.js @@ -1,15 +1,29 @@ // Generated by CoffeeScript 1.12.5 -var Promise, dockerUtils, formatImageName, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, uploadToPromise; +var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, uploadLogs, uploadToPromise; Promise = require('bluebird'); dockerUtils = require('../utils/docker'); getBuilderPushEndpoint = function(baseUrl, owner, app) { - var escApp, escOwner; - escOwner = encodeURIComponent(owner); - escApp = encodeURIComponent(app); - return "https://builder." + baseUrl + "/v1/push?owner=" + escOwner + "&app=" + escApp; + var args, querystring; + querystring = require('querystring'); + args = querystring.stringify({ + owner: owner, + app: app + }); + return "https://builder." + baseUrl + "/v1/push?" + args; +}; + +getBuilderLogPushEndpoint = function(baseUrl, buildId, owner, app) { + var args, querystring; + querystring = require('querystring'); + args = querystring.stringify({ + owner: owner, + app: app, + buildId: buildId + }); + return "https://builder." + baseUrl + "/v1/pushLogs?" + args; }; formatImageName = function(image) { @@ -71,7 +85,6 @@ getBundleInfo = function(options) { performUpload = function(image, token, username, url, size, appName, logStreams) { var post, request; request = require('request'); - url = url || process.env.RESINRC_RESIN_URL; post = request.post({ url: getBuilderPushEndpoint(url, username, appName), auth: { @@ -82,30 +95,47 @@ performUpload = function(image, token, username, url, size, appName, logStreams) return uploadToPromise(post, size, logStreams); }; +uploadLogs = function(logs, token, url, buildId, username, appName) { + var request; + request = require('request'); + return request.post({ + url: getBuilderLogPushEndpoint(url, buildId, username, appName), + auth: { + bearer: token + }, + body: Buffer.from(logs).toString('base64') + }); +}; + uploadToPromise = function(request, size, logStreams) { var logging; logging = require('../utils/logging'); return new Promise(function(resolve, reject) { var handleMessage; handleMessage = function(data) { - var obj; data = data.toString(); logging.logDebug(logStreams, "Received data: " + data); - obj = JSON.parse(data); - if (obj.type != null) { - switch (obj.type) { - case 'error': - return reject(new Error("Remote error: " + obj.error)); - case 'success': - return resolve(obj.image); - case 'status': - return logging.logInfo(logStreams, "Remote: " + obj.message); - default: - return reject(new Error("Received unexpected reply from remote: " + data)); + return Promise["try"](function() { + var obj; + obj = JSON.parse(data); + if (obj.type != null) { + switch (obj.type) { + case 'error': + return reject(new Error("Remote error: " + obj.error)); + case 'success': + return resolve(obj); + case 'status': + return logging.logInfo(logStreams, "Remote: " + obj.message); + default: + return reject(new Error("Received unexpected reply from remote: " + data)); + } + } else { + return reject(new Error("Received unexpected reply from remote: " + data)); } - } else { - return reject(new Error("Received unexpected reply from remote: " + data)); - } + })["catch"](function(e) { + logging.logError(logStreams, 'Error parsing reply from remote side'); + return reject(e); + }); }; request.on('error', reject).on('data', handleMessage); return pushProgress(size, request, logStreams); @@ -128,10 +158,14 @@ module.exports = { parameter: 'source', description: 'The source directory to use when building the image', alias: 's' + }, { + signature: 'nologupload', + description: "Don't upload build logs to the dashboard with image (if building)", + boolean: true } ]), action: function(params, options, done) { - var _, docker, logStreams, logging, resin, tmp, tmpNameAsync; + var _, logStreams, logging, logs, resin, tmp, tmpNameAsync, upload; _ = require('lodash'); tmp = require('tmp'); tmpNameAsync = Promise.promisify(tmp.tmpName); @@ -139,31 +173,57 @@ module.exports = { logging = require('../utils/logging'); logStreams = logging.getLogStreams(); tmp.setGracefulCleanup(); - docker = dockerUtils.getDocker(options); - return parseInput(params, options).then(function(arg) { - var appName, build, imageName, source; - appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3]; - return tmpNameAsync().then(function(tmpPath) { - options = _.assign({}, options, { - appName: appName + logs = ''; + upload = function(token, username, url) { + var docker; + docker = dockerUtils.getDocker(options); + return parseInput(params, options).then(function(arg) { + var appName, build, imageName, source; + appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3]; + return tmpNameAsync().then(function(tmpPath) { + options = _.assign({}, options, { + appName: appName + }); + params = _.assign({}, params, { + source: source + }); + return Promise["try"](function() { + if (build) { + return dockerUtils.runBuild(params, options, getBundleInfo, logStreams); + } else { + return { + image: imageName, + log: '' + }; + } + }).then(function(arg1) { + var buildLogs, imageName; + imageName = arg1.image, buildLogs = arg1.log; + logs = buildLogs; + return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload); + })["finally"](function() { + return require('mz/fs').unlink(tmpPath)["catch"](_.noop); + }); }); - params = _.assign({}, params, { - source: source - }); - return Promise["try"](function() { - if (build) { - return dockerUtils.runBuild(params, options, getBundleInfo, logStreams); - } else { - return imageName; - } - }).then(function(imageName) { - return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), resin.auth.getToken(), resin.auth.whoami(), resin.settings.get('resinUrl'), dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload); - })["finally"](function() { - return require('fs').unlink(tmpPath); - }); - }); - }).then(function(imageName) { - return logging.logSuccess(logStreams, "Successfully deployed image: " + (formatImageName(imageName))); - }).asCallback(done); + }).tap(function(arg) { + var buildId, imageName; + imageName = arg.image, buildId = arg.buildId; + logging.logSuccess(logStreams, "Successfully deployed image: " + (formatImageName(imageName))); + return buildId; + }).then(function(arg) { + var buildId, imageName; + imageName = arg.image, buildId = arg.buildId; + if (logs === '' || (options.nologupload != null)) { + return ''; + } + logging.logInfo(logStreams, 'Uploading logs to dashboard...'); + return Promise.join(logs, token, url, buildId, username, params.appName, uploadLogs)["return"]('Successfully uploaded logs'); + }).then(function(msg) { + if (msg !== '') { + return logging.logSuccess(logStreams, msg); + } + }).asCallback(done); + }; + return Promise.join(resin.auth.getToken(), resin.auth.whoami(), resin.settings.get('resinUrl'), upload); } }; diff --git a/build/utils/docker.js b/build/utils/docker.js index 44213bbf..22168097 100644 --- a/build/utils/docker.js +++ b/build/utils/docker.js @@ -99,14 +99,16 @@ exports.tarDirectory = tarDirectory = function(dir) { }; exports.runBuild = function(params, options, getBundleInfo, logStreams) { - var Promise, dockerBuild, logging, resolver; + var Promise, dockerBuild, es, logging, logs, resolver; Promise = require('bluebird'); dockerBuild = require('resin-docker-build'); resolver = require('resin-bundle-resolve'); + es = require('event-stream'); logging = require('../utils/logging'); if (params.source == null) { params.source = '.'; } + logs = ''; return tarDirectory(params.source).then(function(tarStream) { return new Promise(function(resolve, reject) { var builder, connectOpts, hooks, opts; @@ -115,10 +117,14 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) { if (options.tag != null) { console.log("Tagging image as " + options.tag); } - return resolve(image); + return resolve({ + image: image, + log: logs + }); }, buildFailure: reject, buildStream: function(stream) { + var throughStream; getBundleInfo(options).then(function(info) { var arch, bundle, deviceType; if (info == null) { @@ -133,7 +139,11 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) { }); } })["catch"](reject); - return stream.pipe(logStreams.build); + throughStream = es.through(function(data) { + logs += data.toString(); + return this.emit('data', data); + }); + return stream.pipe(es.pipe(throughStream, logStreams.build)); } }; connectOpts = generateConnectOpts(options); @@ -160,12 +170,7 @@ exports.bufferImage = function(docker, imageId, tmpFile) { image = docker.getImage(imageId); return image.get().then(function(img) { return new Promise(function(resolve, reject) { - return img.on('error', reject).on('data', function(data) { - return stream.write(data); - }).on('end', function() { - stream.close(); - return resolve(); - }); + return img.on('error', reject).on('end', resolve).pipe(stream); }); }).then(function() { return new Promise(function(resolve, reject) { diff --git a/build/utils/logging.js b/build/utils/logging.js index a39b49d2..634fad51 100644 --- a/build/utils/logging.js +++ b/build/utils/logging.js @@ -14,12 +14,14 @@ exports.getLogStreams = function() { logger.addPrefix('debug', colors.magenta('[Debug]')); logger.addPrefix('success', colors.green('[Success]')); logger.addPrefix('warn', colors.yellow('[Warn]')); + logger.addPrefix('error', colors.red('[Error]')); streams = { build: logger.createLogStream('build'), info: logger.createLogStream('info'), debug: logger.createLogStream('debug'), success: logger.createLogStream('success'), - warn: logger.createLogStream('warn') + warn: logger.createLogStream('warn'), + error: logger.createLogStream('error') }; _.mapKeys(streams, function(stream, key) { if (key !== 'debug') { @@ -48,3 +50,7 @@ exports.logSuccess = function(logStreams, msg) { exports.logWarn = function(logStreams, msg) { return logStreams.warn.write(msg + eol); }; + +exports.logError = function(logStreams, msg) { + return logStreams.error.write(msg + eol); +}; diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 55df9206..14fd7ede 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -2,9 +2,14 @@ Promise = require('bluebird') dockerUtils = require('../utils/docker') getBuilderPushEndpoint = (baseUrl, owner, app) -> - escOwner = encodeURIComponent(owner) - escApp = encodeURIComponent(app) - "https://builder.#{baseUrl}/v1/push?owner=#{escOwner}&app=#{escApp}" + querystring = require('querystring') + args = querystring.stringify({ owner, app }) + "https://builder.#{baseUrl}/v1/push?#{args}" + +getBuilderLogPushEndpoint = (baseUrl, buildId, owner, app) -> + querystring = require('querystring') + args = querystring.stringify({ owner, app, buildId }) + "https://builder.#{baseUrl}/v1/pushLogs?#{args}" formatImageName = (image) -> image.split('/').pop() @@ -52,7 +57,6 @@ getBundleInfo = (options) -> performUpload = (image, token, username, url, size, appName, logStreams) -> request = require('request') - url = url || process.env.RESINRC_RESIN_URL post = request.post url: getBuilderPushEndpoint(url, username, appName) auth: @@ -61,19 +65,35 @@ performUpload = (image, token, username, url, size, appName, logStreams) -> uploadToPromise(post, size, logStreams) +uploadLogs = (logs, token, url, buildId, username, appName) -> + request = require('request') + request.post + json: true + url: getBuilderLogPushEndpoint(url, buildId, username, appName) + auth: + bearer: token + body: Buffer.from(logs) + uploadToPromise = (request, size, logStreams) -> logging = require('../utils/logging') + new Promise (resolve, reject) -> handleMessage = (data) -> data = data.toString() logging.logDebug(logStreams, "Received data: #{data}") - obj = JSON.parse(data) + try + obj = JSON.parse(data) + catch e + logging.logError(logStreams, 'Error parsing reply from remote side') + reject(e) + return + if obj.type? switch obj.type when 'error' then reject(new Error("Remote error: #{obj.error}")) - when 'success' then resolve(obj.image) + when 'success' then resolve(obj) when 'status' then logging.logInfo(logStreams, "Remote: #{obj.message}") else reject(new Error("Received unexpected reply from remote: #{data}")) else @@ -115,6 +135,11 @@ module.exports = parameter: 'source' description: 'The source directory to use when building the image' alias: 's' + }, + { + signature: 'nologupload' + description: "Don't upload build logs to the dashboard with image (if building)" + boolean: true } ] action: (params, options, done) -> @@ -130,35 +155,69 @@ module.exports = # Ensure the tmp files gets deleted tmp.setGracefulCleanup() - docker = dockerUtils.getDocker(options) - # Check input parameters - parseInput(params, options) - .then ([appName, build, source, imageName]) -> - tmpNameAsync() - .then (tmpPath) -> + logs = '' - # Setup the build args for how the build routine expects them - options = _.assign({}, options, { appName }) - params = _.assign({}, params, { source }) + upload = (token, username, url) -> + docker = dockerUtils.getDocker(options) + # Check input parameters + parseInput(params, options) + .then ([appName, build, source, imageName]) -> + tmpNameAsync() + .then (tmpPath) -> - Promise.try -> - if build - dockerUtils.runBuild(params, options, getBundleInfo, logStreams) - else - imageName - .then (imageName) -> - Promise.join( - dockerUtils.bufferImage(docker, imageName, tmpPath) - resin.auth.getToken() - resin.auth.whoami() - resin.settings.get('resinUrl') - dockerUtils.getImageSize(docker, imageName) - params.appName - logStreams - performUpload - ) - .finally -> - require('fs').unlink(tmpPath) - .then (imageName) -> - logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}") - .asCallback(done) + # Setup the build args for how the build routine expects them + options = _.assign({}, options, { appName }) + params = _.assign({}, params, { source }) + + Promise.try -> + if build + dockerUtils.runBuild(params, options, getBundleInfo, logStreams) + else + { image: imageName, log: '' } + .then ({ image: imageName, log: buildLogs }) -> + logs = buildLogs + Promise.join( + dockerUtils.bufferImage(docker, imageName, tmpPath) + token + username + url + dockerUtils.getImageSize(docker, imageName) + params.appName + logStreams + performUpload + ) + .finally -> + # If the file was never written to (for instance because an error + # has occured before any data was written) this call will throw an + # ugly error, just suppress it + require('mz/fs').unlink(tmpPath) + .catch(_.noop) + .tap ({ image: imageName, buildId }) -> + logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}") + return buildId + .then ({ image: imageName, buildId }) -> + if logs is '' or options.nologupload? + return '' + + logging.logInfo(logStreams, 'Uploading logs to dashboard...') + + Promise.join( + logs + token + url + buildId + username + params.appName + uploadLogs + ) + .return('Successfully uploaded logs') + .then (msg) -> + logging.logSuccess(logStreams, msg) if msg isnt '' + .asCallback(done) + + Promise.join( + resin.auth.getToken() + resin.auth.whoami() + resin.settings.get('resinUrl') + upload + ) diff --git a/lib/utils/docker.coffee b/lib/utils/docker.coffee index 4bb7147c..f3f54c5e 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker.coffee @@ -117,10 +117,13 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) -> Promise = require('bluebird') dockerBuild = require('resin-docker-build') resolver = require('resin-bundle-resolve') + es = require('event-stream') + logging = require('../utils/logging') # The default build context is the current directory params.source ?= '.' + logs = '' # Tar up the directory, ready for the build stream tarDirectory(params.source) @@ -130,7 +133,7 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) -> buildSuccess: (image) -> if options.tag? console.log("Tagging image as #{options.tag}") - resolve(image) + resolve({ image, log: logs } ) buildFailure: reject buildStream: (stream) -> getBundleInfo(options) @@ -152,7 +155,12 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) -> resolved.tarStream.pipe(stream) .catch(reject) - stream.pipe(logStreams.build) + # And print the output + throughStream = es.through (data) -> + logs += data.toString() + this.emit('data', data) + + stream.pipe(es.pipe(throughStream, logStreams.build)) # Create a builder connectOpts = generateConnectOpts(options) @@ -186,11 +194,8 @@ exports.bufferImage = (docker, imageId, tmpFile) -> new Promise (resolve, reject) -> img .on('error', reject) - .on 'data', (data) -> - stream.write(data) - .on 'end', -> - stream.close() - resolve() + .on('end', resolve) + .pipe(stream) .then -> new Promise (resolve, reject) -> fs.createReadStream(tmpFile) diff --git a/lib/utils/logging.coffee b/lib/utils/logging.coffee index 5166c2c2..271e8e22 100644 --- a/lib/utils/logging.coffee +++ b/lib/utils/logging.coffee @@ -11,13 +11,15 @@ exports.getLogStreams = -> logger.addPrefix('debug', colors.magenta('[Debug]')) logger.addPrefix('success', colors.green('[Success]')) logger.addPrefix('warn', colors.yellow('[Warn]')) + logger.addPrefix('error', colors.red('[Error]')) streams = build: logger.createLogStream('build'), info: logger.createLogStream('info'), debug: logger.createLogStream('debug'), success: logger.createLogStream('success'), - warn: logger.createLogStream('warn') + warn: logger.createLogStream('warn'), + error: logger.createLogStream('error') _.mapKeys streams, (stream, key) -> if key isnt 'debug' @@ -38,3 +40,6 @@ exports.logSuccess = (logStreams, msg) -> exports.logWarn = (logStreams, msg) -> logStreams.warn.write(msg + eol) + +exports.logError = (logStreams, msg) -> + logStreams.error.write(msg + eol)