Allow resin-cli deploy to also upload build logs if present

If build is ran through `resin deploy`, then logs will be stored and
uploaded to the database, where the dashboard can display them

Change-type: minor
Signed-off-by: Cameron Diver <cameron@resin.io>

CHANGELOG.md

@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+### Added
+
+- Add uploading of build logs when present with resin deploy
+
 ## [5.9.1] - 2017-05-01
 
 ### Fixed

build/actions/deploy.js

@@ -1,15 +1,29 @@
 // Generated by CoffeeScript 1.12.5
-var Promise, dockerUtils, formatImageName, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, uploadToPromise;
+var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, uploadLogs, uploadToPromise;
 
 Promise = require('bluebird');
 
 dockerUtils = require('../utils/docker');
 
 getBuilderPushEndpoint = function(baseUrl, owner, app) {
-  var escApp, escOwner;
+  var args, querystring;
-  escOwner = encodeURIComponent(owner);
+  querystring = require('querystring');
-  escApp = encodeURIComponent(app);
+  args = querystring.stringify({
-  return "https://builder." + baseUrl + "/v1/push?owner=" + escOwner + "&app=" + escApp;
+    owner: owner,
+    app: app
+  });
+  return "https://builder." + baseUrl + "/v1/push?" + args;
+};
+
+getBuilderLogPushEndpoint = function(baseUrl, buildId, owner, app) {
+  var args, querystring;
+  querystring = require('querystring');
+  args = querystring.stringify({
+    owner: owner,
+    app: app,
+    buildId: buildId
+  });
+  return "https://builder." + baseUrl + "/v1/pushLogs?" + args;
 };
 
 formatImageName = function(image) {
@@ -71,7 +85,6 @@ getBundleInfo = function(options) {
 performUpload = function(image, token, username, url, size, appName, logStreams) {
   var post, request;
   request = require('request');
-  url = url || process.env.RESINRC_RESIN_URL;
   post = request.post({
     url: getBuilderPushEndpoint(url, username, appName),
     auth: {
@@ -82,30 +95,47 @@ performUpload = function(image, token, username, url, size, appName, logStreams)
   return uploadToPromise(post, size, logStreams);
 };
 
+uploadLogs = function(logs, token, url, buildId, username, appName) {
+  var request;
+  request = require('request');
+  return request.post({
+    url: getBuilderLogPushEndpoint(url, buildId, username, appName),
+    auth: {
+      bearer: token
+    },
+    body: Buffer.from(logs).toString('base64')
+  });
+};
+
 uploadToPromise = function(request, size, logStreams) {
   var logging;
   logging = require('../utils/logging');
   return new Promise(function(resolve, reject) {
     var handleMessage;
     handleMessage = function(data) {
-      var obj;
       data = data.toString();
       logging.logDebug(logStreams, "Received data: " + data);
-      obj = JSON.parse(data);
+      return Promise["try"](function() {
-      if (obj.type != null) {
+        var obj;
-        switch (obj.type) {
+        obj = JSON.parse(data);
-          case 'error':
+        if (obj.type != null) {
-            return reject(new Error("Remote error: " + obj.error));
+          switch (obj.type) {
-          case 'success':
+            case 'error':
-            return resolve(obj.image);
+              return reject(new Error("Remote error: " + obj.error));
-          case 'status':
+            case 'success':
-            return logging.logInfo(logStreams, "Remote: " + obj.message);
+              return resolve(obj);
-          default:
+            case 'status':
-            return reject(new Error("Received unexpected reply from remote: " + data));
+              return logging.logInfo(logStreams, "Remote: " + obj.message);
+            default:
+              return reject(new Error("Received unexpected reply from remote: " + data));
+          }
+        } else {
+          return reject(new Error("Received unexpected reply from remote: " + data));
         }
-      } else {
+      })["catch"](function(e) {
-        return reject(new Error("Received unexpected reply from remote: " + data));
+        logging.logError(logStreams, 'Error parsing reply from remote side');
-      }
+        return reject(e);
+      });
     };
     request.on('error', reject).on('data', handleMessage);
     return pushProgress(size, request, logStreams);
@@ -128,10 +158,14 @@ module.exports = {
       parameter: 'source',
       description: 'The source directory to use when building the image',
       alias: 's'
+    }, {
+      signature: 'nologupload',
+      description: "Don't upload build logs to the dashboard with image (if building)",
+      boolean: true
     }
   ]),
   action: function(params, options, done) {
-    var _, docker, logStreams, logging, resin, tmp, tmpNameAsync;
+    var _, logStreams, logging, logs, resin, tmp, tmpNameAsync, upload;
     _ = require('lodash');
     tmp = require('tmp');
     tmpNameAsync = Promise.promisify(tmp.tmpName);
@@ -139,31 +173,57 @@ module.exports = {
     logging = require('../utils/logging');
     logStreams = logging.getLogStreams();
     tmp.setGracefulCleanup();
-    docker = dockerUtils.getDocker(options);
+    logs = '';
-    return parseInput(params, options).then(function(arg) {
+    upload = function(token, username, url) {
-      var appName, build, imageName, source;
+      var docker;
-      appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3];
+      docker = dockerUtils.getDocker(options);
-      return tmpNameAsync().then(function(tmpPath) {
+      return parseInput(params, options).then(function(arg) {
-        options = _.assign({}, options, {
+        var appName, build, imageName, source;
-          appName: appName
+        appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3];
+        return tmpNameAsync().then(function(tmpPath) {
+          options = _.assign({}, options, {
+            appName: appName
+          });
+          params = _.assign({}, params, {
+            source: source
+          });
+          return Promise["try"](function() {
+            if (build) {
+              return dockerUtils.runBuild(params, options, getBundleInfo, logStreams);
+            } else {
+              return {
+                image: imageName,
+                log: ''
+              };
+            }
+          }).then(function(arg1) {
+            var buildLogs, imageName;
+            imageName = arg1.image, buildLogs = arg1.log;
+            logs = buildLogs;
+            return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload);
+          })["finally"](function() {
+            return require('mz/fs').unlink(tmpPath)["catch"](_.noop);
+          });
         });
-        params = _.assign({}, params, {
+      }).tap(function(arg) {
-          source: source
+        var buildId, imageName;
-        });
+        imageName = arg.image, buildId = arg.buildId;
-        return Promise["try"](function() {
+        logging.logSuccess(logStreams, "Successfully deployed image: " + (formatImageName(imageName)));
-          if (build) {
+        return buildId;
-            return dockerUtils.runBuild(params, options, getBundleInfo, logStreams);
+      }).then(function(arg) {
-          } else {
+        var buildId, imageName;
-            return imageName;
+        imageName = arg.image, buildId = arg.buildId;
-          }
+        if (logs === '' || (options.nologupload != null)) {
-        }).then(function(imageName) {
+          return '';
-          return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), resin.auth.getToken(), resin.auth.whoami(), resin.settings.get('resinUrl'), dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload);
+        }
-        })["finally"](function() {
+        logging.logInfo(logStreams, 'Uploading logs to dashboard...');
-          return require('fs').unlink(tmpPath);
+        return Promise.join(logs, token, url, buildId, username, params.appName, uploadLogs)["return"]('Successfully uploaded logs');
-        });
+      }).then(function(msg) {
-      });
+        if (msg !== '') {
-    }).then(function(imageName) {
+          return logging.logSuccess(logStreams, msg);
-      return logging.logSuccess(logStreams, "Successfully deployed image: " + (formatImageName(imageName)));
+        }
-    }).asCallback(done);
+      }).asCallback(done);
+    };
+    return Promise.join(resin.auth.getToken(), resin.auth.whoami(), resin.settings.get('resinUrl'), upload);
   }
 };

build/utils/docker.js

@@ -99,14 +99,16 @@ exports.tarDirectory = tarDirectory = function(dir) {
 };
 
 exports.runBuild = function(params, options, getBundleInfo, logStreams) {
-  var Promise, dockerBuild, logging, resolver;
+  var Promise, dockerBuild, es, logging, logs, resolver;
   Promise = require('bluebird');
   dockerBuild = require('resin-docker-build');
   resolver = require('resin-bundle-resolve');
+  es = require('event-stream');
   logging = require('../utils/logging');
   if (params.source == null) {
     params.source = '.';
   }
+  logs = '';
   return tarDirectory(params.source).then(function(tarStream) {
     return new Promise(function(resolve, reject) {
       var builder, connectOpts, hooks, opts;
@@ -115,10 +117,14 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) {
           if (options.tag != null) {
             console.log("Tagging image as " + options.tag);
           }
-          return resolve(image);
+          return resolve({
+            image: image,
+            log: logs
+          });
         },
         buildFailure: reject,
         buildStream: function(stream) {
+          var throughStream;
           getBundleInfo(options).then(function(info) {
             var arch, bundle, deviceType;
             if (info == null) {
@@ -133,7 +139,11 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) {
               });
             }
           })["catch"](reject);
-          return stream.pipe(logStreams.build);
+          throughStream = es.through(function(data) {
+            logs += data.toString();
+            return this.emit('data', data);
+          });
+          return stream.pipe(es.pipe(throughStream, logStreams.build));
         }
       };
       connectOpts = generateConnectOpts(options);
@@ -160,12 +170,7 @@ exports.bufferImage = function(docker, imageId, tmpFile) {
   image = docker.getImage(imageId);
   return image.get().then(function(img) {
     return new Promise(function(resolve, reject) {
-      return img.on('error', reject).on('data', function(data) {
+      return img.on('error', reject).on('end', resolve).pipe(stream);
-        return stream.write(data);
-      }).on('end', function() {
-        stream.close();
-        return resolve();
-      });
     });
   }).then(function() {
     return new Promise(function(resolve, reject) {

build/utils/logging.js

@@ -14,12 +14,14 @@ exports.getLogStreams = function() {
   logger.addPrefix('debug', colors.magenta('[Debug]'));
   logger.addPrefix('success', colors.green('[Success]'));
   logger.addPrefix('warn', colors.yellow('[Warn]'));
+  logger.addPrefix('error', colors.red('[Error]'));
   streams = {
     build: logger.createLogStream('build'),
     info: logger.createLogStream('info'),
     debug: logger.createLogStream('debug'),
     success: logger.createLogStream('success'),
-    warn: logger.createLogStream('warn')
+    warn: logger.createLogStream('warn'),
+    error: logger.createLogStream('error')
   };
   _.mapKeys(streams, function(stream, key) {
     if (key !== 'debug') {
@@ -48,3 +50,7 @@ exports.logSuccess = function(logStreams, msg) {
 exports.logWarn = function(logStreams, msg) {
   return logStreams.warn.write(msg + eol);
 };
+
+exports.logError = function(logStreams, msg) {
+  return logStreams.error.write(msg + eol);
+};

lib/actions/deploy.coffee

@@ -2,9 +2,14 @@ Promise = require('bluebird')
 dockerUtils = require('../utils/docker')
 
 getBuilderPushEndpoint = (baseUrl, owner, app) ->
-	escOwner = encodeURIComponent(owner)
+	querystring = require('querystring')
-	escApp = encodeURIComponent(app)
+	args = querystring.stringify({ owner, app })
-	"https://builder.#{baseUrl}/v1/push?owner=#{escOwner}&app=#{escApp}"
+	"https://builder.#{baseUrl}/v1/push?#{args}"
+
+getBuilderLogPushEndpoint = (baseUrl, buildId, owner, app) ->
+	querystring = require('querystring')
+	args = querystring.stringify({ owner, app, buildId })
+	"https://builder.#{baseUrl}/v1/pushLogs?#{args}"
 
 formatImageName = (image) ->
 	image.split('/').pop()
@@ -52,7 +57,6 @@ getBundleInfo = (options) ->
 
 performUpload = (image, token, username, url, size, appName, logStreams) ->
 	request = require('request')
-	url = url || process.env.RESINRC_RESIN_URL
 	post = request.post
 		url: getBuilderPushEndpoint(url, username, appName)
 		auth:
@@ -61,19 +65,35 @@ performUpload = (image, token, username, url, size, appName, logStreams) ->
 
 	uploadToPromise(post, size, logStreams)
 
+uploadLogs = (logs, token, url, buildId, username, appName) ->
+	request = require('request')
+	request.post
+		json: true
+		url: getBuilderLogPushEndpoint(url, buildId, username, appName)
+		auth:
+			bearer: token
+		body: Buffer.from(logs)
+
 uploadToPromise = (request, size, logStreams) ->
 	logging = require('../utils/logging')
+
 	new Promise (resolve, reject) ->
 
 		handleMessage = (data) ->
 			data = data.toString()
 			logging.logDebug(logStreams, "Received data: #{data}")
 
-			obj = JSON.parse(data)
+			try
+				obj = JSON.parse(data)
+			catch e
+				logging.logError(logStreams, 'Error parsing reply from remote side')
+				reject(e)
+				return
+
 			if obj.type?
 				switch obj.type
 					when 'error' then reject(new Error("Remote error: #{obj.error}"))
-					when 'success' then resolve(obj.image)
+					when 'success' then resolve(obj)
 					when 'status' then logging.logInfo(logStreams, "Remote: #{obj.message}")
 					else reject(new Error("Received unexpected reply from remote: #{data}"))
 			else
@@ -115,6 +135,11 @@ module.exports =
 			parameter: 'source'
 			description: 'The source directory to use when building the image'
 			alias: 's'
+		},
+		{
+			signature: 'nologupload'
+			description: "Don't upload build logs to the dashboard with image (if building)"
+			boolean: true
 		}
 	]
 	action: (params, options, done) ->
@@ -130,35 +155,69 @@ module.exports =
 		# Ensure the tmp files gets deleted
 		tmp.setGracefulCleanup()
 
-		docker = dockerUtils.getDocker(options)
+		logs = ''
-		# Check input parameters
-		parseInput(params, options)
-		.then ([appName, build, source, imageName]) ->
-			tmpNameAsync()
-			.then (tmpPath) ->
 
-				# Setup the build args for how the build routine expects them
+		upload = (token, username, url) ->
-				options = _.assign({}, options, { appName })
+			docker = dockerUtils.getDocker(options)
-				params = _.assign({}, params, { source })
+			# Check input parameters
+			parseInput(params, options)
+			.then ([appName, build, source, imageName]) ->
+				tmpNameAsync()
+				.then (tmpPath) ->
 
-				Promise.try ->
+					# Setup the build args for how the build routine expects them
-					if build
+					options = _.assign({}, options, { appName })
-						dockerUtils.runBuild(params, options, getBundleInfo, logStreams)
+					params = _.assign({}, params, { source })
-					else
+
-						imageName
+					Promise.try ->
-				.then (imageName) ->
+						if build
-					Promise.join(
+							dockerUtils.runBuild(params, options, getBundleInfo, logStreams)
-						dockerUtils.bufferImage(docker, imageName, tmpPath)
+						else
-						resin.auth.getToken()
+							{ image: imageName, log: '' }
-						resin.auth.whoami()
+					.then ({ image: imageName, log: buildLogs }) ->
-						resin.settings.get('resinUrl')
+						logs = buildLogs
-						dockerUtils.getImageSize(docker, imageName)
+						Promise.join(
-						params.appName
+							dockerUtils.bufferImage(docker, imageName, tmpPath)
-						logStreams
+							token
-						performUpload
+							username
-					)
+							url
-				.finally ->
+							dockerUtils.getImageSize(docker, imageName)
-					require('fs').unlink(tmpPath)
+							params.appName
-		.then (imageName) ->
+							logStreams
-			logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}")
+							performUpload
-		.asCallback(done)
+						)
+					.finally ->
+						# If the file was never written to (for instance because an error
+						# has occured before any data was written) this call will throw an
+						# ugly error, just suppress it
+						require('mz/fs').unlink(tmpPath)
+						.catch(_.noop)
+			.tap ({ image: imageName, buildId }) ->
+				logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}")
+				return buildId
+			.then ({ image: imageName, buildId }) ->
+				if logs is '' or options.nologupload?
+					return ''
+
+				logging.logInfo(logStreams, 'Uploading logs to dashboard...')
+
+				Promise.join(
+					logs
+					token
+					url
+					buildId
+					username
+					params.appName
+					uploadLogs
+				)
+				.return('Successfully uploaded logs')
+			.then (msg) ->
+				logging.logSuccess(logStreams, msg) if msg isnt ''
+			.asCallback(done)
+
+		Promise.join(
+			resin.auth.getToken()
+			resin.auth.whoami()
+			resin.settings.get('resinUrl')
+			upload
+		)

lib/utils/docker.coffee

@@ -117,10 +117,13 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 	Promise = require('bluebird')
 	dockerBuild = require('resin-docker-build')
 	resolver = require('resin-bundle-resolve')
+	es = require('event-stream')
+
 	logging = require('../utils/logging')
 
 	# The default build context is the current directory
 	params.source ?= '.'
+	logs = ''
 
 	# Tar up the directory, ready for the build stream
 	tarDirectory(params.source)
@@ -130,7 +133,7 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 				buildSuccess: (image) ->
 					if options.tag?
 						console.log("Tagging image as #{options.tag}")
-					resolve(image)
+					resolve({ image, log: logs } )
 				buildFailure: reject
 				buildStream: (stream) ->
 					getBundleInfo(options)
@@ -152,7 +155,12 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 								resolved.tarStream.pipe(stream)
 					.catch(reject)
 
-					stream.pipe(logStreams.build)
+					# And print the output
+					throughStream = es.through (data) ->
+						logs += data.toString()
+						this.emit('data', data)
+
+					stream.pipe(es.pipe(throughStream, logStreams.build))
 
 			# Create a builder
 			connectOpts = generateConnectOpts(options)
@@ -186,11 +194,8 @@ exports.bufferImage = (docker, imageId, tmpFile) ->
 		new Promise (resolve, reject) ->
 			img
 			.on('error', reject)
-			.on 'data', (data) ->
+			.on('end', resolve)
-				stream.write(data)
+			.pipe(stream)
-			.on 'end', ->
-				stream.close()
-				resolve()
 	.then ->
 		new Promise (resolve, reject) ->
 			fs.createReadStream(tmpFile)

lib/utils/logging.coffee

@@ -11,13 +11,15 @@ exports.getLogStreams = ->
 	logger.addPrefix('debug', colors.magenta('[Debug]'))
 	logger.addPrefix('success', colors.green('[Success]'))
 	logger.addPrefix('warn', colors.yellow('[Warn]'))
+	logger.addPrefix('error', colors.red('[Error]'))
 
 	streams =
 		build: logger.createLogStream('build'),
 		info: logger.createLogStream('info'),
 		debug: logger.createLogStream('debug'),
 		success: logger.createLogStream('success'),
-		warn: logger.createLogStream('warn')
+		warn: logger.createLogStream('warn'),
+		error: logger.createLogStream('error')
 
 	_.mapKeys streams, (stream, key) ->
 		if key isnt 'debug'
@@ -38,3 +40,6 @@ exports.logSuccess = (logStreams, msg) ->
 
 exports.logWarn = (logStreams, msg) ->
 	logStreams.warn.write(msg + eol)
+
+exports.logError = (logStreams, msg) ->
+	logStreams.error.write(msg + eol)