Refactor docker stream buffering before start gzipping

build/actions/deploy.js

@@ -1,5 +1,5 @@
 // Generated by CoffeeScript 1.12.6
-var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, renderProgress, uploadLogs, uploadToPromise;
+var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, renderProgress, showPushProgress, uploadLogs, uploadToPromise;
 
 Promise = require('bluebird');
 
@@ -64,7 +64,7 @@ renderProgress = function(percentage, stepCount) {
   return bar + " " + (percentage.toFixed(1)) + "%";
 };
 
-pushProgress = function(imageSize, request, logStreams, timeout) {
+showPushProgress = function(imageSize, request, logStreams, timeout) {
   var ansiEscapes, logging, progressReporter;
   if (timeout == null) {
     timeout = 250;
@@ -95,17 +95,17 @@ getBundleInfo = function(options) {
   });
 };
 
-performUpload = function(image, token, username, url, size, appName, logStreams) {
-  var post, request;
+performUpload = function(imageStream, token, username, url, size, appName, logStreams) {
+  var request, uploadRequest;
   request = require('request');
-  post = request.post({
+  uploadRequest = request.post({
     url: getBuilderPushEndpoint(url, username, appName),
     auth: {
       bearer: token
     },
-    body: image
+    body: imageStream
   });
-  return uploadToPromise(post, size, logStreams);
+  return uploadToPromise(uploadRequest, size, logStreams);
 };
 
 uploadLogs = function(logs, token, url, buildId, username, appName) {
@@ -121,7 +121,7 @@ uploadLogs = function(logs, token, url, buildId, username, appName) {
   });
 };
 
-uploadToPromise = function(request, size, logStreams) {
+uploadToPromise = function(uploadRequest, size, logStreams) {
   var logging;
   logging = require('../utils/logging');
   return new Promise(function(resolve, reject) {
@@ -153,8 +153,8 @@ uploadToPromise = function(request, size, logStreams) {
         return reject(new Error("Received unexpected reply from remote: " + data));
       }
     };
-    request.on('error', reject).on('data', handleMessage);
-    return pushProgress(size, request, logStreams);
+    uploadRequest.on('error', reject).on('data', handleMessage);
+    return showPushProgress(size, uploadRequest, logStreams);
   });
 };
 
@@ -196,7 +196,7 @@ module.exports = {
       return parseInput(params, options).then(function(arg) {
         var appName, build, imageName, source;
         appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3];
-        return tmpNameAsync().then(function(tmpPath) {
+        return tmpNameAsync().then(function(bufferFile) {
           options = _.assign({}, options, {
             appName: appName
           });
@@ -216,9 +216,9 @@ module.exports = {
             var buildLogs, imageName;
             imageName = arg1.image, buildLogs = arg1.log;
             logs = buildLogs;
-            return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload);
+            return Promise.join(dockerUtils.bufferImage(docker, imageName, bufferFile), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload);
           })["finally"](function() {
-            return require('mz/fs').unlink(tmpPath)["catch"](_.noop);
+            return require('mz/fs').unlink(bufferFile)["catch"](_.noop);
           });
         });
       }).tap(function(arg) {

build/utils/docker.js

@@ -273,22 +273,12 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) {
   });
 };
 
-exports.bufferImage = function(docker, imageId, tmpFile) {
-  var Promise, fs, image, stream;
-  Promise = require('bluebird');
-  fs = require('fs');
-  stream = fs.createWriteStream(tmpFile);
+exports.bufferImage = function(docker, imageId, bufferFile) {
+  var image, streamUtils;
+  streamUtils = require('./streams');
   image = docker.getImage(imageId);
   return image.get().then(function(img) {
-    return new Promise(function(resolve, reject) {
-      return img.on('error', reject).on('end', resolve).pipe(stream);
-    });
-  }).then(function() {
-    return new Promise(function(resolve, reject) {
-      return fs.createReadStream(tmpFile).on('open', function() {
-        return resolve(this);
-      }).on('error', reject);
-    });
+    return streamUtils.buffer(img, bufferFile);
   });
 };
 

build/utils/streams.js

@@ -0,0 +1,16 @@
+// Generated by CoffeeScript 1.12.6
+exports.buffer = function(stream, bufferFile) {
+  var Promise, fileWriteStream, fs;
+  Promise = require('bluebird');
+  fs = require('mz/fs');
+  fileWriteStream = fs.createWriteStream(bufferFile);
+  return new Promise(function(resolve, reject) {
+    return stream.on('error', reject).on('end', resolve).pipe(fileWriteStream);
+  }).then(function() {
+    return new Promise(function(resolve, reject) {
+      return fs.createReadStream(bufferFile).on('open', function() {
+        return resolve(this);
+      }).on('error', reject);
+    });
+  });
+};

lib/actions/deploy.coffee

@@ -41,7 +41,7 @@ renderProgress = (percentage, stepCount = 50) ->
 	bar = "[#{_.repeat('=', barCount)}>#{_.repeat(' ', spaceCount)}]"
 	return "#{bar} #{percentage.toFixed(1)}%"
 
-pushProgress = (imageSize, request, logStreams, timeout = 250) ->
+showPushProgress = (imageSize, request, logStreams, timeout = 250) ->
 	logging = require('../utils/logging')
 	ansiEscapes = require('ansi-escapes')
 
@@ -65,15 +65,15 @@ getBundleInfo = (options) ->
 	.then (app) ->
 		[app.arch, app.device_type]
 
-performUpload = (image, token, username, url, size, appName, logStreams) ->
+performUpload = (imageStream, token, username, url, size, appName, logStreams) ->
 	request = require('request')
-	post = request.post
+	uploadRequest = request.post
 		url: getBuilderPushEndpoint(url, username, appName)
 		auth:
 			bearer: token
-		body: image
+		body: imageStream
 
-	uploadToPromise(post, size, logStreams)
+	uploadToPromise(uploadRequest, size, logStreams)
 
 uploadLogs = (logs, token, url, buildId, username, appName) ->
 	request = require('request')
@@ -84,7 +84,7 @@ uploadLogs = (logs, token, url, buildId, username, appName) ->
 			bearer: token
 		body: Buffer.from(logs)
 
-uploadToPromise = (request, size, logStreams) ->
+uploadToPromise = (uploadRequest, size, logStreams) ->
 	logging = require('../utils/logging')
 
 	new Promise (resolve, reject) ->
@@ -109,12 +109,12 @@ uploadToPromise = (request, size, logStreams) ->
 			else
 				reject(new Error("Received unexpected reply from remote: #{data}"))
 
-		request
+		uploadRequest
 		.on('error', reject)
 		.on('data', handleMessage)
 
 		# Set up upload reporting
-		pushProgress(size, request, logStreams)
+		showPushProgress(size, uploadRequest, logStreams)
 
 
 module.exports =
@@ -173,7 +173,7 @@ module.exports =
 			parseInput(params, options)
 			.then ([appName, build, source, imageName]) ->
 				tmpNameAsync()
-				.then (tmpPath) ->
+				.then (bufferFile) ->
 
 					# Setup the build args for how the build routine expects them
 					options = _.assign({}, options, { appName })
@@ -187,7 +187,7 @@ module.exports =
 					.then ({ image: imageName, log: buildLogs }) ->
 						logs = buildLogs
 						Promise.join(
-							dockerUtils.bufferImage(docker, imageName, tmpPath)
+							dockerUtils.bufferImage(docker, imageName, bufferFile)
 							token
 							username
 							url
@@ -200,7 +200,7 @@ module.exports =
 						# If the file was never written to (for instance because an error
 						# has occured before any data was written) this call will throw an
 						# ugly error, just suppress it
-						require('mz/fs').unlink(tmpPath)
+						require('mz/fs').unlink(bufferFile)
 						.catch(_.noop)
 			.tap ({ image: imageName, buildId }) ->
 				logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}")

lib/utils/docker.coffee

@@ -284,26 +284,13 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 # Given an image id or tag, export the image to a tar archive.
 # Also needs the options generated by the appendOptions()
 # function, and a tmpFile to buffer the data into.
-exports.bufferImage = (docker, imageId, tmpFile) ->
-	Promise = require('bluebird')
-	fs = require('fs')
-
-	stream = fs.createWriteStream(tmpFile)
+exports.bufferImage = (docker, imageId, bufferFile) ->
+	streamUtils = require('./streams')
 
 	image = docker.getImage(imageId)
 	image.get()
 	.then (img) ->
-		new Promise (resolve, reject) ->
-			img
-			.on('error', reject)
-			.on('end', resolve)
-			.pipe(stream)
-	.then ->
-		new Promise (resolve, reject) ->
-			fs.createReadStream(tmpFile)
-			.on 'open', ->
-				resolve(this)
-			.on('error', reject)
+		streamUtils.buffer(img, bufferFile)
 
 exports.getDocker = (options) ->
 	Docker = require('dockerode')

lib/utils/streams.coffee

@@ -0,0 +1,17 @@
+exports.buffer = (stream, bufferFile) ->
+	Promise = require('bluebird')
+	fs = require('mz/fs')
+
+	fileWriteStream = fs.createWriteStream(bufferFile)
+
+	new Promise (resolve, reject) ->
+		stream
+		.on('error', reject)
+		.on('end', resolve)
+		.pipe(fileWriteStream)
+	.then ->
+		new Promise (resolve, reject) ->
+			fs.createReadStream(bufferFile)
+			.on 'open', ->
+				resolve(this)
+			.on('error', reject)