From 90a5b15dbcf1326fd98aff59a5020b0c1334b950 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Tue, 13 Jun 2017 19:28:37 +0200 Subject: [PATCH 1/5] Refactor docker stream buffering before start gzipping --- build/actions/deploy.js | 26 +++++++++++++------------- build/utils/docker.js | 18 ++++-------------- build/utils/streams.js | 16 ++++++++++++++++ lib/actions/deploy.coffee | 22 +++++++++++----------- lib/utils/docker.coffee | 19 +++---------------- lib/utils/streams.coffee | 17 +++++++++++++++++ 6 files changed, 64 insertions(+), 54 deletions(-) create mode 100644 build/utils/streams.js create mode 100644 lib/utils/streams.coffee diff --git a/build/actions/deploy.js b/build/actions/deploy.js index 6794d45b..9b626653 100644 --- a/build/actions/deploy.js +++ b/build/actions/deploy.js @@ -1,5 +1,5 @@ // Generated by CoffeeScript 1.12.6 -var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, renderProgress, uploadLogs, uploadToPromise; +var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, renderProgress, showPushProgress, uploadLogs, uploadToPromise; Promise = require('bluebird'); @@ -64,7 +64,7 @@ renderProgress = function(percentage, stepCount) { return bar + " " + (percentage.toFixed(1)) + "%"; }; -pushProgress = function(imageSize, request, logStreams, timeout) { +showPushProgress = function(imageSize, request, logStreams, timeout) { var ansiEscapes, logging, progressReporter; if (timeout == null) { timeout = 250; @@ -95,17 +95,17 @@ getBundleInfo = function(options) { }); }; -performUpload = function(image, token, username, url, size, appName, logStreams) { - var post, request; +performUpload = function(imageStream, token, username, url, size, appName, logStreams) { + var request, uploadRequest; request = require('request'); - post = request.post({ + uploadRequest = request.post({ url: getBuilderPushEndpoint(url, username, appName), auth: { bearer: token }, - body: image + body: imageStream }); - return uploadToPromise(post, size, logStreams); + return uploadToPromise(uploadRequest, size, logStreams); }; uploadLogs = function(logs, token, url, buildId, username, appName) { @@ -121,7 +121,7 @@ uploadLogs = function(logs, token, url, buildId, username, appName) { }); }; -uploadToPromise = function(request, size, logStreams) { +uploadToPromise = function(uploadRequest, size, logStreams) { var logging; logging = require('../utils/logging'); return new Promise(function(resolve, reject) { @@ -153,8 +153,8 @@ uploadToPromise = function(request, size, logStreams) { return reject(new Error("Received unexpected reply from remote: " + data)); } }; - request.on('error', reject).on('data', handleMessage); - return pushProgress(size, request, logStreams); + uploadRequest.on('error', reject).on('data', handleMessage); + return showPushProgress(size, uploadRequest, logStreams); }); }; @@ -196,7 +196,7 @@ module.exports = { return parseInput(params, options).then(function(arg) { var appName, build, imageName, source; appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3]; - return tmpNameAsync().then(function(tmpPath) { + return tmpNameAsync().then(function(bufferFile) { options = _.assign({}, options, { appName: appName }); @@ -216,9 +216,9 @@ module.exports = { var buildLogs, imageName; imageName = arg1.image, buildLogs = arg1.log; logs = buildLogs; - return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload); + return Promise.join(dockerUtils.bufferImage(docker, imageName, bufferFile), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload); })["finally"](function() { - return require('mz/fs').unlink(tmpPath)["catch"](_.noop); + return require('mz/fs').unlink(bufferFile)["catch"](_.noop); }); }); }).tap(function(arg) { diff --git a/build/utils/docker.js b/build/utils/docker.js index accd34b5..4aeb254f 100644 --- a/build/utils/docker.js +++ b/build/utils/docker.js @@ -273,22 +273,12 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) { }); }; -exports.bufferImage = function(docker, imageId, tmpFile) { - var Promise, fs, image, stream; - Promise = require('bluebird'); - fs = require('fs'); - stream = fs.createWriteStream(tmpFile); +exports.bufferImage = function(docker, imageId, bufferFile) { + var image, streamUtils; + streamUtils = require('./streams'); image = docker.getImage(imageId); return image.get().then(function(img) { - return new Promise(function(resolve, reject) { - return img.on('error', reject).on('end', resolve).pipe(stream); - }); - }).then(function() { - return new Promise(function(resolve, reject) { - return fs.createReadStream(tmpFile).on('open', function() { - return resolve(this); - }).on('error', reject); - }); + return streamUtils.buffer(img, bufferFile); }); }; diff --git a/build/utils/streams.js b/build/utils/streams.js new file mode 100644 index 00000000..b446cc91 --- /dev/null +++ b/build/utils/streams.js @@ -0,0 +1,16 @@ +// Generated by CoffeeScript 1.12.6 +exports.buffer = function(stream, bufferFile) { + var Promise, fileWriteStream, fs; + Promise = require('bluebird'); + fs = require('mz/fs'); + fileWriteStream = fs.createWriteStream(bufferFile); + return new Promise(function(resolve, reject) { + return stream.on('error', reject).on('end', resolve).pipe(fileWriteStream); + }).then(function() { + return new Promise(function(resolve, reject) { + return fs.createReadStream(bufferFile).on('open', function() { + return resolve(this); + }).on('error', reject); + }); + }); +}; diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 9530d735..50617cfc 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -41,7 +41,7 @@ renderProgress = (percentage, stepCount = 50) -> bar = "[#{_.repeat('=', barCount)}>#{_.repeat(' ', spaceCount)}]" return "#{bar} #{percentage.toFixed(1)}%" -pushProgress = (imageSize, request, logStreams, timeout = 250) -> +showPushProgress = (imageSize, request, logStreams, timeout = 250) -> logging = require('../utils/logging') ansiEscapes = require('ansi-escapes') @@ -65,15 +65,15 @@ getBundleInfo = (options) -> .then (app) -> [app.arch, app.device_type] -performUpload = (image, token, username, url, size, appName, logStreams) -> +performUpload = (imageStream, token, username, url, size, appName, logStreams) -> request = require('request') - post = request.post + uploadRequest = request.post url: getBuilderPushEndpoint(url, username, appName) auth: bearer: token - body: image + body: imageStream - uploadToPromise(post, size, logStreams) + uploadToPromise(uploadRequest, size, logStreams) uploadLogs = (logs, token, url, buildId, username, appName) -> request = require('request') @@ -84,7 +84,7 @@ uploadLogs = (logs, token, url, buildId, username, appName) -> bearer: token body: Buffer.from(logs) -uploadToPromise = (request, size, logStreams) -> +uploadToPromise = (uploadRequest, size, logStreams) -> logging = require('../utils/logging') new Promise (resolve, reject) -> @@ -109,12 +109,12 @@ uploadToPromise = (request, size, logStreams) -> else reject(new Error("Received unexpected reply from remote: #{data}")) - request + uploadRequest .on('error', reject) .on('data', handleMessage) # Set up upload reporting - pushProgress(size, request, logStreams) + showPushProgress(size, uploadRequest, logStreams) module.exports = @@ -173,7 +173,7 @@ module.exports = parseInput(params, options) .then ([appName, build, source, imageName]) -> tmpNameAsync() - .then (tmpPath) -> + .then (bufferFile) -> # Setup the build args for how the build routine expects them options = _.assign({}, options, { appName }) @@ -187,7 +187,7 @@ module.exports = .then ({ image: imageName, log: buildLogs }) -> logs = buildLogs Promise.join( - dockerUtils.bufferImage(docker, imageName, tmpPath) + dockerUtils.bufferImage(docker, imageName, bufferFile) token username url @@ -200,7 +200,7 @@ module.exports = # If the file was never written to (for instance because an error # has occured before any data was written) this call will throw an # ugly error, just suppress it - require('mz/fs').unlink(tmpPath) + require('mz/fs').unlink(bufferFile) .catch(_.noop) .tap ({ image: imageName, buildId }) -> logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}") diff --git a/lib/utils/docker.coffee b/lib/utils/docker.coffee index 0a798b7d..572dbd3d 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker.coffee @@ -284,26 +284,13 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) -> # Given an image id or tag, export the image to a tar archive. # Also needs the options generated by the appendOptions() # function, and a tmpFile to buffer the data into. -exports.bufferImage = (docker, imageId, tmpFile) -> - Promise = require('bluebird') - fs = require('fs') - - stream = fs.createWriteStream(tmpFile) +exports.bufferImage = (docker, imageId, bufferFile) -> + streamUtils = require('./streams') image = docker.getImage(imageId) image.get() .then (img) -> - new Promise (resolve, reject) -> - img - .on('error', reject) - .on('end', resolve) - .pipe(stream) - .then -> - new Promise (resolve, reject) -> - fs.createReadStream(tmpFile) - .on 'open', -> - resolve(this) - .on('error', reject) + streamUtils.buffer(img, bufferFile) exports.getDocker = (options) -> Docker = require('dockerode') diff --git a/lib/utils/streams.coffee b/lib/utils/streams.coffee new file mode 100644 index 00000000..2f4282b4 --- /dev/null +++ b/lib/utils/streams.coffee @@ -0,0 +1,17 @@ +exports.buffer = (stream, bufferFile) -> + Promise = require('bluebird') + fs = require('mz/fs') + + fileWriteStream = fs.createWriteStream(bufferFile) + + new Promise (resolve, reject) -> + stream + .on('error', reject) + .on('end', resolve) + .pipe(fileWriteStream) + .then -> + new Promise (resolve, reject) -> + fs.createReadStream(bufferFile) + .on 'open', -> + resolve(this) + .on('error', reject) From e584dc43f7226076095b4a0701c2117c78523065 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Tue, 13 Jun 2017 19:28:37 +0200 Subject: [PATCH 2/5] Gzip images when uploading in `resin deploy` Change-Type: minor Connects-To: #549 --- CHANGELOG.md | 3 ++- build/actions/deploy.js | 11 +++++++---- build/utils/docker.js | 21 +++++++++++++++++---- build/utils/streams.js | 2 +- lib/actions/deploy.coffee | 17 +++++++++-------- lib/utils/docker.coffee | 23 +++++++++++++++++------ lib/utils/streams.coffee | 2 +- 7 files changed, 54 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfdb4c92..7ede03f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Added - `package-lock.json` for `npm5` users -- Added ability to run an emulated build silently with resin build +- Added ability to run an emulated build silently with `resin build` +- Gzip images when uploading in `resin deploy` ### Fixed diff --git a/build/actions/deploy.js b/build/actions/deploy.js index 9b626653..0b611152 100644 --- a/build/actions/deploy.js +++ b/build/actions/deploy.js @@ -95,17 +95,20 @@ getBundleInfo = function(options) { }); }; -performUpload = function(imageStream, token, username, url, size, appName, logStreams) { +performUpload = function(gzippedImage, token, username, url, appName, logStreams) { var request, uploadRequest; request = require('request'); uploadRequest = request.post({ url: getBuilderPushEndpoint(url, username, appName), + headers: { + 'Content-Encoding': 'gzip' + }, auth: { bearer: token }, - body: imageStream + body: gzippedImage.stream }); - return uploadToPromise(uploadRequest, size, logStreams); + return uploadToPromise(uploadRequest, gzippedImage.size, logStreams); }; uploadLogs = function(logs, token, url, buildId, username, appName) { @@ -216,7 +219,7 @@ module.exports = { var buildLogs, imageName; imageName = arg1.image, buildLogs = arg1.log; logs = buildLogs; - return Promise.join(dockerUtils.bufferImage(docker, imageName, bufferFile), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload); + return Promise.all([dockerUtils.gzipAndBufferImage(docker, imageName, bufferFile), token, username, url, params.appName, logStreams]).spread(performUpload); })["finally"](function() { return require('mz/fs').unlink(bufferFile)["catch"](_.noop); }); diff --git a/build/utils/docker.js b/build/utils/docker.js index 4aeb254f..60b438c1 100644 --- a/build/utils/docker.js +++ b/build/utils/docker.js @@ -273,12 +273,25 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) { }); }; -exports.bufferImage = function(docker, imageId, bufferFile) { - var image, streamUtils; +exports.gzipAndBufferImage = function(docker, imageId, bufferFile) { + var fs, image, streamUtils, zlib; streamUtils = require('./streams'); + zlib = require('zlib'); + fs = require('mz/fs'); image = docker.getImage(imageId); - return image.get().then(function(img) { - return streamUtils.buffer(img, bufferFile); + return image.get().then(function(imageStream) { + var gzippedStream; + gzippedStream = imageStream.pipe(zlib.createGzip()); + return streamUtils.buffer(gzippedStream, bufferFile); + }).then(function(bufferedStream) { + return fs.stat(bufferFile).then(function(stats) { + var size; + size = stats.size; + return { + stream: bufferedStream, + size: stats.size + }; + }); }); }; diff --git a/build/utils/streams.js b/build/utils/streams.js index b446cc91..48f0ac23 100644 --- a/build/utils/streams.js +++ b/build/utils/streams.js @@ -2,7 +2,7 @@ exports.buffer = function(stream, bufferFile) { var Promise, fileWriteStream, fs; Promise = require('bluebird'); - fs = require('mz/fs'); + fs = require('fs'); fileWriteStream = fs.createWriteStream(bufferFile); return new Promise(function(resolve, reject) { return stream.on('error', reject).on('end', resolve).pipe(fileWriteStream); diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 50617cfc..3efd7443 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -65,15 +65,17 @@ getBundleInfo = (options) -> .then (app) -> [app.arch, app.device_type] -performUpload = (imageStream, token, username, url, size, appName, logStreams) -> +performUpload = (gzippedImage, token, username, url, appName, logStreams) -> request = require('request') uploadRequest = request.post url: getBuilderPushEndpoint(url, username, appName) + headers: + 'Content-Encoding': 'gzip' auth: bearer: token - body: imageStream + body: gzippedImage.stream - uploadToPromise(uploadRequest, size, logStreams) + uploadToPromise(uploadRequest, gzippedImage.size, logStreams) uploadLogs = (logs, token, url, buildId, username, appName) -> request = require('request') @@ -186,16 +188,15 @@ module.exports = { image: imageName, log: '' } .then ({ image: imageName, log: buildLogs }) -> logs = buildLogs - Promise.join( - dockerUtils.bufferImage(docker, imageName, bufferFile) + Promise.all [ + dockerUtils.gzipAndBufferImage(docker, imageName, bufferFile) token username url - dockerUtils.getImageSize(docker, imageName) params.appName logStreams - performUpload - ) + ] + .spread(performUpload) .finally -> # If the file was never written to (for instance because an error # has occured before any data was written) this call will throw an diff --git a/lib/utils/docker.coffee b/lib/utils/docker.coffee index 572dbd3d..cd0a25dd 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker.coffee @@ -281,16 +281,27 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) -> builder.createBuildStream(opts, hooks, reject) -# Given an image id or tag, export the image to a tar archive. -# Also needs the options generated by the appendOptions() -# function, and a tmpFile to buffer the data into. -exports.bufferImage = (docker, imageId, bufferFile) -> +# Given an image id or tag, export the image to a tar archive, +# gzip the result, and buffer it to disk. +# Returns a { stream, size } object +exports.gzipAndBufferImage = (docker, imageId, bufferFile) -> streamUtils = require('./streams') + zlib = require('zlib') + fs = require('mz/fs') image = docker.getImage(imageId) image.get() - .then (img) -> - streamUtils.buffer(img, bufferFile) + .then (imageStream) -> + gzippedStream = imageStream.pipe(zlib.createGzip()) + streamUtils.buffer(gzippedStream, bufferFile) + .then (bufferedStream) -> + fs.stat(bufferFile) + .then (stats) -> + size = stats.size + return { + stream: bufferedStream, + size: stats.size + } exports.getDocker = (options) -> Docker = require('dockerode') diff --git a/lib/utils/streams.coffee b/lib/utils/streams.coffee index 2f4282b4..6ef7f1d0 100644 --- a/lib/utils/streams.coffee +++ b/lib/utils/streams.coffee @@ -1,6 +1,6 @@ exports.buffer = (stream, bufferFile) -> Promise = require('bluebird') - fs = require('mz/fs') + fs = require('fs') fileWriteStream = fs.createWriteStream(bufferFile) From df3c5ca07f015c5a31bf9e04805a9c230f6c7665 Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Tue, 13 Jun 2017 19:28:37 +0200 Subject: [PATCH 3/5] Gzip while streaming, rather than gzipping the buffer up front Connects-To: #549 --- build/actions/deploy.js | 62 +++++++++++++++++++++------------------ build/utils/docker.js | 27 +++++------------ lib/actions/deploy.coffee | 47 +++++++++++++++-------------- lib/utils/docker.coffee | 30 ++++++------------- package.json | 1 + 5 files changed, 76 insertions(+), 91 deletions(-) diff --git a/build/actions/deploy.js b/build/actions/deploy.js index 0b611152..9de3891a 100644 --- a/build/actions/deploy.js +++ b/build/actions/deploy.js @@ -1,5 +1,5 @@ // Generated by CoffeeScript 1.12.6 -var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, renderProgress, showPushProgress, uploadLogs, uploadToPromise; +var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, renderProgress, showPushProgress, updatePushProgress, uploadLogs, uploadToPromise; Promise = require('bluebird'); @@ -64,27 +64,23 @@ renderProgress = function(percentage, stepCount) { return bar + " " + (percentage.toFixed(1)) + "%"; }; -showPushProgress = function(imageSize, request, logStreams, timeout) { - var ansiEscapes, logging, progressReporter; - if (timeout == null) { - timeout = 250; - } +showPushProgress = function(logStreams) { + var logging; + logging = require('../utils/logging'); + return logging.logInfo(logStreams, renderProgress(0)); +}; + +updatePushProgress = function(percentage, logStreams) { + var ansiEscapes, logging; logging = require('../utils/logging'); ansiEscapes = require('ansi-escapes'); - logging.logInfo(logStreams, 'Initializing...'); - return progressReporter = setInterval(function() { - var percent, sent; - sent = request.req.connection._bytesDispatched; - percent = (sent / imageSize) * 100; - if (percent >= 100) { - clearInterval(progressReporter); - percent = 100; - } - process.stdout.write(ansiEscapes.cursorUp(1)); - process.stdout.clearLine(); - process.stdout.cursorTo(0); - return logging.logInfo(logStreams, renderProgress(percent)); - }, timeout); + if (percentage >= 100) { + percentage = 100; + } + process.stdout.write(ansiEscapes.cursorUp(1)); + process.stdout.clearLine(); + process.stdout.cursorTo(0); + return logging.logInfo(logStreams, renderProgress(percentage)); }; getBundleInfo = function(options) { @@ -95,9 +91,20 @@ getBundleInfo = function(options) { }); }; -performUpload = function(gzippedImage, token, username, url, appName, logStreams) { - var request, uploadRequest; +performUpload = function(imageStream, token, username, url, appName, logStreams) { + var progressStream, request, streamWithProgress, uploadRequest, zlib; request = require('request'); + progressStream = require('progress-stream'); + zlib = require('zlib'); + showPushProgress(logStreams); + streamWithProgress = imageStream.pipe(progressStream({ + time: 500, + length: imageStream.length + }, function(arg) { + var percentage; + percentage = arg.percentage; + return updatePushProgress(percentage, logStreams); + })); uploadRequest = request.post({ url: getBuilderPushEndpoint(url, username, appName), headers: { @@ -106,9 +113,9 @@ performUpload = function(gzippedImage, token, username, url, appName, logStreams auth: { bearer: token }, - body: gzippedImage.stream + body: streamWithProgress.pipe(zlib.createGzip()) }); - return uploadToPromise(uploadRequest, gzippedImage.size, logStreams); + return uploadToPromise(uploadRequest, logStreams); }; uploadLogs = function(logs, token, url, buildId, username, appName) { @@ -124,7 +131,7 @@ uploadLogs = function(logs, token, url, buildId, username, appName) { }); }; -uploadToPromise = function(uploadRequest, size, logStreams) { +uploadToPromise = function(uploadRequest, logStreams) { var logging; logging = require('../utils/logging'); return new Promise(function(resolve, reject) { @@ -156,8 +163,7 @@ uploadToPromise = function(uploadRequest, size, logStreams) { return reject(new Error("Received unexpected reply from remote: " + data)); } }; - uploadRequest.on('error', reject).on('data', handleMessage); - return showPushProgress(size, uploadRequest, logStreams); + return uploadRequest.on('error', reject).on('data', handleMessage); }); }; @@ -219,7 +225,7 @@ module.exports = { var buildLogs, imageName; imageName = arg1.image, buildLogs = arg1.log; logs = buildLogs; - return Promise.all([dockerUtils.gzipAndBufferImage(docker, imageName, bufferFile), token, username, url, params.appName, logStreams]).spread(performUpload); + return Promise.all([dockerUtils.bufferImage(docker, imageName, bufferFile), token, username, url, params.appName, logStreams]).spread(performUpload); })["finally"](function() { return require('mz/fs').unlink(bufferFile)["catch"](_.noop); }); diff --git a/build/utils/docker.js b/build/utils/docker.js index 60b438c1..5e4ece2b 100644 --- a/build/utils/docker.js +++ b/build/utils/docker.js @@ -273,24 +273,15 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) { }); }; -exports.gzipAndBufferImage = function(docker, imageId, bufferFile) { - var fs, image, streamUtils, zlib; +exports.bufferImage = function(docker, imageId, bufferFile) { + var Promise, image, imageMetadata, streamUtils; + Promise = require('bluebird'); streamUtils = require('./streams'); - zlib = require('zlib'); - fs = require('mz/fs'); image = docker.getImage(imageId); - return image.get().then(function(imageStream) { - var gzippedStream; - gzippedStream = imageStream.pipe(zlib.createGzip()); - return streamUtils.buffer(gzippedStream, bufferFile); - }).then(function(bufferedStream) { - return fs.stat(bufferFile).then(function(stats) { - var size; - size = stats.size; - return { - stream: bufferedStream, - size: stats.size - }; + imageMetadata = image.inspectAsync(); + return Promise.all([image.get(), imageMetadata.get('Size')]).spread(function(imageStream, imageSize) { + return streamUtils.buffer(imageStream, bufferFile).tap(function(bufferedStream) { + return bufferedStream.length = imageSize; }); }); }; @@ -304,10 +295,6 @@ exports.getDocker = function(options) { return new Docker(connectOpts); }; -exports.getImageSize = function(docker, image) { - return docker.getImage(image).inspectAsync().get('Size'); -}; - hasQemu = function() { var fs; fs = require('mz/fs'); diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 3efd7443..8ae19fb7 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -41,22 +41,20 @@ renderProgress = (percentage, stepCount = 50) -> bar = "[#{_.repeat('=', barCount)}>#{_.repeat(' ', spaceCount)}]" return "#{bar} #{percentage.toFixed(1)}%" -showPushProgress = (imageSize, request, logStreams, timeout = 250) -> +showPushProgress = (logStreams) -> + logging = require('../utils/logging') + logging.logInfo(logStreams, renderProgress(0)) + +updatePushProgress = (percentage, logStreams) -> logging = require('../utils/logging') ansiEscapes = require('ansi-escapes') - logging.logInfo(logStreams, 'Initializing...') - progressReporter = setInterval -> - sent = request.req.connection._bytesDispatched - percent = (sent / imageSize) * 100 - if percent >= 100 - clearInterval(progressReporter) - percent = 100 - process.stdout.write(ansiEscapes.cursorUp(1)) - process.stdout.clearLine() - process.stdout.cursorTo(0) - logging.logInfo(logStreams, renderProgress(percent)) - , timeout + if percentage >= 100 + percentage = 100 + process.stdout.write(ansiEscapes.cursorUp(1)) + process.stdout.clearLine() + process.stdout.cursorTo(0) + logging.logInfo(logStreams, renderProgress(percentage)) getBundleInfo = (options) -> helpers = require('../utils/helpers') @@ -65,17 +63,26 @@ getBundleInfo = (options) -> .then (app) -> [app.arch, app.device_type] -performUpload = (gzippedImage, token, username, url, appName, logStreams) -> +performUpload = (imageStream, token, username, url, appName, logStreams) -> request = require('request') + progressStream = require('progress-stream') + zlib = require('zlib') + + showPushProgress(logStreams) + streamWithProgress = imageStream.pipe(progressStream({ + time: 500, + length: imageStream.length + }, ({ percentage }) -> updatePushProgress(percentage, logStreams))) + uploadRequest = request.post url: getBuilderPushEndpoint(url, username, appName) headers: 'Content-Encoding': 'gzip' auth: bearer: token - body: gzippedImage.stream + body: streamWithProgress.pipe(zlib.createGzip()) - uploadToPromise(uploadRequest, gzippedImage.size, logStreams) + uploadToPromise(uploadRequest, logStreams) uploadLogs = (logs, token, url, buildId, username, appName) -> request = require('request') @@ -86,7 +93,7 @@ uploadLogs = (logs, token, url, buildId, username, appName) -> bearer: token body: Buffer.from(logs) -uploadToPromise = (uploadRequest, size, logStreams) -> +uploadToPromise = (uploadRequest, logStreams) -> logging = require('../utils/logging') new Promise (resolve, reject) -> @@ -115,10 +122,6 @@ uploadToPromise = (uploadRequest, size, logStreams) -> .on('error', reject) .on('data', handleMessage) - # Set up upload reporting - showPushProgress(size, uploadRequest, logStreams) - - module.exports = signature: 'deploy [image]' description: 'Deploy a container to a resin.io application' @@ -189,7 +192,7 @@ module.exports = .then ({ image: imageName, log: buildLogs }) -> logs = buildLogs Promise.all [ - dockerUtils.gzipAndBufferImage(docker, imageName, bufferFile) + dockerUtils.bufferImage(docker, imageName, bufferFile) token username url diff --git a/lib/utils/docker.coffee b/lib/utils/docker.coffee index cd0a25dd..b7c1900b 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker.coffee @@ -283,25 +283,18 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) -> # Given an image id or tag, export the image to a tar archive, # gzip the result, and buffer it to disk. -# Returns a { stream, size } object -exports.gzipAndBufferImage = (docker, imageId, bufferFile) -> +exports.bufferImage = (docker, imageId, bufferFile) -> + Promise = require('bluebird') streamUtils = require('./streams') - zlib = require('zlib') - fs = require('mz/fs') image = docker.getImage(imageId) - image.get() - .then (imageStream) -> - gzippedStream = imageStream.pipe(zlib.createGzip()) - streamUtils.buffer(gzippedStream, bufferFile) - .then (bufferedStream) -> - fs.stat(bufferFile) - .then (stats) -> - size = stats.size - return { - stream: bufferedStream, - size: stats.size - } + imageMetadata = image.inspectAsync() + + Promise.all([image.get(), imageMetadata.get('Size')]) + .spread (imageStream, imageSize) -> + streamUtils.buffer(imageStream, bufferFile) + .tap (bufferedStream) -> + bufferedStream.length = imageSize exports.getDocker = (options) -> Docker = require('dockerode') @@ -311,10 +304,6 @@ exports.getDocker = (options) -> connectOpts['Promise'] = Promise new Docker(connectOpts) -exports.getImageSize = (docker, image) -> - docker.getImage(image).inspectAsync() - .get('Size') - hasQemu = -> fs = require('mz/fs') @@ -381,4 +370,3 @@ copyQemu = (context) -> .then -> fs.chmod(binPath, '755') .return(binPath) - diff --git a/package.json b/package.json index 91b5c47e..cc68d7dd 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "nplugm": "^3.0.0", "president": "^2.0.1", "prettyjson": "^1.1.3", + "progress-stream": "^2.0.0", "raven": "^1.2.0", "reconfix": "^0.0.3", "request": "^2.81.0", From cf7bf2cb7dc581604f6562357ba2d5b6e96696ca Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Wed, 14 Jun 2017 17:42:33 +0200 Subject: [PATCH 4/5] Fix the gzip level for image uploads to a good perf/size balance --- build/actions/deploy.js | 4 +++- lib/actions/deploy.coffee | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build/actions/deploy.js b/build/actions/deploy.js index 9de3891a..67b7d5cf 100644 --- a/build/actions/deploy.js +++ b/build/actions/deploy.js @@ -113,7 +113,9 @@ performUpload = function(imageStream, token, username, url, appName, logStreams) auth: { bearer: token }, - body: streamWithProgress.pipe(zlib.createGzip()) + body: streamWithProgress.pipe(zlib.createGzip({ + level: 6 + })) }); return uploadToPromise(uploadRequest, logStreams); }; diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 8ae19fb7..62765a78 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -80,7 +80,9 @@ performUpload = (imageStream, token, username, url, appName, logStreams) -> 'Content-Encoding': 'gzip' auth: bearer: token - body: streamWithProgress.pipe(zlib.createGzip()) + body: streamWithProgress.pipe(zlib.createGzip({ + level: 6 + })) uploadToPromise(uploadRequest, logStreams) From a29b40eefa526ef101a635e32d8243946a5f5ffc Mon Sep 17 00:00:00 2001 From: Tim Perry Date: Thu, 15 Jun 2017 12:34:26 +0200 Subject: [PATCH 5/5] Move promise.spread to promise.join for clarity (from review) --- build/utils/docker.js | 2 +- lib/utils/docker.coffee | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build/utils/docker.js b/build/utils/docker.js index 5e4ece2b..55274490 100644 --- a/build/utils/docker.js +++ b/build/utils/docker.js @@ -279,7 +279,7 @@ exports.bufferImage = function(docker, imageId, bufferFile) { streamUtils = require('./streams'); image = docker.getImage(imageId); imageMetadata = image.inspectAsync(); - return Promise.all([image.get(), imageMetadata.get('Size')]).spread(function(imageStream, imageSize) { + return Promise.join(image.get(), imageMetadata.get('Size'), function(imageStream, imageSize) { return streamUtils.buffer(imageStream, bufferFile).tap(function(bufferedStream) { return bufferedStream.length = imageSize; }); diff --git a/lib/utils/docker.coffee b/lib/utils/docker.coffee index b7c1900b..9b47e2e8 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker.coffee @@ -290,8 +290,7 @@ exports.bufferImage = (docker, imageId, bufferFile) -> image = docker.getImage(imageId) imageMetadata = image.inspectAsync() - Promise.all([image.get(), imageMetadata.get('Size')]) - .spread (imageStream, imageSize) -> + Promise.join image.get(), imageMetadata.get('Size'), (imageStream, imageSize) -> streamUtils.buffer(imageStream, bufferFile) .tap (bufferedStream) -> bufferedStream.length = imageSize