ExternalVendorCode/balena-cli
1
0
Fork 0
mirror of https://github.com/balena-io/balena-cli.git synced 2025-04-08 20:04:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #554 from resin-io/549-gzip-image-deploy

Browse Source 
Gzip images when deploying
This commit is contained in:
Tim Perry 2017-06-15 12:35:54 +02:00 committed by GitHub
commit 2fce0e964b
8 changed files with 131 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -8,7 +8,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
### Added
- `package-lock.json` for `npm5` users
- Added ability to run an emulated build silently with resin build
- Added ability to run an emulated build silently with `resin build`
- Gzip images when uploading in `resin deploy`
### Fixed

73
build/actions/deploy.js
View File

@ -1,5 +1,5 @@
// Generated by CoffeeScript 1.12.6
var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, pushProgress, renderProgress, uploadLogs, uploadToPromise;
var Promise, dockerUtils, formatImageName, getBuilderLogPushEndpoint, getBuilderPushEndpoint, getBundleInfo, parseInput, performUpload, renderProgress, showPushProgress, updatePushProgress, uploadLogs, uploadToPromise;
Promise = require('bluebird');
@ -64,27 +64,23 @@ renderProgress = function(percentage, stepCount) {
return bar + " " + (percentage.toFixed(1)) + "%";
};
pushProgress = function(imageSize, request, logStreams, timeout) {
var ansiEscapes, logging, progressReporter;
if (timeout == null) {
timeout = 250;
}
showPushProgress = function(logStreams) {
var logging;
logging = require('../utils/logging');
return logging.logInfo(logStreams, renderProgress(0));
};
updatePushProgress = function(percentage, logStreams) {
var ansiEscapes, logging;
logging = require('../utils/logging');
ansiEscapes = require('ansi-escapes');
logging.logInfo(logStreams, 'Initializing...');
return progressReporter = setInterval(function() {
var percent, sent;
sent = request.req.connection._bytesDispatched;
percent = (sent / imageSize) * 100;
if (percent >= 100) {
clearInterval(progressReporter);
percent = 100;
}
process.stdout.write(ansiEscapes.cursorUp(1));
process.stdout.clearLine();
process.stdout.cursorTo(0);
return logging.logInfo(logStreams, renderProgress(percent));
}, timeout);
if (percentage >= 100) {
percentage = 100;
}
process.stdout.write(ansiEscapes.cursorUp(1));
process.stdout.clearLine();
process.stdout.cursorTo(0);
return logging.logInfo(logStreams, renderProgress(percentage));
};
getBundleInfo = function(options) {
@ -95,17 +91,33 @@ getBundleInfo = function(options) {
});
};
performUpload = function(image, token, username, url, size, appName, logStreams) {
var post, request;
performUpload = function(imageStream, token, username, url, appName, logStreams) {
var progressStream, request, streamWithProgress, uploadRequest, zlib;
request = require('request');
post = request.post({
progressStream = require('progress-stream');
zlib = require('zlib');
showPushProgress(logStreams);
streamWithProgress = imageStream.pipe(progressStream({
time: 500,
length: imageStream.length
}, function(arg) {
var percentage;
percentage = arg.percentage;
return updatePushProgress(percentage, logStreams);
}));
uploadRequest = request.post({
url: getBuilderPushEndpoint(url, username, appName),
headers: {
'Content-Encoding': 'gzip'
},
auth: {
bearer: token
},
body: image
body: streamWithProgress.pipe(zlib.createGzip({
level: 6
}))
});
return uploadToPromise(post, size, logStreams);
return uploadToPromise(uploadRequest, logStreams);
};
uploadLogs = function(logs, token, url, buildId, username, appName) {
@ -121,7 +133,7 @@ uploadLogs = function(logs, token, url, buildId, username, appName) {
});
};
uploadToPromise = function(request, size, logStreams) {
uploadToPromise = function(uploadRequest, logStreams) {
var logging;
logging = require('../utils/logging');
return new Promise(function(resolve, reject) {
@ -153,8 +165,7 @@ uploadToPromise = function(request, size, logStreams) {
return reject(new Error("Received unexpected reply from remote: " + data));
}
};
request.on('error', reject).on('data', handleMessage);
return pushProgress(size, request, logStreams);
return uploadRequest.on('error', reject).on('data', handleMessage);
});
};
@ -196,7 +207,7 @@ module.exports = {
return parseInput(params, options).then(function(arg) {
var appName, build, imageName, source;
appName = arg[0], build = arg[1], source = arg[2], imageName = arg[3];
return tmpNameAsync().then(function(tmpPath) {
return tmpNameAsync().then(function(bufferFile) {
options = _.assign({}, options, {
appName: appName
});
@ -216,9 +227,9 @@ module.exports = {
var buildLogs, imageName;
imageName = arg1.image, buildLogs = arg1.log;
logs = buildLogs;
return Promise.join(dockerUtils.bufferImage(docker, imageName, tmpPath), token, username, url, dockerUtils.getImageSize(docker, imageName), params.appName, logStreams, performUpload);
return Promise.all([dockerUtils.bufferImage(docker, imageName, bufferFile), token, username, url, params.appName, logStreams]).spread(performUpload);
})["finally"](function() {
return require('mz/fs').unlink(tmpPath)["catch"](_.noop);
return require('mz/fs').unlink(bufferFile)["catch"](_.noop);
});
});
}).tap(function(arg) {

24
build/utils/docker.js
View File

@ -273,21 +273,15 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) {
});
};
exports.bufferImage = function(docker, imageId, tmpFile) {
var Promise, fs, image, stream;
exports.bufferImage = function(docker, imageId, bufferFile) {
var Promise, image, imageMetadata, streamUtils;
Promise = require('bluebird');
fs = require('fs');
stream = fs.createWriteStream(tmpFile);
streamUtils = require('./streams');
image = docker.getImage(imageId);
return image.get().then(function(img) {
return new Promise(function(resolve, reject) {
return img.on('error', reject).on('end', resolve).pipe(stream);
});
}).then(function() {
return new Promise(function(resolve, reject) {
return fs.createReadStream(tmpFile).on('open', function() {
return resolve(this);
}).on('error', reject);
imageMetadata = image.inspectAsync();
return Promise.join(image.get(), imageMetadata.get('Size'), function(imageStream, imageSize) {
return streamUtils.buffer(imageStream, bufferFile).tap(function(bufferedStream) {
return bufferedStream.length = imageSize;
});
});
};
@ -301,10 +295,6 @@ exports.getDocker = function(options) {
return new Docker(connectOpts);
};
exports.getImageSize = function(docker, image) {
return docker.getImage(image).inspectAsync().get('Size');
};
hasQemu = function() {
var fs;
fs = require('mz/fs');

16
build/utils/streams.js Normal file
View File

@ -0,0 +1,16 @@
// Generated by CoffeeScript 1.12.6
exports.buffer = function(stream, bufferFile) {
var Promise, fileWriteStream, fs;
Promise = require('bluebird');
fs = require('fs');
fileWriteStream = fs.createWriteStream(bufferFile);
return new Promise(function(resolve, reject) {
return stream.on('error', reject).on('end', resolve).pipe(fileWriteStream);
}).then(function() {
return new Promise(function(resolve, reject) {
return fs.createReadStream(bufferFile).on('open', function() {
return resolve(this);
}).on('error', reject);
});
});
};

66
lib/actions/deploy.coffee
View File

@ -41,22 +41,20 @@ renderProgress = (percentage, stepCount = 50) ->
bar = "[#{_.repeat('=', barCount)}>#{_.repeat(' ', spaceCount)}]"
return "#{bar} #{percentage.toFixed(1)}%"
pushProgress = (imageSize, request, logStreams, timeout = 250) ->
showPushProgress = (logStreams) ->
logging = require('../utils/logging')
logging.logInfo(logStreams, renderProgress(0))
updatePushProgress = (percentage, logStreams) ->
logging = require('../utils/logging')
ansiEscapes = require('ansi-escapes')
logging.logInfo(logStreams, 'Initializing...')
progressReporter = setInterval ->
sent = request.req.connection._bytesDispatched
percent = (sent / imageSize) * 100
if percent >= 100
clearInterval(progressReporter)
percent = 100
process.stdout.write(ansiEscapes.cursorUp(1))
process.stdout.clearLine()
process.stdout.cursorTo(0)
logging.logInfo(logStreams, renderProgress(percent))
, timeout
if percentage >= 100
percentage = 100
process.stdout.write(ansiEscapes.cursorUp(1))
process.stdout.clearLine()
process.stdout.cursorTo(0)
logging.logInfo(logStreams, renderProgress(percentage))
getBundleInfo = (options) ->
helpers = require('../utils/helpers')
@ -65,15 +63,28 @@ getBundleInfo = (options) ->
.then (app) ->
[app.arch, app.device_type]
performUpload = (image, token, username, url, size, appName, logStreams) ->
performUpload = (imageStream, token, username, url, appName, logStreams) ->
request = require('request')
post = request.post
progressStream = require('progress-stream')
zlib = require('zlib')
showPushProgress(logStreams)
streamWithProgress = imageStream.pipe(progressStream({
time: 500,
length: imageStream.length
}, ({ percentage }) -> updatePushProgress(percentage, logStreams)))
uploadRequest = request.post
url: getBuilderPushEndpoint(url, username, appName)
headers:
'Content-Encoding': 'gzip'
auth:
bearer: token
body: image
body: streamWithProgress.pipe(zlib.createGzip({
level: 6
}))
uploadToPromise(post, size, logStreams)
uploadToPromise(uploadRequest, logStreams)
uploadLogs = (logs, token, url, buildId, username, appName) ->
request = require('request')
@ -84,7 +95,7 @@ uploadLogs = (logs, token, url, buildId, username, appName) ->
bearer: token
body: Buffer.from(logs)
uploadToPromise = (request, size, logStreams) ->
uploadToPromise = (uploadRequest, logStreams) ->
logging = require('../utils/logging')
new Promise (resolve, reject) ->
@ -109,14 +120,10 @@ uploadToPromise = (request, size, logStreams) ->
else
reject(new Error("Received unexpected reply from remote: #{data}"))
request
uploadRequest
.on('error', reject)
.on('data', handleMessage)
# Set up upload reporting
pushProgress(size, request, logStreams)
module.exports =
signature: 'deploy <appName> [image]'
description: 'Deploy a container to a resin.io application'
@ -173,7 +180,7 @@ module.exports =
parseInput(params, options)
.then ([appName, build, source, imageName]) ->
tmpNameAsync()
.then (tmpPath) ->
.then (bufferFile) ->
# Setup the build args for how the build routine expects them
options = _.assign({}, options, { appName })
@ -186,21 +193,20 @@ module.exports =
{ image: imageName, log: '' }
.then ({ image: imageName, log: buildLogs }) ->
logs = buildLogs
Promise.join(
dockerUtils.bufferImage(docker, imageName, tmpPath)
Promise.all [
dockerUtils.bufferImage(docker, imageName, bufferFile)
token
username
url
dockerUtils.getImageSize(docker, imageName)
params.appName
logStreams
performUpload
)
]
.spread(performUpload)
.finally ->
# If the file was never written to (for instance because an error
# has occured before any data was written) this call will throw an
# ugly error, just suppress it
require('mz/fs').unlink(tmpPath)
require('mz/fs').unlink(bufferFile)
.catch(_.noop)
.tap ({ image: imageName, buildId }) ->
logging.logSuccess(logStreams, "Successfully deployed image: #{formatImageName(imageName)}")

35
lib/utils/docker.coffee
View File

@ -281,29 +281,19 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
builder.createBuildStream(opts, hooks, reject)
# Given an image id or tag, export the image to a tar archive.
# Also needs the options generated by the appendOptions()
# function, and a tmpFile to buffer the data into.
exports.bufferImage = (docker, imageId, tmpFile) ->
# Given an image id or tag, export the image to a tar archive,
# gzip the result, and buffer it to disk.
exports.bufferImage = (docker, imageId, bufferFile) ->
Promise = require('bluebird')
fs = require('fs')
stream = fs.createWriteStream(tmpFile)
streamUtils = require('./streams')
image = docker.getImage(imageId)
image.get()
.then (img) ->
new Promise (resolve, reject) ->
img
.on('error', reject)
.on('end', resolve)
.pipe(stream)
.then ->
new Promise (resolve, reject) ->
fs.createReadStream(tmpFile)
.on 'open', ->
resolve(this)
.on('error', reject)
imageMetadata = image.inspectAsync()
Promise.join image.get(), imageMetadata.get('Size'), (imageStream, imageSize) ->
streamUtils.buffer(imageStream, bufferFile)
.tap (bufferedStream) ->
bufferedStream.length = imageSize
exports.getDocker = (options) ->
Docker = require('dockerode')
@ -313,10 +303,6 @@ exports.getDocker = (options) ->
connectOpts['Promise'] = Promise
new Docker(connectOpts)
exports.getImageSize = (docker, image) ->
docker.getImage(image).inspectAsync()
.get('Size')
hasQemu = ->
fs = require('mz/fs')
@ -383,4 +369,3 @@ copyQemu = (context) ->
.then ->
fs.chmod(binPath, '755')
.return(binPath)

17
lib/utils/streams.coffee Normal file
View File

@ -0,0 +1,17 @@
exports.buffer = (stream, bufferFile) ->
Promise = require('bluebird')
fs = require('fs')
fileWriteStream = fs.createWriteStream(bufferFile)
new Promise (resolve, reject) ->
stream
.on('error', reject)
.on('end', resolve)
.pipe(fileWriteStream)
.then ->
new Promise (resolve, reject) ->
fs.createReadStream(bufferFile)
.on 'open', ->
resolve(this)
.on('error', reject)

1
package.json
View File

@ -63,6 +63,7 @@
"nplugm": "^3.0.0",
"president": "^2.0.1",
"prettyjson": "^1.1.3",
"progress-stream": "^2.0.0",
"raven": "^1.2.0",
"reconfix": "^0.0.3",
"request": "^2.81.0",