Add emulated build option to resin build

This commit adds the ability to run a Docker build for an architecture
which is not the host architecture, using qemu-linux-user. Currently
this is only supported for linux.

Added:
* Installation of qemu which supports propagated execve flags
* Copying of qemu binary into the build context
* Transposing the given Dockerfile to use the qemu binary
* Intercepting of the build stream, so the output looks *almost* exactly
  the same.

Change-type: minor
Signed-off-by: Cameron Diver <cameron@resin.io>

CHANGELOG.md

@@ -8,6 +8,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 
 - `package-lock.json` for `npm5` users
+- Added ability to run an emulated build silently with resin build
 
 ## [5.10.2] - 2017-05-31
 

build/utils/docker.js

@@ -1,5 +1,9 @@
 // Generated by CoffeeScript 1.12.6
-var cacheHighlightStream, generateConnectOpts, parseBuildArgs, tarDirectory;
+var QEMU_BIN_NAME, QEMU_VERSION, cacheHighlightStream, copyQemu, generateConnectOpts, getQemuPath, hasQemu, installQemu, parseBuildArgs, platformNeedsQemu, tarDirectory;
+
+QEMU_VERSION = 'v2.5.50-resin-execve';
+
+QEMU_BIN_NAME = 'qemu-execve';
 
 exports.appendOptions = function(opts) {
   return opts.concat([
@@ -44,6 +48,11 @@ exports.appendOptions = function(opts) {
       signature: 'nocache',
       description: "Don't use docker layer caching when building",
       boolean: true
+    }, {
+      signature: 'emulated',
+      description: 'Run an emulated build using Qemu',
+      boolean: true,
+      alias: 'e'
     }
   ]);
 };
@@ -94,7 +103,8 @@ exports.tarDirectory = tarDirectory = function(dir) {
     return Promise.join(relPath, fs.stat(file), fs.readFile(file), function(filename, stats, data) {
       return pack.entryAsync({
         name: filename,
-        size: stats.size
+        size: stats.size,
+        mode: stats.mode
       }, data);
     });
   }).then(function() {
@@ -147,18 +157,37 @@ parseBuildArgs = function(args, onError) {
 };
 
 exports.runBuild = function(params, options, getBundleInfo, logStreams) {
-  var Promise, dockerBuild, doodles, es, logging, logs, resolver;
+  var Promise, dockerBuild, doodles, es, logging, logs, path, qemuPath, resolver, transpose;
   Promise = require('bluebird');
   dockerBuild = require('resin-docker-build');
   resolver = require('resin-bundle-resolve');
   es = require('event-stream');
   doodles = require('resin-doodles');
+  transpose = require('docker-qemu-transpose');
+  path = require('path');
   logging = require('../utils/logging');
   if (params.source == null) {
     params.source = '.';
   }
   logs = '';
-  return tarDirectory(params.source).then(function(tarStream) {
+  qemuPath = '';
+  return Promise["try"](function() {
+    if (!(options.emulated && platformNeedsQemu())) {
+      return;
+    }
+    return hasQemu().then(function(present) {
+      if (!present) {
+        logging.logInfo(logStreams, 'Installing qemu for ARM emulation...');
+        return installQemu();
+      }
+    }).then(function() {
+      return copyQemu(params.source);
+    }).then(function(binPath) {
+      return qemuPath = binPath.split(path.sep).slice(1).join(path.sep);
+    });
+  }).then(function() {
+    return tarDirectory(params.source);
+  }).then(function(tarStream) {
     return new Promise(function(resolve, reject) {
       var builder, connectOpts, hooks, opts;
       hooks = {
@@ -178,26 +207,49 @@ exports.runBuild = function(params, options, getBundleInfo, logStreams) {
         },
         buildFailure: reject,
         buildStream: function(stream) {
-          var throughStream;
+          var buildThroughStream, logThroughStream, newStream;
+          if (options.emulated) {
+            logging.logInfo(logStreams, 'Running emulated build');
+          }
           getBundleInfo(options).then(function(info) {
             var arch, bundle, deviceType;
             if (info == null) {
               logging.logWarn(logStreams, 'Warning: No architecture/device type or application information provided.\n	Dockerfile/project pre-processing will not be performed.');
-              return tarStream.pipe(stream);
+              return tarStream;
             } else {
               arch = info[0], deviceType = info[1];
               bundle = new resolver.Bundle(tarStream, deviceType, arch);
               return resolver.resolveBundle(bundle, resolver.getDefaultResolvers()).then(function(resolved) {
                 logging.logInfo(logStreams, "Building " + resolved.projectType + " project");
-                return resolved.tarStream.pipe(stream);
+                return resolved.tarStream;
               });
             }
+          }).then(function(buildStream) {
+            if (options.emulated && platformNeedsQemu()) {
+              return transpose.transposeTarStream(buildStream, {
+                hostQemuPath: qemuPath,
+                containerQemuPath: "./" + QEMU_BIN_NAME
+              });
+            } else {
+              return buildStream;
+            }
+          }).then(function(buildStream) {
+            return buildStream.pipe(stream);
           })["catch"](reject);
-          throughStream = es.through(function(data) {
+          logThroughStream = es.through(function(data) {
             logs += data.toString();
             return this.emit('data', data);
           });
-          return stream.pipe(throughStream).pipe(cacheHighlightStream()).pipe(logStreams.build);
+          if (options.emulated && platformNeedsQemu()) {
+            buildThroughStream = transpose.getBuildThroughStream({
+              hostQemuPath: qemuPath,
+              containerQemuPath: "./" + QEMU_BIN_NAME
+            });
+            newStream = stream.pipe(buildThroughStream);
+          } else {
+            newStream = stream;
+          }
+          return newStream.pipe(logThroughStream).pipe(cacheHighlightStream()).pipe(logStreams.build);
         }
       };
       connectOpts = generateConnectOpts(options);
@@ -252,3 +304,70 @@ exports.getDocker = function(options) {
 exports.getImageSize = function(docker, image) {
   return docker.getImage(image).inspectAsync().get('Size');
 };
+
+hasQemu = function() {
+  var fs;
+  fs = require('mz/fs');
+  return getQemuPath().then(fs.stat)["return"](true).catchReturn(false);
+};
+
+getQemuPath = function() {
+  var fs, path, resin;
+  resin = require('resin-sdk-preconfigured');
+  path = require('path');
+  fs = require('mz/fs');
+  return resin.settings.get('binDirectory').then(function(binDir) {
+    return fs.access(binDir)["catch"]({
+      code: 'ENOENT'
+    }, function() {
+      return fs.mkdir(binDir);
+    }).then(function() {
+      return path.join(binDir, QEMU_BIN_NAME);
+    });
+  });
+};
+
+platformNeedsQemu = function() {
+  var os;
+  os = require('os');
+  return os.platform() === 'linux';
+};
+
+installQemu = function() {
+  var fs, request, zlib;
+  request = require('request');
+  fs = require('fs');
+  zlib = require('zlib');
+  return getQemuPath().then(function(qemuPath) {
+    return new Promise(function(resolve, reject) {
+      var installStream, qemuUrl;
+      installStream = fs.createWriteStream(qemuPath);
+      qemuUrl = "https://github.com/resin-io/qemu/releases/download/" + QEMU_VERSION + "/" + QEMU_BIN_NAME + ".gz";
+      return request(qemuUrl).pipe(zlib.createGunzip()).pipe(installStream).on('error', reject).on('finish', resolve);
+    });
+  });
+};
+
+copyQemu = function(context) {
+  var binDir, binPath, fs, path;
+  path = require('path');
+  fs = require('mz/fs');
+  binDir = path.join(context, '.resin');
+  binPath = path.join(binDir, QEMU_BIN_NAME);
+  return fs.access(binDir)["catch"]({
+    code: 'ENOENT'
+  }, function() {
+    return fs.mkdir(binDir);
+  }).then(function() {
+    return getQemuPath();
+  }).then(function(qemu) {
+    return new Promise(function(resolve, reject) {
+      var read, write;
+      read = fs.createReadStream(qemu);
+      write = fs.createWriteStream(binPath);
+      return read.pipe(write).on('error', reject).on('finish', resolve);
+    });
+  }).then(function() {
+    return fs.chmod(binPath, '755');
+  })["return"](binPath);
+};

doc/cli.markdown

@@ -1348,6 +1348,10 @@ Set a build-time variable (eg. "-B 'ARG=value'"). Can be specified multiple time
 
 Don't use docker layer caching when building
 
+#### --emulated, -e
+
+Run an emulated build using Qemu
+
 ## deploy <appName> [image]
 
 Use this command to deploy and optionally build an image to an application.
@@ -1411,3 +1415,7 @@ Set a build-time variable (eg. "-B 'ARG=value'"). Can be specified multiple time
 
 Don't use docker layer caching when building
 
+#### --emulated, -e
+
+Run an emulated build using Qemu
+

lib/utils/docker.coffee

@@ -1,5 +1,8 @@
 # Functions to help actions which rely on using docker
 
+QEMU_VERSION = 'v2.5.50-resin-execve'
+QEMU_BIN_NAME = 'qemu-execve'
+
 # Use this function to seed an action's list of capitano options
 # with the docker options. Using this interface means that
 # all functions using docker will expose the same interface
@@ -58,6 +61,12 @@ exports.appendOptions = (opts) ->
 			description: "Don't use docker layer caching when building"
 			boolean: true
 		},
+		{
+			signature: 'emulated'
+			description: 'Run an emulated build using Qemu'
+			boolean: true
+			alias: 'e'
+		}
 	]
 
 exports.generateConnectOpts = generateConnectOpts = (opts) ->
@@ -111,7 +120,7 @@ exports.tarDirectory = tarDirectory = (dir) ->
 		relPath = path.relative(path.resolve(dir), file)
 		Promise.join relPath, fs.stat(file), fs.readFile(file),
 			(filename, stats, data) ->
-				pack.entryAsync({ name: filename, size: stats.size }, data)
+				pack.entryAsync({ name: filename, size: stats.size, mode: stats.mode }, data)
 	.then ->
 		pack.finalize()
 		return pack
@@ -156,15 +165,33 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 	resolver = require('resin-bundle-resolve')
 	es = require('event-stream')
 	doodles = require('resin-doodles')
+	transpose = require('docker-qemu-transpose')
+	path = require('path')
 
 	logging = require('../utils/logging')
 
 	# The default build context is the current directory
 	params.source ?= '.'
 	logs = ''
+	# Only used in emulated builds
+	qemuPath = ''
 
-	# Tar up the directory, ready for the build stream
-	tarDirectory(params.source)
+	Promise.try ->
+		return if not (options.emulated and platformNeedsQemu())
+
+		hasQemu()
+		.then (present) ->
+			if !present
+				logging.logInfo(logStreams, 'Installing qemu for ARM emulation...')
+				installQemu()
+		.then ->
+			# Copy the qemu binary into the build context
+			copyQemu(params.source)
+		.then (binPath) ->
+			qemuPath = binPath.split(path.sep)[1...].join(path.sep)
+	.then ->
+		# Tar up the directory, ready for the build stream
+		tarDirectory(params.source)
 	.then (tarStream) ->
 		new Promise (resolve, reject) ->
 			hooks =
@@ -182,6 +209,9 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 
 				buildFailure: reject
 				buildStream: (stream) ->
+					if options.emulated
+						logging.logInfo(logStreams, 'Running emulated build')
+
 					getBundleInfo(options)
 					.then (info) ->
 						if !info?
@@ -189,7 +219,7 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 								Warning: No architecture/device type or application information provided.
 									Dockerfile/project pre-processing will not be performed.
 							'''
-							tarStream.pipe(stream)
+							return tarStream
 						else
 							[arch, deviceType] = info
 							# Perform type resolution on the project
@@ -197,17 +227,37 @@ exports.runBuild = (params, options, getBundleInfo, logStreams) ->
 							resolver.resolveBundle(bundle, resolver.getDefaultResolvers())
 							.then (resolved) ->
 								logging.logInfo(logStreams, "Building #{resolved.projectType} project")
-								# Send the resolved tar stream to the docker daemon
-								resolved.tarStream.pipe(stream)
+
+								return resolved.tarStream
+					.then (buildStream) ->
+						# if we need emulation
+						if options.emulated and platformNeedsQemu()
+							return transpose.transposeTarStream buildStream,
+								hostQemuPath: qemuPath
+								containerQemuPath: "./#{QEMU_BIN_NAME}"
+						else
+							return buildStream
+					.then (buildStream) ->
+						# Send the resolved tar stream to the docker daemon
+						buildStream.pipe(stream)
 					.catch(reject)
 
 					# And print the output
-					throughStream = es.through (data) ->
+					logThroughStream = es.through (data) ->
 						logs += data.toString()
 						this.emit('data', data)
 
-					stream
-					.pipe(throughStream)
+					if options.emulated and platformNeedsQemu()
+						buildThroughStream = transpose.getBuildThroughStream
+							hostQemuPath: qemuPath
+							containerQemuPath: "./#{QEMU_BIN_NAME}"
+
+						newStream = stream.pipe(buildThroughStream)
+					else
+						newStream = stream
+
+					newStream
+					.pipe(logThroughStream)
 					.pipe(cacheHighlightStream())
 					.pipe(logStreams.build)
 
@@ -266,3 +316,71 @@ exports.getDocker = (options) ->
 exports.getImageSize = (docker, image) ->
 	docker.getImage(image).inspectAsync()
 	.get('Size')
+
+hasQemu = ->
+	fs = require('mz/fs')
+
+	getQemuPath()
+	.then(fs.stat)
+	.return(true)
+	.catchReturn(false)
+
+getQemuPath = ->
+	resin = require('resin-sdk-preconfigured')
+	path = require('path')
+	fs = require('mz/fs')
+
+	resin.settings.get('binDirectory')
+	.then (binDir) ->
+		# The directory might not be created already,
+		# if not, create it
+		fs.access(binDir)
+		.catch code: 'ENOENT', ->
+			fs.mkdir(binDir)
+		.then ->
+			path.join(binDir, QEMU_BIN_NAME)
+
+platformNeedsQemu = ->
+	os = require('os')
+	os.platform() == 'linux'
+
+installQemu = ->
+	request = require('request')
+	fs = require('fs')
+	zlib = require('zlib')
+
+	getQemuPath()
+	.then (qemuPath) ->
+		new Promise (resolve, reject) ->
+			installStream = fs.createWriteStream(qemuPath)
+			qemuUrl = "https://github.com/resin-io/qemu/releases/download/#{QEMU_VERSION}/#{QEMU_BIN_NAME}.gz"
+			request(qemuUrl)
+			.pipe(zlib.createGunzip())
+			.pipe(installStream)
+			.on('error', reject)
+			.on('finish', resolve)
+
+copyQemu = (context) ->
+	path = require('path')
+	fs = require('mz/fs')
+	# Create a hidden directory in the build context, containing qemu
+	binDir = path.join(context, '.resin')
+	binPath = path.join(binDir, QEMU_BIN_NAME)
+
+	fs.access(binDir)
+	.catch code: 'ENOENT', ->
+		fs.mkdir(binDir)
+	.then ->
+		getQemuPath()
+	.then (qemu) ->
+		new Promise (resolve, reject) ->
+			read = fs.createReadStream(qemu)
+			write = fs.createWriteStream(binPath)
+			read
+			.pipe(write)
+			.on('error', reject)
+			.on('finish', resolve)
+	.then ->
+		fs.chmod(binPath, '755')
+	.return(binPath)
+

package.json

@@ -45,6 +45,7 @@
     "coffee-script": "~1.12.6",
     "columnify": "^1.5.2",
     "denymount": "^2.2.0",
+    "docker-qemu-transpose": "^0.2.1",
     "docker-toolbelt": "^1.3.3",
     "dockerode": "^2.4.2",
     "drivelist": "^5.0.21",