Merge pull request #1403 from balena-io/1355-SecretRemovalError

Fix 'SecretRemovalError' and enable certain emulated builds on a remote device

lib/utils/compose.coffee

@@ -188,7 +188,7 @@ exports.buildProject = (
 	transpose = require('docker-qemu-transpose')
 	{ BALENA_ENGINE_TMP_PATH } = require('../config')
 	{ checkBuildSecretsRequirements, makeBuildTasks } = require('./compose_ts')
-	qemu = require('./qemu')
+	qemu = require('./qemu-ts')
 	{ toPosixPath } = builder.PathUtils
 
 	logger.logInfo("Building for #{arch}/#{deviceType}")
@@ -204,7 +204,7 @@ exports.buildProject = (
 	renderer.start()
 
 	Promise.resolve(checkBuildSecretsRequirements(docker, projectPath))
-	.then -> qemu.installQemuIfNeeded(emulated, logger, arch)
+	.then -> qemu.installQemuIfNeeded(emulated, logger, arch, docker)
 	.tap (needsQemu) ->
 		return if not needsQemu
 		logger.logInfo('Emulation is enabled')

lib/utils/compose_ts.ts

@@ -95,7 +95,7 @@ export async function checkBuildSecretsRequirements(
 	sourceDir: string,
 ) {
 	const [metaObj, metaFilename] = await loadBuildMetatada(sourceDir);
-	if (!_.isEmpty(metaObj['build-secrets'])) {
+	if (metaObj && !_.isEmpty(metaObj['build-secrets'])) {
 		const dockerUtils = await import('./docker');
 		const isBalenaEngine = await dockerUtils.isBalenaEngine(docker);
 		if (!isBalenaEngine) {

lib/utils/qemu-ts.ts

@@ -0,0 +1,61 @@
+/**
+ * @license
+ * Copyright 2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Dockerode = require('dockerode');
+
+import Logger = require('./logger');
+import { getQemuPath, installQemu } from './qemu';
+
+export { QEMU_BIN_NAME } from './qemu';
+export * from './qemu';
+
+export async function installQemuIfNeeded(
+	emulated: boolean,
+	logger: Logger,
+	arch: string,
+	docker: Dockerode,
+): Promise<boolean> {
+	if (!emulated || !(await platformNeedsQemu(docker))) {
+		return false;
+	}
+	const fs = await import('mz/fs');
+	if (!(await fs.exists(await getQemuPath(arch)))) {
+		logger.logInfo(`Installing qemu for ${arch} emulation...`);
+		await installQemu(arch);
+	}
+	return true;
+}
+
+/**
+ * Check whether the Docker daemon (including balenaEngine) requires explicit
+ * QEMU emulation setup. Note that Docker for Mac (macOS), and reportedly also
+ * Docker for Windows, have built-in support for binfmt_misc, so do not require
+ * explicity QEMU setup. References:
+ * - https://en.wikipedia.org/wiki/Binfmt_misc
+ * - https://docs.docker.com/docker-for-mac/multi-arch/
+ * - https://www.ecliptik.com/Cross-Building-and-Running-Multi-Arch-Docker-Images/
+ * - https://stackoverflow.com/questions/55388725/run-linux-arm-container-via-qemu-binfmt-misc-on-docker-lcow
+ *
+ * @param docker Dockerode instance
+ */
+async function platformNeedsQemu(docker: Dockerode): Promise<boolean> {
+	const dockerInfo = await docker.info();
+	// Docker for Mac reports dockerInfo.OSType as 'linux'
+	// https://stackoverflow.com/questions/38223965/how-can-i-detect-if-docker-for-mac-is-installed
+	const isDockerForMac = /Docker for Mac/i.test(dockerInfo.OperatingSystem);
+	return dockerInfo.OSType === 'linux' && !isDockerForMac;
+}

lib/utils/qemu.coffee

@@ -1,18 +1,25 @@
+###*
+# @license
+# Copyright 2017-2019 Balena Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+
 Promise = require('bluebird')
 
 exports.QEMU_VERSION = QEMU_VERSION = 'v4.0.0-balena'
 exports.QEMU_BIN_NAME = QEMU_BIN_NAME = 'qemu-execve'
 
-exports.installQemuIfNeeded = Promise.method (emulated, logger, arch) ->
-	return false if not (emulated and platformNeedsQemu())
-
-	hasQemu(arch)
-	.then (present) ->
-		if !present
-			logger.logInfo("Installing qemu for #{arch} emulation...")
-			installQemu(arch)
-	.return(true)
-
 exports.qemuPathInContext = (context) ->
 	path = require('path')
 	binDir = path.join(context, '.balena')
@@ -45,15 +52,7 @@ exports.copyQemu = (context, arch) ->
 	.then ->
 		path.relative(context, binPath)
 
-hasQemu = (arch) ->
-	fs = require('mz/fs')
-
-	getQemuPath(arch)
-	.then(fs.stat)
-	.return(true)
-	.catchReturn(false)
-
-getQemuPath = (arch) ->
+exports.getQemuPath = getQemuPath = (arch) ->
 	balena = require('balena-sdk').fromSharedOptions()
 	path = require('path')
 	fs = require('mz/fs')
@@ -65,11 +64,7 @@ getQemuPath = (arch) ->
 		.then ->
 			path.join(binDir, "#{QEMU_BIN_NAME}-#{arch}-#{QEMU_VERSION}")
 
-platformNeedsQemu = ->
-	os = require('os')
-	os.platform() == 'linux'
-
-installQemu = (arch) ->
+exports.installQemu = (arch) ->
 	request = require('request')
 	fs = require('fs')
 	zlib = require('zlib')

lib/utils/qemu.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @license
+ * Copyright 2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const QEMU_VERSION: string;
+export const QEMU_BIN_NAME: string;
+
+export async function getQemuPath(arch: string): Promise<string>;
+export async function installQemu(arch: string): Promise<void>;

npm-shrinkwrap.json

@@ -5273,9 +5273,9 @@
       "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
     },
     "fp-ts": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.0.2.tgz",
-      "integrity": "sha512-ZCeu5MkqNDBWe1ewjZQ9Q9JNcPKEKXpitYzJ4ygCWpfJ3skW3imZ45EqsZd+9N8rkBvmsb64ToZTI2xXNO9IcQ=="
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.0.5.tgz",
+      "integrity": "sha512-opI5r+rVlpZE7Rhk0YtqsrmxGkbIw0dRNqGca8FEAMMnjomXotG+R9QkLQg20onx7R8qhepAn4CCOP8usma/Xw=="
     },
     "fragment-cache": {
       "version": "0.2.1",
@@ -7195,9 +7195,9 @@
       "dev": true
     },
     "io-ts": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.0.0.tgz",
-      "integrity": "sha512-6i8PKyNR/dvEbUU9uE+v4iVFU7l674ZEGQsh92y6xEZF/rj46fXbPy+uPPXJEsCP0J0X3UpzXAxp04K4HR2jVw=="
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.0.1.tgz",
+      "integrity": "sha512-RezD+WcCfW4VkMkEcQWL/Nmy/nqsWTvTYg7oUmTGzglvSSV2P9h2z1PVeREPFf0GWNzruYleAt1XCMQZSg1xxQ=="
     },
     "ip": {
       "version": "1.1.5",
@@ -14535,9 +14535,9 @@
           }
         },
         "semver": {
-          "version": "6.2.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz",
-          "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A=="
+          "version": "6.3.0",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
+          "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
         },
         "tar-stream": {
           "version": "2.1.0",
@@ -15066,9 +15066,9 @@
       }
     },
     "resin-multibuild": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.0.1.tgz",
-      "integrity": "sha512-TXWkWsbZMM92Y7Jo+qzRF8qdYeLCIpwQjYrHaAO5TlmUTk/Hb8QZgg1I9pV42mTi54AooXVHb0htE1YYbCxQng==",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.0.2.tgz",
+      "integrity": "sha512-rJkU33ytdxGMn5pPpnpVqaWsI8pETjXYarxdVvU6JPQBHJapKYLfAnMsAkAgFXNxKCs6an8uPIpjzapDJxwrfA==",
       "requires": {
         "@types/bluebird": "3.5.20",
         "@types/dockerode": "2.5.5",

package.json

@@ -199,7 +199,7 @@
     "resin-compose-parse": "^2.1.0",
     "resin-doodles": "0.0.1",
     "resin-image-fs": "^5.0.8",
-    "resin-multibuild": "4.0.1",
+    "resin-multibuild": "4.0.2",
     "resin-release": "^1.2.0",
     "resin-semver": "^1.6.0",
     "resin-stream-logger": "^0.1.2",

tests/utils/qemu.spec.ts

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright 2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect } from 'chai';
+
+describe('resin-multibuild consistency', function() {
+	it('should use the same values for selected constants', async () => {
+		const { QEMU_BIN_NAME: MQEMU_BIN_NAME } = await import('resin-multibuild');
+		const { QEMU_BIN_NAME } = await import('../../build/utils/qemu-ts');
+		expect(QEMU_BIN_NAME).to.equal(MQEMU_BIN_NAME);
+	});
+});