Add "build secrets" and "build variables" support for push/build/deploy

to/on/via balena devices

Change-type: minor
Signed-off-by: Paulo Castro <paulo@balena.io>

lib/config.ts

@@ -1,2 +1,21 @@
+/**
+ * @license
+ * Copyright 2017-2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const BALENA_ENGINE_TMP_PATH = '/var/lib/docker/tmp';
+
 export const sentryDsn =
 	'https://56d2a46124614b01b0f4086897e96110:6e175465accc41b595a96947155f61fb@sentry.io/149239';

lib/utils/compose.coffee

@@ -1,6 +1,6 @@
 ###*
 # @license
-# Copyright 2018 Balena Ltd.
+# Copyright 2017-2019 Balena Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -186,6 +186,8 @@ exports.buildProject = (
 	compose = require('resin-compose-parse')
 	builder = require('resin-multibuild')
 	transpose = require('docker-qemu-transpose')
+	{ BALENA_ENGINE_TMP_PATH } = require('../config')
+	{ checkBuildSecretsRequirements, makeBuildTasks } = require('./compose_ts')
 	qemu = require('./qemu')
 	{ toPosixPath } = builder.PathUtils
 
@@ -201,7 +203,8 @@ exports.buildProject = (
 		renderer = new BuildProgressUI(tty, imageDescriptors)
 	renderer.start()
 
-	qemu.installQemuIfNeeded(emulated, logger, arch)
+	Promise.resolve(checkBuildSecretsRequirements(docker, projectPath))
+	.then -> qemu.installQemuIfNeeded(emulated, logger, arch)
 	.tap (needsQemu) ->
 		return if not needsQemu
 		logger.logInfo('Emulation is enabled')
@@ -213,7 +216,6 @@ exports.buildProject = (
 		# Tar up the directory, ready for the build stream
 		tarDirectory(projectPath)
 		.then (tarStream) ->
-			{ makeBuildTasks } = require('./compose_ts')
 			Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger))
 		.map (task) ->
 			d = imageDescriptorsByServiceName[task.serviceName]
@@ -275,7 +277,7 @@ exports.buildProject = (
 					.pipe(task.logStream)
 	.then (tasks) ->
 		logger.logDebug 'Prepared tasks; building...'
-		builder.performBuilds(tasks, docker)
+		Promise.resolve(builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH))
 		.map (builtImage) ->
 			if not builtImage.successful
 				builtImage.error.serviceName = builtImage.serviceName

lib/utils/compose_ts.ts

@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 import * as Bluebird from 'bluebird';
+import { stripIndent } from 'common-tags';
+import Dockerode = require('dockerode');
 import * as _ from 'lodash';
 import { Composition } from 'resin-compose-parse';
 import * as MultiBuild from 'resin-multibuild';
@@ -24,6 +26,7 @@ import * as tar from 'tar-stream';
 import { BalenaSDK } from 'balena-sdk';
 import { DeviceInfo } from './device/api';
 import Logger = require('./logger');
+import { exitWithExpectedError } from './patterns';
 
 export interface RegistrySecrets {
 	[registryAddress: string]: {
@@ -32,6 +35,82 @@ export interface RegistrySecrets {
 	};
 }
 
+/**
+ * Load the ".balena/balena.yml" file (or resin.yml, or yaml or json),
+ * which contains "build metadata" for features like "build secrets" and
+ * "build variables".
+ * @returns Pair of metadata object and metadata file path
+ */
+export async function loadBuildMetatada(
+	sourceDir: string,
+): Promise<[MultiBuild.ParsedBalenaYml, string]> {
+	const { fs } = await import('mz');
+	const path = await import('path');
+	let metadataPath = '';
+	let rawString = '';
+
+	outer: for (const fName of ['balena', 'resin']) {
+		for (const fExt of ['yml', 'yaml', 'json']) {
+			metadataPath = path.join(sourceDir, `.${fName}`, `${fName}.${fExt}`);
+			try {
+				rawString = await fs.readFile(metadataPath, 'utf8');
+				break outer;
+			} catch (err) {
+				if (err.code === 'ENOENT') {
+					// file not found, try the next name.extension combination
+					continue;
+				} else {
+					throw err;
+				}
+			}
+		}
+	}
+	if (!rawString) {
+		return [{}, ''];
+	}
+	let buildMetadata: MultiBuild.ParsedBalenaYml;
+	try {
+		if (metadataPath.endsWith('json')) {
+			buildMetadata = JSON.parse(rawString);
+		} else {
+			buildMetadata = require('js-yaml').safeLoad(rawString);
+		}
+	} catch (err) {
+		return exitWithExpectedError(
+			`Error parsing file "${metadataPath}":\n ${err.message}`,
+		);
+	}
+	return [buildMetadata, metadataPath];
+}
+
+/**
+ * Check whether the "build secrets" feature is being used and, if so,
+ * verify that the target docker daemon is balenaEngine. If the
+ * requirement is not satisfied, call exitWithExpectedError().
+ * @param docker Dockerode instance
+ * @param sourceDir Project directory where to find .balena/balena.yml
+ */
+export async function checkBuildSecretsRequirements(
+	docker: Dockerode,
+	sourceDir: string,
+) {
+	const [metaObj, metaFilename] = await loadBuildMetatada(sourceDir);
+	if (!_.isEmpty(metaObj['build-secrets'])) {
+		const dockerUtils = await import('./docker');
+		const isBalenaEngine = await dockerUtils.isBalenaEngine(docker);
+		if (!isBalenaEngine) {
+			exitWithExpectedError(stripIndent`
+				The "build secrets" feature currently requires balenaEngine, but a standard Docker
+				daemon was detected. Please use command-line options to specify the hostname and
+				port number (or socket path) of a balenaEngine daemon, running on a balena device
+				or a virtual machine with balenaOS. If the build secrets feature is not required,
+				comment out or delete the 'build-secrets' entry in the file:
+				"${metaFilename}"
+				`);
+		}
+	}
+}
+
 export async function getRegistrySecrets(
 	sdk: BalenaSDK,
 	inputFilename?: string,
@@ -63,7 +142,6 @@ async function parseRegistrySecrets(
 	secretsFilename: string,
 ): Promise<RegistrySecrets> {
 	const { fs } = await import('mz');
-	const { exitWithExpectedError } = await import('../utils/patterns');
 	try {
 		let isYaml = false;
 		if (/.+\.ya?ml$/i.test(secretsFilename)) {
@@ -200,10 +278,8 @@ export function validateSpecifiedDockerfile(
 	if (!dockerfilePath) {
 		return dockerfilePath;
 	}
-	const { exitWithExpectedError } = require('../utils/patterns');
 	const { isAbsolute, join, normalize, parse, posix } = require('path');
 	const { existsSync } = require('fs');
-	const { stripIndent } = require('common-tags');
 	const { contains, toNativePath, toPosixPath } = MultiBuild.PathUtils;
 
 	// reminder: native windows paths may start with a drive specificaton,

lib/utils/device/deploy.ts

@@ -28,7 +28,8 @@ import {
 import * as semver from 'resin-semver';
 import { Readable } from 'stream';
 
-import { makeBuildTasks } from '../compose_ts';
+import { BALENA_ENGINE_TMP_PATH } from '../../config';
+import { checkBuildSecretsRequirements, makeBuildTasks } from '../compose_ts';
 import Logger = require('../logger');
 import { DeviceAPI, DeviceInfo } from './api';
 import * as LocalPushErrors from './errors';
@@ -175,6 +176,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 		opts.devicePort != null ? opts.devicePort : 2375,
 	);
 
+	await checkBuildSecretsRequirements(docker, opts.source);
 	const tarStream = await tarDirectory(opts.source);
 
 	// Try to detect the device information
@@ -310,7 +312,11 @@ export async function performBuilds(
 
 	logger.logDebug('Starting builds...');
 	await assignOutputHandlers(buildTasks, logger, logHandlers);
-	const localImages = await multibuild.performBuilds(buildTasks, docker);
+	const localImages = await multibuild.performBuilds(
+		buildTasks,
+		docker,
+		BALENA_ENGINE_TMP_PATH,
+	);
 
 	// Check for failures
 	await inspectBuildResults(localImages);
@@ -383,7 +389,11 @@ export async function rebuildSingleTask(
 	await assignDockerBuildOpts(docker, [task], opts);
 	await assignOutputHandlers([task], logger, logHandler);
 
-	const [localImage] = await multibuild.performBuilds([task], docker);
+	const [localImage] = await multibuild.performBuilds(
+		[task],
+		docker,
+		BALENA_ENGINE_TMP_PATH,
+	);
 
 	if (!localImage.successful) {
 		throw new LocalPushErrors.BuildError([

lib/utils/docker-coffee.coffee

@@ -1,3 +1,20 @@
+###*
+# @license
+# Copyright 2017-2019 Balena Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+
 # Functions to help actions which rely on using docker
 
 Promise = require('bluebird')

lib/utils/docker-coffee.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @license
+ * Copyright 2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as Bluebird from 'bluebird';
+import DockerToolbelt = require('docker-toolbelt');
+
+export interface BuildDockerOptions {
+	ca?: string; // path to ca (Certificate Authority) file (TLS)
+	cert?: string; // path to cert (Certificate) file (TLS)
+	key?: string; // path to key file (TLS)
+	docker?: string; // dockerode DockerOptions.socketPath
+	dockerHost?: string; // dockerode DockerOptions.host
+	dockerPort?: number; // dockerode DockerOptions.port
+}
+
+export function getDocker(
+	options: BuildDockerOptions,
+): Bluebird<DockerToolbelt>;

lib/utils/docker.ts

@@ -0,0 +1,35 @@
+/**
+ * @license
+ * Copyright 2018-2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Dockerode = require('dockerode');
+import * as dockerode from 'dockerode';
+
+export * from './docker-coffee';
+
+interface BalenaEngineVersion extends dockerode.DockerVersion {
+	Engine?: string;
+}
+
+export async function isBalenaEngine(docker: Dockerode): Promise<boolean> {
+	// dockerVersion.Engine should equal 'balena-engine' for the current/latest
+	// version of balenaEngine, but it was at one point (mis)spelt 'balaena':
+	// https://github.com/balena-os/balena-engine/pull/32/files
+	const dockerVersion = (await docker.version()) as BalenaEngineVersion;
+	return !!(
+		dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/)
+	);
+}

npm-shrinkwrap.json

@@ -363,19 +363,12 @@
       "dev": true
     },
     "@types/dockerode": {
-      "version": "2.5.5",
+      "version": "2.5.6",
-      "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz",
+      "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.6.tgz",
-      "integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==",
+      "integrity": "sha512-toIiWyyZ4tnIkPIehE31l+kzTCLqUX3atqLI6nX3VeEKwEY9tE0mFGaIRPxvLqdW96WVt1/gIiuNwu5oVfLCjg==",
+      "dev": true,
       "requires": {
-        "@types/events": "*",
         "@types/node": "*"
-      },
-      "dependencies": {
-        "@types/node": {
-          "version": "12.0.10",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
-          "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
-        }
       }
     },
     "@types/duplexify": {
@@ -384,13 +377,6 @@
       "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==",
       "requires": {
         "@types/node": "*"
-      },
-      "dependencies": {
-        "@types/node": {
-          "version": "12.0.10",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
-          "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
-        }
       }
     },
     "@types/event-stream": {
@@ -470,13 +456,6 @@
       "integrity": "sha512-KqHs2eAapKL7ZKUiKI/giUYPVgkoDXkVGFehk3goo+3Q8qwxVVRC3iwg+hK/THORbcri4RRxTtlm3JoSY1KZLQ==",
       "requires": {
         "@types/node": "*"
-      },
-      "dependencies": {
-        "@types/node": {
-          "version": "12.0.10",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
-          "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
-        }
       }
     },
     "@types/jwt-decode": {
@@ -536,8 +515,7 @@
     "@types/node": {
       "version": "10.14.5",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.5.tgz",
-      "integrity": "sha512-Ja7d4s0qyGFxjGeDq5S7Si25OFibSAHUi6i17UWnwNnpitADN7hah9q0Tl25gxuV5R1u2Bx+np6w4LHXfHyj/g==",
+      "integrity": "sha512-Ja7d4s0qyGFxjGeDq5S7Si25OFibSAHUi6i17UWnwNnpitADN7hah9q0Tl25gxuV5R1u2Bx+np6w4LHXfHyj/g=="
-      "dev": true
     },
     "@types/optimist": {
       "version": "0.0.29",
@@ -5193,6 +5171,11 @@
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
       "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
     },
+    "fp-ts": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.0.2.tgz",
+      "integrity": "sha512-ZCeu5MkqNDBWe1ewjZQ9Q9JNcPKEKXpitYzJ4ygCWpfJ3skW3imZ45EqsZd+9N8rkBvmsb64ToZTI2xXNO9IcQ=="
+    },
     "fragment-cache": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
@@ -7139,6 +7122,11 @@
       "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=",
       "dev": true
     },
+    "io-ts": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.0.0.tgz",
+      "integrity": "sha512-6i8PKyNR/dvEbUU9uE+v4iVFU7l674ZEGQsh92y6xEZF/rj46fXbPy+uPPXJEsCP0J0X3UpzXAxp04K4HR2jVw=="
+    },
     "ip": {
       "version": "1.1.5",
       "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
@@ -14198,9 +14186,9 @@
           }
         },
         "semver": {
-          "version": "6.1.3",
+          "version": "6.2.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.3.tgz",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz",
-          "integrity": "sha512-aymF+56WJJMyXQHcd4hlK4N75rwj5RQpfW8ePlQnJsTYOBLlLbcIErR/G1s9SkIvKBqOudR3KAx4wEqP+F1hNQ=="
+          "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A=="
         },
         "tar-stream": {
           "version": "2.1.0",
@@ -14607,6 +14595,15 @@
           "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz",
           "integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA=="
         },
+        "@types/dockerode": {
+          "version": "2.5.5",
+          "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz",
+          "integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==",
+          "requires": {
+            "@types/events": "*",
+            "@types/node": "*"
+          }
+        },
         "@types/mz": {
           "version": "0.0.30",
           "resolved": "https://registry.npmjs.org/@types/mz/-/mz-0.0.30.tgz",
@@ -14615,11 +14612,6 @@
             "@types/node": "*"
           }
         },
-        "@types/node": {
-          "version": "10.14.10",
-          "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.10.tgz",
-          "integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q=="
-        },
         "event-stream": {
           "version": "3.3.5",
           "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.5.tgz",
@@ -14725,9 +14717,9 @@
       }
     },
     "resin-multibuild": {
-      "version": "3.1.0",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.0.1.tgz",
-      "integrity": "sha512-GWRYTI3W28HqNidg2B3tOeiAmziSF38xgLxDq6lXa+2tu961kPNXrXqYg7GvpwakpsaC6HESu+mxYw+u+e4Igg==",
+      "integrity": "sha512-TXWkWsbZMM92Y7Jo+qzRF8qdYeLCIpwQjYrHaAO5TlmUTk/Hb8QZgg1I9pV42mTi54AooXVHb0htE1YYbCxQng==",
       "requires": {
         "@types/bluebird": "3.5.20",
         "@types/dockerode": "2.5.5",
@@ -14737,7 +14729,11 @@
         "bluebird": "^3.5.4",
         "docker-progress": "^4.0.0",
         "docker-toolbelt": "^3.3.7",
+        "dockerfile-template": "^0.1.0",
         "dockerode": "^2.5.8",
+        "fp-ts": "^2.0.0",
+        "io-ts": "^2.0.0",
+        "js-yaml": "^3.13.1",
         "lodash": "^4.17.4",
         "resin-bundle-resolve": "^4.1.0",
         "resin-compose-parse": "^2.0.4",
@@ -14752,6 +14748,15 @@
           "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz",
           "integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA=="
         },
+        "@types/dockerode": {
+          "version": "2.5.5",
+          "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz",
+          "integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==",
+          "requires": {
+            "@types/events": "*",
+            "@types/node": "*"
+          }
+        },
         "bl": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",

package.json

@@ -84,7 +84,7 @@
     "@types/bluebird": "3.5.21",
     "@types/chokidar": "^1.7.5",
     "@types/common-tags": "1.4.0",
-    "@types/dockerode": "2.5.5",
+    "@types/dockerode": "2.5.6",
     "@types/fs-extra": "7.0.0",
     "@types/is-root": "1.0.0",
     "@types/lodash": "4.14.112",
@@ -152,7 +152,7 @@
     "docker-progress": "^4.0.0",
     "docker-qemu-transpose": "^0.5.3",
     "docker-toolbelt": "^3.3.7",
-    "dockerode": "^2.5.5",
+    "dockerode": "^2.5.8",
     "dockerode-options": "^0.2.1",
     "ejs": "^2.5.7",
     "etcher-sdk": "^2.0.5",
@@ -189,7 +189,7 @@
     "resin-compose-parse": "^2.1.0",
     "resin-doodles": "0.0.1",
     "resin-image-fs": "^5.0.8",
-    "resin-multibuild": "^3.1.0",
+    "resin-multibuild": "4.0.1",
     "resin-release": "^1.2.0",
     "resin-semver": "^1.6.0",
     "resin-stream-logger": "^0.1.2",