diff --git a/lib/config.ts b/lib/config.ts index 7e6ab7b5..bce3c3ff 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -1,2 +1,21 @@ +/** + * @license + * Copyright 2017-2019 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const BALENA_ENGINE_TMP_PATH = '/var/lib/docker/tmp'; + export const sentryDsn = 'https://56d2a46124614b01b0f4086897e96110:6e175465accc41b595a96947155f61fb@sentry.io/149239'; diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index 0e781ff3..c308c3a9 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -1,6 +1,6 @@ ###* # @license -# Copyright 2018 Balena Ltd. +# Copyright 2017-2019 Balena Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -186,6 +186,8 @@ exports.buildProject = ( compose = require('resin-compose-parse') builder = require('resin-multibuild') transpose = require('docker-qemu-transpose') + { BALENA_ENGINE_TMP_PATH } = require('../config') + { checkBuildSecretsRequirements, makeBuildTasks } = require('./compose_ts') qemu = require('./qemu') { toPosixPath } = builder.PathUtils @@ -201,7 +203,8 @@ exports.buildProject = ( renderer = new BuildProgressUI(tty, imageDescriptors) renderer.start() - qemu.installQemuIfNeeded(emulated, logger, arch) + Promise.resolve(checkBuildSecretsRequirements(docker, projectPath)) + .then -> qemu.installQemuIfNeeded(emulated, logger, arch) .tap (needsQemu) -> return if not needsQemu logger.logInfo('Emulation is enabled') @@ -213,7 +216,6 @@ exports.buildProject = ( # Tar up the directory, ready for the build stream tarDirectory(projectPath) .then (tarStream) -> - { makeBuildTasks } = require('./compose_ts') Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger)) .map (task) -> d = imageDescriptorsByServiceName[task.serviceName] @@ -275,7 +277,7 @@ exports.buildProject = ( .pipe(task.logStream) .then (tasks) -> logger.logDebug 'Prepared tasks; building...' - builder.performBuilds(tasks, docker) + Promise.resolve(builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH)) .map (builtImage) -> if not builtImage.successful builtImage.error.serviceName = builtImage.serviceName diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index c08e3b6a..53d07bf6 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -15,6 +15,8 @@ * limitations under the License. */ import * as Bluebird from 'bluebird'; +import { stripIndent } from 'common-tags'; +import Dockerode = require('dockerode'); import * as _ from 'lodash'; import { Composition } from 'resin-compose-parse'; import * as MultiBuild from 'resin-multibuild'; @@ -24,6 +26,7 @@ import * as tar from 'tar-stream'; import { BalenaSDK } from 'balena-sdk'; import { DeviceInfo } from './device/api'; import Logger = require('./logger'); +import { exitWithExpectedError } from './patterns'; export interface RegistrySecrets { [registryAddress: string]: { @@ -32,6 +35,82 @@ export interface RegistrySecrets { }; } +/** + * Load the ".balena/balena.yml" file (or resin.yml, or yaml or json), + * which contains "build metadata" for features like "build secrets" and + * "build variables". + * @returns Pair of metadata object and metadata file path + */ +export async function loadBuildMetatada( + sourceDir: string, +): Promise<[MultiBuild.ParsedBalenaYml, string]> { + const { fs } = await import('mz'); + const path = await import('path'); + let metadataPath = ''; + let rawString = ''; + + outer: for (const fName of ['balena', 'resin']) { + for (const fExt of ['yml', 'yaml', 'json']) { + metadataPath = path.join(sourceDir, `.${fName}`, `${fName}.${fExt}`); + try { + rawString = await fs.readFile(metadataPath, 'utf8'); + break outer; + } catch (err) { + if (err.code === 'ENOENT') { + // file not found, try the next name.extension combination + continue; + } else { + throw err; + } + } + } + } + if (!rawString) { + return [{}, '']; + } + let buildMetadata: MultiBuild.ParsedBalenaYml; + try { + if (metadataPath.endsWith('json')) { + buildMetadata = JSON.parse(rawString); + } else { + buildMetadata = require('js-yaml').safeLoad(rawString); + } + } catch (err) { + return exitWithExpectedError( + `Error parsing file "${metadataPath}":\n ${err.message}`, + ); + } + return [buildMetadata, metadataPath]; +} + +/** + * Check whether the "build secrets" feature is being used and, if so, + * verify that the target docker daemon is balenaEngine. If the + * requirement is not satisfied, call exitWithExpectedError(). + * @param docker Dockerode instance + * @param sourceDir Project directory where to find .balena/balena.yml + */ +export async function checkBuildSecretsRequirements( + docker: Dockerode, + sourceDir: string, +) { + const [metaObj, metaFilename] = await loadBuildMetatada(sourceDir); + if (!_.isEmpty(metaObj['build-secrets'])) { + const dockerUtils = await import('./docker'); + const isBalenaEngine = await dockerUtils.isBalenaEngine(docker); + if (!isBalenaEngine) { + exitWithExpectedError(stripIndent` + The "build secrets" feature currently requires balenaEngine, but a standard Docker + daemon was detected. Please use command-line options to specify the hostname and + port number (or socket path) of a balenaEngine daemon, running on a balena device + or a virtual machine with balenaOS. If the build secrets feature is not required, + comment out or delete the 'build-secrets' entry in the file: + "${metaFilename}" + `); + } + } +} + export async function getRegistrySecrets( sdk: BalenaSDK, inputFilename?: string, @@ -63,7 +142,6 @@ async function parseRegistrySecrets( secretsFilename: string, ): Promise { const { fs } = await import('mz'); - const { exitWithExpectedError } = await import('../utils/patterns'); try { let isYaml = false; if (/.+\.ya?ml$/i.test(secretsFilename)) { @@ -200,10 +278,8 @@ export function validateSpecifiedDockerfile( if (!dockerfilePath) { return dockerfilePath; } - const { exitWithExpectedError } = require('../utils/patterns'); const { isAbsolute, join, normalize, parse, posix } = require('path'); const { existsSync } = require('fs'); - const { stripIndent } = require('common-tags'); const { contains, toNativePath, toPosixPath } = MultiBuild.PathUtils; // reminder: native windows paths may start with a drive specificaton, diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts index 8e3e1560..1c05d699 100644 --- a/lib/utils/device/deploy.ts +++ b/lib/utils/device/deploy.ts @@ -28,7 +28,8 @@ import { import * as semver from 'resin-semver'; import { Readable } from 'stream'; -import { makeBuildTasks } from '../compose_ts'; +import { BALENA_ENGINE_TMP_PATH } from '../../config'; +import { checkBuildSecretsRequirements, makeBuildTasks } from '../compose_ts'; import Logger = require('../logger'); import { DeviceAPI, DeviceInfo } from './api'; import * as LocalPushErrors from './errors'; @@ -175,6 +176,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise { opts.devicePort != null ? opts.devicePort : 2375, ); + await checkBuildSecretsRequirements(docker, opts.source); const tarStream = await tarDirectory(opts.source); // Try to detect the device information @@ -310,7 +312,11 @@ export async function performBuilds( logger.logDebug('Starting builds...'); await assignOutputHandlers(buildTasks, logger, logHandlers); - const localImages = await multibuild.performBuilds(buildTasks, docker); + const localImages = await multibuild.performBuilds( + buildTasks, + docker, + BALENA_ENGINE_TMP_PATH, + ); // Check for failures await inspectBuildResults(localImages); @@ -383,7 +389,11 @@ export async function rebuildSingleTask( await assignDockerBuildOpts(docker, [task], opts); await assignOutputHandlers([task], logger, logHandler); - const [localImage] = await multibuild.performBuilds([task], docker); + const [localImage] = await multibuild.performBuilds( + [task], + docker, + BALENA_ENGINE_TMP_PATH, + ); if (!localImage.successful) { throw new LocalPushErrors.BuildError([ diff --git a/lib/utils/docker.coffee b/lib/utils/docker-coffee.coffee similarity index 90% rename from lib/utils/docker.coffee rename to lib/utils/docker-coffee.coffee index 47df4dfa..2b84a0e9 100644 --- a/lib/utils/docker.coffee +++ b/lib/utils/docker-coffee.coffee @@ -1,3 +1,20 @@ +###* +# @license +# Copyright 2017-2019 Balena Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + # Functions to help actions which rely on using docker Promise = require('bluebird') diff --git a/lib/utils/docker-coffee.d.ts b/lib/utils/docker-coffee.d.ts new file mode 100644 index 00000000..096ba97c --- /dev/null +++ b/lib/utils/docker-coffee.d.ts @@ -0,0 +1,32 @@ +/** + * @license + * Copyright 2019 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as Bluebird from 'bluebird'; +import DockerToolbelt = require('docker-toolbelt'); + +export interface BuildDockerOptions { + ca?: string; // path to ca (Certificate Authority) file (TLS) + cert?: string; // path to cert (Certificate) file (TLS) + key?: string; // path to key file (TLS) + docker?: string; // dockerode DockerOptions.socketPath + dockerHost?: string; // dockerode DockerOptions.host + dockerPort?: number; // dockerode DockerOptions.port +} + +export function getDocker( + options: BuildDockerOptions, +): Bluebird; diff --git a/lib/utils/docker.ts b/lib/utils/docker.ts new file mode 100644 index 00000000..1aef1563 --- /dev/null +++ b/lib/utils/docker.ts @@ -0,0 +1,35 @@ +/** + * @license + * Copyright 2018-2019 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Dockerode = require('dockerode'); +import * as dockerode from 'dockerode'; + +export * from './docker-coffee'; + +interface BalenaEngineVersion extends dockerode.DockerVersion { + Engine?: string; +} + +export async function isBalenaEngine(docker: Dockerode): Promise { + // dockerVersion.Engine should equal 'balena-engine' for the current/latest + // version of balenaEngine, but it was at one point (mis)spelt 'balaena': + // https://github.com/balena-os/balena-engine/pull/32/files + const dockerVersion = (await docker.version()) as BalenaEngineVersion; + return !!( + dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/) + ); +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index f7c95510..c1cd5560 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -363,19 +363,12 @@ "dev": true }, "@types/dockerode": { - "version": "2.5.5", - "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz", - "integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==", + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.6.tgz", + "integrity": "sha512-toIiWyyZ4tnIkPIehE31l+kzTCLqUX3atqLI6nX3VeEKwEY9tE0mFGaIRPxvLqdW96WVt1/gIiuNwu5oVfLCjg==", + "dev": true, "requires": { - "@types/events": "*", "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "12.0.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz", - "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==" - } } }, "@types/duplexify": { @@ -384,13 +377,6 @@ "integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==", "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "12.0.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz", - "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==" - } } }, "@types/event-stream": { @@ -470,13 +456,6 @@ "integrity": "sha512-KqHs2eAapKL7ZKUiKI/giUYPVgkoDXkVGFehk3goo+3Q8qwxVVRC3iwg+hK/THORbcri4RRxTtlm3JoSY1KZLQ==", "requires": { "@types/node": "*" - }, - "dependencies": { - "@types/node": { - "version": "12.0.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz", - "integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ==" - } } }, "@types/jwt-decode": { @@ -536,8 +515,7 @@ "@types/node": { "version": "10.14.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.5.tgz", - "integrity": "sha512-Ja7d4s0qyGFxjGeDq5S7Si25OFibSAHUi6i17UWnwNnpitADN7hah9q0Tl25gxuV5R1u2Bx+np6w4LHXfHyj/g==", - "dev": true + "integrity": "sha512-Ja7d4s0qyGFxjGeDq5S7Si25OFibSAHUi6i17UWnwNnpitADN7hah9q0Tl25gxuV5R1u2Bx+np6w4LHXfHyj/g==" }, "@types/optimist": { "version": "0.0.29", @@ -5193,6 +5171,11 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, + "fp-ts": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.0.2.tgz", + "integrity": "sha512-ZCeu5MkqNDBWe1ewjZQ9Q9JNcPKEKXpitYzJ4ygCWpfJ3skW3imZ45EqsZd+9N8rkBvmsb64ToZTI2xXNO9IcQ==" + }, "fragment-cache": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", @@ -7139,6 +7122,11 @@ "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=", "dev": true }, + "io-ts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.0.0.tgz", + "integrity": "sha512-6i8PKyNR/dvEbUU9uE+v4iVFU7l674ZEGQsh92y6xEZF/rj46fXbPy+uPPXJEsCP0J0X3UpzXAxp04K4HR2jVw==" + }, "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", @@ -14198,9 +14186,9 @@ } }, "semver": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.1.3.tgz", - "integrity": "sha512-aymF+56WJJMyXQHcd4hlK4N75rwj5RQpfW8ePlQnJsTYOBLlLbcIErR/G1s9SkIvKBqOudR3KAx4wEqP+F1hNQ==" + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz", + "integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A==" }, "tar-stream": { "version": "2.1.0", @@ -14607,6 +14595,15 @@ "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz", "integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA==" }, + "@types/dockerode": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz", + "integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==", + "requires": { + "@types/events": "*", + "@types/node": "*" + } + }, "@types/mz": { "version": "0.0.30", "resolved": "https://registry.npmjs.org/@types/mz/-/mz-0.0.30.tgz", @@ -14615,11 +14612,6 @@ "@types/node": "*" } }, - "@types/node": { - "version": "10.14.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.10.tgz", - "integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q==" - }, "event-stream": { "version": "3.3.5", "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.5.tgz", @@ -14725,9 +14717,9 @@ } }, "resin-multibuild": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-3.1.0.tgz", - "integrity": "sha512-GWRYTI3W28HqNidg2B3tOeiAmziSF38xgLxDq6lXa+2tu961kPNXrXqYg7GvpwakpsaC6HESu+mxYw+u+e4Igg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.0.1.tgz", + "integrity": "sha512-TXWkWsbZMM92Y7Jo+qzRF8qdYeLCIpwQjYrHaAO5TlmUTk/Hb8QZgg1I9pV42mTi54AooXVHb0htE1YYbCxQng==", "requires": { "@types/bluebird": "3.5.20", "@types/dockerode": "2.5.5", @@ -14737,7 +14729,11 @@ "bluebird": "^3.5.4", "docker-progress": "^4.0.0", "docker-toolbelt": "^3.3.7", + "dockerfile-template": "^0.1.0", "dockerode": "^2.5.8", + "fp-ts": "^2.0.0", + "io-ts": "^2.0.0", + "js-yaml": "^3.13.1", "lodash": "^4.17.4", "resin-bundle-resolve": "^4.1.0", "resin-compose-parse": "^2.0.4", @@ -14752,6 +14748,15 @@ "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz", "integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA==" }, + "@types/dockerode": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz", + "integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==", + "requires": { + "@types/events": "*", + "@types/node": "*" + } + }, "bl": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz", diff --git a/package.json b/package.json index 7180409d..5caeba47 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "@types/bluebird": "3.5.21", "@types/chokidar": "^1.7.5", "@types/common-tags": "1.4.0", - "@types/dockerode": "2.5.5", + "@types/dockerode": "2.5.6", "@types/fs-extra": "7.0.0", "@types/is-root": "1.0.0", "@types/lodash": "4.14.112", @@ -152,7 +152,7 @@ "docker-progress": "^4.0.0", "docker-qemu-transpose": "^0.5.3", "docker-toolbelt": "^3.3.7", - "dockerode": "^2.5.5", + "dockerode": "^2.5.8", "dockerode-options": "^0.2.1", "ejs": "^2.5.7", "etcher-sdk": "^2.0.5", @@ -189,7 +189,7 @@ "resin-compose-parse": "^2.1.0", "resin-doodles": "0.0.1", "resin-image-fs": "^5.0.8", - "resin-multibuild": "^3.1.0", + "resin-multibuild": "4.0.1", "resin-release": "^1.2.0", "resin-semver": "^1.6.0", "resin-stream-logger": "^0.1.2",