Add "build secrets" and "build variables" support for push/build/deploy

to/on/via balena devices

Change-type: minor
Signed-off-by: Paulo Castro <paulo@balena.io>
This commit is contained in:
Paulo Castro 2019-07-09 15:43:36 +01:00
parent 8cfacc9cbc
commit 225408c57d
9 changed files with 247 additions and 51 deletions

View File

@ -1,2 +1,21 @@
/**
* @license
* Copyright 2017-2019 Balena Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const BALENA_ENGINE_TMP_PATH = '/var/lib/docker/tmp';
export const sentryDsn =
'https://56d2a46124614b01b0f4086897e96110:6e175465accc41b595a96947155f61fb@sentry.io/149239';

View File

@ -1,6 +1,6 @@
###*
# @license
# Copyright 2018 Balena Ltd.
# Copyright 2017-2019 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -186,6 +186,8 @@ exports.buildProject = (
compose = require('resin-compose-parse')
builder = require('resin-multibuild')
transpose = require('docker-qemu-transpose')
{ BALENA_ENGINE_TMP_PATH } = require('../config')
{ checkBuildSecretsRequirements, makeBuildTasks } = require('./compose_ts')
qemu = require('./qemu')
{ toPosixPath } = builder.PathUtils
@ -201,7 +203,8 @@ exports.buildProject = (
renderer = new BuildProgressUI(tty, imageDescriptors)
renderer.start()
qemu.installQemuIfNeeded(emulated, logger, arch)
Promise.resolve(checkBuildSecretsRequirements(docker, projectPath))
.then -> qemu.installQemuIfNeeded(emulated, logger, arch)
.tap (needsQemu) ->
return if not needsQemu
logger.logInfo('Emulation is enabled')
@ -213,7 +216,6 @@ exports.buildProject = (
# Tar up the directory, ready for the build stream
tarDirectory(projectPath)
.then (tarStream) ->
{ makeBuildTasks } = require('./compose_ts')
Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger))
.map (task) ->
d = imageDescriptorsByServiceName[task.serviceName]
@ -275,7 +277,7 @@ exports.buildProject = (
.pipe(task.logStream)
.then (tasks) ->
logger.logDebug 'Prepared tasks; building...'
builder.performBuilds(tasks, docker)
Promise.resolve(builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH))
.map (builtImage) ->
if not builtImage.successful
builtImage.error.serviceName = builtImage.serviceName

View File

@ -15,6 +15,8 @@
* limitations under the License.
*/
import * as Bluebird from 'bluebird';
import { stripIndent } from 'common-tags';
import Dockerode = require('dockerode');
import * as _ from 'lodash';
import { Composition } from 'resin-compose-parse';
import * as MultiBuild from 'resin-multibuild';
@ -24,6 +26,7 @@ import * as tar from 'tar-stream';
import { BalenaSDK } from 'balena-sdk';
import { DeviceInfo } from './device/api';
import Logger = require('./logger');
import { exitWithExpectedError } from './patterns';
export interface RegistrySecrets {
[registryAddress: string]: {
@ -32,6 +35,82 @@ export interface RegistrySecrets {
};
}
/**
* Load the ".balena/balena.yml" file (or resin.yml, or yaml or json),
* which contains "build metadata" for features like "build secrets" and
* "build variables".
* @returns Pair of metadata object and metadata file path
*/
export async function loadBuildMetatada(
sourceDir: string,
): Promise<[MultiBuild.ParsedBalenaYml, string]> {
const { fs } = await import('mz');
const path = await import('path');
let metadataPath = '';
let rawString = '';
outer: for (const fName of ['balena', 'resin']) {
for (const fExt of ['yml', 'yaml', 'json']) {
metadataPath = path.join(sourceDir, `.${fName}`, `${fName}.${fExt}`);
try {
rawString = await fs.readFile(metadataPath, 'utf8');
break outer;
} catch (err) {
if (err.code === 'ENOENT') {
// file not found, try the next name.extension combination
continue;
} else {
throw err;
}
}
}
}
if (!rawString) {
return [{}, ''];
}
let buildMetadata: MultiBuild.ParsedBalenaYml;
try {
if (metadataPath.endsWith('json')) {
buildMetadata = JSON.parse(rawString);
} else {
buildMetadata = require('js-yaml').safeLoad(rawString);
}
} catch (err) {
return exitWithExpectedError(
`Error parsing file "${metadataPath}":\n ${err.message}`,
);
}
return [buildMetadata, metadataPath];
}
/**
* Check whether the "build secrets" feature is being used and, if so,
* verify that the target docker daemon is balenaEngine. If the
* requirement is not satisfied, call exitWithExpectedError().
* @param docker Dockerode instance
* @param sourceDir Project directory where to find .balena/balena.yml
*/
export async function checkBuildSecretsRequirements(
docker: Dockerode,
sourceDir: string,
) {
const [metaObj, metaFilename] = await loadBuildMetatada(sourceDir);
if (!_.isEmpty(metaObj['build-secrets'])) {
const dockerUtils = await import('./docker');
const isBalenaEngine = await dockerUtils.isBalenaEngine(docker);
if (!isBalenaEngine) {
exitWithExpectedError(stripIndent`
The "build secrets" feature currently requires balenaEngine, but a standard Docker
daemon was detected. Please use command-line options to specify the hostname and
port number (or socket path) of a balenaEngine daemon, running on a balena device
or a virtual machine with balenaOS. If the build secrets feature is not required,
comment out or delete the 'build-secrets' entry in the file:
"${metaFilename}"
`);
}
}
}
export async function getRegistrySecrets(
sdk: BalenaSDK,
inputFilename?: string,
@ -63,7 +142,6 @@ async function parseRegistrySecrets(
secretsFilename: string,
): Promise<RegistrySecrets> {
const { fs } = await import('mz');
const { exitWithExpectedError } = await import('../utils/patterns');
try {
let isYaml = false;
if (/.+\.ya?ml$/i.test(secretsFilename)) {
@ -200,10 +278,8 @@ export function validateSpecifiedDockerfile(
if (!dockerfilePath) {
return dockerfilePath;
}
const { exitWithExpectedError } = require('../utils/patterns');
const { isAbsolute, join, normalize, parse, posix } = require('path');
const { existsSync } = require('fs');
const { stripIndent } = require('common-tags');
const { contains, toNativePath, toPosixPath } = MultiBuild.PathUtils;
// reminder: native windows paths may start with a drive specificaton,

View File

@ -28,7 +28,8 @@ import {
import * as semver from 'resin-semver';
import { Readable } from 'stream';
import { makeBuildTasks } from '../compose_ts';
import { BALENA_ENGINE_TMP_PATH } from '../../config';
import { checkBuildSecretsRequirements, makeBuildTasks } from '../compose_ts';
import Logger = require('../logger');
import { DeviceAPI, DeviceInfo } from './api';
import * as LocalPushErrors from './errors';
@ -175,6 +176,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
opts.devicePort != null ? opts.devicePort : 2375,
);
await checkBuildSecretsRequirements(docker, opts.source);
const tarStream = await tarDirectory(opts.source);
// Try to detect the device information
@ -310,7 +312,11 @@ export async function performBuilds(
logger.logDebug('Starting builds...');
await assignOutputHandlers(buildTasks, logger, logHandlers);
const localImages = await multibuild.performBuilds(buildTasks, docker);
const localImages = await multibuild.performBuilds(
buildTasks,
docker,
BALENA_ENGINE_TMP_PATH,
);
// Check for failures
await inspectBuildResults(localImages);
@ -383,7 +389,11 @@ export async function rebuildSingleTask(
await assignDockerBuildOpts(docker, [task], opts);
await assignOutputHandlers([task], logger, logHandler);
const [localImage] = await multibuild.performBuilds([task], docker);
const [localImage] = await multibuild.performBuilds(
[task],
docker,
BALENA_ENGINE_TMP_PATH,
);
if (!localImage.successful) {
throw new LocalPushErrors.BuildError([

View File

@ -1,3 +1,20 @@
###*
# @license
# Copyright 2017-2019 Balena Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###
# Functions to help actions which rely on using docker
Promise = require('bluebird')

32
lib/utils/docker-coffee.d.ts vendored Normal file
View File

@ -0,0 +1,32 @@
/**
* @license
* Copyright 2019 Balena Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as Bluebird from 'bluebird';
import DockerToolbelt = require('docker-toolbelt');
export interface BuildDockerOptions {
ca?: string; // path to ca (Certificate Authority) file (TLS)
cert?: string; // path to cert (Certificate) file (TLS)
key?: string; // path to key file (TLS)
docker?: string; // dockerode DockerOptions.socketPath
dockerHost?: string; // dockerode DockerOptions.host
dockerPort?: number; // dockerode DockerOptions.port
}
export function getDocker(
options: BuildDockerOptions,
): Bluebird<DockerToolbelt>;

35
lib/utils/docker.ts Normal file
View File

@ -0,0 +1,35 @@
/**
* @license
* Copyright 2018-2019 Balena Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Dockerode = require('dockerode');
import * as dockerode from 'dockerode';
export * from './docker-coffee';
interface BalenaEngineVersion extends dockerode.DockerVersion {
Engine?: string;
}
export async function isBalenaEngine(docker: Dockerode): Promise<boolean> {
// dockerVersion.Engine should equal 'balena-engine' for the current/latest
// version of balenaEngine, but it was at one point (mis)spelt 'balaena':
// https://github.com/balena-os/balena-engine/pull/32/files
const dockerVersion = (await docker.version()) as BalenaEngineVersion;
return !!(
dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/)
);
}

81
npm-shrinkwrap.json generated
View File

@ -363,19 +363,12 @@
"dev": true
},
"@types/dockerode": {
"version": "2.5.5",
"resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz",
"integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==",
"version": "2.5.6",
"resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.6.tgz",
"integrity": "sha512-toIiWyyZ4tnIkPIehE31l+kzTCLqUX3atqLI6nX3VeEKwEY9tE0mFGaIRPxvLqdW96WVt1/gIiuNwu5oVfLCjg==",
"dev": true,
"requires": {
"@types/events": "*",
"@types/node": "*"
},
"dependencies": {
"@types/node": {
"version": "12.0.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
"integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
}
}
},
"@types/duplexify": {
@ -384,13 +377,6 @@
"integrity": "sha512-5zOA53RUlzN74bvrSGwjudssD9F3a797sDZQkiYpUOxW+WHaXTCPz4/d5Dgi6FKnOqZ2CpaTo0DhgIfsXAOE/A==",
"requires": {
"@types/node": "*"
},
"dependencies": {
"@types/node": {
"version": "12.0.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
"integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
}
}
},
"@types/event-stream": {
@ -470,13 +456,6 @@
"integrity": "sha512-KqHs2eAapKL7ZKUiKI/giUYPVgkoDXkVGFehk3goo+3Q8qwxVVRC3iwg+hK/THORbcri4RRxTtlm3JoSY1KZLQ==",
"requires": {
"@types/node": "*"
},
"dependencies": {
"@types/node": {
"version": "12.0.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.10.tgz",
"integrity": "sha512-LcsGbPomWsad6wmMNv7nBLw7YYYyfdYcz6xryKYQhx89c3XXan+8Q6AJ43G5XDIaklaVkK3mE4fCb0SBvMiPSQ=="
}
}
},
"@types/jwt-decode": {
@ -536,8 +515,7 @@
"@types/node": {
"version": "10.14.5",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.5.tgz",
"integrity": "sha512-Ja7d4s0qyGFxjGeDq5S7Si25OFibSAHUi6i17UWnwNnpitADN7hah9q0Tl25gxuV5R1u2Bx+np6w4LHXfHyj/g==",
"dev": true
"integrity": "sha512-Ja7d4s0qyGFxjGeDq5S7Si25OFibSAHUi6i17UWnwNnpitADN7hah9q0Tl25gxuV5R1u2Bx+np6w4LHXfHyj/g=="
},
"@types/optimist": {
"version": "0.0.29",
@ -5193,6 +5171,11 @@
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"fp-ts": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.0.2.tgz",
"integrity": "sha512-ZCeu5MkqNDBWe1ewjZQ9Q9JNcPKEKXpitYzJ4ygCWpfJ3skW3imZ45EqsZd+9N8rkBvmsb64ToZTI2xXNO9IcQ=="
},
"fragment-cache": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
@ -7139,6 +7122,11 @@
"integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=",
"dev": true
},
"io-ts": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.0.0.tgz",
"integrity": "sha512-6i8PKyNR/dvEbUU9uE+v4iVFU7l674ZEGQsh92y6xEZF/rj46fXbPy+uPPXJEsCP0J0X3UpzXAxp04K4HR2jVw=="
},
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
@ -14198,9 +14186,9 @@
}
},
"semver": {
"version": "6.1.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.1.3.tgz",
"integrity": "sha512-aymF+56WJJMyXQHcd4hlK4N75rwj5RQpfW8ePlQnJsTYOBLlLbcIErR/G1s9SkIvKBqOudR3KAx4wEqP+F1hNQ=="
"version": "6.2.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.2.0.tgz",
"integrity": "sha512-jdFC1VdUGT/2Scgbimf7FSx9iJLXoqfglSF+gJeuNWVpiE37OIbc1jywR/GJyFdz3mnkz2/id0L0J/cr0izR5A=="
},
"tar-stream": {
"version": "2.1.0",
@ -14607,6 +14595,15 @@
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz",
"integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA=="
},
"@types/dockerode": {
"version": "2.5.5",
"resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz",
"integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==",
"requires": {
"@types/events": "*",
"@types/node": "*"
}
},
"@types/mz": {
"version": "0.0.30",
"resolved": "https://registry.npmjs.org/@types/mz/-/mz-0.0.30.tgz",
@ -14615,11 +14612,6 @@
"@types/node": "*"
}
},
"@types/node": {
"version": "10.14.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.10.tgz",
"integrity": "sha512-V8wj+w2YMNvGuhgl/MA5fmTxgjmVHVoasfIaxMMZJV6Y8Kk+Ydpi1z2whoShDCJ2BuNVoqH/h1hrygnBxkrw/Q=="
},
"event-stream": {
"version": "3.3.5",
"resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.5.tgz",
@ -14725,9 +14717,9 @@
}
},
"resin-multibuild": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-3.1.0.tgz",
"integrity": "sha512-GWRYTI3W28HqNidg2B3tOeiAmziSF38xgLxDq6lXa+2tu961kPNXrXqYg7GvpwakpsaC6HESu+mxYw+u+e4Igg==",
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.0.1.tgz",
"integrity": "sha512-TXWkWsbZMM92Y7Jo+qzRF8qdYeLCIpwQjYrHaAO5TlmUTk/Hb8QZgg1I9pV42mTi54AooXVHb0htE1YYbCxQng==",
"requires": {
"@types/bluebird": "3.5.20",
"@types/dockerode": "2.5.5",
@ -14737,7 +14729,11 @@
"bluebird": "^3.5.4",
"docker-progress": "^4.0.0",
"docker-toolbelt": "^3.3.7",
"dockerfile-template": "^0.1.0",
"dockerode": "^2.5.8",
"fp-ts": "^2.0.0",
"io-ts": "^2.0.0",
"js-yaml": "^3.13.1",
"lodash": "^4.17.4",
"resin-bundle-resolve": "^4.1.0",
"resin-compose-parse": "^2.0.4",
@ -14752,6 +14748,15 @@
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.20.tgz",
"integrity": "sha512-Wk41MVdF+cHBfVXj/ufUHJeO3BlIQr1McbHZANErMykaCWeDSZbH5erGjNBw2/3UlRdSxZbLfSuQTzFmPOYFsA=="
},
"@types/dockerode": {
"version": "2.5.5",
"resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.5.tgz",
"integrity": "sha512-NSDdW9JUia7bLba87jTfZR/o1YcIFDcSC9HcdrXGJluHVfl8hXiUBFJwB9z3LSk/xmvv/pmEKzHFq6JmpNhlfg==",
"requires": {
"@types/events": "*",
"@types/node": "*"
}
},
"bl": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",

View File

@ -84,7 +84,7 @@
"@types/bluebird": "3.5.21",
"@types/chokidar": "^1.7.5",
"@types/common-tags": "1.4.0",
"@types/dockerode": "2.5.5",
"@types/dockerode": "2.5.6",
"@types/fs-extra": "7.0.0",
"@types/is-root": "1.0.0",
"@types/lodash": "4.14.112",
@ -152,7 +152,7 @@
"docker-progress": "^4.0.0",
"docker-qemu-transpose": "^0.5.3",
"docker-toolbelt": "^3.3.7",
"dockerode": "^2.5.5",
"dockerode": "^2.5.8",
"dockerode-options": "^0.2.1",
"ejs": "^2.5.7",
"etcher-sdk": "^2.0.5",
@ -189,7 +189,7 @@
"resin-compose-parse": "^2.1.0",
"resin-doodles": "0.0.1",
"resin-image-fs": "^5.0.8",
"resin-multibuild": "^3.1.0",
"resin-multibuild": "4.0.1",
"resin-release": "^1.2.0",
"resin-semver": "^1.6.0",
"resin-stream-logger": "^0.1.2",