From db25a65753b4bd9901e46be6013b101f813aa035 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Thu, 11 Apr 2019 12:49:19 +0100 Subject: [PATCH] Add --dockerfile option to the build, deploy and push commands It allows the selection of an alternative Dockerfile in single- container projects that do not include a docker-compose file. Change-type: minor Signed-off-by: Paulo Castro --- doc/cli.markdown | 49 ++++++++++++++++++---------- lib/actions/build.coffee | 24 +++++++------- lib/actions/deploy.coffee | 17 ++++++---- lib/actions/push.ts | 18 +++++++++- lib/utils/compose.coffee | 30 ++++++++++++----- lib/utils/compose.d.ts | 1 + lib/utils/compose_ts.ts | 67 ++++++++++++++++++++++++++++++++++++++ lib/utils/device/deploy.ts | 9 ++++- lib/utils/helpers.ts | 6 ---- lib/utils/ignore.ts | 19 ++++++++++- lib/utils/remote-build.ts | 2 ++ package.json | 11 +++---- 12 files changed, 193 insertions(+), 60 deletions(-) diff --git a/doc/cli.markdown b/doc/cli.markdown index 004fc2c3..9f523de2 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -1442,6 +1442,10 @@ The source that should be sent to the balena builder to be built (defaults to th Force an emulated build to occur on the remote builder +#### --dockerfile <Dockerfile> + +Alternative Dockerfile name/path, relative to the source folder + #### --nocache, -c Don't use cache when building this project @@ -1712,20 +1716,20 @@ name of container to stop ## build [source] -Use this command to build an image or a complete multicontainer project -with the provided docker daemon in your development machine or balena -device. (See also the `balena push` command for the option of building -images in the balenaCloud build servers.) +Use this command to build an image or a complete multicontainer project with +the provided docker daemon in your development machine or balena device. +(See also the `balena push` command for the option of building images in the +balenaCloud build servers.) -You must provide either an application or a device-type/architecture -pair to use the balena Dockerfile pre-processor -(e.g. Dockerfile.template -> Dockerfile). +You must provide either an application or a device-type/architecture pair to use +the balena Dockerfile pre-processor (e.g. Dockerfile.template -> Dockerfile). This command will look into the given source directory (or the current working -directory if one isn't specified) for a compose file. If one is found, this -command will build each service defined in the compose file. If a compose file -isn't found, the command will look for a Dockerfile, and if yet that isn't found, -it will try to generate one. +directory if one isn't specified) for a docker-compose.yml file. If it is found, +this command will build each service defined in the compose file. If a compose +file isn't found, the command will look for a Dockerfile[.template] file (or +alternative Dockerfile specified with the `-f` option), and if yet that isn't +found, it will try to generate one. The --registry-secrets option specifies a JSON or YAML file containing private Docker registry usernames and passwords to be used when pulling base images. @@ -1772,6 +1776,10 @@ Specify an alternate project name; default is the directory name Run an emulated build using Qemu +#### --dockerfile <Dockerfile> + +Alternative Dockerfile name/path, relative to the source folder + #### --logs Display full log output @@ -1831,17 +1839,18 @@ balena device. (See also the `balena push` command for the option of building the image in the balenaCloud build servers.) Unless an image is specified, this command will look into the current directory -(or the one specified by --source) for a compose file. If one is found, this -command will deploy each service defined in the compose file, building it first -if an image for it doesn't exist. If a compose file isn't found, the command -will look for a Dockerfile, and if yet that isn't found, it will try to -generate one. +(or the one specified by --source) for a docker-compose.yml file. If one is +found, this command will deploy each service defined in the compose file, +building it first if an image for it doesn't exist. If a compose file isn't +found, the command will look for a Dockerfile[.template] file (or alternative +Dockerfile specified with the `-f` option), and if yet that isn't found, it +will try to generate one. To deploy to an app on which you're a collaborator, use `balena deploy /`. -When --build is used, all options supported by `balena build` are also -supported by this command. +When --build is used, all options supported by `balena build` are also supported +by this command. The --registry-secrets option specifies a JSON or YAML file containing private Docker registry usernames and passwords to be used when pulling base images. @@ -1885,6 +1894,10 @@ Specify an alternate project name; default is the directory name Run an emulated build using Qemu +#### --dockerfile <Dockerfile> + +Alternative Dockerfile name/path, relative to the source folder + #### --logs Display full log output diff --git a/lib/actions/build.coffee b/lib/actions/build.coffee index 6152a397..ec6db920 100644 --- a/lib/actions/build.coffee +++ b/lib/actions/build.coffee @@ -19,6 +19,8 @@ buildProject = (docker, logger, composeOpts, opts) -> logger composeOpts.projectPath composeOpts.projectName + undefined # image: name of pre-built image + composeOpts.dockerfilePath # ok if undefined ) .then (project) -> appType = opts.app?.application_type?[0] @@ -50,20 +52,20 @@ module.exports = description: 'Build a single image or a multicontainer project locally' primary: true help: """ - Use this command to build an image or a complete multicontainer project - with the provided docker daemon in your development machine or balena - device. (See also the `balena push` command for the option of building - images in the balenaCloud build servers.) + Use this command to build an image or a complete multicontainer project with + the provided docker daemon in your development machine or balena device. + (See also the `balena push` command for the option of building images in the + balenaCloud build servers.) - You must provide either an application or a device-type/architecture - pair to use the balena Dockerfile pre-processor - (e.g. Dockerfile.template -> Dockerfile). + You must provide either an application or a device-type/architecture pair to use + the balena Dockerfile pre-processor (e.g. Dockerfile.template -> Dockerfile). This command will look into the given source directory (or the current working - directory if one isn't specified) for a compose file. If one is found, this - command will build each service defined in the compose file. If a compose file - isn't found, the command will look for a Dockerfile, and if yet that isn't found, - it will try to generate one. + directory if one isn't specified) for a docker-compose.yml file. If it is found, + this command will build each service defined in the compose file. If a compose + file isn't found, the command will look for a Dockerfile[.template] file (or + alternative Dockerfile specified with the `-f` option), and if yet that isn't + found, it will try to generate one. #{registrySecretsHelp} diff --git a/lib/actions/deploy.coffee b/lib/actions/deploy.coffee index 2adb21b3..2bb0cd25 100644 --- a/lib/actions/deploy.coffee +++ b/lib/actions/deploy.coffee @@ -10,6 +10,7 @@ Opts must be an object with the following keys: app: the application instance to deploy to image: the image to deploy; optional + dockerfilePath: name of an alternative Dockerfile; optional shouldPerformBuild shouldUploadLogs buildEmulated @@ -25,6 +26,7 @@ deployProject = (docker, logger, composeOpts, opts) -> composeOpts.projectPath composeOpts.projectName opts.image + composeOpts.dockerfilePath # ok if undefined ) .then (project) -> if project.descriptors.length > 1 and !opts.app.application_type?[0]?.supports_multicontainer @@ -133,17 +135,18 @@ module.exports = the image in the balenaCloud build servers.) Unless an image is specified, this command will look into the current directory - (or the one specified by --source) for a compose file. If one is found, this - command will deploy each service defined in the compose file, building it first - if an image for it doesn't exist. If a compose file isn't found, the command - will look for a Dockerfile, and if yet that isn't found, it will try to - generate one. + (or the one specified by --source) for a docker-compose.yml file. If one is + found, this command will deploy each service defined in the compose file, + building it first if an image for it doesn't exist. If a compose file isn't + found, the command will look for a Dockerfile[.template] file (or alternative + Dockerfile specified with the `-f` option), and if yet that isn't found, it + will try to generate one. To deploy to an app on which you're a collaborator, use `balena deploy /`. - When --build is used, all options supported by `balena build` are also - supported by this command. + When --build is used, all options supported by `balena build` are also supported + by this command. #{registrySecretsHelp} diff --git a/lib/actions/push.ts b/lib/actions/push.ts index 8701a59f..81e09bb4 100644 --- a/lib/actions/push.ts +++ b/lib/actions/push.ts @@ -104,6 +104,7 @@ export const push: CommandDefinition< { source: string; emulated: boolean; + dockerfile: string; // DeviceDeployOptions.dockerfilePath (alternative Dockerfile) nocache: boolean; 'registry-secrets': string; live: boolean; @@ -158,6 +159,12 @@ export const push: CommandDefinition< description: 'Force an emulated build to occur on the remote builder', boolean: true, }, + { + signature: 'dockerfile', + parameter: 'Dockerfile', + description: + 'Alternative Dockerfile name/path, relative to the source folder', + }, { signature: 'nocache', alias: 'c', @@ -194,7 +201,9 @@ export const push: CommandDefinition< const { exitIfNotLoggedIn, exitWithExpectedError } = await import( '../utils/patterns' ); - const { parseRegistrySecrets } = await import('../utils/compose_ts'); + const { validateSpecifiedDockerfile, parseRegistrySecrets } = await import( + '../utils/compose_ts' + ); const { BuildError } = await import('../utils/device/errors'); const appOrDevice: string | null = params.applicationOrDevice; @@ -207,6 +216,11 @@ export const push: CommandDefinition< console.log(`[debug] Using ${source} as build source`); } + const dockerfilePath = validateSpecifiedDockerfile( + source, + options.dockerfile, + ); + const registrySecrets = options['registry-secrets'] ? await parseRegistrySecrets(options['registry-secrets']) : {}; @@ -229,6 +243,7 @@ export const push: CommandDefinition< getAppOwner(sdk, app), async (token, baseUrl, owner) => { const opts = { + dockerfilePath, emulated: options.emulated, nocache: options.nocache, registrySecrets, @@ -254,6 +269,7 @@ export const push: CommandDefinition< deviceDeploy.deployToDevice({ source, deviceHost: device, + dockerfilePath, registrySecrets, nocache: options.nocache || false, live: options.live || false, diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index 00bd88ba..aa717f8b 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -36,16 +36,21 @@ exports.appendOptions = (opts) -> boolean: true alias: 'e' }, + { + signature: 'dockerfile' + parameter: 'Dockerfile' + description: 'Alternative Dockerfile name/path, relative to the source folder' + }, { signature: 'logs' description: 'Display full log output' boolean: true }, { - signature: 'registry-secrets', - alias: 'R', - parameter: 'secrets.yml|.json', - description: 'Path to a YAML or JSON file with passwords for a private Docker registry', + signature: 'registry-secrets' + alias: 'R' + parameter: 'secrets.yml|.json' + description: 'Path to a YAML or JSON file with passwords for a private Docker registry' }, ] @@ -55,6 +60,7 @@ exports.generateOpts = (options) -> projectName: options.projectName projectPath: projectPath inlineLogs: !!options.logs + dockerfilePath: options.dockerfile compositionFileNames = [ 'docker-compose.yml' @@ -97,11 +103,14 @@ createProject = (composePath, composeStr, projectName = null) -> # of it in one go. if image is given, it'll create a default project for # that without looking for a project. falls back to creating a default # project if none is found at the given projectPath. -exports.loadProject = (logger, projectPath, projectName, image) -> +exports.loadProject = (logger, projectPath, projectName, image, dockerfilePath) -> + { validateSpecifiedDockerfile } = require('./compose_ts') compose = require('resin-compose-parse') logger.logDebug('Loading project...') Promise.try -> + dockerfilePath = validateSpecifiedDockerfile(projectPath, dockerfilePath) + if image? logger.logInfo("Creating default composition with image: #{image}") return compose.defaultComposition(image) @@ -110,11 +119,14 @@ exports.loadProject = (logger, projectPath, projectName, image) -> resolveProject(projectPath) .tap -> - logger.logInfo('Compose file detected') + if dockerfilePath + logger.logWarn("Ignoring alternative dockerfile \"#{dockerfilePath}\"\ because a docker-compose file exists") + else + logger.logInfo('Compose file detected') .catch (e) -> logger.logDebug("Failed to resolve project: #{e}") logger.logInfo("Creating default composition with source: #{projectPath}") - return compose.defaultComposition() + return compose.defaultComposition(undefined, dockerfilePath) .then (composeStr) -> logger.logDebug('Creating project...') createProject(projectPath, composeStr, projectName) @@ -126,7 +138,7 @@ exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) -> fs = require('mz/fs') streamToPromise = require('stream-to-promise') { FileIgnorer } = require('./ignore') - { toPosixPath } = require('./helpers') + { toPosixPath } = require('resin-multibuild').PathUtils getFiles = -> streamToPromise(klaw(dir)) @@ -175,7 +187,7 @@ exports.buildProject = ( builder = require('resin-multibuild') transpose = require('docker-qemu-transpose') qemu = require('./qemu') - { toPosixPath } = require('./helpers') + { toPosixPath } = builder.PathUtils logger.logInfo("Building for #{arch}/#{deviceType}") diff --git a/lib/utils/compose.d.ts b/lib/utils/compose.d.ts index ad523597..58ca97bd 100644 --- a/lib/utils/compose.d.ts +++ b/lib/utils/compose.d.ts @@ -46,6 +46,7 @@ export function loadProject( projectPath: string, projectName: string, image?: string, + dockerfilePath?: string, ): Bluebird; export function tarDirectory( diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index dd747dee..d25d67ef 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -155,3 +155,70 @@ async function performResolution( }).then(resolve, reject); }); } + +/** + * Enforce that, for example, if 'myProject/MyDockerfile.template' is specified + * as an alternativate Dockerfile name, then 'myProject/MyDockerfile' must not + * exist. + * @param projectPath The project source folder (-s command-line option) + * @param dockerfilePath The alternative Dockerfile specified by the user + */ +export function validateSpecifiedDockerfile( + projectPath: string, + dockerfilePath: string = '', +): string { + if (!dockerfilePath) { + return dockerfilePath; + } + const { exitWithExpectedError } = require('../utils/patterns'); + const { isAbsolute, join, normalize, parse, posix } = require('path'); + const { existsSync } = require('fs'); + const { stripIndent } = require('common-tags'); + const { contains, toNativePath, toPosixPath } = MultiBuild.PathUtils; + + // reminder: native windows paths may start with a drive specificaton, + // e.g. 'C:\absolute' or 'C:relative'. + if (isAbsolute(dockerfilePath) || posix.isAbsolute(dockerfilePath)) { + exitWithExpectedError(stripIndent` + Error: absolute Dockerfile path detected: + "${dockerfilePath}" + The Dockerfile path should be relative to the source folder. + `); + } + const nativeProjectPath = normalize(projectPath); + const nativeDockerfilePath = join(projectPath, toNativePath(dockerfilePath)); + + if (!contains(nativeProjectPath, nativeDockerfilePath)) { + // Note that testing the existence of nativeDockerfilePath in the + // filesystem (after joining its path to the source folder) is not + // sufficient, because the user could have added '../' to the path. + exitWithExpectedError(stripIndent` + Error: the specified Dockerfile must be in a subfolder of the source folder: + Specified dockerfile: "${nativeDockerfilePath}" + Source folder: "${nativeProjectPath}" + `); + } + + if (!existsSync(nativeDockerfilePath)) { + exitWithExpectedError(stripIndent` + Error: Dockerfile not found: "${nativeDockerfilePath}" + `); + } + + const { dir, ext, name } = parse(nativeDockerfilePath); + if (ext) { + const nativePathMinusExt = join(dir, name); + + if (existsSync(nativePathMinusExt)) { + exitWithExpectedError(stripIndent` + Error: "${name}" exists on the same folder as "${dockerfilePath}". + When an alternative Dockerfile name is specified, a file with the same + base name (minus the file extension) must not exist in the same folder. + This is because the base name file will be auto generated and added to + the tar stream that is sent to the docker daemon, resulting in duplicate + Dockerfiles and undefined behavior. + `); + } + } + return posix.normalize(toPosixPath(dockerfilePath)); +} diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts index 708bdea2..ac6e6878 100644 --- a/lib/utils/device/deploy.ts +++ b/lib/utils/device/deploy.ts @@ -43,6 +43,7 @@ export interface DeviceDeployOptions { source: string; deviceHost: string; devicePort?: number; + dockerfilePath?: string; registrySecrets: RegistrySecrets; nocache: boolean; live: boolean; @@ -99,7 +100,13 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise { globalLogger.logInfo(`Starting build on device ${opts.deviceHost}`); - const project = await loadProject(globalLogger, opts.source, 'local'); + const project = await loadProject( + globalLogger, + opts.source, // project path + 'local', // project name + undefined, // name of a pre-built image + opts.dockerfilePath, // alternative Dockerfile; OK to be undefined + ); // Attempt to attach to the device's docker daemon const docker = connectToDocker( diff --git a/lib/utils/helpers.ts b/lib/utils/helpers.ts index 10ddac6b..a387a488 100644 --- a/lib/utils/helpers.ts +++ b/lib/utils/helpers.ts @@ -19,20 +19,14 @@ import Bluebird = require('bluebird'); import chalk from 'chalk'; import _ = require('lodash'); import os = require('os'); -import path = require('path'); import visuals = require('resin-cli-visuals'); import rindle = require('rindle'); import { InitializeEmitter, OperationState } from 'balena-device-init'; const waitStreamAsync = Bluebird.promisify(rindle.wait); - const balena = BalenaSdk.fromSharedOptions(); -export function toPosixPath(p: string): string { - return p.replace(new RegExp('\\' + path.sep, 'g'), '/'); -} - export function getGroupDefaults(group: { options: Array<{ name: string; default?: string }>; }): { [name: string]: string | undefined } { diff --git a/lib/utils/ignore.ts b/lib/utils/ignore.ts index 76305063..e1ad325a 100644 --- a/lib/utils/ignore.ts +++ b/lib/utils/ignore.ts @@ -1,11 +1,28 @@ +/** + * @license + * Copyright 2019 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import * as _ from 'lodash'; import { fs } from 'mz'; import * as path from 'path'; +import * as MultiBuild from 'resin-multibuild'; import dockerIgnore = require('@zeit/dockerignore'); import ignore from 'ignore'; -import { toPosixPath } from './helpers'; +const { toPosixPath } = MultiBuild.PathUtils; export enum IgnoreFileType { DockerIgnore, diff --git a/lib/utils/remote-build.ts b/lib/utils/remote-build.ts index c7035404..9854ce96 100644 --- a/lib/utils/remote-build.ts +++ b/lib/utils/remote-build.ts @@ -32,6 +32,7 @@ const CURSOR_METADATA_REGEX = /([a-z]+)([0-9]+)?/; const TRIM_REGEX = /\n+$/; export interface BuildOpts { + dockerfilePath: string; emulated: boolean; nocache: boolean; registrySecrets: RegistrySecrets; @@ -78,6 +79,7 @@ async function getBuilderEndpoint( const args = querystring.stringify({ owner, app, + dockerfilePath: opts.dockerfilePath, emulated: opts.emulated, nocache: opts.nocache, }); diff --git a/package.json b/package.json index 1ff61e06..060287a6 100644 --- a/package.json +++ b/package.json @@ -72,11 +72,15 @@ "@types/fs-extra": "5.0.4", "@types/is-root": "1.0.0", "@types/lodash": "4.14.112", + "@types/mixpanel": "2.14.0", "@types/mkdirp": "0.5.2", "@types/node": "6.14.2", "@types/prettyjson": "0.0.28", + "@types/stream-to-promise": "2.2.0", "@types/raven": "2.5.1", + "@types/request": "2.48.1", "@types/tar-stream": "1.6.0", + "@types/through2": "2.0.33", "catch-uncommitted": "^1.0.0", "ent": "^2.2.0", "filehound": "^1.16.2", @@ -98,11 +102,6 @@ }, "dependencies": { "@resin.io/valid-email": "^0.1.0", - "@types/dockerode": "2.5.5", - "@types/mixpanel": "2.14.0", - "@types/request": "2.48.1", - "@types/stream-to-promise": "2.2.0", - "@types/through2": "^2.0.33", "@zeit/dockerignore": "0.0.3", "JSONStream": "^1.0.3", "ansi-escapes": "^2.0.0", @@ -162,7 +161,7 @@ "request": "^2.81.0", "resin-cli-form": "^2.0.1", "resin-cli-visuals": "^1.4.0", - "resin-compose-parse": "^2.0.4", + "resin-compose-parse": "^2.1.0", "resin-doodles": "0.0.1", "resin-image-fs": "^5.0.2", "resin-multibuild": "^3.1.0",