Add --dockerfile option to the build, deploy and push commands

It allows the selection of an alternative Dockerfile in single-
container projects that do not include a docker-compose file.

Change-type: minor
Signed-off-by: Paulo Castro <paulo@balena.io>

doc/cli.markdown

@@ -1442,6 +1442,10 @@ The source that should be sent to the balena builder to be built (defaults to th
 
 Force an emulated build to occur on the remote builder
 
+#### --dockerfile <Dockerfile>
+
+Alternative Dockerfile name/path, relative to the source folder
+
 #### --nocache, -c
 
 Don't use cache when building this project
@@ -1712,20 +1716,20 @@ name of container to stop
 
 ## build [source]
 
-Use this command to build an image or a complete multicontainer project
+Use this command to build an image or a complete multicontainer project with
-with the provided docker daemon in your development machine or balena
+the provided docker daemon in your development machine or balena device.
-device. (See also the `balena push` command for the option of building
+(See also the `balena push` command for the option of building images in the
-images in the balenaCloud build servers.)
+balenaCloud build servers.)
 
-You must provide either an application or a device-type/architecture
+You must provide either an application or a device-type/architecture pair to use
-pair to use the balena Dockerfile pre-processor
+the balena Dockerfile pre-processor (e.g. Dockerfile.template -> Dockerfile).
-(e.g. Dockerfile.template -> Dockerfile).
 
 This command will look into the given source directory (or the current working
-directory if one isn't specified) for a compose file. If one is found, this
+directory if one isn't specified) for a docker-compose.yml file. If it is found,
-command will build each service defined in the compose file. If a compose file
+this command will build each service defined in the compose file. If a compose
-isn't found, the command will look for a Dockerfile, and if yet that isn't found,
+file isn't found, the command will look for a Dockerfile[.template] file (or
-it will try to generate one.
+alternative Dockerfile specified with the `-f` option), and if yet that isn't
+found, it will try to generate one.
 
 The --registry-secrets option specifies a JSON or YAML file containing private
 Docker registry usernames and passwords to be used when pulling base images.
@@ -1772,6 +1776,10 @@ Specify an alternate project name; default is the directory name
 
 Run an emulated build using Qemu
 
+#### --dockerfile <Dockerfile>
+
+Alternative Dockerfile name/path, relative to the source folder
+
 #### --logs
 
 Display full log output
@@ -1831,17 +1839,18 @@ balena device. (See also the `balena push` command for the option of building
 the image in the balenaCloud build servers.)
 
 Unless an image is specified, this command will look into the current directory
-(or the one specified by --source) for a compose file. If one is found, this
+(or the one specified by --source) for a docker-compose.yml file.  If one is
-command will deploy each service defined in the compose file, building it first
+found, this command will deploy each service defined in the compose file,
-if an image for it doesn't exist. If a compose file isn't found, the command
+building it first if an image for it doesn't exist. If a compose file isn't
-will look for a Dockerfile, and if yet that isn't found, it will try to
+found, the command will look for a Dockerfile[.template] file (or alternative
-generate one.
+Dockerfile specified with the `-f` option), and if yet that isn't found, it
+will try to generate one.
 
 To deploy to an app on which you're a collaborator, use
 `balena deploy <appOwnerUsername>/<appName>`.
 
-When --build is used, all options supported by `balena build` are also
+When --build is used, all options supported by `balena build` are also supported
-supported by this command.
+by this command.
 
 The --registry-secrets option specifies a JSON or YAML file containing private
 Docker registry usernames and passwords to be used when pulling base images.
@@ -1885,6 +1894,10 @@ Specify an alternate project name; default is the directory name
 
 Run an emulated build using Qemu
 
+#### --dockerfile &#60;Dockerfile&#62;
+
+Alternative Dockerfile name/path, relative to the source folder
+
 #### --logs
 
 Display full log output

lib/actions/build.coffee

@@ -19,6 +19,8 @@ buildProject = (docker, logger, composeOpts, opts) ->
 		logger
 		composeOpts.projectPath
 		composeOpts.projectName
+		undefined  # image: name of pre-built image
+		composeOpts.dockerfilePath  # ok if undefined
 	)
 	.then (project) ->
 		appType = opts.app?.application_type?[0]
@@ -50,20 +52,20 @@ module.exports =
 	description: 'Build a single image or a multicontainer project locally'
 	primary: true
 	help: """
-		Use this command to build an image or a complete multicontainer project
+		Use this command to build an image or a complete multicontainer project with
-		with the provided docker daemon in your development machine or balena
+		the provided docker daemon in your development machine or balena device.
-		device. (See also the `balena push` command for the option of building
+		(See also the `balena push` command for the option of building images in the
-		images in the balenaCloud build servers.)
+		balenaCloud build servers.)
 
-		You must provide either an application or a device-type/architecture
+		You must provide either an application or a device-type/architecture pair to use
-		pair to use the balena Dockerfile pre-processor
+		the balena Dockerfile pre-processor (e.g. Dockerfile.template -> Dockerfile).
-		(e.g. Dockerfile.template -> Dockerfile).
 
 		This command will look into the given source directory (or the current working
-		directory if one isn't specified) for a compose file. If one is found, this
+		directory if one isn't specified) for a docker-compose.yml file. If it is found,
-		command will build each service defined in the compose file. If a compose file
+		this command will build each service defined in the compose file. If a compose
-		isn't found, the command will look for a Dockerfile, and if yet that isn't found,
+		file isn't found, the command will look for a Dockerfile[.template] file (or
-		it will try to generate one.
+		alternative Dockerfile specified with the `-f` option), and if yet that isn't
+		found, it will try to generate one.
 
 		#{registrySecretsHelp}
 

lib/actions/deploy.coffee

@@ -10,6 +10,7 @@ Opts must be an object with the following keys:
 
 	app: the application instance to deploy to
 	image: the image to deploy; optional
+	dockerfilePath: name of an alternative Dockerfile; optional
 	shouldPerformBuild
 	shouldUploadLogs
 	buildEmulated
@@ -25,6 +26,7 @@ deployProject = (docker, logger, composeOpts, opts) ->
 		composeOpts.projectPath
 		composeOpts.projectName
 		opts.image
+		composeOpts.dockerfilePath  # ok if undefined
 	)
 	.then (project) ->
 		if project.descriptors.length > 1 and !opts.app.application_type?[0]?.supports_multicontainer
@@ -133,17 +135,18 @@ module.exports =
 		the image in the balenaCloud build servers.)
 
 		Unless an image is specified, this command will look into the current directory
-		(or the one specified by --source) for a compose file. If one is found, this
+		(or the one specified by --source) for a docker-compose.yml file.  If one is
-		command will deploy each service defined in the compose file, building it first
+		found, this command will deploy each service defined in the compose file,
-		if an image for it doesn't exist. If a compose file isn't found, the command
+		building it first if an image for it doesn't exist. If a compose file isn't
-		will look for a Dockerfile, and if yet that isn't found, it will try to
+		found, the command will look for a Dockerfile[.template] file (or alternative
-		generate one.
+		Dockerfile specified with the `-f` option), and if yet that isn't found, it
+		will try to generate one.
 
 		To deploy to an app on which you're a collaborator, use
 		`balena deploy <appOwnerUsername>/<appName>`.
 
-		When --build is used, all options supported by `balena build` are also
+		When --build is used, all options supported by `balena build` are also supported
-		supported by this command.
+		by this command.
 
 		#{registrySecretsHelp}
 

lib/actions/push.ts

@@ -104,6 +104,7 @@ export const push: CommandDefinition<
 	{
 		source: string;
 		emulated: boolean;
+		dockerfile: string; // DeviceDeployOptions.dockerfilePath (alternative Dockerfile)
 		nocache: boolean;
 		'registry-secrets': string;
 		live: boolean;
@@ -158,6 +159,12 @@ export const push: CommandDefinition<
 			description: 'Force an emulated build to occur on the remote builder',
 			boolean: true,
 		},
+		{
+			signature: 'dockerfile',
+			parameter: 'Dockerfile',
+			description:
+				'Alternative Dockerfile name/path, relative to the source folder',
+		},
 		{
 			signature: 'nocache',
 			alias: 'c',
@@ -194,7 +201,9 @@ export const push: CommandDefinition<
 		const { exitIfNotLoggedIn, exitWithExpectedError } = await import(
 			'../utils/patterns'
 		);
-		const { parseRegistrySecrets } = await import('../utils/compose_ts');
+		const { validateSpecifiedDockerfile, parseRegistrySecrets } = await import(
+			'../utils/compose_ts'
+		);
 		const { BuildError } = await import('../utils/device/errors');
 
 		const appOrDevice: string | null = params.applicationOrDevice;
@@ -207,6 +216,11 @@ export const push: CommandDefinition<
 			console.log(`[debug] Using ${source} as build source`);
 		}
 
+		const dockerfilePath = validateSpecifiedDockerfile(
+			source,
+			options.dockerfile,
+		);
+
 		const registrySecrets = options['registry-secrets']
 			? await parseRegistrySecrets(options['registry-secrets'])
 			: {};
@@ -229,6 +243,7 @@ export const push: CommandDefinition<
 					getAppOwner(sdk, app),
 					async (token, baseUrl, owner) => {
 						const opts = {
+							dockerfilePath,
 							emulated: options.emulated,
 							nocache: options.nocache,
 							registrySecrets,
@@ -254,6 +269,7 @@ export const push: CommandDefinition<
 					deviceDeploy.deployToDevice({
 						source,
 						deviceHost: device,
+						dockerfilePath,
 						registrySecrets,
 						nocache: options.nocache || false,
 						live: options.live || false,

lib/utils/compose.coffee

@@ -36,16 +36,21 @@ exports.appendOptions = (opts) ->
 			boolean: true
 			alias: 'e'
 		},
+		{
+			signature: 'dockerfile'
+			parameter: 'Dockerfile'
+			description: 'Alternative Dockerfile name/path, relative to the source folder'
+		},
 		{
 			signature: 'logs'
 			description: 'Display full log output'
 			boolean: true
 		},
 		{
-			signature: 'registry-secrets',
+			signature: 'registry-secrets'
-			alias: 'R',
+			alias: 'R'
-			parameter: 'secrets.yml|.json',
+			parameter: 'secrets.yml|.json'
-			description: 'Path to a YAML or JSON file with passwords for a private Docker registry',
+			description: 'Path to a YAML or JSON file with passwords for a private Docker registry'
 		},
 	]
 
@@ -55,6 +60,7 @@ exports.generateOpts = (options) ->
 		projectName: options.projectName
 		projectPath: projectPath
 		inlineLogs: !!options.logs
+		dockerfilePath: options.dockerfile
 
 compositionFileNames = [
 	'docker-compose.yml'
@@ -97,11 +103,14 @@ createProject = (composePath, composeStr, projectName = null) ->
 # of it in one go. if image is given, it'll create a default project for
 # that without looking for a project. falls back to creating a default
 # project if none is found at the given projectPath.
-exports.loadProject = (logger, projectPath, projectName, image) ->
+exports.loadProject = (logger, projectPath, projectName, image, dockerfilePath) ->
+	{ validateSpecifiedDockerfile } = require('./compose_ts')
 	compose = require('resin-compose-parse')
 	logger.logDebug('Loading project...')
 
 	Promise.try ->
+		dockerfilePath = validateSpecifiedDockerfile(projectPath, dockerfilePath)
+
 		if image?
 			logger.logInfo("Creating default composition with image: #{image}")
 			return compose.defaultComposition(image)
@@ -110,11 +119,14 @@ exports.loadProject = (logger, projectPath, projectName, image) ->
 
 		resolveProject(projectPath)
 		.tap ->
-			logger.logInfo('Compose file detected')
+			if dockerfilePath
+				logger.logWarn("Ignoring alternative dockerfile \"#{dockerfilePath}\"\ because a docker-compose file exists")
+			else
+				logger.logInfo('Compose file detected')
 		.catch (e) ->
 			logger.logDebug("Failed to resolve project: #{e}")
 			logger.logInfo("Creating default composition with source: #{projectPath}")
-			return compose.defaultComposition()
+			return compose.defaultComposition(undefined, dockerfilePath)
 	.then (composeStr) ->
 		logger.logDebug('Creating project...')
 		createProject(projectPath, composeStr, projectName)
@@ -126,7 +138,7 @@ exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) ->
 	fs = require('mz/fs')
 	streamToPromise = require('stream-to-promise')
 	{ FileIgnorer } = require('./ignore')
-	{ toPosixPath } = require('./helpers')
+	{ toPosixPath } = require('resin-multibuild').PathUtils
 
 	getFiles = ->
 		streamToPromise(klaw(dir))
@@ -175,7 +187,7 @@ exports.buildProject = (
 	builder = require('resin-multibuild')
 	transpose = require('docker-qemu-transpose')
 	qemu = require('./qemu')
-	{ toPosixPath } = require('./helpers')
+	{ toPosixPath } = builder.PathUtils
 
 	logger.logInfo("Building for #{arch}/#{deviceType}")
 

lib/utils/compose.d.ts

@@ -46,6 +46,7 @@ export function loadProject(
 	projectPath: string,
 	projectName: string,
 	image?: string,
+	dockerfilePath?: string,
 ): Bluebird<ComposeProject>;
 
 export function tarDirectory(

lib/utils/compose_ts.ts

@@ -155,3 +155,70 @@ async function performResolution(
 		}).then(resolve, reject);
 	});
 }
+
+/**
+ * Enforce that, for example, if 'myProject/MyDockerfile.template' is specified
+ * as an alternativate Dockerfile name, then 'myProject/MyDockerfile' must not
+ * exist.
+ * @param projectPath The project source folder (-s command-line option)
+ * @param dockerfilePath The alternative Dockerfile specified by the user
+ */
+export function validateSpecifiedDockerfile(
+	projectPath: string,
+	dockerfilePath: string = '',
+): string {
+	if (!dockerfilePath) {
+		return dockerfilePath;
+	}
+	const { exitWithExpectedError } = require('../utils/patterns');
+	const { isAbsolute, join, normalize, parse, posix } = require('path');
+	const { existsSync } = require('fs');
+	const { stripIndent } = require('common-tags');
+	const { contains, toNativePath, toPosixPath } = MultiBuild.PathUtils;
+
+	// reminder: native windows paths may start with a drive specificaton,
+	// e.g. 'C:\absolute' or 'C:relative'.
+	if (isAbsolute(dockerfilePath) || posix.isAbsolute(dockerfilePath)) {
+		exitWithExpectedError(stripIndent`
+			Error: absolute Dockerfile path detected:
+			"${dockerfilePath}"
+			The Dockerfile path should be relative to the source folder.
+		`);
+	}
+	const nativeProjectPath = normalize(projectPath);
+	const nativeDockerfilePath = join(projectPath, toNativePath(dockerfilePath));
+
+	if (!contains(nativeProjectPath, nativeDockerfilePath)) {
+		// Note that testing the existence of nativeDockerfilePath in the
+		// filesystem (after joining its path to the source folder) is not
+		// sufficient, because the user could have added '../' to the path.
+		exitWithExpectedError(stripIndent`
+			Error: the specified Dockerfile must be in a subfolder of the source folder:
+			Specified dockerfile: "${nativeDockerfilePath}"
+			Source folder: "${nativeProjectPath}"
+		`);
+	}
+
+	if (!existsSync(nativeDockerfilePath)) {
+		exitWithExpectedError(stripIndent`
+			Error: Dockerfile not found: "${nativeDockerfilePath}"
+		`);
+	}
+
+	const { dir, ext, name } = parse(nativeDockerfilePath);
+	if (ext) {
+		const nativePathMinusExt = join(dir, name);
+
+		if (existsSync(nativePathMinusExt)) {
+			exitWithExpectedError(stripIndent`
+				Error: "${name}" exists on the same folder as "${dockerfilePath}".
+				When an alternative Dockerfile name is specified, a file with the same
+				base name (minus the file extension) must not exist in the same folder.
+				This is because the base name file will be auto generated and added to
+				the tar stream that is sent to the docker daemon, resulting in duplicate
+				Dockerfiles and undefined behavior.
+			`);
+		}
+	}
+	return posix.normalize(toPosixPath(dockerfilePath));
+}

lib/utils/device/deploy.ts

@@ -43,6 +43,7 @@ export interface DeviceDeployOptions {
 	source: string;
 	deviceHost: string;
 	devicePort?: number;
+	dockerfilePath?: string;
 	registrySecrets: RegistrySecrets;
 	nocache: boolean;
 	live: boolean;
@@ -99,7 +100,13 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 
 	globalLogger.logInfo(`Starting build on device ${opts.deviceHost}`);
 
-	const project = await loadProject(globalLogger, opts.source, 'local');
+	const project = await loadProject(
+		globalLogger,
+		opts.source, // project path
+		'local', // project name
+		undefined, // name of a pre-built image
+		opts.dockerfilePath, // alternative Dockerfile; OK to be undefined
+	);
 
 	// Attempt to attach to the device's docker daemon
 	const docker = connectToDocker(

lib/utils/helpers.ts

@@ -19,20 +19,14 @@ import Bluebird = require('bluebird');
 import chalk from 'chalk';
 import _ = require('lodash');
 import os = require('os');
-import path = require('path');
 import visuals = require('resin-cli-visuals');
 import rindle = require('rindle');
 
 import { InitializeEmitter, OperationState } from 'balena-device-init';
 
 const waitStreamAsync = Bluebird.promisify(rindle.wait);
-
 const balena = BalenaSdk.fromSharedOptions();
 
-export function toPosixPath(p: string): string {
-	return p.replace(new RegExp('\\' + path.sep, 'g'), '/');
-}
-
 export function getGroupDefaults(group: {
 	options: Array<{ name: string; default?: string }>;
 }): { [name: string]: string | undefined } {

lib/utils/ignore.ts

@@ -1,11 +1,28 @@
+/**
+ * @license
+ * Copyright 2019 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 import * as _ from 'lodash';
 import { fs } from 'mz';
 import * as path from 'path';
+import * as MultiBuild from 'resin-multibuild';
 
 import dockerIgnore = require('@zeit/dockerignore');
 import ignore from 'ignore';
 
-import { toPosixPath } from './helpers';
+const { toPosixPath } = MultiBuild.PathUtils;
 
 export enum IgnoreFileType {
 	DockerIgnore,

lib/utils/remote-build.ts

@@ -32,6 +32,7 @@ const CURSOR_METADATA_REGEX = /([a-z]+)([0-9]+)?/;
 const TRIM_REGEX = /\n+$/;
 
 export interface BuildOpts {
+	dockerfilePath: string;
 	emulated: boolean;
 	nocache: boolean;
 	registrySecrets: RegistrySecrets;
@@ -78,6 +79,7 @@ async function getBuilderEndpoint(
 	const args = querystring.stringify({
 		owner,
 		app,
+		dockerfilePath: opts.dockerfilePath,
 		emulated: opts.emulated,
 		nocache: opts.nocache,
 	});

package.json

@@ -72,11 +72,15 @@
     "@types/fs-extra": "5.0.4",
     "@types/is-root": "1.0.0",
     "@types/lodash": "4.14.112",
+    "@types/mixpanel": "2.14.0",
     "@types/mkdirp": "0.5.2",
     "@types/node": "6.14.2",
     "@types/prettyjson": "0.0.28",
+    "@types/stream-to-promise": "2.2.0",
     "@types/raven": "2.5.1",
+    "@types/request": "2.48.1",
     "@types/tar-stream": "1.6.0",
+    "@types/through2": "2.0.33",
     "catch-uncommitted": "^1.0.0",
     "ent": "^2.2.0",
     "filehound": "^1.16.2",
@@ -98,11 +102,6 @@
   },
   "dependencies": {
     "@resin.io/valid-email": "^0.1.0",
-    "@types/dockerode": "2.5.5",
-    "@types/mixpanel": "2.14.0",
-    "@types/request": "2.48.1",
-    "@types/stream-to-promise": "2.2.0",
-    "@types/through2": "^2.0.33",
     "@zeit/dockerignore": "0.0.3",
     "JSONStream": "^1.0.3",
     "ansi-escapes": "^2.0.0",
@@ -162,7 +161,7 @@
     "request": "^2.81.0",
     "resin-cli-form": "^2.0.1",
     "resin-cli-visuals": "^1.4.0",
-    "resin-compose-parse": "^2.0.4",
+    "resin-compose-parse": "^2.1.0",
     "resin-doodles": "0.0.1",
     "resin-image-fs": "^5.0.2",
     "resin-multibuild": "^3.1.0",