push/build/deploy: add --nogitignore option and update dockerignore filter library

Connects-to: #1032
Connects-to: #1148
Change-type: minor

doc/cli.markdown

@@ -1805,6 +1805,7 @@ containers. The synchronization is only in one direction, from this machine to
 the device, and changes made on the device itself may be overwritten.
 This feature requires a device running supervisor version v9.7.0 or greater.
 
+REGISTRY SECRETS
 The --registry-secrets option specifies a JSON or YAML file containing private
 Docker registry usernames and passwords to be used when pulling base images.
 Sample registry-secrets YAML file:
@@ -1826,6 +1827,41 @@ If the --registry-secrets option is not specified, and a secrets.yml or
 secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.
 
+DOCKERIGNORE AND GITIGNORE FILES
+By default, both '.dockerignore' and '.gitignore' files are taken into account
+in order to prevent files from being sent to the balenaCloud builder or Docker
+or balenaEngine (balenaOS device).
+
+However, this behavior has been DEPRECATED and will change in an upcoming major
+version release. The --nogitignore (-G) option should be used to enable the new
+behavior already now. This option will cause the CLI to:
+
+* Disregard all '.gitignore' files at the source directory and subdirectories,
+  and consider only the '.dockerignore' file (if any) at the source directory.
+* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine
+  even if they are listed in '.gitignore' files (a longstanding feature request).
+* Use a new '.dockerignore' parser and filter library that improves compatibility
+  with "docker build" and fixes several issues (mainly on Windows).
+* Prevent a warning message from being printed.
+
+When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are
+also used and "merged" (in memory) with the patterns found in the '.dockerignore'
+file (if any), in the following order:
+
+    **/.git
+    < user's patterns from the '.dockerignore' file, if any >
+    !**/.balena
+    !**/.resin
+    !**/Dockerfile
+    !**/Dockerfile.*
+    !**/docker-compose.yml
+
+If necessary, the effect of the '**/.git' pattern may be modified by adding
+"counter patterns" to the '.dockerignore' file, for example '!service1/.git'.
+For documentation on pattern format, see:
+- https://docs.docker.com/engine/reference/builder/#dockerignore-file
+- https://www.npmjs.com/package/@balena/dockerignore
+
 Examples:
 
 	$ balena push myApp
@@ -1845,7 +1881,7 @@ Examples:
 
 #### --source, -s &#60;source&#62;
 
-The source that should be sent to the balena builder to be built (defaults to the current directory)
+Source directory to be sent to balenaCloud or balenaOS device (default: current working dir)
 
 #### --emulated, -e
 
@@ -1909,6 +1945,12 @@ left hand side of the = character will be treated as the variable name.
 On Windows only, convert line endings from CRLF (Windows format) to LF (Unix format).
 Source files are not modified.
 
+#### --nogitignore, -G
+
+Disregard all .gitignore files, and consider only the .dockerignore file (if any)
+at the source directory. This will be the default behavior in an upcoming major
+version release. For more information, see 'balena help push'.
+
 # Settings
 
 ## settings
@@ -1971,6 +2013,7 @@ found, it will look for a Dockerfile[.template] file (or alternative Dockerfile
 specified with the `--dockerfile` option), and if no dockerfile is found, it
 will try to generate one.
 
+REGISTRY SECRETS
 The --registry-secrets option specifies a JSON or YAML file containing private
 Docker registry usernames and passwords to be used when pulling base images.
 Sample registry-secrets YAML file:
@@ -1992,6 +2035,41 @@ If the --registry-secrets option is not specified, and a secrets.yml or
 secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.
 
+DOCKERIGNORE AND GITIGNORE FILES
+By default, both '.dockerignore' and '.gitignore' files are taken into account
+in order to prevent files from being sent to the balenaCloud builder or Docker
+or balenaEngine (balenaOS device).
+
+However, this behavior has been DEPRECATED and will change in an upcoming major
+version release. The --nogitignore (-G) option should be used to enable the new
+behavior already now. This option will cause the CLI to:
+
+* Disregard all '.gitignore' files at the source directory and subdirectories,
+  and consider only the '.dockerignore' file (if any) at the source directory.
+* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine
+  even if they are listed in '.gitignore' files (a longstanding feature request).
+* Use a new '.dockerignore' parser and filter library that improves compatibility
+  with "docker build" and fixes several issues (mainly on Windows).
+* Prevent a warning message from being printed.
+
+When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are
+also used and "merged" (in memory) with the patterns found in the '.dockerignore'
+file (if any), in the following order:
+
+    **/.git
+    < user's patterns from the '.dockerignore' file, if any >
+    !**/.balena
+    !**/.resin
+    !**/Dockerfile
+    !**/Dockerfile.*
+    !**/docker-compose.yml
+
+If necessary, the effect of the '**/.git' pattern may be modified by adding
+"counter patterns" to the '.dockerignore' file, for example '!service1/.git'.
+For documentation on pattern format, see:
+- https://docs.docker.com/engine/reference/builder/#dockerignore-file
+- https://www.npmjs.com/package/@balena/dockerignore
+
 Examples:
 
 	$ balena build
@@ -2032,6 +2110,12 @@ Alternative Dockerfile name/path, relative to the source folder
 
 Display full log output
 
+#### --nogitignore, -G
+
+Disregard all .gitignore files, and consider only the .dockerignore file (if any)
+at the source directory. This will be the default behavior in an upcoming major
+version release. For more information, see 'balena help undefined'.
+
 #### --noparent-check
 
 Disable project validation check of 'docker-compose.yml' file in parent folder
@@ -2112,6 +2196,7 @@ To deploy to an app on which you're a collaborator, use
 When --build is used, all options supported by `balena build` are also supported
 by this command.
 
+REGISTRY SECRETS
 The --registry-secrets option specifies a JSON or YAML file containing private
 Docker registry usernames and passwords to be used when pulling base images.
 Sample registry-secrets YAML file:
@@ -2133,6 +2218,41 @@ If the --registry-secrets option is not specified, and a secrets.yml or
 secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.
 
+DOCKERIGNORE AND GITIGNORE FILES
+By default, both '.dockerignore' and '.gitignore' files are taken into account
+in order to prevent files from being sent to the balenaCloud builder or Docker
+or balenaEngine (balenaOS device).
+
+However, this behavior has been DEPRECATED and will change in an upcoming major
+version release. The --nogitignore (-G) option should be used to enable the new
+behavior already now. This option will cause the CLI to:
+
+* Disregard all '.gitignore' files at the source directory and subdirectories,
+  and consider only the '.dockerignore' file (if any) at the source directory.
+* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine
+  even if they are listed in '.gitignore' files (a longstanding feature request).
+* Use a new '.dockerignore' parser and filter library that improves compatibility
+  with "docker build" and fixes several issues (mainly on Windows).
+* Prevent a warning message from being printed.
+
+When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are
+also used and "merged" (in memory) with the patterns found in the '.dockerignore'
+file (if any), in the following order:
+
+    **/.git
+    < user's patterns from the '.dockerignore' file, if any >
+    !**/.balena
+    !**/.resin
+    !**/Dockerfile
+    !**/Dockerfile.*
+    !**/docker-compose.yml
+
+If necessary, the effect of the '**/.git' pattern may be modified by adding
+"counter patterns" to the '.dockerignore' file, for example '!service1/.git'.
+For documentation on pattern format, see:
+- https://docs.docker.com/engine/reference/builder/#dockerignore-file
+- https://www.npmjs.com/package/@balena/dockerignore
+
 Examples:
 
 	$ balena deploy myApp
@@ -2169,6 +2289,12 @@ Alternative Dockerfile name/path, relative to the source folder
 
 Display full log output
 
+#### --nogitignore, -G
+
+Disregard all .gitignore files, and consider only the .dockerignore file (if any)
+at the source directory. This will be the default behavior in an upcoming major
+version release. For more information, see 'balena help undefined'.
+
 #### --noparent-check
 
 Disable project validation check of 'docker-compose.yml' file in parent folder

lib/actions/build.js

@@ -21,7 +21,7 @@ import * as Promise from 'bluebird';
 
 import * as dockerUtils from '../utils/docker';
 import * as compose from '../utils/compose';
-import { registrySecretsHelp } from '../utils/messages';
+import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import { getBalenaSdk } from '../utils/lazy';
 
 /*
@@ -62,6 +62,7 @@ const buildProject = function(docker, logger, composeOpts, opts) {
 				composeOpts.inlineLogs,
 				opts.convertEol,
 				composeOpts.dockerfilePath,
+				composeOpts.nogitignore,
 			);
 		})
 		.then(function() {
@@ -95,6 +96,8 @@ will try to generate one.
 
 ${registrySecretsHelp}
 
+${dockerignoreHelp}
+
 Examples:
 
 	$ balena build

lib/actions/deploy.js

@@ -21,7 +21,7 @@ import * as Promise from 'bluebird';
 
 import * as dockerUtils from '../utils/docker';
 import * as compose from '../utils/compose';
-import { registrySecretsHelp } from '../utils/messages';
+import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import { ExpectedError } from '../errors';
 import { getBalenaSdk, getChalk } from '../utils/lazy';
 
@@ -95,6 +95,7 @@ const deployProject = function(docker, logger, composeOpts, opts) {
 							composeOpts.inlineLogs,
 							opts.convertEol,
 							composeOpts.dockerfilePath,
+							composeOpts.nogitignore,
 						)
 						.then(builtImages => _.keyBy(builtImages, 'serviceName'));
 				})
@@ -200,6 +201,8 @@ by this command.
 
 ${registrySecretsHelp}
 
+${dockerignoreHelp}
+
 Examples:
 
 	$ balena deploy myApp

lib/actions/push.ts

@@ -20,7 +20,7 @@ import { stripIndent } from 'common-tags';
 
 import { ExpectedError } from '../errors';
 import { getBalenaSdk } from '../utils/lazy';
-import { registrySecretsHelp } from '../utils/messages';
+import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import {
 	validateApplicationName,
 	validateDotLocalUrl,
@@ -109,6 +109,7 @@ export const push: CommandDefinition<
 		nocache?: boolean;
 		'noparent-check'?: boolean;
 		'registry-secrets'?: string;
+		nogitignore?: boolean;
 		nolive?: boolean;
 		detached?: boolean;
 		service?: string | string[];
@@ -148,6 +149,8 @@ export const push: CommandDefinition<
 
 		${registrySecretsHelp.split('\n').join('\n\t\t')}
 
+		${dockerignoreHelp.split('\n').join('\n\t\t')}
+
 		Examples:
 
 			$ balena push myApp
@@ -168,7 +171,7 @@ export const push: CommandDefinition<
 			signature: 'source',
 			alias: 's',
 			description:
-				'The source that should be sent to the balena builder to be built (defaults to the current directory)',
+				'Source directory to be sent to balenaCloud or balenaOS device (default: current working dir)',
 			parameter: 'source',
 		},
 		{
@@ -259,6 +262,16 @@ export const push: CommandDefinition<
 				Source files are not modified.`,
 			boolean: true,
 		},
+		{
+			signature: 'nogitignore',
+			alias: 'G',
+			description: stripIndent`
+				Disregard all .gitignore files, and consider only the .dockerignore file (if any)
+				at the source directory. This will be the default behavior in an upcoming major
+				version release. For more information, see 'balena help push'.
+			`,
+			boolean: true,
+		},
 	],
 	async action(params, options) {
 		const sdk = getBalenaSdk();
@@ -336,6 +349,7 @@ export const push: CommandDefinition<
 							source,
 							auth: token,
 							baseUrl,
+							nogitignore: !!options.nogitignore,
 							sdk,
 							opts,
 						};
@@ -359,6 +373,7 @@ export const push: CommandDefinition<
 						dockerfilePath,
 						registrySecrets,
 						nocache: options.nocache || false,
+						nogitignore: options.nogitignore || false,
 						noParentCheck: options['noparent-check'] || false,
 						nolive: options.nolive || false,
 						detached: options.detached || false,

lib/preparser.ts

@@ -61,6 +61,9 @@ export async function routeCliFramework(argv: string[], options: AppOptions) {
 		}
 	}
 
+	const Logger = await import('./utils/logger');
+	Logger.command = cmdSlice[0];
+
 	const [isOclif, isTopic] = isOclifCommand(cmdSlice);
 
 	if (isOclif) {
@@ -68,6 +71,7 @@ export async function routeCliFramework(argv: string[], options: AppOptions) {
 		if (isTopic) {
 			// convert space-separated commands to oclif's topic:command syntax
 			oclifArgs = [cmdSlice[0] + ':' + cmdSlice[1], ...cmdSlice.slice(2)];
+			Logger.command = `${cmdSlice[0]} ${cmdSlice[1]}`;
 		}
 		if (process.env.DEBUG) {
 			console.log(

lib/utils/compose-types.d.ts

@@ -41,6 +41,7 @@ export interface ComposeProject {
 }
 
 interface TarDirectoryOptions {
-	preFinalizeCallback?: (pack: Pack) => void;
 	convertEol?: boolean;
+	preFinalizeCallback?: (pack: Pack) => void | Promise<void>;
+	nogitignore: boolean;
 }

lib/utils/compose.js

@@ -16,8 +16,11 @@
  */
 
 import * as Promise from 'bluebird';
+import { stripIndent } from 'common-tags';
 import * as path from 'path';
+
 import { getBalenaSdk, getChalk } from './lazy';
+import { IgnoreFileType } from './ignore';
 
 export const appendProjectOptions = opts =>
 	opts.concat([
@@ -31,6 +34,7 @@ export const appendProjectOptions = opts =>
 	]);
 
 export function appendOptions(opts) {
+	const Logger = require('./logger');
 	return appendProjectOptions(opts).concat([
 		{
 			signature: 'emulated',
@@ -49,6 +53,16 @@ export function appendOptions(opts) {
 			description: 'Display full log output',
 			boolean: true,
 		},
+		{
+			signature: 'nogitignore',
+			description: stripIndent`
+				Disregard all .gitignore files, and consider only the .dockerignore file (if any)
+				at the source directory. This will be the default behavior in an upcoming major
+				version release. For more information, see 'balena help ${Logger.command}'.
+			`,
+			boolean: true,
+			alias: 'G',
+		},
 		{
 			signature: 'noparent-check',
 			description:
@@ -83,6 +97,7 @@ export function generateOpts(options) {
 		projectPath,
 		inlineLogs: !!options.logs,
 		dockerfilePath: options.dockerfile,
+		nogitignore: !!options.nogitignore,
 		noParentCheck: options['noparent-check'],
 	}));
 }
@@ -132,15 +147,33 @@ export function createProject(composePath, composeStr, projectName = null) {
 }
 
 /**
- * @param {string} dir
+ * Create a tar stream out of the local filesystem at the given directory,
- * @param {import('./compose-types').TarDirectoryOptions} [param]
+ * while optionally applying file filters such as '.dockerignore' and
+ * optionally converting text file line endings (CRLF to LF).
+ * @param {string} dir Source directory
+ * @param {import('./compose-types').TarDirectoryOptions} param
  * @returns {Promise<import('stream').Readable>}
  */
-export const tarDirectory = function(dir, param) {
+export function tarDirectory(dir, param) {
-	if (param == null) {
+	let { nogitignore = false } = param;
-		param = {};
+	if (nogitignore) {
+		return Promise.resolve(require('./compose_ts').tarDirectory(dir, param));
+	} else {
+		return originalTarDirectory(dir, param);
 	}
-	let { preFinalizeCallback = null, convertEol = false } = param;
+}
+
+/**
+ * @param {string} dir Source directory
+ * @param {import('./compose-types').TarDirectoryOptions} param
+ * @returns {Promise<import('stream').Readable>}
+ */
+function originalTarDirectory(dir, param) {
+	let {
+		preFinalizeCallback = null,
+		convertEol = false,
+		nogitignore = false,
+	} = param;
 	if (convertEol == null) {
 		convertEol = false;
 	}
@@ -149,6 +182,7 @@ export const tarDirectory = function(dir, param) {
 	const klaw = require('klaw');
 	const fs = require('mz/fs');
 	const streamToPromise = require('stream-to-promise');
+	const { printGitignoreWarn } = require('./compose_ts');
 	const { FileIgnorer } = require('./ignore');
 	const { toPosixPath } = require('resin-multibuild').PathUtils;
 	let readFile;
@@ -167,13 +201,24 @@ export const tarDirectory = function(dir, param) {
 
 	const ignore = new FileIgnorer(dir);
 	const pack = tar.pack();
+	const ignoreFiles = {};
 	return getFiles()
 		.each(function(file) {
 			const type = ignore.getIgnoreFileType(path.relative(dir, file));
 			if (type != null) {
+				ignoreFiles[type] = ignoreFiles[type] || [];
+				ignoreFiles[type].push(path.resolve(dir, file));
 				return ignore.addIgnoreFile(file, type);
 			}
 		})
+		.tap(() => {
+			if (!nogitignore) {
+				printGitignoreWarn(
+					(ignoreFiles[IgnoreFileType.DockerIgnore] || [])[0] || '',
+					ignoreFiles[IgnoreFileType.GitIgnore] || [],
+				);
+			}
+		})
 		.filter(ignore.filter)
 		.map(function(file) {
 			const relPath = path.relative(path.resolve(dir), file);
@@ -193,7 +238,7 @@ export const tarDirectory = function(dir, param) {
 			pack.finalize();
 			return pack;
 		});
-};
+}
 
 const truncateString = function(str, len) {
 	if (str.length < len) {
@@ -221,6 +266,7 @@ export function buildProject(
 	inlineLogs,
 	convertEol,
 	dockerfilePath,
+	nogitignore,
 ) {
 	const _ = require('lodash');
 	const humanize = require('humanize');
@@ -274,7 +320,7 @@ export function buildProject(
 		.then((
 			needsQemu, // Tar up the directory, ready for the build stream
 		) =>
-			tarDirectory(projectPath, { convertEol })
+			tarDirectory(projectPath, { convertEol, nogitignore })
 				.then(tarStream =>
 					makeBuildTasks(
 						composition,

lib/utils/compose_ts.ts

@@ -156,6 +156,93 @@ async function loadBuildMetatada(
 	return [buildMetadata, metadataPath];
 }
 
+/**
+ * Create a tar stream out of the local filesystem at the given directory,
+ * while optionally applying file filters such as '.dockerignore' and
+ * optionally converting text file line endings (CRLF to LF).
+ * @param dir Source directory
+ * @param param Options
+ * @returns {Promise<import('stream').Readable>}
+ */
+export async function tarDirectory(
+	dir: string,
+	{
+		preFinalizeCallback,
+		convertEol = false,
+		nogitignore = false,
+	}: import('./compose-types').TarDirectoryOptions,
+): Promise<import('stream').Readable> {
+	(await import('assert')).strict.strictEqual(nogitignore, true);
+	const { filterFilesWithDockerignore } = await import('./ignore');
+	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
+
+	let readFile: (file: string) => Promise<Buffer>;
+	if (process.platform === 'win32') {
+		const { readFileWithEolConversion } = require('./eol-conversion');
+		readFile = file => readFileWithEolConversion(file, convertEol);
+	} else {
+		readFile = fs.readFile;
+	}
+	const pack = tar.pack();
+	const fileStatsList = await filterFilesWithDockerignore(dir);
+	for (const fileStats of fileStatsList) {
+		pack.entry(
+			{
+				name: toPosixPath(fileStats.relPath),
+				size: fileStats.stats.size,
+				mode: fileStats.stats.mode,
+			},
+			await readFile(fileStats.filePath),
+		);
+	}
+	if (preFinalizeCallback) {
+		await preFinalizeCallback(pack);
+	}
+	pack.finalize();
+	return pack;
+}
+
+/**
+ * Print a deprecation warning if any '.gitignore' or '.dockerignore' file is
+ * found and the --nogitignore (-G) option has not been provided.
+ * @param dockerignoreFile Absolute path to a .dockerignore file
+ * @param gitignoreFiles Array of absolute paths to .gitginore files
+ */
+export function printGitignoreWarn(
+	dockerignoreFile: string,
+	gitignoreFiles: string[],
+) {
+	const ignoreFiles = [dockerignoreFile, ...gitignoreFiles].filter(e => e);
+	if (ignoreFiles.length === 0) {
+		return;
+	}
+	const hr =
+		'-------------------------------------------------------------------------------';
+	const msg = [' ', hr, 'Using file ignore patterns from:'];
+	msg.push(...ignoreFiles);
+	if (gitignoreFiles.length) {
+		msg.push(stripIndent`
+			balena CLI currently uses gitgnore and dockerignore files, but an upcoming major
+			version release will disregard gitignore files and use a dockerignore file only.
+			Use the --nogitignore (-G) option to enable the new behavior already now and
+			suppress this warning. For more information, see 'balena help ${Logger.command}'.
+		`);
+		msg.push(hr);
+		Logger.getLogger().logWarn(msg.join('\n'));
+	} else if (dockerignoreFile && process.platform === 'win32') {
+		msg.push(stripIndent`
+			Use the --nogitignore (-G) option to suppress this warning and enable the use
+			of a better dockerignore parser and filter library that fixes several issues
+			on Windows and improves compatibility with "docker build", but which may also
+			cause a different set of files to be filtered out (because of the bug fixes).
+			The --nogitignore option will be the default behavior in an upcoming balena CLI
+			major version release. For more information, see 'balena help ${Logger.command}'.
+		`);
+		msg.push(hr);
+		Logger.getLogger().logWarn(msg.join('\n'));
+	}
+}
+
 /**
  * Check whether the "build secrets" feature is being used and, if so,
  * verify that the target docker daemon is balenaEngine. If the

lib/utils/device/deploy.ts

@@ -56,6 +56,7 @@ export interface DeviceDeployOptions {
 	dockerfilePath?: string;
 	registrySecrets: RegistrySecrets;
 	nocache: boolean;
+	nogitignore: boolean;
 	noParentCheck: boolean;
 	nolive: boolean;
 	detached: boolean;
@@ -186,6 +187,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 	globalLogger.logDebug('Tarring all non-ignored files...');
 	const tarStream = await tarDirectory(opts.source, {
 		convertEol: opts.convertEol,
+		nogitignore: opts.nogitignore,
 	});
 
 	// Try to detect the device information
@@ -400,7 +402,10 @@ export async function rebuildSingleTask(
 		}
 	};
 
-	const tarStream = await tarDirectory(source);
+	const tarStream = await tarDirectory(source, {
+		convertEol: opts.convertEol,
+		nogitignore: opts.nogitignore,
+	});
 
 	const task = _.find(
 		await makeBuildTasks(

lib/utils/ignore.ts

@@ -22,6 +22,8 @@ import * as MultiBuild from 'resin-multibuild';
 import dockerIgnore = require('@zeit/dockerignore');
 import ignore from 'ignore';
 
+import { ExpectedError } from '../errors';
+
 const { toPosixPath } = MultiBuild.PathUtils;
 
 export enum IgnoreFileType {
@@ -182,3 +184,92 @@ export class FileIgnorer {
 		return !/^\.\.\//.test(path.posix.relative(path1, path2));
 	}
 }
+
+interface FileStats {
+	filePath: string;
+	relPath: string;
+	stats: fs.Stats;
+}
+
+/**
+ * Create a list of files (FileStats[]) for the filesystem subtree rooted at
+ * projectDir, listing each file with both a full path and a relative path,
+ * but excluding entries for directories themselves.
+ * @param projectDir Source directory (root of subtree to be listed)
+ * @param dir Used for recursive calls only (omit on first function call)
+ */
+async function listFiles(
+	projectDir: string,
+	dir: string = projectDir,
+): Promise<FileStats[]> {
+	const files: FileStats[] = [];
+	const dirEntries = await fs.readdir(dir);
+	await Promise.all(
+		dirEntries.map(async entry => {
+			const filePath = path.join(dir, entry);
+			const stats = await fs.stat(filePath);
+			if (stats.isDirectory()) {
+				files.push(...(await listFiles(projectDir, filePath)));
+			} else if (stats.isFile()) {
+				files.push({
+					filePath,
+					relPath: path.relative(projectDir, filePath),
+					stats,
+				});
+			}
+		}),
+	);
+	return files;
+}
+
+/**
+ * Return the contents of a .dockerignore file at projectDir, as a string.
+ * Return an empty string if a .dockerignore file does not exist.
+ * @param projectDir Source directory
+ * @returns Contents of the .dockerignore file, as a UTF-8 string
+ */
+async function readDockerIgnoreFile(projectDir: string): Promise<string> {
+	const dockerIgnorePath = path.join(projectDir, '.dockerignore');
+	let dockerIgnoreStr = '';
+	try {
+		dockerIgnoreStr = await fs.readFile(dockerIgnorePath, 'utf8');
+	} catch (err) {
+		if (err.code !== 'ENOENT') {
+			throw new ExpectedError(
+				`Error reading file "${dockerIgnorePath}": ${err.message}`,
+			);
+		}
+	}
+	return dockerIgnoreStr;
+}
+
+/**
+ * Create a list of files (FileStats[]) for the filesystem subtree rooted at
+ * projectDir, filtered against a .dockerignore file (if any) also at projectDir,
+ * plus a few hardcoded dockerignore patterns.
+ * @param projectDir Source directory to
+ */
+export async function filterFilesWithDockerignore(
+	projectDir: string,
+): Promise<FileStats[]> {
+	// path.resolve() also converts forward slashes to backslashes on Windows
+	projectDir = path.resolve(projectDir);
+	const dockerIgnoreStr = await readDockerIgnoreFile(projectDir);
+	const $dockerIgnore = (await import('@balena/dockerignore')).default;
+	const ig = $dockerIgnore({ ignorecase: false });
+
+	ig.add(['**/.git']);
+	if (dockerIgnoreStr) {
+		ig.add(dockerIgnoreStr);
+	}
+	ig.add([
+		'!**/.balena',
+		'!**/.resin',
+		'!**/Dockerfile',
+		'!**/Dockerfile.*',
+		'!**/docker-compose.yml',
+	]);
+
+	const files = await listFiles(projectDir);
+	return files.filter((file: FileStats) => !ig.ignores(file.relPath));
+}

lib/utils/logger.ts

@@ -39,6 +39,8 @@ enum Level {
  */
 class Logger {
 	public static readonly Level = Level;
+	// `Logger.command` is currently set in `preparser.ts`
+	public static command: string; // CLI cmd, e.g. 'push', 'env add', ...
 
 	public streams: {
 		build: NodeJS.ReadWriteStream;

lib/utils/messages.ts

@@ -1,3 +1,20 @@
+/**
+ * @license
+ * Copyright 2017-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 const DEBUG_MODE = !!process.env.DEBUG;
 
 export const reachingOut = `\
@@ -28,6 +45,7 @@ export const balenaAsciiArt = `\
 `;
 
 export const registrySecretsHelp = `\
+REGISTRY SECRETS
 The --registry-secrets option specifies a JSON or YAML file containing private
 Docker registry usernames and passwords to be used when pulling base images.
 Sample registry-secrets YAML file:
@@ -48,3 +66,39 @@ check: https://github.com/balena-io-playground/sample-gcr-registry-secrets
 If the --registry-secrets option is not specified, and a secrets.yml or
 secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.`;
+
+export const dockerignoreHelp = `\
+DOCKERIGNORE AND GITIGNORE FILES
+By default, both '.dockerignore' and '.gitignore' files are taken into account
+in order to prevent files from being sent to the balenaCloud builder or Docker
+or balenaEngine (balenaOS device).
+
+However, this behavior has been DEPRECATED and will change in an upcoming major
+version release. The --nogitignore (-G) option should be used to enable the new
+behavior already now. This option will cause the CLI to:
+
+* Disregard all '.gitignore' files at the source directory and subdirectories,
+  and consider only the '.dockerignore' file (if any) at the source directory.
+* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine
+  even if they are listed in '.gitignore' files (a longstanding feature request).
+* Use a new '.dockerignore' parser and filter library that improves compatibility
+  with "docker build" and fixes several issues (mainly on Windows).
+* Prevent a warning message from being printed.
+
+When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are
+also used and "merged" (in memory) with the patterns found in the '.dockerignore'
+file (if any), in the following order:
+
+    **/.git
+    < user's patterns from the '.dockerignore' file, if any >
+    !**/.balena
+    !**/.resin
+    !**/Dockerfile
+    !**/Dockerfile.*
+    !**/docker-compose.yml
+
+If necessary, the effect of the '**/.git' pattern may be modified by adding
+"counter patterns" to the '.dockerignore' file, for example '!service1/.git'.
+For documentation on pattern format, see:
+- https://docs.docker.com/engine/reference/builder/#dockerignore-file
+- https://www.npmjs.com/package/@balena/dockerignore`;

lib/utils/remote-build.ts

@@ -51,6 +51,7 @@ export interface RemoteBuild {
 	source: string;
 	auth: string;
 	baseUrl: string;
+	nogitignore: boolean;
 	opts: BuildOpts;
 
 	sdk: BalenaSDK;
@@ -302,6 +303,7 @@ async function getTarStream(build: RemoteBuild): Promise<Stream.Readable> {
 		return await tarDirectory(path.resolve(build.source), {
 			preFinalizeCallback: preFinalizeCb,
 			convertEol: build.opts.convertEol,
+			nogitignore: build.nogitignore,
 		});
 	} finally {
 		tarSpinner.stop();

npm-shrinkwrap.json

@@ -104,6 +104,11 @@
         }
       }
     },
+    "@balena/dockerignore": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz",
+      "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q=="
+    },
     "@balena/lint": {
       "version": "4.1.1",
       "resolved": "https://registry.npmjs.org/@balena/lint/-/lint-4.1.1.tgz",

package.json

@@ -164,6 +164,7 @@
     "typescript": "^3.8.3"
   },
   "dependencies": {
+    "@balena/dockerignore": "^1.0.2",
     "@oclif/command": "^1.5.19",
     "@resin.io/valid-email": "^0.1.0",
     "@sentry/node": "^5.13.2",

tests/commands/build.spec.ts

@@ -25,14 +25,13 @@ import { fs } from 'mz';
 import * as path from 'path';
 
 import { BalenaAPIMock } from '../balena-api-mock';
+import { expectStreamNoCRLF, testDockerBuildStream } from '../docker-build';
+import { DockerMock, dockerResponsePath } from '../docker-mock';
+import { cleanOutput, runCommand } from '../helpers';
 import {
 	ExpectedTarStreamFiles,
 	ExpectedTarStreamFilesByService,
-	expectStreamNoCRLF,
+} from '../projects';
-	testDockerBuildStream,
-} from '../docker-build';
-import { DockerMock, dockerResponsePath } from '../docker-mock';
-import { cleanOutput, runCommand } from '../helpers';
 
 const repoPath = path.normalize(path.join(__dirname, '..', '..'));
 const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects');
@@ -53,6 +52,15 @@ const commonQueryParams = [
 	['labels', ''],
 ];
 
+const commonComposeQueryParams = [
+	['t', '${tag}'],
+	[
+		'buildargs',
+		'{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable"}',
+	],
+	['labels', ''],
+];
+
 describe('balena build', function() {
 	let api: BalenaAPIMock;
 	let docker: DockerMock;
@@ -104,7 +112,7 @@ describe('balena build', function() {
 		}
 		docker.expectGetInfo({});
 		await testDockerBuildStream({
-			commandLine: `build ${projectPath} --deviceType nuc --arch amd64`,
+			commandLine: `build ${projectPath} --deviceType nuc --arch amd64 -G`,
 			dockerMock: docker,
 			expectedFilesByService: { main: expectedFiles },
 			expectedQueryParamsByService: { main: commonQueryParams },
@@ -178,7 +186,7 @@ describe('balena build', function() {
 			mock.reRequire('../../build/utils/qemu');
 			docker.expectGetInfo({ OperatingSystem: 'balenaOS 2.44.0+rev1' });
 			await testDockerBuildStream({
-				commandLine: `build ${projectPath} --emulated --deviceType ${deviceType} --arch ${arch}`,
+				commandLine: `build ${projectPath} --emulated --deviceType ${deviceType} --arch ${arch} --nogitignore`,
 				dockerMock: docker,
 				expectedFilesByService: { main: expectedFiles },
 				expectedQueryParamsByService: { main: commonQueryParams },
@@ -273,8 +281,15 @@ describe('balena build', function() {
 			'utf8',
 		);
 		const expectedQueryParamsByService = {
-			service1: commonQueryParams,
+			service1: [
-			service2: [...commonQueryParams, ['dockerfile', 'Dockerfile-alt']],
+				['t', '${tag}'],
+				[
+					'buildargs',
+					'{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable","SERVICE1_VAR":"This is a service specific variable"}',
+				],
+				['labels', ''],
+			],
+			service2: [...commonComposeQueryParams, ['dockerfile', 'Dockerfile-alt']],
 		};
 		const expectedResponseLines: string[] = [
 			...commonResponseLines[responseFilename],
@@ -292,7 +307,7 @@ describe('balena build', function() {
 		}
 		docker.expectGetInfo({});
 		await testDockerBuildStream({
-			commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol`,
+			commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -G`,
 			dockerMock: docker,
 			expectedFilesByService,
 			expectedQueryParamsByService,

tests/commands/deploy.spec.ts

@@ -23,9 +23,10 @@ import { fs } from 'mz';
 import * as path from 'path';
 
 import { BalenaAPIMock } from '../balena-api-mock';
-import { ExpectedTarStreamFiles, testDockerBuildStream } from '../docker-build';
+import { testDockerBuildStream } from '../docker-build';
 import { DockerMock, dockerResponsePath } from '../docker-mock';
 import { cleanOutput, runCommand } from '../helpers';
+import { ExpectedTarStreamFiles } from '../projects';
 
 const repoPath = path.normalize(path.join(__dirname, '..', '..'));
 const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects');
@@ -119,7 +120,7 @@ describe('balena deploy', function() {
 		}
 
 		await testDockerBuildStream({
-			commandLine: `deploy testApp --build --source ${projectPath}`,
+			commandLine: `deploy testApp --build --source ${projectPath} -G`,
 			dockerMock: docker,
 			expectedFilesByService: { main: expectedFiles },
 			expectedQueryParamsByService: { main: commonQueryParams },

tests/commands/push.spec.ts

@@ -24,12 +24,13 @@ import * as path from 'path';
 
 import { BalenaAPIMock } from '../balena-api-mock';
 import { BuilderMock, builderResponsePath } from '../builder-mock';
-import {
+import { expectStreamNoCRLF, testPushBuildStream } from '../docker-build';
-	ExpectedTarStreamFiles,
-	expectStreamNoCRLF,
-	testPushBuildStream,
-} from '../docker-build';
 import { cleanOutput, runCommand } from '../helpers';
+import {
+	addRegSecretsEntries,
+	ExpectedTarStreamFiles,
+	setupDockerignoreTestData,
+} from '../projects';
 
 const repoPath = path.normalize(path.join(__dirname, '..', '..'));
 const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects');
@@ -76,6 +77,8 @@ const commonQueryParams = [
 	['headless', 'false'],
 ];
 
+const itSkipWindows = process.platform === 'win32' ? it.skip : it;
+
 describe('balena push', function() {
 	let api: BalenaAPIMock;
 	let builder: BuilderMock;
@@ -95,6 +98,14 @@ describe('balena push', function() {
 		builder.done();
 	});
 
+	this.beforeAll(async () => {
+		await setupDockerignoreTestData();
+	});
+
+	this.afterAll(async () => {
+		await setupDockerignoreTestData({ cleanup: true });
+	});
+
 	it('should create the expected tar stream (single container)', async () => {
 		const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic');
 		const expectedFiles: ExpectedTarStreamFiles = {
@@ -103,6 +114,7 @@ describe('balena push', function() {
 			Dockerfile: { fileSize: 88, type: 'file' },
 			'Dockerfile-alt': { fileSize: 30, type: 'file' },
 		};
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
 		const responseFilename = 'build-POST-v3.json';
 		const responseBody = await fs.readFile(
 			path.join(builderResponsePath, responseFilename),
@@ -122,7 +134,7 @@ describe('balena push', function() {
 
 		await testPushBuildStream({
 			builderMock: builder,
-			commandLine: `push testApp --source ${projectPath}`,
+			commandLine: `push testApp --source ${projectPath} -R ${regSecretsPath} -G`,
 			expectedFiles,
 			expectedQueryParams: commonQueryParams,
 			expectedResponseLines,
@@ -140,6 +152,7 @@ describe('balena push', function() {
 			Dockerfile: { fileSize: 88, type: 'file' },
 			'Dockerfile-alt': { fileSize: 30, type: 'file' },
 		};
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
 		const responseFilename = 'build-POST-v3.json';
 		const responseBody = await fs.readFile(
 			path.join(builderResponsePath, responseFilename),
@@ -151,7 +164,7 @@ describe('balena push', function() {
 
 		await testPushBuildStream({
 			builderMock: builder,
-			commandLine: `push testApp --source ${projectPath} --dockerfile Dockerfile-alt`,
+			commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} --dockerfile Dockerfile-alt --nogitignore`,
 			expectedFiles,
 			expectedQueryParams,
 			expectedResponseLines: commonResponseLines[responseFilename],
@@ -173,6 +186,7 @@ describe('balena push', function() {
 			Dockerfile: { fileSize: 88, type: 'file' },
 			'Dockerfile-alt': { fileSize: 30, type: 'file' },
 		};
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
 		const responseFilename = 'build-POST-v3.json';
 		const responseBody = await fs.readFile(
 			path.join(builderResponsePath, responseFilename),
@@ -191,7 +205,182 @@ describe('balena push', function() {
 
 		await testPushBuildStream({
 			builderMock: builder,
-			commandLine: `push testApp --source ${projectPath} --convert-eol`,
+			commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l`,
+			expectedFiles,
+			expectedQueryParams: commonQueryParams,
+			expectedResponseLines,
+			projectPath,
+			responseBody,
+			responseCode: 200,
+		});
+	});
+
+	// Skip Windows because the old tarDirectory() implementation (still used when
+	// '--nogitignore' is not provided) uses the old `zeit/dockerignore` npm package
+	// that is broken on Windows (reason why we created `@balena/dockerignore`).
+	itSkipWindows(
+		'should create the expected tar stream (single container, with gitignore)',
+		async () => {
+			const projectPath = path.join(
+				projectsPath,
+				'no-docker-compose',
+				'dockerignore1',
+			);
+			const expectedFiles: ExpectedTarStreamFiles = {
+				'.balena/balena.yml': { fileSize: 12, type: 'file' },
+				'.dockerignore': { fileSize: 438, type: 'file' },
+				'.gitignore': { fileSize: 20, type: 'file' },
+				'.git/bar.txt': { fileSize: 4, type: 'file' },
+				'.git/foo.txt': { fileSize: 4, type: 'file' },
+				'c.txt': { fileSize: 1, type: 'file' },
+				Dockerfile: { fileSize: 13, type: 'file' },
+				'src/.balena/balena.yml': { fileSize: 16, type: 'file' },
+				'src/.gitignore': { fileSize: 10, type: 'file' },
+				'vendor/.git/vendor-git-contents': { fileSize: 20, type: 'file' },
+			};
+			const regSecretsPath = await addRegSecretsEntries(expectedFiles);
+			const responseFilename = 'build-POST-v3.json';
+			const responseBody = await fs.readFile(
+				path.join(builderResponsePath, responseFilename),
+				'utf8',
+			);
+			const expectedResponseLines = [
+				'[Warn] Using file ignore patterns from:',
+				`[Warn] ${path.join(projectPath, '.dockerignore')}`,
+				`[Warn] ${path.join(projectPath, '.gitignore')}`,
+				`[Warn] ${path.join(projectPath, 'src', '.gitignore')}`,
+				'[Warn] balena CLI currently uses gitgnore and dockerignore files, but an upcoming major',
+				'[Warn] version release will disregard gitignore files and use a dockerignore file only.',
+				'[Warn] Use the --nogitignore (-G) option to enable the new behavior already now and',
+				"[Warn] suppress this warning. For more information, see 'balena help push'.",
+				...commonResponseLines[responseFilename],
+			];
+
+			await testPushBuildStream({
+				builderMock: builder,
+				commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l`,
+				expectedFiles,
+				expectedQueryParams: commonQueryParams,
+				expectedResponseLines,
+				projectPath,
+				responseBody,
+				responseCode: 200,
+			});
+		},
+	);
+
+	it('should create the expected tar stream (single container, --nogitignore)', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore1',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'.balena/balena.yml': { fileSize: 12, type: 'file' },
+			'.dockerignore': { fileSize: 438, type: 'file' },
+			'.gitignore': { fileSize: 20, type: 'file' },
+			'.git/foo.txt': { fileSize: 4, type: 'file' },
+			'a.txt': { fileSize: 1, type: 'file' },
+			'c.txt': { fileSize: 1, type: 'file' },
+			Dockerfile: { fileSize: 13, type: 'file' },
+			'src/.balena/balena.yml': { fileSize: 16, type: 'file' },
+			'src/.gitignore': { fileSize: 10, type: 'file' },
+			'src/src-a.txt': { fileSize: 5, type: 'file' },
+			'src/src-c.txt': { fileSize: 5, type: 'file' },
+			'vendor/.git/vendor-git-contents': { fileSize: 20, type: 'file' },
+		};
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
+		const responseFilename = 'build-POST-v3.json';
+		const responseBody = await fs.readFile(
+			path.join(builderResponsePath, responseFilename),
+			'utf8',
+		);
+
+		await testPushBuildStream({
+			builderMock: builder,
+			commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l -G`,
+			expectedFiles,
+			expectedQueryParams: commonQueryParams,
+			expectedResponseLines: commonResponseLines[responseFilename],
+			projectPath,
+			responseBody,
+			responseCode: 200,
+		});
+	});
+
+	it('should create the expected tar stream (single container, symbolic links, --nogitignore)', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore2',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'.dockerignore': { fileSize: 34, type: 'file' },
+			'b.txt': { fileSize: 1, type: 'file' },
+			Dockerfile: { fileSize: 13, type: 'file' },
+			'src/src-b.txt': { fileSize: 5, type: 'file' },
+			'symlink-a.txt': { fileSize: 5, type: 'file' },
+		};
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
+		const responseFilename = 'build-POST-v3.json';
+		const responseBody = await fs.readFile(
+			path.join(builderResponsePath, responseFilename),
+			'utf8',
+		);
+
+		await testPushBuildStream({
+			builderMock: builder,
+			commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l -G`,
+			expectedFiles,
+			expectedQueryParams: commonQueryParams,
+			expectedResponseLines: commonResponseLines[responseFilename],
+			projectPath,
+			responseBody,
+			responseCode: 200,
+		});
+	});
+
+	it('should create the expected tar stream (single container, dockerignore warn)', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore2',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'.dockerignore': { fileSize: 34, type: 'file' },
+			'b.txt': { fileSize: 1, type: 'file' },
+			Dockerfile: { fileSize: 13, type: 'file' },
+			'src/src-b.txt': { fileSize: 5, type: 'file' },
+			'symlink-a.txt': { fileSize: 5, type: 'file' },
+		};
+		if (isWindows) {
+			// this test uses the old tarDirectory implementation, which uses
+			// the zeit/dockerignore library that has bugs on Windows
+			expectedFiles['src/src-a.txt'] = { fileSize: 5, type: 'file' };
+		}
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
+		const responseFilename = 'build-POST-v3.json';
+		const responseBody = await fs.readFile(
+			path.join(builderResponsePath, responseFilename),
+			'utf8',
+		);
+		const expectedResponseLines = isWindows
+			? [
+					'[Warn] Using file ignore patterns from:',
+					`[Warn] ${path.join(projectPath, '.dockerignore')}`,
+					'[Warn] Use the --nogitignore (-G) option to suppress this warning and enable the use',
+					'[Warn] of a better dockerignore parser and filter library that fixes several issues',
+					'[Warn] on Windows and improves compatibility with "docker build", but which may also',
+					'[Warn] cause a different set of files to be filtered out (because of the bug fixes).',
+					'[Warn] The --nogitignore option will be the default behavior in an upcoming balena CLI',
+					"[Warn] major version release. For more information, see 'balena help push'.",
+					...commonResponseLines[responseFilename],
+			  ]
+			: commonResponseLines[responseFilename];
+
+		await testPushBuildStream({
+			builderMock: builder,
+			commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l`,
 			expectedFiles,
 			expectedQueryParams: commonQueryParams,
 			expectedResponseLines,
@@ -204,6 +393,8 @@ describe('balena push', function() {
 	it('should create the expected tar stream (docker-compose)', async () => {
 		const projectPath = path.join(projectsPath, 'docker-compose', 'basic');
 		const expectedFiles: ExpectedTarStreamFiles = {
+			'.balena/balena.yml': { fileSize: 197, type: 'file' },
+			'.dockerignore': { fileSize: 22, type: 'file' },
 			'docker-compose.yml': { fileSize: 245, type: 'file' },
 			'service1/Dockerfile.template': { fileSize: 144, type: 'file' },
 			'service1/file1.sh': { fileSize: 12, type: 'file' },
@@ -214,6 +405,7 @@ describe('balena push', function() {
 				type: 'file',
 			},
 		};
+		const regSecretsPath = await addRegSecretsEntries(expectedFiles);
 		const responseFilename = 'build-POST-v3.json';
 		const responseBody = await fs.readFile(
 			path.join(builderResponsePath, responseFilename),
@@ -234,7 +426,7 @@ describe('balena push', function() {
 
 		await testPushBuildStream({
 			builderMock: builder,
-			commandLine: `push testApp --source ${projectPath} --convert-eol`,
+			commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l -G`,
 			expectedFiles,
 			expectedQueryParams: commonQueryParams,
 			expectedResponseLines,
@@ -258,7 +450,7 @@ describe('balena push: project validation', function() {
 		];
 
 		const { out, err } = await runCommand(
-			`push testApp --source ${projectPath}`,
+			`push testApp --source ${projectPath} --nogitignore`,
 		);
 		expect(
 			cleanOutput(err).map(line => line.replace(/\s{2,}/g, ' ')),

tests/docker-build.ts

@@ -29,25 +29,11 @@ import { URL } from 'url';
 import { BuilderMock } from './builder-mock';
 import { DockerMock } from './docker-mock';
 import { cleanOutput, fillTemplateArray, runCommand } from './helpers';
-
+import {
-export interface ExpectedTarStreamFile {
+	ExpectedTarStreamFile,
-	contents?: string;
+	ExpectedTarStreamFiles,
-	fileSize: number;
+	ExpectedTarStreamFilesByService,
-	testStream?: (
+} from './projects';
-		header: tar.Headers,
-		stream: Readable,
-		expected?: ExpectedTarStreamFile,
-	) => Promise<void>;
-	type: tar.Headers['type'];
-}
-
-export interface ExpectedTarStreamFiles {
-	[filePath: string]: ExpectedTarStreamFile;
-}
-
-export interface ExpectedTarStreamFilesByService {
-	[service: string]: ExpectedTarStreamFiles;
-}
 
 /**
  * Run a few chai.expect() test assertions on a tar stream/buffer produced by
@@ -77,10 +63,6 @@ export async function inspectTarStream(
 			'entry',
 			async (header: tar.Headers, stream: Readable, next: tar.Callback) => {
 				try {
-					// TODO: test the .balena folder instead of ignoring it
-					if (header.name.startsWith('.balena/')) {
-						stream.resume();
-					} else {
 					expect(foundFiles).to.not.have.property(header.name);
 					foundFiles[header.name] = {
 						fileSize: header.size || 0,
@@ -92,7 +74,6 @@ export async function inspectTarStream(
 					} else {
 						await defaultTestStream(header, stream, expected, projectPath);
 					}
-					}
 				} catch (err) {
 					reject(err);
 				}
@@ -122,6 +103,11 @@ async function defaultTestStream(
 	if (expected?.contents) {
 		expectedContents = Buffer.from(expected.contents);
 	}
+	if (header.name === '.balena/registry-secrets.json') {
+		expectedContents = await fs.readFile(
+			path.join(__dirname, 'test-data', 'projects', 'registry-secrets.json'),
+		);
+	}
 	const [buf, buf2] = await Promise.all([
 		streamToBuffer(stream),
 		expectedContents ||

tests/projects.ts

@@ -0,0 +1,85 @@
+/**
+ * @license
+ * Copyright 2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { Headers } from 'tar-stream';
+import { promisify } from 'util';
+
+const statAsync = promisify(fs.stat);
+
+export interface ExpectedTarStreamFile {
+	contents?: string;
+	fileSize: number;
+	testStream?: (
+		header: Headers,
+		stream: import('stream').Readable,
+		expected?: ExpectedTarStreamFile,
+	) => Promise<void>;
+	type: Headers['type'];
+}
+
+export interface ExpectedTarStreamFiles {
+	[filePath: string]: ExpectedTarStreamFile;
+}
+
+export interface ExpectedTarStreamFilesByService {
+	[service: string]: ExpectedTarStreamFiles;
+}
+
+export const repoPath = path.normalize(path.join(__dirname, '..'));
+export const projectsPath = path.join(
+	repoPath,
+	'tests',
+	'test-data',
+	'projects',
+);
+
+export async function setupDockerignoreTestData({ cleanup = false } = {}) {
+	const { copy, remove } = await import('fs-extra');
+	const dockerignoreProjDir = path.join(
+		__dirname,
+		'test-data',
+		'projects',
+		'no-docker-compose',
+		'dockerignore1',
+	);
+	const subdirs = ['', 'vendor'];
+	for (const subdir of subdirs) {
+		// A git repo cannot store a '.git' subfolder, even under tests/test-data/,
+		// so we store a 'dot.git' folder instead, and copy it as '.git' before
+		// running the tests. (Interestingly, 'git status' also ignores the '.git'
+		// folder, and shows a "clean repo" even after this copy is executed.)
+		const aliasDir = path.join(dockerignoreProjDir, subdir, 'dot.git');
+		const gitDir = path.join(dockerignoreProjDir, subdir, '.git');
+		await remove(gitDir);
+		if (!cleanup) {
+			await copy(aliasDir, gitDir);
+		}
+	}
+}
+
+export async function addRegSecretsEntries(
+	expectedFiles: ExpectedTarStreamFiles,
+): Promise<string> {
+	const regSecretsPath = path.join(projectsPath, 'registry-secrets.json');
+	expectedFiles['.balena/registry-secrets.json'] = {
+		fileSize: (await statAsync(regSecretsPath)).size,
+		type: 'file',
+	};
+	return regSecretsPath;
+}

tests/test-data/projects/docker-compose/basic/.balena/balena.yml

@@ -0,0 +1,7 @@
+build-variables:
+    global:
+        - MY_VAR_1=This is a variable
+        - MY_VAR_2=Also a variable
+    services:
+        service1:
+            - SERVICE1_VAR=This is a service specific variable

tests/test-data/projects/docker-compose/basic/.dockerignore

@@ -0,0 +1 @@
+service1/test-ignore*

tests/test-data/projects/docker-compose/basic/service1/test-ignore.txt

@@ -0,0 +1 @@
+test-ignore

tests/test-data/projects/no-docker-compose/dockerignore1/.balena/balena.yml

@@ -0,0 +1 @@
+"balena.yml"

tests/test-data/projects/no-docker-compose/dockerignore1/.dockerignore

@@ -0,0 +1,19 @@
+# Note that the CLI "hardcodes" some dockerignore patterns in its source code,
+# which get "merged" with the user's patterns in their .dockerginore file as follows:
+#
+#   **/.git
+#   <user's patterns from their .dockerignore file go here>
+#   !**/.balena
+#   !**/.resin
+#   !**/Dockerfile
+#   !**/Dockerfile.*
+#   !**/docker-compose.yml
+#
+b.txt
+Dockerfile
+registry-secrets.json
+src/*b.txt
+**/.balena
+**/dot.git
+!.git/foo.txt
+!vendor/.git

tests/test-data/projects/no-docker-compose/dockerignore1/.gitignore

@@ -0,0 +1,2 @@
+a.txt
+src/src-a.txt

tests/test-data/projects/no-docker-compose/dockerignore1/Dockerfile

@@ -0,0 +1 @@
+FROM busybox

tests/test-data/projects/no-docker-compose/dockerignore1/a.txt

@@ -0,0 +1 @@
+a

tests/test-data/projects/no-docker-compose/dockerignore1/b.txt

@@ -0,0 +1 @@
+b

tests/test-data/projects/no-docker-compose/dockerignore1/c.txt

@@ -0,0 +1 @@
+c

tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/bar.txt

@@ -0,0 +1 @@
+bar

tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/foo.txt

@@ -0,0 +1 @@
+foo

tests/test-data/projects/no-docker-compose/dockerignore1/src/.balena/balena.yml

@@ -0,0 +1 @@
+"src-balena.yml"

tests/test-data/projects/no-docker-compose/dockerignore1/src/.gitignore

@@ -0,0 +1 @@
+src-c.txt

tests/test-data/projects/no-docker-compose/dockerignore1/src/src-a.txt

@@ -0,0 +1 @@
+src-a

tests/test-data/projects/no-docker-compose/dockerignore1/src/src-b.txt

@@ -0,0 +1 @@
+src-b

tests/test-data/projects/no-docker-compose/dockerignore1/src/src-c.txt

@@ -0,0 +1 @@
+src-c

tests/test-data/projects/no-docker-compose/dockerignore1/vendor/dot.git/vendor-git-contents

@@ -0,0 +1 @@
+vendor-git-contents

tests/test-data/projects/no-docker-compose/dockerignore2/.dockerignore

@@ -0,0 +1,3 @@
+a.txt
+src/src-a.txt
+symlink-b.txt

tests/test-data/projects/no-docker-compose/dockerignore2/Dockerfile

@@ -0,0 +1 @@
+FROM busybox

tests/test-data/projects/no-docker-compose/dockerignore2/a.txt

@@ -0,0 +1 @@
+a

tests/test-data/projects/no-docker-compose/dockerignore2/b.txt

@@ -0,0 +1 @@
+b

tests/test-data/projects/no-docker-compose/dockerignore2/src/src-a.txt

@@ -0,0 +1 @@
+src-a

tests/test-data/projects/no-docker-compose/dockerignore2/src/src-b.txt

@@ -0,0 +1 @@
+src-b

tests/test-data/projects/no-docker-compose/dockerignore2/symlink-a.txt

@@ -0,0 +1 @@
+src/src-a.txt

tests/test-data/projects/no-docker-compose/dockerignore2/symlink-b.txt

@@ -0,0 +1 @@
+src/src-b.txt

tests/test-data/projects/registry-secrets.json

@@ -0,0 +1 @@
+{"https://index.docker.io/v1/":{"username":"test","password":"test"}}

tests/utils/tarDirectory.spec.ts

@@ -0,0 +1,192 @@
+/**
+ * @license
+ * Copyright 2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// tslint:disable-next-line:no-var-requires
+require('./../config-tests'); // required for side effects
+
+import { expect } from 'chai';
+import * as _ from 'lodash';
+import * as path from 'path';
+import * as tar from 'tar-stream';
+
+import { tarDirectory } from '../../build/utils/compose';
+import { setupDockerignoreTestData } from '../projects';
+
+const repoPath = path.normalize(path.join(__dirname, '..', '..'));
+const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects');
+
+interface TarFiles {
+	[name: string]: {
+		fileSize?: number;
+		type?: string | null;
+	};
+}
+
+const itSkipWindows = process.platform === 'win32' ? it.skip : it;
+
+describe('compare new and old tarDirectory implementations', async function() {
+	const extraContent = 'extra';
+	const extraEntry: tar.Headers = {
+		name: 'extra.txt',
+		size: extraContent.length,
+		type: 'file',
+	};
+	const preFinalizeCallback = (pack: tar.Pack) => {
+		pack.entry(extraEntry, extraContent);
+	};
+
+	this.beforeAll(async () => {
+		await setupDockerignoreTestData();
+	});
+
+	this.afterAll(async () => {
+		await setupDockerignoreTestData({ cleanup: true });
+	});
+
+	it('should produce the expected file list', async function() {
+		const dockerignoreProjDir = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore1',
+		);
+		const expectedFiles = {
+			'.balena/balena.yml': { fileSize: 12, type: 'file' },
+			'.dockerignore': { fileSize: 438, type: 'file' },
+			'.gitignore': { fileSize: 20, type: 'file' },
+			'.git/foo.txt': { fileSize: 4, type: 'file' },
+			'a.txt': { fileSize: 1, type: 'file' },
+			'c.txt': { fileSize: 1, type: 'file' },
+			Dockerfile: { fileSize: 13, type: 'file' },
+			'extra.txt': { fileSize: 5, type: 'file' },
+			'src/.balena/balena.yml': { fileSize: 16, type: 'file' },
+			'src/.gitignore': { fileSize: 10, type: 'file' },
+			'src/src-a.txt': { fileSize: 5, type: 'file' },
+			'src/src-c.txt': { fileSize: 5, type: 'file' },
+			'vendor/.git/vendor-git-contents': { fileSize: 20, type: 'file' },
+		};
+
+		const tarPack = await tarDirectory(dockerignoreProjDir, {
+			preFinalizeCallback,
+			nogitignore: true,
+		});
+		const fileList = await getTarPackFiles(tarPack);
+
+		expect(fileList).to.deep.equal(expectedFiles);
+	});
+
+	it('should produce the expected file list (symbolic links)', async function() {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore2',
+		);
+		const expectedFiles = {
+			'.dockerignore': { fileSize: 34, type: 'file' },
+			'b.txt': { fileSize: 1, type: 'file' },
+			Dockerfile: { fileSize: 13, type: 'file' },
+			'src/src-b.txt': { fileSize: 5, type: 'file' },
+			'symlink-a.txt': { fileSize: 5, type: 'file' },
+		};
+
+		const tarPack = await tarDirectory(projectPath, { nogitignore: true });
+		const fileList = await getTarPackFiles(tarPack);
+
+		expect(fileList).to.deep.equal(expectedFiles);
+	});
+
+	// Skip Windows because the old tarDirectory() implementation (still used when
+	// '--nogitignore' is not provided) uses the old `zeit/dockerignore` npm package
+	// that is broken on Windows (reason why we created `@balena/dockerignore`).
+	itSkipWindows('should produce a compatible tar stream', async function() {
+		const dockerignoreProjDir = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore1',
+		);
+		const oldTarPack = await tarDirectory(dockerignoreProjDir, {
+			preFinalizeCallback,
+			nogitignore: false,
+		});
+		const oldFileList = await getTarPackFiles(oldTarPack);
+
+		const newTarPack = await tarDirectory(dockerignoreProjDir, {
+			preFinalizeCallback,
+			nogitignore: true,
+		});
+		const newFileList = await getTarPackFiles(newTarPack);
+
+		const gitIgnored = ['a.txt', 'src/src-a.txt', 'src/src-c.txt'];
+
+		expect({
+			...newFileList,
+			..._.pick(oldFileList, ['.git/bar.txt']),
+		}).to.deep.equal({
+			...oldFileList,
+			..._.pick(newFileList, gitIgnored),
+		});
+	});
+
+	itSkipWindows(
+		'should produce a compatible tar stream (symbolic links)',
+		async function() {
+			const dockerignoreProjDir = path.join(
+				projectsPath,
+				'no-docker-compose',
+				'dockerignore2',
+			);
+			const oldTarPack = await tarDirectory(dockerignoreProjDir, {
+				preFinalizeCallback,
+				nogitignore: false,
+			});
+			const oldFileList = await getTarPackFiles(oldTarPack);
+
+			const newTarPack = await tarDirectory(dockerignoreProjDir, {
+				preFinalizeCallback,
+				nogitignore: true,
+			});
+			const newFileList = await getTarPackFiles(newTarPack);
+
+			expect(newFileList).to.deep.equal(oldFileList);
+		},
+	);
+});
+
+async function getTarPackFiles(
+	pack: import('stream').Readable,
+): Promise<TarFiles> {
+	const { drainStream } = await import('tar-utils');
+	const fileList: TarFiles = {};
+	const extract = tar.extract();
+
+	return await new Promise((resolve, reject) => {
+		extract
+			.on('error', reject)
+			.on('entry', async function(header, stream, next) {
+				expect(fileList).to.not.have.property(header.name);
+				fileList[header.name] = {
+					fileSize: header.size,
+					type: header.type,
+				};
+				await drainStream(stream);
+				next();
+			})
+			.on('finish', function() {
+				resolve(fileList);
+			});
+		pack.pipe(extract);
+	});
+}