Fix Windows installer build in new balena CI workers (qqjs patch)

Change-type: patch

.mocharc-standalone.js

@@ -0,0 +1,6 @@
+const commonConfig = require('./.mocharc.js');
+
+module.exports = {
+	...commonConfig,
+	spec: ['tests/auth/*.spec.ts', 'tests/commands/**/*.spec.ts'],
+};

.mocharc.js

@@ -0,0 +1,8 @@
+module.exports = {
+	spec: 'tests/commands/app/create.spec.ts',
+	reporter: 'spec',
+	require: 'ts-node/register/transpile-only',
+	file: './tests/config-tests',
+	timeout: 12000,
+	spec: 'tests/**/*.spec.ts',
+};

.versionbot/CHANGELOG.yml

@@ -1,3 +1,151 @@
+- commits:
+    - subject: 'scan: Print production devices'' info on scan'
+      hash: 7d5a64f59a47c3a051fb2cbe9e45a71029cca694
+      body: ''
+      footer:
+        Resolves: '#1713'
+        resolves: '#1713'
+        Change-type: minor
+        change-type: minor
+        Signed-off-by: Marios Balamatsias <mbalamatsias@gmail.com>
+        signed-off-by: Marios Balamatsias <mbalamatsias@gmail.com>
+      author: Marios Balamatsias
+      nested: []
+  version: 12.29.0
+  date: 2020-12-01T13:33:03.012Z
+- commits:
+    - subject: Add ability to disable analytics for performance testing
+      hash: 950d173d276f97cb3fcfd4bb9b578b5888572a69
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Connects-to: '#1708'
+        connects-to: '#1708'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.28.3
+  date: 2020-11-26T12:55:50.969Z
+- commits:
+    - subject: 'docs: Add references to the masterclasses in the CLI help and README'
+      hash: 2bf59530c4aebdd302814e9f59d41c7b9d2672c3
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: >-
+        Fix debug message logic (don't suggest --debug if it is already being
+        used)
+      hash: a4fd7d6118a04e8e9f0e718a765b508fb11209e6
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Fix unhandled promise rejection when ~/.balena is not accessible
+      hash: 65f053dd6e2d6e212b90e905be7af5d13772c7e6
+      body: ''
+      footer:
+        Resolves: '#2096'
+        resolves: '#2096'
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.28.2
+  date: 2020-11-20T12:07:27.250Z
+- commits:
+    - subject: 'scan: Prevent spinner animation output to stdout when --json is used'
+      hash: 2f9212d622f7affe4391e0d6bac1a06e859b7488
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.28.1
+  date: 2020-11-20T00:26:37.290Z
+- commits:
+    - subject: 'push: Reduce memory usage when filtering files with dockerignore'
+      hash: 4768f763856aa9f761988477f97ec872d226b004
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'Livepush: Refactor dockerignore filtering and add test cases'
+      hash: d6b324927481ce03217c15509db2e046d74cb208
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'Livepush: Ignore paths set in .dockerignore files'
+      hash: 02a54667469982ba676da5e0b8a0f0f379b320e5
+      body: ''
+      footer:
+        Change-type: minor
+        change-type: minor
+        Signed-off-by: Josh Bowling <josh@balena.io>
+        signed-off-by: Josh Bowling <josh@balena.io>
+      author: Josh Bowling
+      nested: []
+  version: 12.28.0
+  date: 2020-11-19T17:07:59.865Z
+- commits:
+    - subject: 'Test code optimization: avoid running ~70 test cases twice'
+      hash: 6a556131995eebc318f5a02831a1cc1e2fb03b36
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'docs: Add note about macOS Big Sur notarization workaround'
+      hash: 893a39e8918756db8dd4cdd5135f430a405a409e
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.27.4
+  date: 2020-11-15T23:42:22.338Z
+- commits:
+    - subject: Avoid reporting balenarc parsing errors
+      hash: 1b0c14feab4e3c12d459d26539f65895519f89cf
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Connects-to: '#1100'
+        connects-to: '#1100'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.27.3
+  date: 2020-11-11T16:54:11.888Z
+- commits:
+    - subject: Modify `os download` help to mention dev images
+      hash: 7da9a800ccd2af0fbb051f1d95707e5d8623e227
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Signed-off-by: Thomas Manning <thomasm@balena.io>
+        signed-off-by: Thomas Manning <thomasm@balena.io>
+      author: Thomas Manning
+      nested: []
+  version: 12.27.2
+  date: 2020-11-09T12:08:41.495Z
 - commits:
     - subject: Improve application-identifier disambiguation
       hash: 46249e319ba3ee8aa1b951f5241dafa625175045

CHANGELOG.md

@@ -4,6 +4,43 @@ All notable changes to this project will be documented in this file
 automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 12.29.0 - 2020-12-01
+
+* scan: Print production devices' info on scan [Marios Balamatsias]
+
+## 12.28.3 - 2020-11-26
+
+* Add ability to disable analytics for performance testing [Scott Lowe]
+
+## 12.28.2 - 2020-11-20
+
+* docs: Add references to the masterclasses in the CLI help and README [Paulo Castro]
+* Fix debug message logic (don't suggest --debug if it is already being used) [Paulo Castro]
+* Fix unhandled promise rejection when ~/.balena is not accessible [Paulo Castro]
+
+## 12.28.1 - 2020-11-20
+
+* scan: Prevent spinner animation output to stdout when --json is used [Paulo Castro]
+
+## 12.28.0 - 2020-11-19
+
+* push: Reduce memory usage when filtering files with dockerignore [Paulo Castro]
+* Livepush: Refactor dockerignore filtering and add test cases [Paulo Castro]
+* Livepush: Ignore paths set in .dockerignore files [Josh Bowling]
+
+## 12.27.4 - 2020-11-15
+
+* Test code optimization: avoid running ~70 test cases twice [Paulo Castro]
+* docs: Add note about macOS Big Sur notarization workaround [Paulo Castro]
+
+## 12.27.3 - 2020-11-11
+
+* Avoid reporting balenarc parsing errors [Scott Lowe]
+
+## 12.27.2 - 2020-11-09
+
+* Modify `os download` help to mention dev images [Thomas Manning]
+
 ## 12.27.1 - 2020-11-06
 
 * Improve application-identifier disambiguation [Scott Lowe]

INSTALL-ADVANCED.md

@@ -59,7 +59,7 @@ macOS:   | `/usr/local/lib/balena-cli/` <br> `/usr/local/bin/balena`
    [macOS](https://www.architectryan.com/2012/10/02/add-to-the-path-on-mac-os-x-mountain-lion/#.Uydjga1dXDg) |
    [Windows](https://www.computerhope.com/issues/ch000549.htm)
 
-> * If you are using macOS Catalina (10.15), [check this known issue and
+> * If you are using macOS 10.15 or later (Catalina, Big Sur), [check this known issue and
 >   workaround](https://github.com/balena-io/balena-cli/issues/1479).
 > * **Linux Alpine** and **Busybox:** the standalone zip package is not currently compatible with
 >   these "compact" Linux distributions, because of the alternative C libraries they ship with.

README.md

@@ -139,12 +139,14 @@ The full CLI command reference is available [on the web](https://www.balena.io/d
 
 ## Support, FAQ and troubleshooting
 
-If you come across any problems or would like to get in touch:
+To learn more, troubleshoot issues, or to contact us for support:
 
-* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md).
-* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud).
-* For bug reports or feature requests,
-  [have a look at the GitHub issues or create a new one](https://github.com/balena-io/balena-cli/issues/).
+* Check the [masterclass tutorials](https://www.balena.io/docs/learn/more/masterclasses/overview/)
+* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md)
+* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud)
+
+For CLI bug reports or feature requests, check the
+[CLI GitHub issues](https://github.com/balena-io/balena-cli/issues/).
 
 ## Deprecation policy
 

doc/cli.markdown

@@ -127,12 +127,14 @@ address like `192.168.1.2`.
 
 ## Support, FAQ and troubleshooting
 
-If you come across any problems or would like to get in touch:
+To learn more, troubleshoot issues, or to contact us for support:
 
-* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md).
-* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud).
-* For bug reports or feature requests,
-  [have a look at the GitHub issues or create a new one](https://github.com/balena-io/balena-cli/issues/).
+* Check the [masterclass tutorials](https://www.balena.io/docs/learn/more/masterclasses/overview/)
+* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md)
+* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud)
+
+For CLI bug reports or feature requests, check the
+[CLI GitHub issues](https://github.com/balena-io/balena-cli/issues/).
 
 ## Deprecation policy
 
@@ -1522,6 +1524,11 @@ Only show system logs. This can be used in combination with --service.
 
 Scan for balenaOS devices on your local network.
 
+The output includes device information collected through balenaEngine for
+devices running a development image of balenaOS. Devices running a production
+image do not expose balenaEngine (on TCP port 2375), which is why less
+information is printed about them.
+
 Examples:
 
 	$ balena scan
@@ -1714,11 +1721,15 @@ versions for the given device type are pre-release).
 You can pass `--version menu` to pick the OS version from the interactive menu
 of all available versions.
 
+To download a development image append `.dev` to the version or select from
+the interactive menu.
+
 Examples:
 
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img
-	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 1.24.1
-	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^1.20.0
+	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1
+	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1.dev
+	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^2.60.0
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version latest
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version default
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version menu

lib/commands/os/download.ts

@@ -46,11 +46,15 @@ export default class OsDownloadCmd extends Command {
 
 		You can pass \`--version menu\` to pick the OS version from the interactive menu
 		of all available versions.
+
+		To download a development image append \`.dev\` to the version or select from
+		the interactive menu.
 `;
 	public static examples = [
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img',
-		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 1.24.1',
-		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^1.20.0',
+		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1',
+		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1.dev',
+		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^2.60.0',
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version latest',
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version default',
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version menu',

lib/commands/scan.ts

@@ -19,7 +19,7 @@ import { flags } from '@oclif/command';
 import type { LocalBalenaOsDevice } from 'balena-sync';
 import Command from '../command';
 import * as cf from '../utils/common-flags';
-import { getVisuals, stripIndent } from '../utils/lazy';
+import { getCliUx, stripIndent } from '../utils/lazy';
 
 interface FlagsDef {
 	json?: boolean;
@@ -33,6 +33,11 @@ export default class ScanCmd extends Command {
 		Scan for balenaOS devices on your local network.
 
 		Scan for balenaOS devices on your local network.
+
+		The output includes device information collected through balenaEngine for
+		devices running a development image of balenaOS. Devices running a production
+		image do not expose balenaEngine (on TCP port 2375), which is why less
+		information is printed about them.
 `;
 
 	public static examples = [
@@ -65,10 +70,8 @@ export default class ScanCmd extends Command {
 
 	public async run() {
 		const _ = await import('lodash');
-		const { SpinnerPromise } = getVisuals();
 		const { discover } = await import('balena-sync');
 		const prettyjson = await import('prettyjson');
-		const { ExpectedError } = await import('../errors');
 		const dockerUtils = await import('../utils/docker');
 
 		const dockerPort = 2375;
@@ -80,37 +83,54 @@ export default class ScanCmd extends Command {
 			options.timeout != null ? options.timeout * 1000 : undefined;
 
 		// Find active local devices
-		const activeLocalDevices: LocalBalenaOsDevice[] = await new SpinnerPromise({
-			promise: discover.discoverLocalBalenaOsDevices(discoverTimeout),
-			startMessage: 'Scanning for local balenaOS devices..',
-			stopMessage: options.json ? '' : 'Reporting scan results',
-		}).filter(async ({ address }: { address: string }) => {
-			const docker = dockerUtils.createClient({
-				host: address,
-				port: dockerPort,
-				timeout: dockerTimeout,
-			}) as any;
-			try {
-				await docker.pingAsync();
-				return true;
-			} catch (err) {
-				return false;
-			}
-		});
+		const ux = getCliUx();
+		ux.action.start('Scanning for local balenaOS devices');
 
-		// Exit with message if no devices found
-		if (_.isEmpty(activeLocalDevices)) {
-			// TODO: Consider whether this should really be an error
-			throw new ExpectedError(
-				process.platform === 'win32'
-					? ScanCmd.noDevicesFoundMessage + ScanCmd.windowsTipMessage
-					: ScanCmd.noDevicesFoundMessage,
-			);
-		}
+		const localDevices: LocalBalenaOsDevice[] = await discover.discoverLocalBalenaOsDevices(
+			discoverTimeout,
+		);
+		const engineReachableDevices: boolean[] = await Promise.all(
+			localDevices.map(async ({ address }: { address: string }) => {
+				const docker = dockerUtils.createClient({
+					host: address,
+					port: dockerPort,
+					timeout: dockerTimeout,
+				}) as any;
+				try {
+					await docker.pingAsync();
+					return true;
+				} catch (err) {
+					return false;
+				}
+			}),
+		);
+
+		const developmentDevices: LocalBalenaOsDevice[] = localDevices.filter(
+			(_localDevice, index) => engineReachableDevices[index],
+		);
+
+		const productionDevices = _.differenceWith(
+			localDevices,
+			developmentDevices,
+			_.isEqual,
+		);
+
+		const productionDevicesInfo = _.map(
+			productionDevices,
+			(device: LocalBalenaOsDevice) => {
+				return {
+					host: device.host,
+					address: device.address,
+					osVariant: 'production',
+					dockerInfo: undefined,
+					dockerVersion: undefined,
+				};
+			},
+		);
 
 		// Query devices for info
 		const devicesInfo = await Promise.all(
-			activeLocalDevices.map(async ({ host, address }) => {
+			developmentDevices.map(async ({ host, address }) => {
 				const docker = dockerUtils.createClient({
 					host: address,
 					port: dockerPort,
@@ -123,12 +143,15 @@ export default class ScanCmd extends Command {
 				return {
 					host,
 					address,
+					osVariant: 'development',
 					dockerInfo,
 					dockerVersion,
 				};
 			}),
 		);
 
+		ux.action.stop('Reporting scan results');
+
 		// Reduce properties if not --verbose
 		if (!options.verbose) {
 			devicesInfo.forEach((d: any) => {
@@ -141,11 +164,21 @@ export default class ScanCmd extends Command {
 			});
 		}
 
+		const cmdOutput = productionDevicesInfo.concat(devicesInfo);
+
 		// Output results
+		if (!options.json && cmdOutput.length === 0) {
+			console.error(
+				process.platform === 'win32'
+					? ScanCmd.noDevicesFoundMessage + ScanCmd.windowsTipMessage
+					: ScanCmd.noDevicesFoundMessage,
+			);
+			return;
+		}
 		console.log(
 			options.json
-				? JSON.stringify(devicesInfo, null, 4)
-				: prettyjson.render(devicesInfo, { noColor: true }),
+				? JSON.stringify(cmdOutput, null, 4)
+				: prettyjson.render(cmdOutput, { noColor: true }),
 		);
 	}
 

lib/errors.ts

@@ -156,6 +156,7 @@ const EXPECTED_ERROR_REGEXES = [
 	/must also be provided when using/, // oclif parser (depends-on)
 	/^Expected an integer/, // oclif parser (flags.integer)
 	/^Flag .* expects a value/, // oclif parser
+	/^Error parsing config file.*balenarc\.yml/,
 ];
 
 // Support unit testing of handleError
@@ -223,7 +224,7 @@ export const printErrorMessage = function (message: string) {
 		console.error(line);
 	});
 
-	console.error(`\n${getHelp}\n`);
+	console.error(`\n${getHelp()}\n`);
 };
 
 export const printExpectedErrorMessage = function (message: string) {

lib/events.ts

@@ -84,7 +84,7 @@ export async function trackCommand(commandSignature: string) {
 			try {
 				const balena = getBalenaSdk();
 				const $username = await balena.auth.whoami();
-				storage.set('cachedUsername', {
+				await storage.set('cachedUsername', {
 					token,
 					username: $username,
 				} as CachedUsername);
@@ -104,8 +104,11 @@ export async function trackCommand(commandSignature: string) {
 				});
 			});
 		}
-		// Don't actually call mixpanel.track() while running test cases
-		if (!process.env.BALENA_CLI_TEST_TYPE) {
+		// Don't actually call mixpanel.track() while running test cases, or if suppressed
+		if (
+			!process.env.BALENA_CLI_TEST_TYPE &&
+			!process.env.BALENARC_NO_ANALYTICS
+		) {
 			await mixpanel.track(`[CLI] ${commandSignature}`, {
 				distinct_id: username,
 				version: packageJSON.version,

lib/help.ts

@@ -1,3 +1,19 @@
+/**
+ * @license
+ * Copyright 2017-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 import Help from '@oclif/plugin-help';
 import * as indent from 'indent-string';
 import { getChalk } from './utils/lazy';
@@ -123,16 +139,10 @@ export default class BalenaHelp extends Help {
 		console.log('  --help, -h');
 		console.log('  --debug\n');
 
-		console.log(
-			`For help, visit our support forums: ${chalk.grey(
-				'https://forums.balena.io',
-			)}`,
-		);
-		console.log(
-			`For bug reports or feature requests, see: ${chalk.grey(
-				'https://github.com/balena-io/balena-cli/issues/',
-			)}\n`,
-		);
+		const {
+			reachingOut,
+		} = require('./utils/messages') as typeof import('./utils/messages');
+		console.log(reachingOut);
 	}
 
 	protected formatCommands(commands: any[]): string {

lib/utils/compose_ts.ts

@@ -543,16 +543,14 @@ async function loadBuildMetatada(
 }
 
 /**
- * Return a map of service name to service subdirectory, obtained from the given
- * composition object. If a composition object is not provided, an attempt will
- * be made to parse a 'docker-compose.yml' file at the given sourceDir.
- * Entries will be NOT be returned for subdirectories equal to '.' (e.g. the
- * 'main' "service" of a single-container application).
- *
+ * Return a map of service name to service subdirectory (relative to sourceDir),
+ * obtained from the given composition object. If a composition object is not
+ * provided, an attempt will be made to parse a 'docker-compose.yml' file at
+ * the given sourceDir.
  * @param sourceDir Project source directory (project root)
  * @param composition Optional previously parsed composition object
  */
-async function getServiceDirsFromComposition(
+export async function getServiceDirsFromComposition(
 	sourceDir: string,
 	composition?: Composition,
 ): Promise<Dictionary<string>> {
@@ -585,11 +583,8 @@ async function getServiceDirsFromComposition(
 			dir = dir.endsWith(path.sep) ? dir.slice(0, -1) : dir;
 			// remove './' prefix (or '.\\' on Windows)
 			dir = dir.startsWith(relPrefix) ? dir.slice(2) : dir;
-			// filter out a '.' service directory (e.g. for the 'main' service
-			// of a single-container application)
-			if (dir && dir !== '.') {
-				serviceDirs[serviceName] = dir;
-			}
+
+			serviceDirs[serviceName] = dir || '.';
 		}
 	}
 	return serviceDirs;
@@ -660,10 +655,6 @@ async function newTarDirectory(
 	const { filterFilesWithDockerignore } = await import('./ignore');
 	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
 
-	const serviceDirs = multiDockerignore
-		? await getServiceDirsFromComposition(dir, composition)
-		: {};
-
 	let readFile: (file: string) => Promise<Buffer>;
 	if (process.platform === 'win32') {
 		const { readFileWithEolConversion } = require('./eol-conversion');
@@ -673,10 +664,11 @@ async function newTarDirectory(
 	}
 	const tar = await import('tar-stream');
 	const pack = tar.pack();
+	const serviceDirs = await getServiceDirsFromComposition(dir, composition);
 	const {
 		filteredFileList,
 		dockerignoreFiles,
-	} = await filterFilesWithDockerignore(dir, serviceDirs);
+	} = await filterFilesWithDockerignore(dir, multiDockerignore, serviceDirs);
 	printDockerignoreWarn(dockerignoreFiles, serviceDirs, multiDockerignore);
 	for (const fileStats of filteredFileList) {
 		pack.entry(
@@ -703,7 +695,7 @@ async function newTarDirectory(
  * @param serviceDirsByService Map of service names to service subdirectories
  * @param multiDockerignore Whether --multi-dockerignore (-m) was provided
  */
-export function printDockerignoreWarn(
+function printDockerignoreWarn(
 	dockerignoreFiles: Array<import('./ignore').FileStats>,
 	serviceDirsByService: Dictionary<string>,
 	multiDockerignore: boolean,

lib/utils/device/live.ts

@@ -1,5 +1,23 @@
+/**
+ * @license
+ * Copyright 2019-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import * as chokidar from 'chokidar';
 import type * as Dockerode from 'dockerode';
+import * as fs from 'fs';
 import Livepush, { ContainerNotRunningError } from 'livepush';
 import * as _ from 'lodash';
 import * as path from 'path';
@@ -92,8 +110,22 @@ export class LivepushManager {
 		// Split the composition into a load of differents paths
 		// which we can
 		this.logger.logLivepush('Device state settled');
-		// create livepush instances for
 
+		// Prepare dockerignore data for file watcher
+		const { getDockerignoreByService } = await import('../ignore');
+		const { getServiceDirsFromComposition } = await import('../compose_ts');
+		const rootContext = path.resolve(this.buildContext);
+		const serviceDirsByService = await getServiceDirsFromComposition(
+			this.deployOpts.source,
+			this.composition,
+		);
+		const dockerignoreByService = await getDockerignoreByService(
+			this.deployOpts.source,
+			this.deployOpts.multiDockerignore,
+			serviceDirsByService,
+		);
+
+		// create livepush instances for each service
 		for (const serviceName of _.keys(this.composition.services)) {
 			const service = this.composition.services[serviceName];
 			const buildTask = _.find(this.buildTasks, { serviceName });
@@ -106,7 +138,6 @@ export class LivepushManager {
 
 			// We only care about builds
 			if (service.build != null) {
-				const context = path.join(this.buildContext, service.build.context);
 				if (buildTask.dockerfile == null) {
 					throw new Error(
 						`Could not detect dockerfile for service: ${serviceName}`,
@@ -137,6 +168,10 @@ export class LivepushManager {
 					return;
 				}
 
+				// path.resolve() converts to an absolute path, removes trailing slashes,
+				// and also converts forward slashes to backslashes on Windows.
+				const context = path.resolve(rootContext, service.build.context);
+
 				const livepush = await Livepush.init({
 					dockerfile,
 					context,
@@ -155,27 +190,22 @@ export class LivepushManager {
 
 				this.updateEventsWaiting[serviceName] = [];
 				this.deleteEventsWaiting[serviceName] = [];
-				const addEvent = (eventQueue: string[], changedPath: string) => {
+				const addEvent = ($serviceName: string, changedPath: string) => {
 					this.logger.logDebug(
-						`Got an add filesystem event for service: ${serviceName}. File: ${changedPath}`,
+						`Got an add filesystem event for service: ${$serviceName}. File: ${changedPath}`,
 					);
+					const eventQueue = this.updateEventsWaiting[$serviceName];
 					eventQueue.push(changedPath);
-					this.getDebouncedEventHandler(serviceName)();
+					this.getDebouncedEventHandler($serviceName)();
 				};
-				// TODO: Memoize this for containers which share a context
-				const monitor = chokidar.watch('.', {
-					cwd: context,
-					ignoreInitial: true,
-					ignored: '.git',
-				});
-				monitor.on('add', (changedPath: string) =>
-					addEvent(this.updateEventsWaiting[serviceName], changedPath),
-				);
-				monitor.on('change', (changedPath: string) =>
-					addEvent(this.updateEventsWaiting[serviceName], changedPath),
-				);
-				monitor.on('unlink', (changedPath: string) =>
-					addEvent(this.deleteEventsWaiting[serviceName], changedPath),
+
+				const monitor = this.setupFilesystemWatcher(
+					serviceName,
+					rootContext,
+					context,
+					addEvent,
+					dockerignoreByService,
+					this.deployOpts.multiDockerignore,
 				);
 
 				this.containers[serviceName] = {
@@ -209,6 +239,57 @@ export class LivepushManager {
 		});
 	}
 
+	protected setupFilesystemWatcher(
+		serviceName: string,
+		rootContext: string,
+		serviceContext: string,
+		changedPathHandler: (serviceName: string, changedPath: string) => void,
+		dockerignoreByService: {
+			[serviceName: string]: import('@balena/dockerignore').Ignore;
+		},
+		multiDockerignore: boolean,
+	): chokidar.FSWatcher {
+		const contextForDockerignore = multiDockerignore
+			? serviceContext
+			: rootContext;
+		const dockerignore = dockerignoreByService[serviceName];
+		// TODO: Memoize this for services that share a context
+		const monitor = chokidar.watch('.', {
+			cwd: serviceContext,
+			followSymlinks: true,
+			ignoreInitial: true,
+			ignored: (filePath: string, stats: fs.Stats | undefined) => {
+				if (!stats) {
+					try {
+						// sync because chokidar defines a sync interface
+						stats = fs.lstatSync(filePath);
+					} catch (err) {
+						// OK: the file may have been deleted. See also:
+						// https://github.com/paulmillr/chokidar/blob/3.4.3/lib/fsevents-handler.js#L326-L328
+						// https://github.com/paulmillr/chokidar/blob/3.4.3/lib/nodefs-handler.js#L364
+					}
+				}
+				if (stats && !stats.isFile() && !stats.isSymbolicLink()) {
+					// never ignore directories for compatibility with
+					// dockerignore exclusion patterns
+					return !stats.isDirectory();
+				}
+				const relPath = path.relative(contextForDockerignore, filePath);
+				return dockerignore.ignores(relPath);
+			},
+		});
+		monitor.on('add', (changedPath: string) =>
+			changedPathHandler(serviceName, changedPath),
+		);
+		monitor.on('change', (changedPath: string) =>
+			changedPathHandler(serviceName, changedPath),
+		);
+		monitor.on('unlink', (changedPath: string) =>
+			changedPathHandler(serviceName, changedPath),
+		);
+		return monitor;
+	}
+
 	public static preprocessDockerfile(content: string): string {
 		return new Dockerfile(content).generateLiveDockerfile();
 	}

lib/utils/ignore.ts

@@ -21,6 +21,7 @@ import * as MultiBuild from 'resin-multibuild';
 
 import dockerIgnore = require('@zeit/dockerignore');
 import ignore from 'ignore';
+import type { Ignore } from '@balena/dockerignore';
 
 import { ExpectedError } from '../errors';
 
@@ -196,37 +197,6 @@ export interface FileStats {
 	stats: Stats;
 }
 
-/**
- * Create a list of files (FileStats[]) for the filesystem subtree rooted at
- * projectDir, listing each file with both a full path and a relative path,
- * but excluding entries for directories themselves.
- * @param projectDir Source directory (root of subtree to be listed)
- * @param dir Used for recursive calls only (omit on first function call)
- */
-async function listFiles(
-	projectDir: string,
-	dir: string = projectDir,
-): Promise<FileStats[]> {
-	const files: FileStats[] = [];
-	const dirEntries = await fs.readdir(dir);
-	await Promise.all(
-		dirEntries.map(async (entry) => {
-			const filePath = path.join(dir, entry);
-			const stats = await fs.stat(filePath);
-			if (stats.isDirectory()) {
-				files.push(...(await listFiles(projectDir, filePath)));
-			} else if (stats.isFile()) {
-				files.push({
-					filePath,
-					relPath: path.relative(projectDir, filePath),
-					stats,
-				});
-			}
-		}),
-	);
-	return files;
-}
-
 /**
  * Return the contents of a .dockerignore file at projectDir, as a string.
  * Return an empty string if a .dockerignore file does not exist.
@@ -254,9 +224,9 @@ async function readDockerIgnoreFile(projectDir: string): Promise<string> {
  * a set of default/hardcoded patterns.
  * @param directory Directory where to look for a .dockerignore file
  */
-async function getDockerIgnoreInstance(
+export async function getDockerIgnoreInstance(
 	directory: string,
-): Promise<import('@balena/dockerignore').Ignore> {
+): Promise<Ignore> {
 	const dockerIgnoreStr = await readDockerIgnoreFile(directory);
 	const $dockerIgnore = (await import('@balena/dockerignore')).default;
 	const ig = $dockerIgnore({ ignorecase: false });
@@ -283,7 +253,8 @@ export interface ServiceDirs {
  * Create a list of files (FileStats[]) for the filesystem subtree rooted at
  * projectDir, filtered against the applicable .dockerignore files, including
  * a few default/hardcoded dockerignore patterns.
- * @param projectDir Source directory to
+ * @param projectDir Source directory
+ * @param multiDockerignore The --multi-dockerignore (-m) option
  * @param serviceDirsByService Map of service names to their subdirectories.
  * The service directory names/paths must be relative to the project root dir
  * and be "normalized" (path.normalize()) before the call to this function:
@@ -293,39 +264,106 @@ export interface ServiceDirs {
  */
 export async function filterFilesWithDockerignore(
 	projectDir: string,
-	serviceDirsByService?: ServiceDirs,
+	multiDockerignore: boolean,
+	serviceDirsByService: ServiceDirs,
 ): Promise<{ filteredFileList: FileStats[]; dockerignoreFiles: FileStats[] }> {
 	// path.resolve() also converts forward slashes to backslashes on Windows
 	projectDir = path.resolve(projectDir);
-	// ignoreByDir stores an instance of the dockerignore filter for each service dir
-	const ignoreByDir: {
-		[serviceDir: string]: import('@balena/dockerignore').Ignore;
-	} = {
-		'.': await getDockerIgnoreInstance(projectDir),
-	};
-	const serviceDirs: string[] = Object.values(serviceDirsByService || {})
-		// filter out the project source/root dir
-		.filter((dir) => dir && dir !== '.')
+	const root = '.' + path.sep;
+	const ignoreByService = await getDockerignoreByService(
+		projectDir,
+		multiDockerignore,
+		serviceDirsByService,
+	);
+	// Sample contents of ignoreByDir:
+	// { './': (dockerignore instance), 'foo/': (dockerignore instance) }
+	const ignoreByDir: { [serviceDir: string]: Ignore } = {};
+	for (let [serviceName, dir] of Object.entries(serviceDirsByService)) {
+		// convert slashes to backslashes on Windows, resolve '..' segments
+		dir = path.normalize(dir);
 		// add a trailing '/' (or '\' on Windows) to the path
-		.map((dir) => (dir.endsWith(path.sep) ? dir : dir + path.sep));
-
-	for (const serviceDir of serviceDirs) {
-		ignoreByDir[serviceDir] = await getDockerIgnoreInstance(
-			path.join(projectDir, serviceDir),
-		);
+		dir = dir.endsWith(path.sep) ? dir : dir + path.sep;
+		ignoreByDir[dir] = ignoreByService[serviceName];
 	}
-	const files = await listFiles(projectDir);
+	if (!ignoreByDir[root]) {
+		ignoreByDir[root] = await getDockerIgnoreInstance(projectDir);
+	}
+	const dockerignoreServiceDirs: string[] = multiDockerignore
+		? Object.keys(ignoreByDir).filter((dir) => dir && dir !== root)
+		: [];
 	const dockerignoreFiles: FileStats[] = [];
-	const filteredFileList = files.filter((file: FileStats) => {
-		if (path.basename(file.relPath) === '.dockerignore') {
-			dockerignoreFiles.push(file);
-		}
-		for (const dir of serviceDirs) {
-			if (file.relPath.startsWith(dir)) {
-				return !ignoreByDir[dir].ignores(file.relPath.substring(dir.length));
-			}
-		}
-		return !ignoreByDir['.'].ignores(file.relPath);
+	const filteredFileList: FileStats[] = [];
+	const klaw = await import('klaw');
+
+	await new Promise((resolve, reject) => {
+		// Looking at klaw's source code, `preserveSymlinks` appears to only
+		// afect the `stats` argument to the `data` event handler
+		klaw(projectDir, { preserveSymlinks: false })
+			.on('error', reject)
+			.on('end', resolve)
+			.on('data', (item: { path: string; stats: Stats }) => {
+				const { path: filePath, stats } = item;
+				// With `preserveSymlinks: false`, filePath cannot be a symlink.
+				// filePath may be a directory or a regular or special file
+				if (!stats.isFile()) {
+					return;
+				}
+				const relPath = path.relative(projectDir, filePath);
+				const fileInfo = {
+					filePath,
+					relPath,
+					stats,
+				};
+				if (path.basename(relPath) === '.dockerignore') {
+					dockerignoreFiles.push(fileInfo);
+				}
+				for (const dir of dockerignoreServiceDirs) {
+					if (relPath.startsWith(dir)) {
+						if (!ignoreByDir[dir].ignores(relPath.substring(dir.length))) {
+							filteredFileList.push(fileInfo);
+						}
+						return;
+					}
+				}
+				if (!ignoreByDir[root].ignores(relPath)) {
+					filteredFileList.push(fileInfo);
+				}
+			});
 	});
 	return { filteredFileList, dockerignoreFiles };
 }
+
+let dockerignoreByService: { [serviceName: string]: Ignore } | null = null;
+
+/**
+ * Get dockerignore instances for each service in serviceDirsByService.
+ * Dockerignore instances are cached and may be shared between services.
+ * @param projectDir Source directory
+ * @param multiDockerignore The --multi-dockerignore (-m) option
+ * @param serviceDirsByService Map of service names to their subdirectories
+ */
+export async function getDockerignoreByService(
+	projectDir: string,
+	multiDockerignore: boolean,
+	serviceDirsByService: ServiceDirs,
+): Promise<{ [serviceName: string]: Ignore }> {
+	if (dockerignoreByService) {
+		return dockerignoreByService;
+	}
+	const cachedDirs: { [dir: string]: Ignore } = {};
+	// path.resolve() converts to an absolute path, removes trailing slashes,
+	// and also converts forward slashes to backslashes on Windows.
+	projectDir = path.resolve(projectDir);
+	dockerignoreByService = {};
+
+	for (let [serviceName, dir] of Object.entries(serviceDirsByService)) {
+		dir = multiDockerignore ? dir : '.';
+		const absDir = path.resolve(projectDir, dir);
+		if (!cachedDirs[absDir]) {
+			cachedDirs[absDir] = await getDockerIgnoreInstance(absDir);
+		}
+		dockerignoreByService[serviceName] = cachedDirs[absDir];
+	}
+
+	return dockerignoreByService;
+}

lib/utils/messages.ts

@@ -15,26 +15,20 @@
  * limitations under the License.
  */
 
-const DEBUG_MODE = !!process.env.DEBUG;
-
 export const reachingOut = `\
-If you need help, or just want to say hi, don't hesitate in reaching out
-through our discussion and support forums at https://forums.balena.io
-
-For bug reports or feature requests, have a look at the GitHub issues or
-create a new one at: https://github.com/balena-io/balena-cli/issues/\
+For further help or support, visit:
+https://www.balena.io/docs/reference/balena-cli/#support-faq-and-troubleshooting
 `;
 
 const debugHint = `\
 Additional information may be available with the \`--debug\` flag.
-`;
+\n`;
 
-export const help = `\
-For help, visit our support forums: https://forums.balena.io
-For bug reports or feature requests, see: https://github.com/balena-io/balena-cli/issues/
-`;
+export const help = reachingOut;
 
-export const getHelp = (DEBUG_MODE ? '' : debugHint) + help;
+// Note that the value of process.env.DEBUG may change after the --debug flag
+// is parsed, so its evaluation cannot happen at module loading time.
+export const getHelp = () => (process.env.DEBUG ? '' : debugHint) + help;
 
 export const balenaAsciiArt = `\
  _            _

package.json

@@ -1,6 +1,6 @@
 {
   "name": "balena-cli",
-  "version": "12.27.1",
+  "version": "12.29.0",
   "description": "The official balena Command Line Interface",
   "main": "./build/app.js",
   "homepage": "https://github.com/balena-io/balena-cli",
@@ -61,13 +61,13 @@
     "test:shrinkwrap": "ts-node --transpile-only automation/run.ts test-shrinkwrap",
     "test:source": "cross-env BALENA_CLI_TEST_TYPE=source mocha",
     "test:standalone": "npm run build:standalone && npm run test:standalone:fast",
-    "test:standalone:fast": "cross-env BALENA_CLI_TEST_TYPE=standalone mocha",
+    "test:standalone:fast": "cross-env BALENA_CLI_TEST_TYPE=standalone mocha --config .mocharc-standalone.js",
     "test:fast": "npm run build:fast && npm run test:source",
     "test:only": "npm run build:fast && cross-env BALENA_CLI_TEST_TYPE=source mocha \"tests/**/${npm_config_test}.spec.ts\"",
     "catch-uncommitted": "ts-node --transpile-only automation/run.ts catch-uncommitted",
     "ci": "npm run test && npm run catch-uncommitted",
     "watch": "gulp watch",
-    "lint": "balena-lint -e ts -e js --typescript --fix automation/ lib/ typings/ tests/ bin/balena bin/balena-dev gulpfile.js",
+    "lint": "balena-lint -e ts -e js --typescript --fix automation/ lib/ typings/ tests/ bin/balena bin/balena-dev gulpfile.js .mocharc.js .mocharc-standalone.js",
     "update": "ts-node --transpile-only ./automation/update-module.ts",
     "prepare": "echo {} > bin/.fast-boot.json",
     "prepublishOnly": "npm run build"
@@ -91,13 +91,6 @@
       "pre-commit": "node automation/check-npm-version.js && node automation/check-doc.js"
     }
   },
-  "mocha": {
-    "reporter": "spec",
-    "require": "ts-node/register/transpile-only",
-    "file": "./tests/config-tests",
-    "timeout": 12000,
-    "_": "tests/**/*.spec.ts"
-  },
   "oclif": {
     "bin": "balena",
     "commands": "./build/commands",
@@ -142,7 +135,7 @@
     "@types/klaw": "^3.0.1",
     "@types/lodash": "^4.14.159",
     "@types/mixpanel": "^2.14.2",
-    "@types/mocha": "^5.2.7",
+    "@types/mocha": "^8.0.4",
     "@types/mock-require": "^2.0.0",
     "@types/moment-duration-format": "^2.2.2",
     "@types/net-keepalive": "^0.4.1",
@@ -156,7 +149,7 @@
     "@types/rewire": "^2.5.28",
     "@types/rimraf": "^3.0.0",
     "@types/shell-escape": "^0.2.0",
-    "@types/sinon": "^9.0.4",
+    "@types/sinon": "^9.0.8",
     "@types/split": "^1.0.0",
     "@types/stream-to-promise": "2.2.0",
     "@types/tar-stream": "^2.1.0",
@@ -180,15 +173,15 @@
     "intercept-stdout": "^0.1.2",
     "jsonwebtoken": "^8.5.1",
     "mkdirp": "^0.5.5",
-    "mocha": "^6.2.3",
+    "mocha": "^8.2.1",
     "mock-require": "^3.0.3",
     "nock": "^12.0.3",
     "parse-link-header": "~1.0.1",
     "pkg": "^4.4.9",
     "publish-release": "^1.6.1",
-    "rewire": "^4.0.1",
+    "rewire": "^5.0.0",
     "simple-git": "^1.132.0",
-    "sinon": "^9.0.3",
+    "sinon": "^9.2.1",
     "ts-node": "^8.10.2",
     "typescript": "^4.0.2"
   },
@@ -216,7 +209,7 @@
     "bluebird": "^3.7.2",
     "body-parser": "^1.19.0",
     "chalk": "^3.0.0",
-    "chokidar": "^3.3.1",
+    "chokidar": "^3.4.3",
     "cli-truncate": "^2.1.0",
     "color-hash": "^1.0.3",
     "columnify": "^1.5.2",

tests/commands/help.spec.ts

@@ -19,6 +19,7 @@ import { expect } from 'chai';
 
 import { BalenaAPIMock } from '../balena-api-mock';
 import { cleanOutput, runCommand } from '../helpers';
+import * as messages from '../../build/utils/messages';
 
 const SIMPLE_HELP = `
 USAGE
@@ -103,10 +104,7 @@ GLOBAL OPTIONS
 
 `;
 
-const ONLINE_RESOURCES = `
-For help, visit our support forums: https://forums.balena.io
-For bug reports or feature requests, see: https://github.com/balena-io/balena-cli/issues/
-`;
+const ONLINE_RESOURCES = messages.reachingOut;
 
 describe.skip('balena help', function () {
 	let api: BalenaAPIMock;

tests/commands/ssh.spec.ts

@@ -120,6 +120,13 @@ describe('balena ssh', function () {
 async function checkSsh(): Promise<boolean> {
 	const { which } = await import('../../build/utils/helpers');
 	const sshPath = await which('ssh', false);
+	if ((sshPath || '').includes('\\Windows\\System32\\OpenSSH\\ssh')) {
+		// don't use Windows' built-in ssh tool for these test cases
+		// because it messes up with the terminal window such that
+		// "line breaks stop working" (and not even '\033c' fixes it)
+		// and all mocha output gets printed on a single very long line...
+		return false;
+	}
 	return !!sshPath;
 }
 
@@ -127,11 +134,13 @@ async function checkSsh(): Promise<boolean> {
 async function startMockSshServer(): Promise<[Server, number]> {
 	const server = createServer((c) => {
 		// 'connection' listener
-		c.on('end', () => {
+		const handler = (msg: string) => {
 			if (process.env.DEBUG) {
-				console.error('[debug] mock ssh server: client disconnected');
+				console.error(`[debug] mock ssh server: ${msg}`);
 			}
-		});
+		};
+		c.on('error', (err) => handler(err.message));
+		c.on('end', () => handler('client disconnected'));
 		c.end();
 	});
 	server.on('error', (err) => {

tests/docker-build.ts

@@ -20,6 +20,7 @@ import * as _ from 'lodash';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 import { PathUtils } from 'resin-multibuild';
+import rewire = require('rewire');
 import * as sinon from 'sinon';
 import { Readable } from 'stream';
 import * as tar from 'tar-stream';
@@ -199,6 +200,8 @@ export async function testDockerBuildStream(o: {
 		}
 	}
 
+	resetDockerignoreCache();
+
 	const { exitCode, out, err } = await runCommand(o.commandLine);
 
 	if (expectedErrorLines.length) {
@@ -249,8 +252,20 @@ export async function testPushBuildStream(o: {
 			inspectTarStream(buildRequestBody, o.expectedFiles, o.projectPath),
 	});
 
+	resetDockerignoreCache();
+
 	const { out, err } = await runCommand(o.commandLine);
 
 	expect(err).to.be.empty;
 	expect(cleanOutput(out, true)).to.include.members(expectedResponseLines);
 }
+
+export function resetDockerignoreCache() {
+	if (process.env.BALENA_CLI_TEST_TYPE !== 'source') {
+		return;
+	}
+	const ignorePath = '../build/utils/ignore';
+	delete require.cache[require.resolve(ignorePath)];
+	const ignoreMod = rewire(ignorePath);
+	ignoreMod.__set__('dockerignoreByService', null);
+}

tests/errors.spec.ts

@@ -187,7 +187,7 @@ describe('printErrorMessage() function', () => {
 		expect(consoleError.getCall(0).args[0]).to.equal(expectedOutputMessages[0]);
 		expect(consoleError.getCall(1).args[0]).to.equal(expectedOutputMessages[1]);
 		expect(consoleError.getCall(2).args[0]).to.equal(expectedOutputMessages[2]);
-		expect(consoleError.getCall(3).args[0]).to.equal(`\n${getHelp}\n`);
+		expect(consoleError.getCall(3).args[0]).to.equal(`\n${getHelp()}\n`);
 
 		consoleError.restore();
 	});

tests/utils/device/live.spec.ts

@@ -0,0 +1,374 @@
+/**
+ * @license
+ * Copyright 2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect } from 'chai';
+import * as chokidar from 'chokidar';
+import { promises as fs } from 'fs';
+import * as _ from 'lodash';
+import * as path from 'path';
+import { promisify } from 'util';
+
+import { LivepushManager } from '../../../lib/utils/device/live';
+import { resetDockerignoreCache } from '../../docker-build';
+import { setupDockerignoreTestData } from '../../projects';
+
+const delay = promisify(setTimeout);
+const FS_WATCH_DURATION_MS = 500;
+
+const repoPath = path.normalize(path.join(__dirname, '..', '..', '..'));
+const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects');
+
+interface ByService<T> {
+	[serviceName: string]: T;
+}
+
+class MockLivepushManager extends LivepushManager {
+	public constructor() {
+		super({
+			buildContext: '',
+			composition: { version: '2.1', services: {} },
+			buildTasks: [],
+			docker: {} as import('dockerode'),
+			api: {} as import('../../../lib/utils/device/api').DeviceAPI,
+			logger: {} as import('../../../lib/utils/logger'),
+			buildLogs: {},
+			deployOpts: {} as import('../../../lib/utils/device/deploy').DeviceDeployOptions,
+		});
+	}
+
+	public testSetupFilesystemWatcher(
+		serviceName: string,
+		rootContext: string,
+		serviceContext: string,
+		changedPathHandler: (serviceName: string, changedPath: string) => void,
+		dockerignoreByService: ByService<import('@balena/dockerignore').Ignore>,
+		multiDockerignore: boolean,
+	): import('chokidar').FSWatcher {
+		return super.setupFilesystemWatcher(
+			serviceName,
+			rootContext,
+			serviceContext,
+			changedPathHandler,
+			dockerignoreByService,
+			multiDockerignore,
+		);
+	}
+}
+
+// "describeSS" stands for "describe Skip Standalone"
+const describeSS =
+	process.env.BALENA_CLI_TEST_TYPE === 'standalone' ? describe.skip : describe;
+
+describeSS('LivepushManager::setupFilesystemWatcher', function () {
+	const manager = new MockLivepushManager();
+
+	async function createMonitors(
+		projectPath: string,
+		composition: import('resin-compose-parse').Composition,
+		multiDockerignore: boolean,
+		changedPathHandler: (serviceName: string, changedPath: string) => void,
+	): Promise<ByService<chokidar.FSWatcher>> {
+		const { getServiceDirsFromComposition } = await import(
+			'../../../build/utils/compose_ts'
+		);
+		const { getDockerignoreByService } = await import(
+			'../../../build/utils/ignore'
+		);
+		const rootContext = path.resolve(projectPath);
+
+		const monitors: ByService<chokidar.FSWatcher> = {};
+
+		const serviceDirsByService = await getServiceDirsFromComposition(
+			projectPath,
+			composition,
+		);
+		const dockerignoreByService = await getDockerignoreByService(
+			projectPath,
+			multiDockerignore,
+			serviceDirsByService,
+		);
+
+		for (const serviceName of Object.keys(composition.services)) {
+			const service = composition.services[serviceName];
+			const serviceContext = path.resolve(rootContext, service.build!.context);
+
+			const monitor = manager.testSetupFilesystemWatcher(
+				serviceName,
+				rootContext,
+				serviceContext,
+				changedPathHandler,
+				dockerignoreByService,
+				multiDockerignore,
+			);
+			monitors[serviceName] = monitor;
+
+			await new Promise((resolve, reject) => {
+				monitor.on('error', reject);
+				monitor.on('ready', resolve);
+			});
+		}
+		return monitors;
+	}
+
+	this.beforeAll(async () => {
+		await setupDockerignoreTestData();
+	});
+
+	this.afterAll(async () => {
+		await setupDockerignoreTestData({ cleanup: true });
+	});
+
+	this.beforeEach(() => {
+		resetDockerignoreCache();
+	});
+
+	describe('for project no-docker-compose/basic', function () {
+		const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic');
+		const composition = {
+			version: '2.1',
+			services: {
+				main: { build: { context: '.' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored', async () => {
+			const changedPaths: ByService<string[]> = { main: [] };
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'Dockerfile')),
+				touch(path.join(projectPath, 'src', 'start.sh')),
+				touch(path.join(projectPath, 'src', 'windows-crlf.sh')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['main']).to.have.members([
+				'Dockerfile',
+				path.join('src', 'start.sh'),
+				path.join('src', 'windows-crlf.sh'),
+			]);
+		});
+	});
+
+	describe('for project no-docker-compose/dockerignore1', function () {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore1',
+		);
+		const composition = {
+			version: '2.1',
+			services: {
+				main: { build: { context: '.' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored', async () => {
+			const changedPaths: ByService<string[]> = { main: [] };
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'a.txt')),
+				touch(path.join(projectPath, 'b.txt')),
+				touch(path.join(projectPath, 'vendor', '.git', 'vendor-git-contents')),
+				touch(path.join(projectPath, 'src', 'src-a.txt')),
+				touch(path.join(projectPath, 'src', 'src-b.txt')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['main']).to.have.members([
+				'a.txt',
+				path.join('src', 'src-a.txt'),
+				path.join('vendor', '.git', 'vendor-git-contents'),
+			]);
+		});
+	});
+
+	describe('for project no-docker-compose/dockerignore2', function () {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore2',
+		);
+		const composition = {
+			version: '2.1',
+			services: {
+				main: { build: { context: '.' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored', async () => {
+			const changedPaths: ByService<string[]> = { main: [] };
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'a.txt')),
+				touch(path.join(projectPath, 'b.txt')),
+				touch(path.join(projectPath, 'lib', 'src-a.txt')),
+				touch(path.join(projectPath, 'lib', 'src-b.txt')),
+				touch(path.join(projectPath, 'src', 'src-a.txt')),
+				touch(path.join(projectPath, 'src', 'src-b.txt')),
+				touch(path.join(projectPath, 'symlink-a.txt')),
+				touch(path.join(projectPath, 'symlink-b.txt')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			// chokidar appears to treat symbolic links differently on different
+			// platforms like Linux and macOS. On Linux only, change events are
+			// reported for symlinks when the target file they point to is changed.
+			// We tolerate this difference in this test case.
+			const expectedNoSymlink = [
+				'b.txt',
+				path.join('lib', 'src-b.txt'),
+				path.join('src', 'src-b.txt'),
+			];
+			const expectedWithSymlink = [...expectedNoSymlink, 'symlink-a.txt'];
+			expect(changedPaths['main']).to.include.members(expectedNoSymlink);
+			expect(expectedWithSymlink).to.include.members(changedPaths['main']);
+		});
+	});
+
+	describe('for project docker-compose/basic', function () {
+		const projectPath = path.join(projectsPath, 'docker-compose', 'basic');
+		const composition = {
+			version: '2.1',
+			services: {
+				service1: { build: { context: 'service1' } },
+				service2: { build: { context: 'service2' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored (docker-compose)', async () => {
+			const changedPaths: ByService<string[]> = {
+				service1: [],
+				service2: [],
+			};
+			const multiDockerignore = false;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'service1', 'test-ignore.txt')),
+				touch(path.join(projectPath, 'service1', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'src', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'file2-crlf.sh')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['service1']).to.have.members(['file1.sh']);
+			expect(changedPaths['service2']).to.have.members([
+				path.join('src', 'file1.sh'),
+				'file2-crlf.sh',
+			]);
+		});
+
+		it('should trigger change events for paths that are not ignored (docker-compose, multi-dockerignore)', async () => {
+			const changedPaths: ByService<string[]> = {
+				service1: [],
+				service2: [],
+			};
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'service1', 'test-ignore.txt')),
+				touch(path.join(projectPath, 'service1', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'src', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'file2-crlf.sh')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['service1']).to.have.members([
+				'file1.sh',
+				'test-ignore.txt',
+			]);
+			expect(changedPaths['service2']).to.have.members(['file2-crlf.sh']);
+		});
+	});
+});
+
+async function touch(filePath: string) {
+	const time = new Date();
+	return fs.utimes(filePath, time, time);
+}

typings/balena-sync/index.d.ts

@@ -23,6 +23,7 @@ declare module 'balena-sync' {
 	export interface LocalBalenaOsDevice {
 		address: string;
 		host: string;
+		osVariant: string;
 		port: number;
 	}