Fix Windows installer build in new balena CI workers (qqjs patch)

Change-type: patch

.github/ISSUE_TEMPLATE.md

@@ -1,7 +1,7 @@
 
 # About this issue tracker
 
-*balenaCLI (Command Line Interface) is a tool used to interact with the balena platform.
+*The balena CLI (Command Line Interface) is a tool used to interact with the balena platform.
 This GitHub issue tracker is used for bug reports and feature requests regarding the CLI
 tool. General and troubleshooting questions (such as setting up your project to work with a
 balenalib base image) are encouraged to be posted to the [balena
@@ -12,7 +12,7 @@ community can both contribute and benefit from the answers.*
 the same problem or feature please add comments to the existing issue.*
 
 *Thank you for your time and effort creating the issue report, and helping us improve
-balenaCLI!*
+the balena CLI!*
 
 ---
 
@@ -63,7 +63,7 @@ fixed it.
 
 # Specifications
 
-- **balenaCLI version:** e.g. 1.2.3 (output of the `"balena version -a"` command)
+- **balena CLI version:** e.g. 1.2.3 (output of the `"balena version -a"` command)
 - **Cloud backend: openBalena or balenaCloud?** If unsure, it will be balenaCloud
 - **Operating system version:** e.g. Windows 10, Ubuntu 18.04, macOS 10.14.5
 - **32/64 bit OS and processor:** e.g. 32-bit Windows on 64-bit Intel processor

.mocharc-standalone.js

@@ -0,0 +1,6 @@
+const commonConfig = require('./.mocharc.js');
+
+module.exports = {
+	...commonConfig,
+	spec: ['tests/auth/*.spec.ts', 'tests/commands/**/*.spec.ts'],
+};

.mocharc.js

@@ -0,0 +1,8 @@
+module.exports = {
+	spec: 'tests/commands/app/create.spec.ts',
+	reporter: 'spec',
+	require: 'ts-node/register/transpile-only',
+	file: './tests/config-tests',
+	timeout: 12000,
+	spec: 'tests/**/*.spec.ts',
+};

.versionbot/CHANGELOG.yml

@@ -1,3 +1,446 @@
+- commits:
+    - subject: 'scan: Print production devices'' info on scan'
+      hash: 7d5a64f59a47c3a051fb2cbe9e45a71029cca694
+      body: ''
+      footer:
+        Resolves: '#1713'
+        resolves: '#1713'
+        Change-type: minor
+        change-type: minor
+        Signed-off-by: Marios Balamatsias <mbalamatsias@gmail.com>
+        signed-off-by: Marios Balamatsias <mbalamatsias@gmail.com>
+      author: Marios Balamatsias
+      nested: []
+  version: 12.29.0
+  date: 2020-12-01T13:33:03.012Z
+- commits:
+    - subject: Add ability to disable analytics for performance testing
+      hash: 950d173d276f97cb3fcfd4bb9b578b5888572a69
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Connects-to: '#1708'
+        connects-to: '#1708'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.28.3
+  date: 2020-11-26T12:55:50.969Z
+- commits:
+    - subject: 'docs: Add references to the masterclasses in the CLI help and README'
+      hash: 2bf59530c4aebdd302814e9f59d41c7b9d2672c3
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: >-
+        Fix debug message logic (don't suggest --debug if it is already being
+        used)
+      hash: a4fd7d6118a04e8e9f0e718a765b508fb11209e6
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Fix unhandled promise rejection when ~/.balena is not accessible
+      hash: 65f053dd6e2d6e212b90e905be7af5d13772c7e6
+      body: ''
+      footer:
+        Resolves: '#2096'
+        resolves: '#2096'
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.28.2
+  date: 2020-11-20T12:07:27.250Z
+- commits:
+    - subject: 'scan: Prevent spinner animation output to stdout when --json is used'
+      hash: 2f9212d622f7affe4391e0d6bac1a06e859b7488
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.28.1
+  date: 2020-11-20T00:26:37.290Z
+- commits:
+    - subject: 'push: Reduce memory usage when filtering files with dockerignore'
+      hash: 4768f763856aa9f761988477f97ec872d226b004
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'Livepush: Refactor dockerignore filtering and add test cases'
+      hash: d6b324927481ce03217c15509db2e046d74cb208
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'Livepush: Ignore paths set in .dockerignore files'
+      hash: 02a54667469982ba676da5e0b8a0f0f379b320e5
+      body: ''
+      footer:
+        Change-type: minor
+        change-type: minor
+        Signed-off-by: Josh Bowling <josh@balena.io>
+        signed-off-by: Josh Bowling <josh@balena.io>
+      author: Josh Bowling
+      nested: []
+  version: 12.28.0
+  date: 2020-11-19T17:07:59.865Z
+- commits:
+    - subject: 'Test code optimization: avoid running ~70 test cases twice'
+      hash: 6a556131995eebc318f5a02831a1cc1e2fb03b36
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'docs: Add note about macOS Big Sur notarization workaround'
+      hash: 893a39e8918756db8dd4cdd5135f430a405a409e
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.27.4
+  date: 2020-11-15T23:42:22.338Z
+- commits:
+    - subject: Avoid reporting balenarc parsing errors
+      hash: 1b0c14feab4e3c12d459d26539f65895519f89cf
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Connects-to: '#1100'
+        connects-to: '#1100'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.27.3
+  date: 2020-11-11T16:54:11.888Z
+- commits:
+    - subject: Modify `os download` help to mention dev images
+      hash: 7da9a800ccd2af0fbb051f1d95707e5d8623e227
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Signed-off-by: Thomas Manning <thomasm@balena.io>
+        signed-off-by: Thomas Manning <thomasm@balena.io>
+      author: Thomas Manning
+      nested: []
+  version: 12.27.2
+  date: 2020-11-09T12:08:41.495Z
+- commits:
+    - subject: Improve application-identifier disambiguation
+      hash: 46249e319ba3ee8aa1b951f5241dafa625175045
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Resolves: '#2077'
+        resolves: '#2077'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.27.1
+  date: 2020-11-06T09:01:07.764Z
+- commits:
+    - subject: Add command app purge
+      hash: 1e18096873bf35c016a5812f91c0bf4e8ce743ba
+      body: ''
+      footer:
+        Change-type: minor
+        change-type: minor
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.27.0
+  date: 2020-11-05T16:11:52.151Z
+- commits:
+    - subject: >-
+        config generate + openBalena v3: Fix "Cannot read property '__id' of
+        undefined"
+      hash: 6b4c28a0268c61413633af7351cc3f35b346d123
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.26.2
+  date: 2020-11-05T13:29:37.895Z
+- commits:
+    - subject: 'devices: Fix "TypeError: Cannot read property ''slug'' of undefined"'
+      hash: 9d177609f5adfa357363ba5356f51027a388f635
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.26.1
+  date: 2020-10-31T00:34:10.685Z
+- commits:
+    - subject: Add command device purge
+      hash: 47fcffe36813dfdbe59986ed6ae0e4a1a441ea63
+      body: ''
+      footer:
+        Change-type: minor
+        change-type: minor
+        Resolves: '#1547'
+        resolves: '#1547'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.26.0
+  date: 2020-10-29T10:06:17.048Z
+- commits:
+    - subject: 'ssh: Fix "Found more than one container with a service name <name>"'
+      hash: 7f000ee8c338c88af4a41dee1a2fb924c2c2ee00
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.25.6
+  date: 2020-10-28T01:11:21.452Z
+- commits:
+    - subject: Remove need for hardcoded list of command ids
+      hash: c48564e85a618af64c5f63722f0039f6e75862ba
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.25.5
+  date: 2020-10-27T09:39:31.007Z
+- commits:
+    - subject: Update Contributing document re commit messages / versionbot / changelog
+      hash: 077e25ebc4e1b3f1cf5aefefcc8601f9dbe38d1f
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'config generate: Fix "Application is ambiguous" when app slug is used'
+      hash: 709f009f9b0014acc087f154bca2f5f3ac7dec71
+      body: ''
+      footer:
+        Connects-to: '#1893'
+        connects-to: '#1893'
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'config generate: Fix device type compatibility check'
+      hash: 116ab1fbc16be7ebc0e66192779a4f44d248d502
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.25.4
+  date: 2020-10-25T17:29:17.581Z
+- commits:
+    - subject: 'build/deploy: Add more test cases (--buildArg option)'
+      hash: 6b208ec2abde887ffd11cfdfb624382ea7bfc049
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Fix typing (don't assume that 'docker-toolbelt' uses Bluebird promises)
+      hash: 099d755900ff9e3d994b517225da872025c3f445
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'build/deploy: Fix --buildArg option with docker-compose.yml projects'
+      hash: 3199f15662373ca53fe1c7541259b31799e42315
+      body: ''
+      footer:
+        Resolves: '#1053'
+        resolves: '#1053'
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: 'build/deploy: Fix image size notice at end of build'
+      hash: 4c8dc29946067aeaf46789058ecb310d0d862750
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Convert more code to Typescript (compose.js)
+      hash: 2b22fb89f1a2b25c532a5ec278c800e83cdcfeac
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.25.3
+  date: 2020-10-21T13:37:13.594Z
+- commits:
+    - subject: Revert styling of "balena CLI" as "balenaCLI"
+      hash: 478b5dd363288e2556e26b86d91d08e923788bae
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: >-
+        Add help note regarding the version of Node.js printed by `balena
+        version -a`
+      hash: 0708608c7eb710c8cd7749845384e897ba39c741
+      body: ''
+      footer:
+        Connects-to: '#2068'
+        connects-to: '#2068'
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: >-
+        preload: Fix parsing of `--add-certificate` option, amend help for
+        `--app`
+      hash: c245dc70c244f82ee20f8c50110d302a0e86824d
+      body: ''
+      footer:
+        Connects-to: '#2063'
+        connects-to: '#2063'
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.25.2
+  date: 2020-10-21T11:24:21.642Z
+- commits:
+    - subject: Treat authorization errors as expected
+      hash: 592efd0a2e5c0c46f4cbf0b1107b135ccab55211
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Resolves: '#2035'
+        resolves: '#2035'
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.25.1
+  date: 2020-10-13T08:21:31.203Z
+- commits:
+    - subject: Refactor initialization code (delete app-oclif.ts and app-common.ts)
+      hash: d0e4fa0e59b9c6a4c3f414361c089c68f1dfd872
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Support BALENARC_NO_SENTRY env var to disable Sentry.io error reporting
+      hash: cf376316bc4863b98223bac9c81697c2245341ae
+      body: ''
+      footer:
+        Change-type: minor
+        change-type: minor
+      author: Paulo Castro
+      nested: []
+    - subject: Update Sentry package (may fix "Maximum call stack size exceeded")
+      hash: 8f0f3bda294acda75be15d630967b526581a3c1f
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+  version: 12.25.0
+  date: 2020-10-11T00:06:46.011Z
+- commits:
+    - subject: 'login: sign up at the configured balena instance'
+      hash: af70f16a9b8de7150bcdd8d76a89f72c59614526
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Signed-off-by: Matthew McGinn <matthew@balena.io>
+        signed-off-by: Matthew McGinn <matthew@balena.io>
+      author: Matthew McGinn
+      nested: []
+  version: 12.24.1
+  date: 2020-10-07T13:28:42.250Z
+- commits:
+    - subject: 'scan: Add ''--json'' option to help with scripting'
+      hash: 8a506bc4c01e2082073b7858ab79874e707cf59d
+      body: ''
+      footer:
+        Change-type: minor
+        change-type: minor
+      author: Paulo Castro
+      nested: []
+  version: 12.24.0
+  date: 2020-10-06T17:09:31.036Z
+- commits:
+    - subject: Update CONTRIBUTING.md re balena-dev workflow
+      hash: 89782218666af4db1297b8672560913d3de8fd8c
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Add bin/balena* scripts to linter paths
+      hash: 2974c203b5f4a1339e0feea8bf127cecbe9f13c3
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Paulo Castro
+      nested: []
+    - subject: Workaround balena-dev/oclif compatibility issues
+      hash: c85acbd90bbb5de3dae13638555365b7a1cb472c
+      body: ''
+      footer:
+        Change-type: patch
+        change-type: patch
+        Signed-off-by: Scott Lowe <scott@balena.io>
+        signed-off-by: Scott Lowe <scott@balena.io>
+      author: Scott Lowe
+      nested: []
+  version: 12.23.4
+  date: 2020-10-05T21:13:25.405Z
 - commits:
     - subject: Rename actions-oclif/ to commands/
       hash: eddbdfe0dcea8df801d11d398e72f34c8354f7e1

CHANGELOG.md

@@ -4,6 +4,115 @@ All notable changes to this project will be documented in this file
 automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 12.29.0 - 2020-12-01
+
+* scan: Print production devices' info on scan [Marios Balamatsias]
+
+## 12.28.3 - 2020-11-26
+
+* Add ability to disable analytics for performance testing [Scott Lowe]
+
+## 12.28.2 - 2020-11-20
+
+* docs: Add references to the masterclasses in the CLI help and README [Paulo Castro]
+* Fix debug message logic (don't suggest --debug if it is already being used) [Paulo Castro]
+* Fix unhandled promise rejection when ~/.balena is not accessible [Paulo Castro]
+
+## 12.28.1 - 2020-11-20
+
+* scan: Prevent spinner animation output to stdout when --json is used [Paulo Castro]
+
+## 12.28.0 - 2020-11-19
+
+* push: Reduce memory usage when filtering files with dockerignore [Paulo Castro]
+* Livepush: Refactor dockerignore filtering and add test cases [Paulo Castro]
+* Livepush: Ignore paths set in .dockerignore files [Josh Bowling]
+
+## 12.27.4 - 2020-11-15
+
+* Test code optimization: avoid running ~70 test cases twice [Paulo Castro]
+* docs: Add note about macOS Big Sur notarization workaround [Paulo Castro]
+
+## 12.27.3 - 2020-11-11
+
+* Avoid reporting balenarc parsing errors [Scott Lowe]
+
+## 12.27.2 - 2020-11-09
+
+* Modify `os download` help to mention dev images [Thomas Manning]
+
+## 12.27.1 - 2020-11-06
+
+* Improve application-identifier disambiguation [Scott Lowe]
+
+## 12.27.0 - 2020-11-05
+
+* Add command app purge [Scott Lowe]
+
+## 12.26.2 - 2020-11-05
+
+* config generate + openBalena v3: Fix "Cannot read property '__id' of undefined" [Paulo Castro]
+
+## 12.26.1 - 2020-10-31
+
+* devices: Fix "TypeError: Cannot read property 'slug' of undefined" [Paulo Castro]
+
+## 12.26.0 - 2020-10-29
+
+* Add command device purge [Scott Lowe]
+
+## 12.25.6 - 2020-10-28
+
+* ssh: Fix "Found more than one container with a service name <name>" [Paulo Castro]
+
+## 12.25.5 - 2020-10-27
+
+* Remove need for hardcoded list of command ids [Scott Lowe]
+
+## 12.25.4 - 2020-10-25
+
+* Update Contributing document re commit messages / versionbot / changelog [Paulo Castro]
+* config generate: Fix "Application is ambiguous" when app slug is used [Paulo Castro]
+* config generate: Fix device type compatibility check [Paulo Castro]
+
+## 12.25.3 - 2020-10-21
+
+* build/deploy: Add more test cases (--buildArg option) [Paulo Castro]
+* Fix typing (don't assume that 'docker-toolbelt' uses Bluebird promises) [Paulo Castro]
+* build/deploy: Fix --buildArg option with docker-compose.yml projects [Paulo Castro]
+* build/deploy: Fix image size notice at end of build [Paulo Castro]
+* Convert more code to Typescript (compose.js) [Paulo Castro]
+
+## 12.25.2 - 2020-10-21
+
+* Revert styling of "balena CLI" as "balenaCLI" [Paulo Castro]
+* Add help note regarding the version of Node.js printed by `balena version -a` [Paulo Castro]
+* preload: Fix parsing of `--add-certificate` option, amend help for `--app` [Paulo Castro]
+
+## 12.25.1 - 2020-10-13
+
+* Treat authorization errors as expected [Scott Lowe]
+
+## 12.25.0 - 2020-10-11
+
+* Refactor initialization code (delete app-oclif.ts and app-common.ts) [Paulo Castro]
+* Support BALENARC_NO_SENTRY env var to disable Sentry.io error reporting [Paulo Castro]
+* Update Sentry package (may fix "Maximum call stack size exceeded") [Paulo Castro]
+
+## 12.24.1 - 2020-10-07
+
+* login: sign up at the configured balena instance [Matthew McGinn]
+
+## 12.24.0 - 2020-10-06
+
+* scan: Add '--json' option to help with scripting [Paulo Castro]
+
+## 12.23.4 - 2020-10-05
+
+* Update CONTRIBUTING.md re balena-dev workflow [Paulo Castro]
+* Add bin/balena* scripts to linter paths [Paulo Castro]
+* Workaround balena-dev/oclif compatibility issues [Scott Lowe]
+
 ## 12.23.3 - 2020-10-02
 
 * Rename actions-oclif/ to commands/ [Scott Lowe]

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing
 
-balenaCLI is an open source project and your contribution is welcome!
+The balena CLI is an open source project and your contribution is welcome!
 
 * Install the dependencies listed in the [NPM Installation
   section](./INSTALL-ADVANCED.md#npm-installation) section of the installation instructions. Check
@@ -21,52 +21,75 @@ Before opening a PR, test your changes with `npm test`. Keep compatibility in mi
 meant to run on Linux, macOS and Windows. balena CI will run test code on all three platforms, but
 this will only help if you add some test cases for your new code!
 
-## ./bin/balena-dev and oclif
-
-When using `./bin/balena-dev`, it is currently necessary to manually edit the `oclif` section of
-`package.json` to replace `./build` with `./lib` as follows:
-
-Change from:
-```
-  "oclif": {
-    "commands": "./build/commands",
-    "hooks": {
-      "prerun": "./build/hooks/prerun/track"
-```
-
-To:
-```
-  "oclif": {
-    "commands": "./lib/commands",
-    "hooks": {
-      "prerun": "./lib/hooks/prerun/track"
-```
-
-And then remember to change it back before pushing the pull request. This is obviously error prone
-and inconvenient, and improvement suggestions are welcome: is there a better solution than
-automatically editing `package.json`? It is doable, if it is what needs to be done.
-
 ## Semantic versioning, commit messages and the ChangeLog
 
-The CLI version numbering adheres to [Semantic Versioning](http://semver.org/). The following
-header/row is required in the body of a commit message, and will cause the CI build to fail if absent:
+When a pull request is merged, Balena's versionbot / Continuous Integration system takes care of
+automatically creating a new CLI release on both the [npm
+registry](https://www.npmjs.com/package/balena-cli) and the GitHub [releases
+page](https://github.com/balena-io/balena-cli/releases). The release version numbering adheres to
+the [Semantic Versioning's](http://semver.org/) concept of patch, minor and major releases.
+Generally, bug fixes and documentation changes are classed as patch changes, while new features are
+classed as minor changes. If a change breaks backwards compatibility, it is a major change.
+
+A new version entry is also automatically added to the
+[CHANGELOG.md](https://github.com/balena-io/balena-cli/blob/master/CHANGELOG.md) file when a pull
+request is merged. Each pull request corresponds to a single version / release. Each commit in the
+pull request becomes a bullet point entry in the Changelog. The Changelog file should not be
+manually edited.
+
+To support this automation, a commit message should be structured as follows:
+
+```text
+The first line becomes a bullet point in the CHANGELOG file
+
+Optionally, a more detailed description in one or more paragraphs.
+The detailed description can be seen with `git log`, but it is not copied
+to the CHANGELOG file.
 
-```
 Change-type: patch|minor|major
 ```
 
-Version numbers and commit messages are automatically added to the `CHANGELOG.md` file by the CI
-build flow, after a pull request is merged. It should not be manually edited.
+Only the first line of the commit message is copied to the Changelog file. The `Change-type` footer
+must be preceded by a blank line, and indicates the commit's semver change type. When a PR consists
+of multiple commits, the commits may have different change type values. As a whole, the PR will
+produce a release of the "highest" change type. For example, two commits mixing patch and minor
+change types will produce a minor CLI release, while two commits mixing minor and major change
+types will produce a major CLI release.
+
+The commit message is parsed / checked by versionbot with the
+[resin-commit-lint](https://github.com/balena-io-modules/resin-commit-lint#resin-commit-lint)
+package.
+
+Because of the way that the Changelog file is automatically updated from commit messages, which
+become the source of "what's new" for CLI end users, we advocate "meaningful commits" and
+user-focused commit messages. A meaningful commit is one that, in isolation, introduces a fix or
+feature (or part of a fix or feature) that makes sense at the Changelog level, and which leaves the
+CLI in a non-broken state. Sometimes, in the course of preparing a single pull request, a developer
+creates several commits as a way of saving their "work in progress", which may even fail to build
+(e.g. `npm run build` fails), and which is then fixed or undone by further commits in the same PR.
+In this situation, the recommendation is to "squash" or "fixup" the work-in-progress commits into
+fewer, meaningful commits. Interactive rebase is a good tool to achieve this:
+[blog](https://thoughtbot.com/blog/git-interactive-rebase-squash-amend-rewriting-history),
+[docs](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History).
+
+Mixing multiple distinct features or bug fixes in a single commit is discouraged, because the
+description will likely not fit in the single-line Changelog bullet point and also because it
+makes it harder to review the pull request (especially a large one) and harder to isolate and
+revert individual changes in case a bug is found later on. Create a separate commit for each
+feature / bug fix, or even separate pull requests.
+
+If you need to catch up with changes to the master branch while working on a pull request,
+use rebase instead of merge: [docs](https://git-scm.com/book/en/v2/Git-Branching-Rebasing).
 
 If `package.json` is updated for dependencies listed in the `repo.yml` file (like `balena-sdk`),
-the commit message body should include a line in the following format:
+the commit message body should also include a line in the following format:
 ```
 Update balena-sdk from 12.0.0 to 12.1.0
 ```
 
-This allows the CI to produce nested change logs (with expandable arrows), pulling in commit
-messages from the upstream repositories. The following npm script can be used to automatically
-produce a commit with a suitable commit message:
+This allows versionbot to produce nested Changelog entries (with expandable arrows), pulling in
+commit messages from the upstream repositories. The following npm script can be used to
+automatically produce a commit with a suitable commit message:
 ```
 npm run update balena-sdk ^12.1.0
 ```
@@ -74,7 +97,7 @@ npm run update balena-sdk ^12.1.0
 The script will create a new branch (only if `master` is currently checked out), run `npm update`
 with the given target version and commit the `package.json` and `npm-shrinkwrap.json` files. The
 script by default will set the `Change-type` to `patch` or `minor`, depending on the semver change
-of the updated dependency. For a `major` change type, it can specified as an extra argument:
+of the updated dependency. A `major` change type can specified as an extra argument:
 ```
 npm run update balena-sdk ^12.14.0 patch
 npm run update balena-sdk ^13.0.0 major

INSTALL-ADVANCED.md

@@ -1,4 +1,4 @@
-# balenaCLI Advanced Installation Options
+# balena CLI Advanced Installation Options
 
 **These are alternative, advanced installation options. Most users would prefer the [recommended,
 streamlined installation
@@ -8,11 +8,11 @@ There are 3 options to choose from to install balena's CLI:
 
 * [Executable Installer](#executable-installer): the easiest method on Windows and macOS, using the
   traditional graphical desktop application installers.
-* [Standalone Zip Package](#standalone-zip-package): these are plain zip files with the balenaCLI
+* [Standalone Zip Package](#standalone-zip-package): these are plain zip files with the balena CLI
   executable in them: extract and run. Available for all platforms: Linux, Windows, macOS.
   Recommended also for scripted installation in CI (continuous integration) environments.
 * [NPM Installation](#npm-installation): recommended for Node.js developers who may be interested
-  in integrating balenaCLI in their existing projects or workflow.
+  in integrating the balena CLI in their existing projects or workflow.
 
 Some specific CLI commands have a few extra installation steps: see section [Additional
 Dependencies](#additional-dependencies).
@@ -59,7 +59,7 @@ macOS:   | `/usr/local/lib/balena-cli/` <br> `/usr/local/bin/balena`
    [macOS](https://www.architectryan.com/2012/10/02/add-to-the-path-on-mac-os-x-mountain-lion/#.Uydjga1dXDg) |
    [Windows](https://www.computerhope.com/issues/ch000549.htm)
 
-> * If you are using macOS Catalina (10.15), [check this known issue and
+> * If you are using macOS 10.15 or later (Catalina, Big Sur), [check this known issue and
 >   workaround](https://github.com/balena-io/balena-cli/issues/1479).
 > * **Linux Alpine** and **Busybox:** the standalone zip package is not currently compatible with
 >   these "compact" Linux distributions, because of the alternative C libraries they ship with.
@@ -74,7 +74,7 @@ as described above.
 
 ## NPM Installation
 
-If you are a Node.js developer, you may wish to install balenaCLI via [npm](https://www.npmjs.com).
+If you are a Node.js developer, you may wish to install the balena CLI via [npm](https://www.npmjs.com).
 The npm installation involves building native (platform-specific) binary modules, which require
 some additional development tools to be installed first:
 
@@ -112,7 +112,7 @@ On **Windows (not WSL),** the dependencies above and additional ones can be met
   
   `npm install -g --production windows-build-tools`
 
-With these dependencies in place, the balenaCLI installation command is:
+With these dependencies in place, the balena CLI installation command is:
 
 ```sh
 $ npm install balena-cli -g --production --unsafe-perm

INSTALL-LINUX.md

@@ -1,4 +1,4 @@
-# balenaCLI Installation Instructions for Linux
+# balena CLI Installation Instructions for Linux
 
 These instructions are for the recommended installation option. They are suitable for most Linux
 distributions, except notably for **Linux Alpine** or **Busybox**. For these distros, see [advanced
@@ -20,14 +20,14 @@ Selected operating system: **Linux**
 
 4. Check that the installation was successful by running the following commands on a
    command terminal:  
-   * `balena version` - should print balenaCLI's version
+   * `balena version` - should print the CLI's version
    * `balena help` - should print a list of available commands
 
-No further steps are required to run most balenaCLI commands. The `balena ssh`, `scan`, `build`,
+No further steps are required to run most CLI commands. The `balena ssh`, `scan`, `build`,
 `deploy` and `preload` commands may require additional software to be installed, as described
 below.
 
-To update balenaCLI to a new version, download a new release zip file and replace the previous
+To update the balena CLI to a new version, download a new release zip file and replace the previous
 installation folder. To uninstall, simply delete the folder and edit the PATH environment variable
 as described above.
 
@@ -39,7 +39,7 @@ These commands require [Docker](https://docs.docker.com/install/overview/) or
 [balenaEngine](https://www.balena.io/engine/) to be available (on a local or remote machine). Most
 users will simply follow [Docker's installation
 instructions](https://docs.docker.com/install/overview/) to install Docker on the same laptop (dev
-machine) where balenaCLI is installed. The [advanced installation
+machine) where the balena CLI is installed. The [advanced installation
 options](./INSTALL-ADVANCED.md) document describes other possibilities.
 
 ### balena ssh

INSTALL-MAC.md

@@ -1,4 +1,4 @@
-# balenaCLI Installation Instructions for macOS
+# balena CLI Installation Instructions for macOS
 
 These instructions are for the recommended installation option. Advanced users may also be
 interested in [advanced installation options](./INSTALL-ADVANCED.md).
@@ -18,10 +18,10 @@ Selected operating system: **macOS**
 
 3. Check that the installation was successful by running the following commands on a
    command terminal:  
-   * `balena version` - should print balenaCLI's version
+   * `balena version` - should print the CLI's version
    * `balena help` - should print a list of available commands
 
-No further steps are required to run most balenaCLI commands. The `balena ssh`, `build`, `deploy`
+No further steps are required to run most CLI commands. The `balena ssh`, `build`, `deploy`
 and `preload` commands may require additional software to be installed, as described below.
 
 ## Additional Dependencies
@@ -32,7 +32,7 @@ These commands require [Docker](https://docs.docker.com/install/overview/) or
 [balenaEngine](https://www.balena.io/engine/) to be available (on a local or remote machine). Most
 users will simply follow [Docker's installation
 instructions](https://docs.docker.com/install/overview/) to install Docker on the same laptop (dev
-machine) where balenaCLI is installed. The [advanced installation
+machine) where the balena CLI is installed. The [advanced installation
 options](./INSTALL-ADVANCED.md) document describes other possibilities.
 
 ### balena ssh
@@ -62,7 +62,7 @@ present workaround is to either:
 
 * Downgrade Docker Desktop to version 18.06.1. Link: [Docker CE for
   Mac](https://docs.docker.com/docker-for-mac/release-notes/#docker-community-edition-18061-ce-mac73-2018-08-29)
-* Install balenaCLI on a Linux machine (as Docker for Linux still supports AUFS). A Linux Virtual
-  Machine also works, but a Docker container is _not_ recommended.
+* Install the balena CLI on a Linux machine (as Docker for Linux still supports AUFS). A Linux
+  Virtual Machine also works, but a Docker container is _not_ recommended.
 
 Long term, we are working on replacing AUFS with overlay2 for the affected device types.

INSTALL-WINDOWS.md

@@ -1,4 +1,4 @@
-# balenaCLI Installation Instructions for Windows
+# balena CLI Installation Instructions for Windows
 
 These instructions are for the recommended installation option. Advanced users may also be
 interested in [advanced installation options](./INSTALL-ADVANCED.md).
@@ -18,10 +18,10 @@ Selected operating system: **Windows**
 
 3. Check that the installation was successful by running the following commands on a
    command terminal:  
-   * `balena version` - should print balenaCLI's version
+   * `balena version` - should print the CLI's version
    * `balena help` - should print a list of available commands
 
-No further steps are required to run most balenaCLI commands. The `balena ssh`, `scan`, `build`,
+No further steps are required to run most CLI commands. The `balena ssh`, `scan`, `build`,
 `deploy`, `preload` and `os configure` commands may require additional software to be installed, as
 described below.
 
@@ -33,7 +33,7 @@ These commands require [Docker](https://docs.docker.com/install/overview/) or
 [balenaEngine](https://www.balena.io/engine/) to be available (on a local or remote machine). Most
 users will simply follow [Docker's installation
 instructions](https://docs.docker.com/install/overview/) to install Docker on the same laptop (dev
-machine) where balenaCLI is installed. The [advanced installation
+machine) where the balena CLI is installed. The [advanced installation
 options](./INSTALL-ADVANCED.md) document describes other possibilities.
 
 ### balena ssh
@@ -69,8 +69,8 @@ present workaround is to either:
 
 * Downgrade Docker Desktop to version 18.06.1. Link: [Docker CE for
   Windows](https://docs.docker.com/docker-for-windows/release-notes/#docker-community-edition-18061-ce-win73-2018-08-29)
-* Install balenaCLI on a Linux machine (as Docker for Linux still supports AUFS). A Linux Virtual
-  Machine also works, but a Docker container is _not_ recommended.
+* Install the balena CLI on a Linux machine (as Docker for Linux still supports AUFS). A Linux
+  Virtual Machine also works, but a Docker container is _not_ recommended.
 
 Long term, we are working on replacing AUFS with overlay2 for the affected device types.
 
@@ -78,5 +78,5 @@ Long term, we are working on replacing AUFS with overlay2 for the affected devic
 
 * The `balena os configure` command is currently not supported on Windows natively, but works with
   the [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/about) (WSL). When
-  using WSL, [install balenaCLI for
+  using WSL, [install the balena CLI for
   Linux](https://github.com/balena-io/balena-cli/blob/master/INSTALL-LINUX.md).

INSTALL.md

@@ -1,4 +1,4 @@
-# balenaCLI Installation Instructions
+# balena CLI Installation Instructions
 
 Please select your operating system:
 

README.md

@@ -1,4 +1,4 @@
-# balenaCLI
+# balena CLI
 
 The official balena Command Line Interface.
 
@@ -7,17 +7,17 @@ The official balena Command Line Interface.
 
 ## About
 
-balenaCLI is a Command Line Interface for [balenaCloud](https://www.balena.io/cloud/) or
+The balena CLI is a Command Line Interface for [balenaCloud](https://www.balena.io/cloud/) or
 [openBalena](https://www.balena.io/open/). It is a software tool available for Windows, macOS and
 Linux, used through a command prompt / terminal window. It can be used interactively or invoked in
-scripts. balenaCLI builds on the [balena API](https://www.balena.io/docs/reference/api/overview/)
+scripts. The balena CLI builds on the [balena API](https://www.balena.io/docs/reference/api/overview/)
 and the [balena SDK](https://www.balena.io/docs/reference/sdk/node-sdk/), and can also be directly
-imported in Node.js applications. balenaCLI is an [open-source project on
+imported in Node.js applications. The balena CLI is an [open-source project on
 GitHub](https://github.com/balena-io/balena-cli/), and your contribution is also welcome!
 
 ## Installation
 
-Check the [balenaCLI installation instructions on
+Check the [balena CLI installation instructions on
 GitHub](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md).
 
 ## Choosing a shell (command prompt/terminal)
@@ -42,9 +42,9 @@ are supported. Alternative shells include:
     [comment](https://github.com/balena-io/balena-cli/issues/598#issuecomment-556513098).
 * Microsoft's [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/about)
   (WSL). In this case, a Linux distribution like Ubuntu is installed via the Microsoft Store, and a
-  balenaCLI release **for Linux** should be selected. See
-  [FAQ](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md) for using balenaCLI
-  with WSL and Docker Desktop for Windows.
+  balena CLI release **for Linux** should be selected. See
+  [FAQ](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md) for using the
+  balena CLI with WSL and Docker Desktop for Windows.
 
 On **macOS** and **Linux,** the standard terminal window is supported. Optionally, `bash` command
 auto completion may be enabled by copying the
@@ -106,7 +106,7 @@ server, it should be configured with the following rules in the `squid.conf` fil
 
 The `BALENARC_NO_PROXY` variable may be used to exclude specified destinations from proxying.
 
-> * This feature requires balenaCLI version 11.30.8 or later. In the case of the npm [installation
+> * This feature requires CLI version 11.30.8 or later. In the case of the npm [installation
 >   option](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md), it also requires
 >   Node.js version 10.16.0 or later.
 > * To exclude a `balena ssh` target from proxying (IP address or `.local` hostname), the
@@ -139,28 +139,30 @@ The full CLI command reference is available [on the web](https://www.balena.io/d
 
 ## Support, FAQ and troubleshooting
 
-If you come across any problems or would like to get in touch:
+To learn more, troubleshoot issues, or to contact us for support:
 
-* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md).
-* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud).
-* For bug reports or feature requests,
-  [have a look at the GitHub issues or create a new one](https://github.com/balena-io/balena-cli/issues/).
+* Check the [masterclass tutorials](https://www.balena.io/docs/learn/more/masterclasses/overview/)
+* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md)
+* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud)
+
+For CLI bug reports or feature requests, check the
+[CLI GitHub issues](https://github.com/balena-io/balena-cli/issues/).
 
 ## Deprecation policy
 
-balenaCLI uses [semver versioning](https://semver.org/), with the concepts
+The balena CLI uses [semver versioning](https://semver.org/), with the concepts
 of major, minor and patch version releases.
 
-The latest release of a major version of balenaCLI will remain compatible with
+The latest release of a major version of the balena CLI will remain compatible with
 the balenaCloud backend services for at least one year from the date when the
-following major version is released. For example, balenaCLI v10.17.5, as the
+following major version is released. For example, balena CLI v10.17.5, as the
 latest v10 release, would remain compatible with the balenaCloud backend for one
 year from the date when v11.0.0 is released.
 
 At the end of this period, the older major version is considered deprecated and
 some of the functionality that depends on balenaCloud services may stop working
 at any time.
-Users are encouraged to regularly update balenaCLI to the latest version.
+Users are encouraged to regularly update the balena CLI to the latest version.
 
 ## Contributing (including editing documentation files)
 

TROUBLESHOOTING.md

@@ -1,23 +1,23 @@
-# balenaCLI FAQ & Troubleshooting
+# balena CLI FAQ & Troubleshooting
 
-## Where is balenaCLI's configuration file located?
+## Where is the balena CLI's configuration file located?
 
 The per-user configuration file lives in `$HOME/.balenarc.yml` or `%UserProfile%\_balenarc.yml`, in
 Unix based operating systems and Windows respectively.
 
-balenaCLI also attempts to read a `balenarc.yml` file in the current directory, which takes
+The balena CLI also attempts to read a `balenarc.yml` file in the current directory, which takes
 precedence over the per-user configuration file.
 
-## How do I point balenaCLI to the staging environment?
+## How do I point the balena CLI to the staging environment?
 
 Set the `BALENARC_BALENA_URL=balena-staging.com` environment variable, or add
-`balenaUrl: balena-staging.com` to balenaCLI's configuration file.
+`balenaUrl: balena-staging.com` to the balena CLI's configuration file.
 
-## How do I make balenaCLI persist data in another directory?
+## How do I make the balena CLI persist data in another directory?
 
-balenaCLI persists the session token, as well as cached assets, to `$HOME/.balena` or
+The balena CLI persists the session token, as well as cached assets, to `$HOME/.balena` or
 `%UserProfile%\_balena`. This directory can be changed by setting an environment variable,
-`BALENARC_DATA_DIRECTORY=/opt/balena`, or by adding `dataDirectory: /opt/balena` to balenaCLI's
+`BALENARC_DATA_DIRECTORY=/opt/balena`, or by adding `dataDirectory: /opt/balena` to the CLI's
 configuration file, replacing `/opt/balena` with the desired directory.
 
 ## After burning to an SD card, my device doesn't boot
@@ -64,9 +64,9 @@ Or in Windows:
 
 ## I get `EACCES: permission denied` when logging in
 
-balenaCLI stores the session token in `$HOME/.balena` or `C:\Users\<user>\_balena` in UNIX based
+The balena CLI stores the session token in `$HOME/.balena` or `C:\Users\<user>\_balena` in UNIX based
 operating systems and Windows respectively. This error usually indicates that the user doesn't have
-permissions over that directory, which can happen if balenaCLI was executed as the `root` user.
+permissions over that directory, which can happen if the CLI was executed as the `root` user.
 
 Try resetting the ownership by running:
 
@@ -76,7 +76,15 @@ $ sudo chown -R <user> $HOME/.balena
 
 ## Broken line wrapping / cursor behavior with `balena ssh`
 
-Users sometimes come across broken line wrapping or cursor behavior in text terminals, for example when long command lines are typed in a `balena ssh` session, or when using text editors like `vim` or `nano`. This is not something specific to balenaCLI, being also a commonly reported issue with standard remote terminal tools like `ssh` or `telnet`. It is often a remote shell configuration issue (files like `/etc/profile`, `~/.bash_profile`, `~/.bash_login`, `~/.profile` and the like), including UTF-8 misconfiguration, the use of unsupported ASCII control characters in shell prompt formatting (e.g. the `$PS1` env var) or the output of tools or log files that use colored text. The issue can sometimes be fixed by resizing the client terminal window, or by running one or more of the following commands on the shell:
+Users sometimes come across broken line wrapping or cursor behavior in text terminals, for example
+when long command lines are typed in a `balena ssh` session, or when using text editors like `vim`
+or `nano`. This is not something specific to the balena CLI, being also a commonly reported issue
+with standard remote terminal tools like `ssh` or `telnet`.  It is often a remote shell
+configuration issue (files like `/etc/profile`, `~/.bash_profile`, `~/.bash_login`, `~/.profile`
+and the like on the remote machine), including UTF-8 misconfiguration, the use of unsupported ASCII
+control characters in shell prompt formatting (e.g. the `$PS1` env var) or the output of tools or
+log files that use colored text. The issue can sometimes be fixed by simply resizing the client
+terminal window, or by running one or more of the following commands on the shell:
 
 ```sh
 export TERMINAL=linux

automation/capitanodoc/capitanodoc.ts

@@ -26,7 +26,7 @@ import { MarkdownFileParser } from './utils';
  * some content to this object.
  */
 const capitanoDoc = {
-	title: 'balenaCLI Documentation',
+	title: 'balena CLI Documentation',
 	introduction: '',
 	categories: [
 		{
@@ -39,9 +39,10 @@ const capitanoDoc = {
 				'build/commands/apps.js',
 				'build/commands/app/index.js',
 				'build/commands/app/create.js',
-				'build/commands/app/rm.js',
+				'build/commands/app/purge.js',
 				'build/commands/app/rename.js',
 				'build/commands/app/restart.js',
+				'build/commands/app/rm.js',
 			],
 		},
 		{
@@ -55,20 +56,21 @@ const capitanoDoc = {
 		{
 			title: 'Device',
 			files: [
+				'build/commands/devices/index.js',
+				'build/commands/devices/supported.js',
+				'build/commands/device/index.js',
 				'build/commands/device/identify.js',
 				'build/commands/device/init.js',
-				'build/commands/device/index.js',
 				'build/commands/device/move.js',
+				'build/commands/device/os-update.js',
+				'build/commands/device/public-url.js',
+				'build/commands/device/purge.js',
 				'build/commands/device/reboot.js',
 				'build/commands/device/register.js',
 				'build/commands/device/rename.js',
-				'build/commands/device/rm.js',
 				'build/commands/device/restart.js',
+				'build/commands/device/rm.js',
 				'build/commands/device/shutdown.js',
-				'build/commands/devices/index.js',
-				'build/commands/devices/supported.js',
-				'build/commands/device/os-update.js',
-				'build/commands/device/public-url.js',
 			],
 		},
 		{

automation/capitanodoc/utils.ts

@@ -57,7 +57,7 @@ export class MarkdownFileParser {
 	 * Extract the lines of a markdown document section with the given title.
 	 * For example, consider this sample markdown document:
 	 * ```
-	 *     # balenaCLI
+	 *     # balena CLI
 	 *
 	 *     ## Introduction
 	 *     Lorem ipsum dolor sit amet, consectetur adipiscing elit,

bin/balena

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
 
+// tslint:disable:no-var-requires
+
 // We boost the threadpool size as ext2fs can deadlock with some
 // operations otherwise, if the pool runs out.
 process.env.UV_THREADPOOL_SIZE = '64';
@@ -10,7 +12,7 @@ process.env.OCLIF_TS_NODE = 0;
 // Use fast-boot to cache require lookups, speeding up startup
 require('fast-boot2').start({
 	cacheScope: __dirname + '/..',
-	cacheFile: __dirname + '/.fast-boot.json'
+	cacheFile: __dirname + '/.fast-boot.json',
 });
 
 // Set the desired es version for downstream modules that support it

bin/balena-dev

@@ -1,14 +1,31 @@
 #!/usr/bin/env node
 
 // ****************************************************************************
-// THIS IS FOR DEV PERROSES ONLY AND WILL NOT BE PART OF THE PUBLISHED PACKAGE
+// THIS IS FOR DEV PURPOSES ONLY AND WILL NOT BE PART OF THE PUBLISHED PACKAGE
 // Before opening a PR you should build and test your changes using bin/balena
 // ****************************************************************************
 
+// tslint:disable:no-var-requires
+
 // We boost the threadpool size as ext2fs can deadlock with some
 // operations otherwise, if the pool runs out.
 process.env.UV_THREADPOOL_SIZE = '64';
 
+const path = require('path');
+const rootDir = path.join(__dirname, '..');
+
+// Allow balena-dev to work with oclif by temporarily
+// pointing oclif config options to lib/ instead of build/
+modifyOclifPaths();
+// Undo changes on exit
+process.on('exit', function () {
+	modifyOclifPaths(true);
+});
+// Undo changes in case of ctrl-v
+process.on('SIGINT', function () {
+	modifyOclifPaths(true);
+});
+
 // Use fast-boot to cache require lookups, speeding up startup
 require('fast-boot2').start({
 	cacheScope: __dirname + '/..',
@@ -18,8 +35,6 @@ require('fast-boot2').start({
 // Set the desired es version for downstream modules that support it
 require('@balena/es-version').set('es2018');
 
-const path = require('path');
-const rootDir = path.join(__dirname, '..');
 // Note: before ts-node v6.0.0, 'transpile-only' (no type checking) was the
 // default option. We upgraded ts-node and found that adding 'transpile-only'
 // was necessary to avoid a mysterious 'null' error message. On the plus side,
@@ -30,3 +45,30 @@ require('ts-node').register({
 	transpileOnly: true,
 });
 require('../lib/app').run();
+
+// Modify package.json oclif paths from build/ -> lib/, or vice versa
+function modifyOclifPaths(revert) {
+	const fs = require('fs');
+	const packageJsonPath = path.join(rootDir, 'package.json');
+
+	const packageJson = fs.readFileSync(packageJsonPath, 'utf8');
+	const packageObj = JSON.parse(packageJson);
+
+	if (!packageObj.oclif) {
+		return;
+	}
+
+	let oclifSectionText = JSON.stringify(packageObj.oclif);
+	if (!revert) {
+		oclifSectionText = oclifSectionText.replace(/\/build\//g, '/lib/');
+	} else {
+		oclifSectionText = oclifSectionText.replace(/\/lib\//g, '/build/');
+	}
+
+	packageObj.oclif = JSON.parse(oclifSectionText);
+	fs.writeFileSync(
+		packageJsonPath,
+		`${JSON.stringify(packageObj, null, 2)}\n`,
+		'utf8',
+	);
+}

doc/cli.markdown

@@ -1,16 +1,16 @@
-# balenaCLI Documentation
+# balena CLI Documentation
 
-balenaCLI is a Command Line Interface for [balenaCloud](https://www.balena.io/cloud/) or
+The balena CLI is a Command Line Interface for [balenaCloud](https://www.balena.io/cloud/) or
 [openBalena](https://www.balena.io/open/). It is a software tool available for Windows, macOS and
 Linux, used through a command prompt / terminal window. It can be used interactively or invoked in
-scripts. balenaCLI builds on the [balena API](https://www.balena.io/docs/reference/api/overview/)
+scripts. The balena CLI builds on the [balena API](https://www.balena.io/docs/reference/api/overview/)
 and the [balena SDK](https://www.balena.io/docs/reference/sdk/node-sdk/), and can also be directly
-imported in Node.js applications. balenaCLI is an [open-source project on
+imported in Node.js applications. The balena CLI is an [open-source project on
 GitHub](https://github.com/balena-io/balena-cli/), and your contribution is also welcome!
 
 ## Installation
 
-Check the [balenaCLI installation instructions on
+Check the [balena CLI installation instructions on
 GitHub](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md).
 
 ## Choosing a shell (command prompt/terminal)
@@ -35,9 +35,9 @@ are supported. Alternative shells include:
     [comment](https://github.com/balena-io/balena-cli/issues/598#issuecomment-556513098).
 * Microsoft's [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/about)
   (WSL). In this case, a Linux distribution like Ubuntu is installed via the Microsoft Store, and a
-  balenaCLI release **for Linux** should be selected. See
-  [FAQ](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md) for using balenaCLI
-  with WSL and Docker Desktop for Windows.
+  balena CLI release **for Linux** should be selected. See
+  [FAQ](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md) for using the
+  balena CLI with WSL and Docker Desktop for Windows.
 
 On **macOS** and **Linux,** the standard terminal window is supported. Optionally, `bash` command
 auto completion may be enabled by copying the
@@ -99,7 +99,7 @@ server, it should be configured with the following rules in the `squid.conf` fil
 
 The `BALENARC_NO_PROXY` variable may be used to exclude specified destinations from proxying.
 
-> * This feature requires balenaCLI version 11.30.8 or later. In the case of the npm [installation
+> * This feature requires CLI version 11.30.8 or later. In the case of the npm [installation
 >   option](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md), it also requires
 >   Node.js version 10.16.0 or later.
 > * To exclude a `balena ssh` target from proxying (IP address or `.local` hostname), the
@@ -127,28 +127,30 @@ address like `192.168.1.2`.
 
 ## Support, FAQ and troubleshooting
 
-If you come across any problems or would like to get in touch:
+To learn more, troubleshoot issues, or to contact us for support:
 
-* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md).
-* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud).
-* For bug reports or feature requests,
-  [have a look at the GitHub issues or create a new one](https://github.com/balena-io/balena-cli/issues/).
+* Check the [masterclass tutorials](https://www.balena.io/docs/learn/more/masterclasses/overview/)
+* Check our [FAQ / troubleshooting document](https://github.com/balena-io/balena-cli/blob/master/TROUBLESHOOTING.md)
+* Ask us a question through the [balenaCloud forum](https://forums.balena.io/c/balena-cloud)
+
+For CLI bug reports or feature requests, check the
+[CLI GitHub issues](https://github.com/balena-io/balena-cli/issues/).
 
 ## Deprecation policy
 
-balenaCLI uses [semver versioning](https://semver.org/), with the concepts
+The balena CLI uses [semver versioning](https://semver.org/), with the concepts
 of major, minor and patch version releases.
 
-The latest release of a major version of balenaCLI will remain compatible with
+The latest release of a major version of the balena CLI will remain compatible with
 the balenaCloud backend services for at least one year from the date when the
-following major version is released. For example, balenaCLI v10.17.5, as the
+following major version is released. For example, balena CLI v10.17.5, as the
 latest v10 release, would remain compatible with the balenaCloud backend for one
 year from the date when v11.0.0 is released.
 
 At the end of this period, the older major version is considered deprecated and
 some of the functionality that depends on balenaCloud services may stop working
 at any time.
-Users are encouraged to regularly update balenaCLI to the latest version.
+Users are encouraged to regularly update the balena CLI to the latest version.
 
 
 # CLI Command Reference
@@ -162,9 +164,10 @@ Users are encouraged to regularly update balenaCLI to the latest version.
 	- [apps](#apps)
 	- [app <name>](#app-name)
 	- [app create <name>](#app-create-name)
-	- [app rm <name>](#app-rm-name)
+	- [app purge <name>](#app-purge-name)
 	- [app rename <name> [newname]](#app-rename-name-newname)
 	- [app restart <name>](#app-restart-name)
+	- [app rm <name>](#app-rm-name)
 
 - Authentication
 
@@ -174,20 +177,21 @@ Users are encouraged to regularly update balenaCLI to the latest version.
 
 - Device
 
+	- [devices](#devices)
+	- [devices supported](#devices-supported)
+	- [device <uuid>](#device-uuid)
 	- [device identify <uuid>](#device-identify-uuid)
 	- [device init](#device-init)
-	- [device <uuid>](#device-uuid)
 	- [device move <uuid(s)>](#device-move-uuid-s)
+	- [device os-update <uuid>](#device-os-update-uuid)
+	- [device public-url <uuid>](#device-public-url-uuid)
+	- [device purge <uuid>](#device-purge-uuid)
 	- [device reboot <uuid>](#device-reboot-uuid)
 	- [device register <application>](#device-register-application)
 	- [device rename <uuid> [newname]](#device-rename-uuid-newname)
-	- [device rm <uuid(s)>](#device-rm-uuid-s)
 	- [device restart <uuid>](#device-restart-uuid)
+	- [device rm <uuid(s)>](#device-rm-uuid-s)
 	- [device shutdown <uuid>](#device-shutdown-uuid)
-	- [devices](#devices)
-	- [devices supported](#devices-supported)
-	- [device os-update <uuid>](#device-os-update-uuid)
-	- [device public-url <uuid>](#device-public-url-uuid)
 
 - Environment Variables
 
@@ -364,16 +368,14 @@ application name
 
 application device type (Check available types with `balena devices supported`)
 
-## app rm <name>
+## app purge <name>
 
-Permanently remove a balena application.
-
-The --yes option may be used to avoid interactive confirmation.
+Purge data from all devices belonging to an application.
+This will clear the application's /data directory.
 
 Examples:
 
-	$ balena app rm MyApp
-	$ balena app rm MyApp --yes
+	$ balena app purge MyApp
 
 ### Arguments
 
@@ -383,10 +385,6 @@ application name or numeric ID
 
 ### Options
 
-#### -y, --yes
-
-answer "yes" to all questions (non interactive use)
-
 ## app rename <name> [newName]
 
 Rename an application.
@@ -413,7 +411,7 @@ the new name for the application
 
 ## app restart <name>
 
-Restart all devices that belongs to a certain application.
+Restart all devices belonging to an application.
 
 Examples:
 
@@ -427,6 +425,29 @@ application name or numeric ID
 
 ### Options
 
+## app rm <name>
+
+Permanently remove a balena application.
+
+The --yes option may be used to avoid interactive confirmation.
+
+Examples:
+
+	$ balena app rm MyApp
+	$ balena app rm MyApp --yes
+
+### Arguments
+
+#### NAME
+
+application name or numeric ID
+
+### Options
+
+#### -y, --yes
+
+answer "yes" to all questions (non interactive use)
+
 # Authentication
 
 ## login
@@ -504,6 +525,90 @@ Examples:
 
 # Device
 
+## devices
+
+list all devices that belong to you.
+
+You can filter the devices by application by using the `--application` option.
+
+The --json option is recommended when scripting the output of this command,
+because field names are less likely to change in JSON format and because it
+better represents data types like arrays, empty strings and null values.
+The 'jq' utility may be helpful for querying JSON fields in shell scripts
+(https://stedolan.github.io/jq/manual/).
+
+Examples:
+
+	$ balena devices
+	$ balena devices --application MyApp
+	$ balena devices --app MyApp
+	$ balena devices -a MyApp
+
+### Options
+
+#### -a, --application APPLICATION
+
+application name
+
+#### --app APP
+
+same as '--application'
+
+#### -j, --json
+
+produce JSON output instead of tabular output
+
+## devices supported
+
+List the supported device types (like 'raspberrypi3' or 'intel-nuc').
+
+The --verbose option adds extra columns/fields to the output, including the
+"STATE" column whose values are one of 'new', 'released' or 'discontinued'.
+However, 'discontinued' device types are only listed if the '--discontinued'
+option is used.
+
+The --json option is recommended when scripting the output of this command,
+because the JSON format is less likely to change and it better represents data
+types like lists and empty strings (for example, the ALIASES column contains a
+list of zero or more values). The 'jq' utility may be helpful in shell scripts
+(https://stedolan.github.io/jq/manual/).
+
+Examples:
+
+	$ balena devices supported
+	$ balena devices supported --verbose
+	$ balena devices supported -vj
+
+### Options
+
+#### --discontinued
+
+include "discontinued" device types
+
+#### -j, --json
+
+produce JSON output instead of tabular output
+
+#### -v, --verbose
+
+add extra columns in the tabular output (ALIASES, ARCH, STATE)
+
+## device <uuid>
+
+Show information about a single device.
+
+Examples:
+
+	$ balena device 7cf02a6
+
+### Arguments
+
+#### UUID
+
+the device uuid
+
+### Options
+
 ## device identify <uuid>
 
 Identify a device by making the ACT LED blink (Raspberry Pi).
@@ -569,22 +674,6 @@ Check `balena util available-drives` for available options.
 
 path to the config JSON file, see `balena os build-config`
 
-## device <uuid>
-
-Show information about a single device.
-
-Examples:
-
-	$ balena device 7cf02a6
-
-### Arguments
-
-#### UUID
-
-the device uuid
-
-### Options
-
 ## device move <uuid(s)>
 
 Move one or more devices to another application.
@@ -614,212 +703,6 @@ application name
 
 same as '--application'
 
-## device reboot <uuid>
-
-Remotely reboot a device.
-
-Examples:
-
-	$ balena device reboot 23c73a1
-
-### Arguments
-
-#### UUID
-
-the uuid of the device to reboot
-
-### Options
-
-#### -f, --force
-
-force action if the update lock is set
-
-## device register <application>
-
-Register a device to an application.
-
-Examples:
-
-	$ balena device register MyApp
-	$ balena device register MyApp --uuid <uuid>
-
-### Arguments
-
-#### APPLICATION
-
-the name or id of application to register device with
-
-### Options
-
-#### -u, --uuid UUID
-
-custom uuid
-
-## device rename &#60;uuid&#62; [newName]
-
-Rename a device.
-
-Note, if the name is omitted, it will be prompted for interactively.
-
-Examples:
-
-	$ balena device rename 7cf02a6
-	$ balena device rename 7cf02a6 MyPi
-
-### Arguments
-
-#### UUID
-
-the uuid of the device to rename
-
-#### NEWNAME
-
-the new name for the device
-
-### Options
-
-## device rm &#60;uuid(s)&#62;
-
-Remove one or more devices from balena.
-
-Note this command asks for confirmation interactively.
-You can avoid this by passing the `--yes` option.
-
-Examples:
-
-	$ balena device rm 7cf02a6
-	$ balena device rm 7cf02a6,dc39e52
-	$ balena device rm 7cf02a6 --yes
-
-### Arguments
-
-#### UUID
-
-comma-separated list (no blank spaces) of device UUIDs to be removed
-
-### Options
-
-#### -y, --yes
-
-answer "yes" to all questions (non interactive use)
-
-## device restart &#60;uuid&#62;
-
-Restart containers on a device.
-If the --service flag is provided, then only those services' containers
-will be restarted, otherwise all containers on the device will be restarted.
-
-Multiple devices and services may be specified with a comma-separated list
-of values (no spaces).
-
-Note this does not reboot the device, to do so use instead `balena device reboot`.
-
-Examples:
-
-	$ balena device restart 23c73a1
-	$ balena device restart 55d43b3,23c73a1
-	$ balena device restart 23c73a1 --service myService
-	$ balena device restart 23c73a1 -s myService1,myService2
-
-### Arguments
-
-#### UUID
-
-comma-separated list (no blank spaces) of device UUIDs to restart
-
-### Options
-
-#### -s, --service SERVICE
-
-comma-separated list (no blank spaces) of service names to restart
-
-## device shutdown &#60;uuid&#62;
-
-Remotely shutdown a device.
-
-Examples:
-
-	$ balena device shutdown 23c73a1
-
-### Arguments
-
-#### UUID
-
-the uuid of the device to shutdown
-
-### Options
-
-#### -f, --force
-
-force action if the update lock is set
-
-## devices
-
-list all devices that belong to you.
-
-You can filter the devices by application by using the `--application` option.
-
-The --json option is recommended when scripting the output of this command,
-because field names are less likely to change in JSON format and because it
-better represents data types like arrays and empty strings. The 'jq' utility
-may also be helpful in shell scripts (https://stedolan.github.io/jq/manual/).
-
-Examples:
-
-	$ balena devices
-	$ balena devices --application MyApp
-	$ balena devices --app MyApp
-	$ balena devices -a MyApp
-
-### Options
-
-#### -a, --application APPLICATION
-
-application name
-
-#### --app APP
-
-same as '--application'
-
-#### -j, --json
-
-produce JSON output instead of tabular output
-
-## devices supported
-
-List the supported device types (like 'raspberrypi3' or 'intel-nuc').
-
-The --verbose option adds extra columns/fields to the output, including the
-"STATE" column whose values are one of 'new', 'released' or 'discontinued'.
-However, 'discontinued' device types are only listed if the '--discontinued'
-option is used.
-
-The --json option is recommended when scripting the output of this command,
-because the JSON format is less likely to change and it better represents data
-types like lists and empty strings (for example, the ALIASES column contains a
-list of zero or more values). The 'jq' utility may be helpful in shell scripts
-(https://stedolan.github.io/jq/manual/).
-
-Examples:
-
-	$ balena devices supported
-	$ balena devices supported --verbose
-	$ balena devices supported -vj
-
-### Options
-
-#### --discontinued
-
-include "discontinued" device types
-
-#### -j, --json
-
-produce JSON output instead of tabular output
-
-#### -v, --verbose
-
-add extra columns in the tabular output (ALIASES, ARCH, STATE)
-
 ## device os-update &#60;uuid&#62;
 
 Start a Host OS update for a device.
@@ -890,6 +773,166 @@ disable the public URL
 
 determine if public URL is enabled
 
+## device purge &#60;uuid&#62;
+
+Purge application data from a device.
+This will clear the application's /data directory.
+
+Multiple devices may be specified with a comma-separated list
+of values (no spaces).
+
+Examples:
+
+	$ balena device purge 23c73a1
+	$ balena device purge 55d43b3,23c73a1
+
+### Arguments
+
+#### UUID
+
+comma-separated list (no blank spaces) of device UUIDs
+
+### Options
+
+## device reboot &#60;uuid&#62;
+
+Remotely reboot a device.
+
+Examples:
+
+	$ balena device reboot 23c73a1
+
+### Arguments
+
+#### UUID
+
+the uuid of the device to reboot
+
+### Options
+
+#### -f, --force
+
+force action if the update lock is set
+
+## device register &#60;application&#62;
+
+Register a device to an application.
+
+Examples:
+
+	$ balena device register MyApp
+	$ balena device register MyApp --uuid <uuid>
+
+### Arguments
+
+#### APPLICATION
+
+the name or id of application to register device with
+
+### Options
+
+#### -u, --uuid UUID
+
+custom uuid
+
+## device rename &#60;uuid&#62; [newName]
+
+Rename a device.
+
+Note, if the name is omitted, it will be prompted for interactively.
+
+Examples:
+
+	$ balena device rename 7cf02a6
+	$ balena device rename 7cf02a6 MyPi
+
+### Arguments
+
+#### UUID
+
+the uuid of the device to rename
+
+#### NEWNAME
+
+the new name for the device
+
+### Options
+
+## device restart &#60;uuid&#62;
+
+Restart containers on a device.
+If the --service flag is provided, then only those services' containers
+will be restarted, otherwise all containers on the device will be restarted.
+
+Multiple devices and services may be specified with a comma-separated list
+of values (no spaces).
+
+Note this does not reboot the device, to do so use instead `balena device reboot`.
+
+Examples:
+
+	$ balena device restart 23c73a1
+	$ balena device restart 55d43b3,23c73a1
+	$ balena device restart 23c73a1 --service myService
+	$ balena device restart 23c73a1 -s myService1,myService2
+
+### Arguments
+
+#### UUID
+
+comma-separated list (no blank spaces) of device UUIDs to restart
+
+### Options
+
+#### -s, --service SERVICE
+
+comma-separated list (no blank spaces) of service names to restart
+
+## device rm &#60;uuid(s)&#62;
+
+Remove one or more devices from balena.
+
+Note this command asks for confirmation interactively.
+You can avoid this by passing the `--yes` option.
+
+Examples:
+
+	$ balena device rm 7cf02a6
+	$ balena device rm 7cf02a6,dc39e52
+	$ balena device rm 7cf02a6 --yes
+
+### Arguments
+
+#### UUID
+
+comma-separated list (no blank spaces) of device UUIDs to be removed
+
+### Options
+
+#### -y, --yes
+
+answer "yes" to all questions (non interactive use)
+
+## device shutdown &#60;uuid&#62;
+
+Remotely shutdown a device.
+
+Examples:
+
+	$ balena device shutdown 23c73a1
+
+### Arguments
+
+#### UUID
+
+the uuid of the device to shutdown
+
+### Options
+
+#### -f, --force
+
+force action if the update lock is set
+
 # Environment Variables
 
 ## envs
@@ -941,7 +984,7 @@ Examples:
 
 #### --all
 
-No-op since balenaCLI v12.0.0.
+No-op since balena CLI v12.0.0.
 
 #### -a, --application APPLICATION
 
@@ -1302,7 +1345,11 @@ show additional commands
 
 ## version
 
-Display version information for balenaCLI and/or Node.js.
+Display version information for the balena CLI and/or Node.js. Note that the
+balena CLI executable installers for Windows and macOS, and the standalone
+zip packages, ship with a built-in copy of Node.js.  In this case, the
+reported version of Node.js regards this built-in copy, rather than any
+other `node` engine that may also be available on the command prompt.
 
 The --json option is recommended when scripting the output of this command,
 because the JSON format is less likely to change and it better represents
@@ -1477,6 +1524,11 @@ Only show system logs. This can be used in combination with --service.
 
 Scan for balenaOS devices on your local network.
 
+The output includes device information collected through balenaEngine for
+devices running a development image of balenaOS. Devices running a production
+image do not expose balenaEngine (on TCP port 2375), which is why less
+information is printed about them.
+
 Examples:
 
 	$ balena scan
@@ -1493,6 +1545,10 @@ display full info
 
 scan timeout in seconds
 
+#### -j, --json
+
+produce JSON output instead of tabular output
+
 ## ssh &#60;applicationOrDevice&#62; [service]
 
 Start a shell on a local or remote device. If a service name is not provided,
@@ -1665,11 +1721,15 @@ versions for the given device type are pre-release).
 You can pass `--version menu` to pick the OS version from the interactive menu
 of all available versions.
 
+To download a development image append `.dev` to the version or select from
+the interactive menu.
+
 Examples:
 
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img
-	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 1.24.1
-	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^1.20.0
+	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1
+	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1.dev
+	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^2.60.0
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version latest
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version default
 	$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version menu
@@ -1750,7 +1810,7 @@ A suitable key is automatically generated or fetched if this option is omitted.
 
 Note: This command is currently not supported on Windows natively. Windows users
 are advised to install the Windows Subsystem for Linux (WSL) with Ubuntu, and use
-the Linux release of balenaCLI:
+the Linux release of the balena CLI:
 https://docs.microsoft.com/en-us/windows/wsl/about
 
 Examples:
@@ -2068,7 +2128,7 @@ the image file path
 
 #### -a, --app APP
 
-name, slug or numeric ID of the application to preload
+name of the application to preload
 
 #### -c, --commit COMMIT
 
@@ -2169,7 +2229,7 @@ secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.
 
 DOCKERIGNORE AND GITIGNORE FILES  
-By default, balenaCLI will use a single ".dockerignore" file (if any) at
+By default, the balena CLI will use a single ".dockerignore" file (if any) at
 the project root (--source directory) in order to decide which source files to
 exclude from the "build context" (tar stream) sent to balenaCloud, Docker
 daemon or balenaEngine. In a microservices (multicontainer) application, the
@@ -2194,7 +2254,7 @@ compatibility with the standard docker-compose tool, while still allowing a
 root .dockerignore file (at the overall project root) to filter files and
 folders that are outside service subdirectories.
 
-balenaCLI releases older than v12.0.0 also took .gitignore files into account.
+balena CLI releases older than v12.0.0 also took .gitignore files into account.
 This behavior is deprecated, but may still be enabled with the --gitignore (-g)
 option if compatibility is required. This option is mutually exclusive with
 --multi-dockerignore (-m) and will be removed in the CLI's next major version
@@ -2322,7 +2382,7 @@ left hand side of the = character will be treated as the variable name.
 
 #### -l, --convert-eol
 
-No-op and deprecated since balenaCLI v12.0.0
+No-op and deprecated since balena CLI v12.0.0
 
 #### --noconvert-eol
 
@@ -2334,7 +2394,7 @@ Have each service use its own .dockerignore file. See "balena help push".
 
 #### -G, --nogitignore
 
-No-op (default behavior) since balenaCLI v12.0.0. See "balena help push".
+No-op (default behavior) since balena CLI v12.0.0. See "balena help push".
 
 #### -g, --gitignore
 
@@ -2346,7 +2406,7 @@ required until your project can be adapted.
 
 ## settings
 
-Use this command to display current balenaCLI settings.
+Use this command to display the current balena CLI settings.
 
 Examples:
 
@@ -2446,7 +2506,7 @@ secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.
 
 DOCKERIGNORE AND GITIGNORE FILES  
-By default, balenaCLI will use a single ".dockerignore" file (if any) at
+By default, the balena CLI will use a single ".dockerignore" file (if any) at
 the project root (--source directory) in order to decide which source files to
 exclude from the "build context" (tar stream) sent to balenaCloud, Docker
 daemon or balenaEngine. In a microservices (multicontainer) application, the
@@ -2471,7 +2531,7 @@ compatibility with the standard docker-compose tool, while still allowing a
 root .dockerignore file (at the overall project root) to filter files and
 folders that are outside service subdirectories.
 
-balenaCLI releases older than v12.0.0 also took .gitignore files into account.
+balena CLI releases older than v12.0.0 also took .gitignore files into account.
 This behavior is deprecated, but may still be enabled with the --gitignore (-g)
 option if compatibility is required. This option is mutually exclusive with
 --multi-dockerignore (-m) and will be removed in the CLI's next major version
@@ -2536,7 +2596,7 @@ Alternative Dockerfile name/path, relative to the source folder
 
 #### --logs
 
-No-op and deprecated since balenaCLI v12.0.0. Build logs are now shown by default.
+No-op and deprecated since balena CLI v12.0.0. Build logs are now shown by default.
 
 #### --nologs
 
@@ -2554,7 +2614,7 @@ Have each service use its own .dockerignore file. See "balena help build".
 
 #### -G, --nogitignore
 
-No-op (default behavior) since balenaCLI v12.0.0. See "balena help build".
+No-op (default behavior) since balena CLI v12.0.0. See "balena help build".
 
 #### --noparent-check
 
@@ -2566,7 +2626,7 @@ Path to a YAML or JSON file with passwords for a private Docker registry
 
 #### -l, --convert-eol
 
-No-op and deprecated since balenaCLI v12.0.0
+No-op and deprecated since balena CLI v12.0.0
 
 #### --noconvert-eol
 
@@ -2668,7 +2728,7 @@ secrets.json file exists in the balena directory (usually $HOME/.balena),
 this file will be used instead.
 
 DOCKERIGNORE AND GITIGNORE FILES  
-By default, balenaCLI will use a single ".dockerignore" file (if any) at
+By default, the balena CLI will use a single ".dockerignore" file (if any) at
 the project root (--source directory) in order to decide which source files to
 exclude from the "build context" (tar stream) sent to balenaCloud, Docker
 daemon or balenaEngine. In a microservices (multicontainer) application, the
@@ -2693,7 +2753,7 @@ compatibility with the standard docker-compose tool, while still allowing a
 root .dockerignore file (at the overall project root) to filter files and
 folders that are outside service subdirectories.
 
-balenaCLI releases older than v12.0.0 also took .gitignore files into account.
+balena CLI releases older than v12.0.0 also took .gitignore files into account.
 This behavior is deprecated, but may still be enabled with the --gitignore (-g)
 option if compatibility is required. This option is mutually exclusive with
 --multi-dockerignore (-m) and will be removed in the CLI's next major version
@@ -2759,7 +2819,7 @@ Alternative Dockerfile name/path, relative to the source folder
 
 #### --logs
 
-No-op and deprecated since balenaCLI v12.0.0. Build logs are now shown by default.
+No-op and deprecated since balena CLI v12.0.0. Build logs are now shown by default.
 
 #### --nologs
 
@@ -2777,7 +2837,7 @@ Have each service use its own .dockerignore file. See "balena help build".
 
 #### -G, --nogitignore
 
-No-op (default behavior) since balenaCLI v12.0.0. See "balena help build".
+No-op (default behavior) since balena CLI v12.0.0. See "balena help build".
 
 #### --noparent-check
 
@@ -2789,7 +2849,7 @@ Path to a YAML or JSON file with passwords for a private Docker registry
 
 #### -l, --convert-eol
 
-No-op and deprecated since balenaCLI v12.0.0
+No-op and deprecated since balena CLI v12.0.0
 
 #### --noconvert-eol
 

lib/app-oclif.ts

@@ -1,61 +0,0 @@
-/**
- * @license
- * Copyright 2019 Balena Ltd.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Main } from '@oclif/command';
-
-import { trackPromise } from './hooks/prerun/track';
-
-class CustomMain extends Main {
-	protected _helpOverride(): boolean {
-		// Disable oclif's default handler for the 'version' command
-		if (['-v', '--version', 'version'].includes(this.argv[0])) {
-			return false;
-		} else {
-			return super._helpOverride();
-		}
-	}
-}
-
-import type { AppOptions } from './preparser';
-
-/**
- * oclif CLI entrypoint
- */
-export async function run(command: string[], options: AppOptions) {
-	const runPromise = CustomMain.run(command).then(
-		() => {
-			if (!options.noFlush) {
-				return require('@oclif/command/flush');
-			}
-		},
-		(error) => {
-			// oclif sometimes exits with ExitError code 0 (not an error)
-			// (Avoid `error instanceof ExitError` here for the reasons explained
-			// in the CONTRIBUTING.md file regarding the `instanceof` operator.)
-			if (error.oclif?.exit === 0) {
-				return;
-			} else {
-				throw error;
-			}
-		},
-	);
-	try {
-		await Promise.all([trackPromise, runPromise]);
-	} catch (err) {
-		await (await import('./errors')).handleError(err);
-	}
-}

lib/app.ts

@@ -1,6 +1,6 @@
 /**
  * @license
- * Copyright 2019 Balena Ltd.
+ * Copyright 2019-2020 Balena Ltd.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,78 +15,138 @@
  * limitations under the License.
  */
 
+import * as packageJSON from '../package.json';
+import { CliSettings } from './utils/bootstrap';
+import { onceAsync, stripIndent } from './utils/lazy';
+
 /**
- * CLI entrypoint, but see also `bin/balena` and `bin/balena-dev` which
- * call this function.
+ * Sentry.io setup
+ * @see https://docs.sentry.io/error-reporting/quickstart/?platform=node
  */
+export const setupSentry = onceAsync(async () => {
+	const config = await import('./config');
+	const Sentry = await import('@sentry/node');
+	Sentry.init({
+		dsn: config.sentryDsn,
+		release: packageJSON.version,
+	});
+	Sentry.configureScope((scope) => {
+		scope.setExtras({
+			is_pkg: !!(process as any).pkg,
+			node_version: process.version,
+			platform: process.platform,
+		});
+	});
+	return Sentry.getCurrentHub();
+});
+
+async function checkNodeVersion() {
+	const validNodeVersions = packageJSON.engines.node;
+	if (!(await import('semver')).satisfies(process.version, validNodeVersions)) {
+		console.warn(stripIndent`
+			------------------------------------------------------------------------------
+			Warning: Node version "${process.version}" does not match required versions "${validNodeVersions}".
+			This may cause unexpected behavior. To upgrade Node, visit:
+			https://nodejs.org/en/download/
+			------------------------------------------------------------------------------
+			`);
+	}
+}
+
+/** Setup balena-sdk options that are shared with imported packages */
+function setupBalenaSdkSharedOptions(settings: CliSettings) {
+	const BalenaSdk = require('balena-sdk') as typeof import('balena-sdk');
+	BalenaSdk.setSharedOptions({
+		apiUrl: settings.get<string>('apiUrl'),
+		dataDirectory: settings.get<string>('dataDirectory'),
+	});
+}
+
+/**
+ * Addresses the console warning:
+ * (node:49500) MaxListenersExceededWarning: Possible EventEmitter memory
+ * leak detected. 11 error listeners added. Use emitter.setMaxListeners() to
+ * increase limit
+ */
+export function setMaxListeners(maxListeners: number) {
+	require('events').EventEmitter.defaultMaxListeners = maxListeners;
+}
+
+/** Selected CLI initialization steps */
+async function init() {
+	if (process.env.BALENARC_NO_SENTRY) {
+		console.error(`WARN: disabling Sentry.io error reporting`);
+	} else {
+		await setupSentry();
+	}
+	checkNodeVersion();
+
+	const settings = new CliSettings();
+
+	// Proxy setup should be done early on, before loading balena-sdk
+	await (await import('./utils/proxy')).setupGlobalHttpProxy(settings);
+
+	setupBalenaSdkSharedOptions(settings);
+
+	// check for CLI updates once a day
+	(await import('./utils/update')).notify();
+}
+
+/** Execute the oclif parser and the CLI command. */
+async function oclifRun(
+	command: string[],
+	options: import('./preparser').AppOptions,
+) {
+	const { CustomMain } = await import('./utils/oclif-utils');
+	const runPromise = CustomMain.run(command).then(
+		() => {
+			if (!options.noFlush) {
+				return require('@oclif/command/flush');
+			}
+		},
+		(error) => {
+			// oclif sometimes exits with ExitError code 0 (not an error)
+			// (Avoid `error instanceof ExitError` here for the reasons explained
+			// in the CONTRIBUTING.md file regarding the `instanceof` operator.)
+			if (error.oclif?.exit === 0) {
+				return;
+			} else {
+				throw error;
+			}
+		},
+	);
+	const { trackPromise } = await import('./hooks/prerun/track');
+	await Promise.all([trackPromise, runPromise]);
+}
+
+/** CLI entrypoint. Called by the `bin/balena` and `bin/balena-dev` scripts. */
 export async function run(
 	cliArgs = process.argv,
 	options: import('./preparser').AppOptions = {},
 ) {
-	// DEBUG set to falsy for negative values else is truthy
-	process.env.DEBUG = ['0', 'no', 'false', '', undefined].includes(
-		process.env.DEBUG?.toLowerCase(),
-	)
-		? ''
-		: '1';
-
-	// The 'pkgExec' special/internal command provides a Node.js interpreter
-	// for use of the standalone zip package. See pkgExec function.
-	if (cliArgs.length > 3 && cliArgs[2] === 'pkgExec') {
-		return pkgExec(cliArgs[3], cliArgs.slice(4));
-	}
-
-	const { globalInit } = await import('./app-common');
-	const { preparseArgs, checkDeletedCommand } = await import('./preparser');
-
-	// globalInit() must be called very early on (before other imports) because
-	// it sets up Sentry error reporting, global HTTP proxy settings, balena-sdk
-	// shared options, and performs node version requirement checks.
-	await globalInit();
-
-	// Look for commands that have been removed and if so, exit with a notice
-	checkDeletedCommand(cliArgs.slice(2));
-
-	const args = await preparseArgs(cliArgs);
-	await (await import('./app-oclif')).run(args, options);
-
-	// Windows fix: reading from stdin prevents the process from exiting
-	process.stdin.pause();
-}
-
-/**
- * Implements the 'pkgExec' command, used as a way to provide a Node.js
- * interpreter for child_process.spawn()-like operations when the CLI is
- * executing as a standalone zip package (built-in Node interpreter) and
- * the system may not have a separate Node.js installation. A present use
- * case is a patched version of the 'windosu' package that requires a
- * Node.js interpreter to spawn a privileged child process.
- *
- * @param modFunc Path to a JS module that will be executed via require().
- * The modFunc argument may optionally contain a function name separated
- * by '::', for example '::main' in:
- * 'C:\\snapshot\\balena-cli\\node_modules\\windosu\\lib\\pipe.js::main'
- * in which case that function is executed in the require'd module.
- * @param args Optional arguments to passed through process.argv and as
- * arguments to the function specified via modFunc.
- */
-async function pkgExec(modFunc: string, args: string[]) {
-	const [modPath, funcName] = modFunc.split('::');
-	let replacedModPath = modPath;
-	const match = modPath
-		.replace(/\\/g, '/')
-		.match(/\/snapshot\/balena-cli\/(.+)/);
-	if (match) {
-		replacedModPath = `../${match[1]}`;
-	}
-	process.argv = [process.argv[0], process.argv[1], ...args];
 	try {
-		const mod: any = await import(replacedModPath);
-		if (funcName) {
-			await mod[funcName](...args);
+		const { normalizeEnvVars, pkgExec } = await import('./utils/bootstrap');
+		normalizeEnvVars();
+
+		// The 'pkgExec' special/internal command provides a Node.js interpreter
+		// for use of the standalone zip package. See pkgExec function.
+		if (cliArgs.length > 3 && cliArgs[2] === 'pkgExec') {
+			return pkgExec(cliArgs[3], cliArgs.slice(4));
 		}
+
+		await init();
+
+		const { preparseArgs, checkDeletedCommand } = await import('./preparser');
+
+		// Look for commands that have been removed and if so, exit with a notice
+		checkDeletedCommand(cliArgs.slice(2));
+
+		const args = await preparseArgs(cliArgs);
+		await oclifRun(args, options);
 	} catch (err) {
-		console.error(`Error executing pkgExec "${modFunc}" [${args.join()}]`);
-		console.error(err);
+		await (await import('./errors')).handleError(err);
+	} finally {
+		// Windows fix: reading from stdin prevents the process from exiting
+		process.stdin.pause();
 	}
 }

lib/auth/index.ts

@@ -22,7 +22,7 @@ import { LoginServer } from './server';
  */
 
 /**
- * @summary Login to balenaCLI using the web dashboard
+ * @summary Login to the balena CLI using the web dashboard
  * @function
  * @public
  *

lib/auth/pages/error.ejs

@@ -3,7 +3,7 @@
 <head>
 	<meta charset="utf-8">
 	<meta http-equiv="x-ua-compatible" content="ie=edge">
-	<title>balenaCLI - Error</title>
+	<title>balena CLI - Error</title>
 	<meta name="description" content="">
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<link rel="stylesheet" type="text/css" href="./static/style.css" inline>
@@ -13,7 +13,7 @@
 		<img class="icon" src="./static/images/sad.png" inline>
 		<h1>Something went wrong</h1>
 		<br>
-		<p>The balenaCLI login was not successful.</p>
+		<p>The balena CLI login was not successful.</p>
 		<br>
 		<br>
 		<a href="https://forums.balena.io/" class="button danger">Get help in our forums</a>

lib/auth/pages/success.ejs

@@ -3,7 +3,7 @@
 <head>
 	<meta charset="utf-8">
 	<meta http-equiv="x-ua-compatible" content="ie=edge">
-	<title>balenaCLI - Success</title>
+	<title>balena CLI - Success</title>
 	<meta name="description" content="">
 	<meta name="viewport" content="width=device-width, initial-scale=1">
 	<link rel="stylesheet" type="text/css" href="./static/style.css" inline>
@@ -13,7 +13,7 @@
 		<img class="icon" src="./static/images/happy.png" inline>
 		<h1>Success!</h1>
 		<br>
-		<p>The balenaCLI login was successful.</p>
+		<p>The balena CLI login was successful.</p>
 		<p>You may now close this page and return to the command prompt.</p>
 	</div>
 </body>

lib/commands/app/index.ts

@@ -19,8 +19,7 @@ import { flags } from '@oclif/command';
 import Command from '../../command';
 import * as cf from '../../utils/common-flags';
 import { getBalenaSdk, getVisuals, stripIndent } from '../../utils/lazy';
-import { tryAsInteger } from '../../utils/validation';
-import { Release } from 'balena-sdk';
+import type { Release } from 'balena-sdk';
 
 interface FlagsDef {
 	help: void;
@@ -58,15 +57,14 @@ export default class AppCmd extends Command {
 	public async run() {
 		const { args: params } = this.parse<FlagsDef, ArgsDef>(AppCmd);
 
-		const application = (await getBalenaSdk().models.application.get(
-			tryAsInteger(params.name),
-			{
-				$expand: {
-					is_for__device_type: { $select: 'slug' },
-					should_be_running__release: { $select: 'commit' },
-				},
+		const { getApplication } = await import('../../utils/sdk');
+
+		const application = (await getApplication(getBalenaSdk(), params.name, {
+			$expand: {
+				is_for__device_type: { $select: 'slug' },
+				should_be_running__release: { $select: 'commit' },
 			},
-		)) as ApplicationWithDeviceType & {
+		})) as ApplicationWithDeviceType & {
 			should_be_running__release: [Release?];
 		};
 

lib/commands/app/purge.ts

@@ -0,0 +1,82 @@
+/**
+ * @license
+ * Copyright 2016-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { flags } from '@oclif/command';
+import Command from '../../command';
+import * as cf from '../../utils/common-flags';
+import { getBalenaSdk, stripIndent } from '../../utils/lazy';
+import { tryAsInteger } from '../../utils/validation';
+
+interface FlagsDef {
+	help: void;
+}
+
+interface ArgsDef {
+	name: string;
+}
+
+export default class AppRestartCmd extends Command {
+	public static description = stripIndent`
+		Purge data from an application.
+
+		Purge data from all devices belonging to an application.
+		This will clear the application's /data directory.
+`;
+	public static examples = ['$ balena app purge MyApp'];
+
+	public static args = [
+		{
+			name: 'name',
+			description: 'application name or numeric ID',
+			required: true,
+		},
+	];
+
+	public static usage = 'app purge <name>';
+
+	public static flags: flags.Input<FlagsDef> = {
+		help: cf.help,
+	};
+
+	public static authenticated = true;
+
+	public async run() {
+		const { args: params } = this.parse<FlagsDef, ArgsDef>(AppRestartCmd);
+
+		const balena = getBalenaSdk();
+
+		// balena.models.application.purge only accepts a numeric id
+		// so we must first fetch the app to get it's id, if we have been given a name
+		let nameOrId = tryAsInteger(params.name);
+
+		if (typeof nameOrId === 'string') {
+			const app = await balena.models.application.get(nameOrId);
+			nameOrId = app.id;
+		}
+
+		try {
+			await balena.models.application.purge(nameOrId);
+		} catch (e) {
+			if (e.message.toLowerCase().includes('no online device(s) found')) {
+				// application.purge throws an error if no devices are online
+				// ignore in this case.
+			} else {
+				throw e;
+			}
+		}
+	}
+}

lib/commands/app/rename.ts

@@ -69,12 +69,13 @@ export default class AppRenameCmd extends Command {
 		const { args: params } = this.parse<FlagsDef, ArgsDef>(AppRenameCmd);
 
 		const { ExpectedError, instanceOf } = await import('../../errors');
+		const { getApplication } = await import('../../utils/sdk');
 		const balena = getBalenaSdk();
 
 		// Get app
 		let app;
 		try {
-			app = await balena.models.application.get(params.name, {
+			app = await getApplication(balena, params.name, {
 				$expand: {
 					application_type: {
 						$select: ['is_legacy'],

lib/commands/app/restart.ts

@@ -33,7 +33,7 @@ export default class AppRestartCmd extends Command {
 	public static description = stripIndent`
 		Restart an application.
 
-		Restart all devices that belongs to a certain application.
+		Restart all devices belonging to an application.
 `;
 	public static examples = ['$ balena app restart MyApp'];
 

lib/commands/build.ts

@@ -22,8 +22,8 @@ import * as compose from '../utils/compose';
 import type { Application, ApplicationType, BalenaSDK } from 'balena-sdk';
 import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import type { ComposeCliFlags, ComposeOpts } from '../utils/compose-types';
-import { composeCliFlags } from '../utils/compose_ts';
-import type { DockerCliFlags } from '../utils/docker';
+import { buildProject, composeCliFlags } from '../utils/compose_ts';
+import type { BuildOpts, DockerCliFlags } from '../utils/docker';
 import { dockerCliFlags } from '../utils/docker';
 
 interface FlagsDef extends ComposeCliFlags, DockerCliFlags {
@@ -219,7 +219,7 @@ ${dockerignoreHelp}
 			arch: string;
 			deviceType: string;
 			buildEmulated: boolean;
-			buildOpts: any;
+			buildOpts: BuildOpts;
 		},
 	) {
 		const { loadProject } = await import('../utils/compose_ts');
@@ -238,21 +238,21 @@ ${dockerignoreHelp}
 			);
 		}
 
-		await compose.buildProject(
+		await buildProject({
 			docker,
 			logger,
-			project.path,
-			project.name,
-			project.composition,
-			opts.arch,
-			opts.deviceType,
-			opts.buildEmulated,
-			opts.buildOpts,
-			composeOpts.inlineLogs,
-			composeOpts.convertEol,
-			composeOpts.dockerfilePath,
-			composeOpts.nogitignore,
-			composeOpts.multiDockerignore,
-		);
+			projectPath: project.path,
+			projectName: project.name,
+			composition: project.composition,
+			arch: opts.arch,
+			deviceType: opts.deviceType,
+			emulated: opts.buildEmulated,
+			buildOpts: opts.buildOpts,
+			inlineLogs: composeOpts.inlineLogs,
+			convertEol: composeOpts.convertEol,
+			dockerfilePath: composeOpts.dockerfilePath,
+			nogitignore: composeOpts.nogitignore,
+			multiDockerignore: composeOpts.multiDockerignore,
+		});
 	}
 }

lib/commands/config/generate.ts

@@ -19,7 +19,7 @@ import { flags } from '@oclif/command';
 import Command from '../../command';
 import * as cf from '../../utils/common-flags';
 import { getBalenaSdk, getCliForm, stripIndent } from '../../utils/lazy';
-import type { Application, PineDeferred } from 'balena-sdk';
+import type { PineDeferred } from 'balena-sdk';
 
 interface FlagsDef {
 	version: string; // OS version
@@ -126,38 +126,52 @@ export default class ConfigGenerateCmd extends Command {
 	public async run() {
 		const { flags: options } = this.parse<FlagsDef, {}>(ConfigGenerateCmd);
 
+		const { getApplication } = await import('../../utils/sdk');
+
 		const balena = getBalenaSdk();
 
 		await this.validateOptions(options);
 
-		let deviceType = options.deviceType;
-		// Get device | application
-		let resource;
+		let resourceDeviceType: string;
+		let application: ApplicationWithDeviceType | null = null;
+		let device:
+			| (DeviceWithDeviceType & { belongs_to__application: PineDeferred })
+			| null = null;
 		if (options.device != null) {
 			const { tryAsInteger } = await import('../../utils/validation');
-			resource = (await balena.models.device.get(tryAsInteger(options.device), {
-				$expand: {
-					is_of__device_type: { $select: 'slug' },
-				},
-			})) as DeviceWithDeviceType & { belongs_to__application: PineDeferred };
-			deviceType = deviceType || resource.is_of__device_type[0].slug;
+			const rawDevice = await balena.models.device.get(
+				tryAsInteger(options.device),
+				{ $expand: { is_of__device_type: { $select: 'slug' } } },
+			);
+			if (!rawDevice.belongs_to__application) {
+				const { ExpectedError } = await import('../../errors');
+				throw new ExpectedError(stripIndent`
+					Device ${options.device} does not appear to belong to an accessible application.
+					Try with a different device, or use '--application' instead of '--device'.`);
+			}
+			device = rawDevice as DeviceWithDeviceType & {
+				belongs_to__application: PineDeferred;
+			};
+			resourceDeviceType = device.is_of__device_type[0].slug;
 		} else {
-			resource = (await balena.models.application.get(options.application!, {
+			application = (await getApplication(balena, options.application!, {
 				$expand: {
 					is_for__device_type: { $select: 'slug' },
 				},
 			})) as ApplicationWithDeviceType;
-			deviceType = deviceType || resource.is_for__device_type[0].slug;
+			resourceDeviceType = application.is_for__device_type[0].slug;
 		}
 
+		const deviceType = options.deviceType || resourceDeviceType;
+
 		const deviceManifest = await balena.models.device.getManifestBySlug(
-			deviceType!,
+			deviceType,
 		);
 
 		// Check compatibility if application and deviceType provided
 		if (options.application && options.deviceType) {
 			const appDeviceManifest = await balena.models.device.getManifestBySlug(
-				deviceType!,
+				resourceDeviceType,
 			);
 
 			const helpers = await import('../../utils/helpers');
@@ -184,18 +198,15 @@ export default class ConfigGenerateCmd extends Command {
 		);
 
 		let config;
-		if ('uuid' in resource && resource.uuid != null) {
+		if (device) {
 			config = await generateDeviceConfig(
-				resource,
+				device,
 				options.deviceApiKey || options['generate-device-api-key'] || undefined,
 				answers,
 			);
-		} else {
+		} else if (application) {
 			answers.deviceType = deviceType;
-			config = await generateApplicationConfig(
-				resource as Application,
-				answers,
-			);
+			config = await generateApplicationConfig(application, answers);
 		}
 
 		// Output

lib/commands/deploy.ts

@@ -16,14 +16,24 @@
  */
 
 import { flags } from '@oclif/command';
+import type { ImageDescriptor } from 'resin-compose-parse';
+
 import Command from '../command';
 import { ExpectedError } from '../errors';
 import { getBalenaSdk, getChalk } from '../utils/lazy';
 import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
 import * as compose from '../utils/compose';
-import type { ComposeCliFlags, ComposeOpts } from '../utils/compose-types';
+import type {
+	BuiltImage,
+	ComposeCliFlags,
+	ComposeOpts,
+} from '../utils/compose-types';
 import type { DockerCliFlags } from '../utils/docker';
-import { composeCliFlags } from '../utils/compose_ts';
+import {
+	buildProject,
+	composeCliFlags,
+	isBuildConfig,
+} from '../utils/compose_ts';
 import { dockerCliFlags } from '../utils/docker';
 import type { Application, ApplicationType, DeviceType } from 'balena-sdk';
 
@@ -214,22 +224,21 @@ ${dockerignoreHelp}
 			}
 
 			// find which services use images that already exist locally
-			let servicesToSkip = await Promise.all(
-				project.descriptors.map(async function (d: any) {
+			let servicesToSkip: string[] = await Promise.all(
+				project.descriptors.map(async function (d: ImageDescriptor) {
 					// unconditionally build (or pull) if explicitly requested
 					if (opts.shouldPerformBuild) {
-						return d;
+						return '';
 					}
 					try {
 						await docker
-							.getImage(
-								(typeof d.image === 'string' ? d.image : d.image.tag) || '',
-							)
+							.getImage((isBuildConfig(d.image) ? d.image.tag : d.image) || '')
 							.inspect();
 
 						return d.serviceName;
 					} catch {
 						// Ignore
+						return '';
 					}
 				}),
 			);
@@ -243,35 +252,35 @@ ${dockerignoreHelp}
 				compositionToBuild.services,
 				servicesToSkip,
 			);
-			let builtImagesByService: Dictionary<any> = {};
+			let builtImagesByService: Dictionary<BuiltImage> = {};
 			if (_.size(compositionToBuild.services) === 0) {
 				logger.logInfo(
 					'Everything is up to date (use --build to force a rebuild)',
 				);
 			} else {
-				const builtImages = await compose.buildProject(
+				const builtImages = await buildProject({
 					docker,
 					logger,
-					project.path,
-					project.name,
-					compositionToBuild,
-					opts.app.arch,
-					(opts.app?.is_for__device_type as DeviceType[])?.[0].slug,
-					opts.buildEmulated,
-					opts.buildOpts,
-					composeOpts.inlineLogs,
-					composeOpts.convertEol,
-					composeOpts.dockerfilePath,
-					composeOpts.nogitignore,
-					composeOpts.multiDockerignore,
-				);
+					projectPath: project.path,
+					projectName: project.name,
+					composition: compositionToBuild,
+					arch: opts.app.arch,
+					deviceType: (opts.app?.is_for__device_type as DeviceType[])?.[0].slug,
+					emulated: opts.buildEmulated,
+					buildOpts: opts.buildOpts,
+					inlineLogs: composeOpts.inlineLogs,
+					convertEol: composeOpts.convertEol,
+					dockerfilePath: composeOpts.dockerfilePath,
+					nogitignore: composeOpts.nogitignore,
+					multiDockerignore: composeOpts.multiDockerignore,
+				});
 				builtImagesByService = _.keyBy(builtImages, 'serviceName');
 			}
-			const images = project.descriptors.map(
+			const images: BuiltImage[] = project.descriptors.map(
 				(d) =>
 					builtImagesByService[d.serviceName] ?? {
 						serviceName: d.serviceName,
-						name: typeof d.image === 'string' ? d.image : d.image.tag,
+						name: (isBuildConfig(d.image) ? d.image.tag : d.image) || '',
 						logs: 'Build skipped; image for service already exists.',
 						props: {},
 					},

lib/commands/device/init.ts

@@ -84,18 +84,19 @@ export default class DeviceInitCmd extends Command {
 		const tmp = await import('tmp');
 		const tmpNameAsync = promisify(tmp.tmpName);
 		tmp.setGracefulCleanup();
-		const balena = getBalenaSdk();
 		const { downloadOSImage } = await import('../../utils/cloud');
-		const Logger = await import('../../utils/logger');
+		const { getApplication } = await import('../../utils/sdk');
 
-		const logger = Logger.getLogger();
+		const logger = await Command.getLogger();
+		const balena = getBalenaSdk();
 
 		// Consolidate application options
 		options.application = options.application || options.app;
 		delete options.app;
 
 		// Get application and
-		const application = (await balena.models.application.get(
+		const application = (await getApplication(
+			balena,
 			options['application'] ||
 				(await (await import('../../utils/patterns')).selectApplication()),
 			{

lib/commands/device/purge.ts

@@ -0,0 +1,80 @@
+/**
+ * @license
+ * Copyright 2016-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { flags } from '@oclif/command';
+import type { IArg } from '@oclif/parser/lib/args';
+import Command from '../../command';
+import * as cf from '../../utils/common-flags';
+import { getBalenaSdk, getCliUx, stripIndent } from '../../utils/lazy';
+
+interface FlagsDef {
+	help: void;
+}
+
+interface ArgsDef {
+	uuid: string;
+}
+
+export default class DevicePurgeCmd extends Command {
+	public static description = stripIndent`
+		Purge application data from a device.
+
+		Purge application data from a device.
+		This will clear the application's /data directory.
+
+		Multiple devices may be specified with a comma-separated list
+		of values (no spaces).
+		`;
+	public static examples = [
+		'$ balena device purge 23c73a1',
+		'$ balena device purge 55d43b3,23c73a1',
+	];
+
+	public static usage = 'device purge <uuid>';
+
+	public static args: Array<IArg<any>> = [
+		{
+			name: 'uuid',
+			description: 'comma-separated list (no blank spaces) of device UUIDs',
+			required: true,
+		},
+	];
+
+	public static flags: flags.Input<FlagsDef> = {
+		help: cf.help,
+	};
+
+	public static authenticated = true;
+
+	public async run() {
+		const { args: params } = this.parse<FlagsDef, ArgsDef>(DevicePurgeCmd);
+
+		const { tryAsInteger } = await import('../../utils/validation');
+		const balena = getBalenaSdk();
+		const ux = getCliUx();
+
+		const deviceIds = params.uuid.split(',').map((id) => {
+			return tryAsInteger(id);
+		});
+
+		for (const deviceId of deviceIds) {
+			ux.action.start(`Purging data from device ${deviceId}`);
+			await balena.models.device.purge(deviceId);
+			ux.action.stop();
+		}
+	}
+}

lib/commands/device/register.ts

@@ -20,7 +20,6 @@ import type { IArg } from '@oclif/parser/lib/args';
 import Command from '../../command';
 import * as cf from '../../utils/common-flags';
 import { getBalenaSdk, stripIndent } from '../../utils/lazy';
-import { tryAsInteger } from '../../utils/validation';
 
 interface FlagsDef {
 	uuid?: string;
@@ -46,7 +45,6 @@ export default class DeviceRegisterCmd extends Command {
 		{
 			name: 'application',
 			description: 'the name or id of application to register device with',
-			parse: (app) => tryAsInteger(app),
 			required: true,
 		},
 	];
@@ -68,9 +66,11 @@ export default class DeviceRegisterCmd extends Command {
 			DeviceRegisterCmd,
 		);
 
+		const { getApplication } = await import('../../utils/sdk');
+
 		const balena = getBalenaSdk();
 
-		const application = await balena.models.application.get(params.application);
+		const application = await getApplication(balena, params.application);
 		const uuid = options.uuid ?? balena.models.device.generateUniqueKey();
 
 		console.info(`Registering to ${application.app_name}: ${uuid}`);

lib/commands/devices/index.ts

@@ -25,7 +25,8 @@ import type { Application } from 'balena-sdk';
 
 interface ExtendedDevice extends DeviceWithDeviceType {
 	dashboard_url?: string;
-	application_name?: string;
+	application_name?: string | null;
+	device_type?: string | null;
 }
 
 interface FlagsDef {
@@ -45,8 +46,9 @@ export default class DevicesCmd extends Command {
 
 		The --json option is recommended when scripting the output of this command,
 		because field names are less likely to change in JSON format and because it
-		better represents data types like arrays and empty strings. The 'jq' utility
-		may also be helpful in shell scripts (https://stedolan.github.io/jq/manual/).
+		better represents data types like arrays, empty strings and null values.
+		The 'jq' utility may be helpful for querying JSON fields in shell scripts
+		(https://stedolan.github.io/jq/manual/).
 	`;
 	public static examples = [
 		'$ balena devices',
@@ -97,14 +99,11 @@ export default class DevicesCmd extends Command {
 			device.dashboard_url = balena.models.device.getDashboardUrl(device.uuid);
 
 			const belongsToApplication = device.belongs_to__application as Application[];
-			device.application_name = belongsToApplication?.[0]
-				? belongsToApplication[0].app_name
-				: 'N/a';
+			device.application_name = belongsToApplication?.[0]?.app_name || null;
 
 			device.uuid = device.uuid.slice(0, 7);
 
-			// @ts-ignore
-			device.device_type = device.is_of__device_type[0].slug;
+			device.device_type = device.is_of__device_type?.[0]?.slug || null;
 			return device;
 		});
 
@@ -120,8 +119,8 @@ export default class DevicesCmd extends Command {
 			'os_version',
 			'dashboard_url',
 		];
+		const _ = await import('lodash');
 		if (options.json) {
-			const _ = await import('lodash');
 			console.log(
 				JSON.stringify(
 					devices.map((device) => _.pick(device, fields)),
@@ -130,7 +129,12 @@ export default class DevicesCmd extends Command {
 				),
 			);
 		} else {
-			console.log(getVisuals().table.horizontal(devices, fields));
+			console.log(
+				getVisuals().table.horizontal(
+					devices.map((dev) => _.mapValues(dev, (val) => val ?? 'N/a')),
+					fields,
+				),
+			);
 		}
 	}
 }

lib/commands/envs.ts

@@ -161,7 +161,7 @@ export default class EnvsCmd extends Command {
 			? {
 					all: flags.boolean({
 						description: stripIndent`
-				No-op since balenaCLI v12.0.0.`,
+				No-op since balena CLI v12.0.0.`,
 						hidden: true,
 					}),
 			  }

lib/commands/login.ts

@@ -132,7 +132,7 @@ export default class LoginCmd extends Command {
 
 		console.log(messages.balenaAsciiArt);
 		console.log(`\nLogging in to ${balenaUrl}`);
-		await this.doLogin(options, params.token);
+		await this.doLogin(options, balenaUrl, params.token);
 
 		const username = await balena.auth.whoami();
 
@@ -146,7 +146,11 @@ Find out about the available commands by running:
 ${messages.reachingOut}`);
 	}
 
-	async doLogin(loginOptions: FlagsDef, token?: string): Promise<void> {
+	async doLogin(
+		loginOptions: FlagsDef,
+		balenaUrl: string = 'balena-cloud.com',
+		token?: string,
+	): Promise<void> {
 		// Token
 		if (loginOptions.token) {
 			if (!token) {
@@ -178,8 +182,8 @@ ${messages.reachingOut}`);
 			// User had not selected login preference, prompt interactively
 			const loginType = await patterns.askLoginType();
 			if (loginType === 'register') {
-				const signupUrl = 'https://dashboard.balena-cloud.com/signup';
 				const open = await import('open');
+				const signupUrl = `https://dashboard.${balenaUrl}/signup`;
 				open(signupUrl, { wait: false });
 				throw new ExpectedError(`Please sign up at ${signupUrl}`);
 			}

lib/commands/os/configure.ts

@@ -90,7 +90,7 @@ export default class OsConfigureCmd extends Command {
 
 		Note: This command is currently not supported on Windows natively. Windows users
 		are advised to install the Windows Subsystem for Linux (WSL) with Ubuntu, and use
-		the Linux release of balenaCLI:
+		the Linux release of the balena CLI:
 		https://docs.microsoft.com/en-us/windows/wsl/about
 	`;
 	public static examples = [
@@ -184,6 +184,8 @@ export default class OsConfigureCmd extends Command {
 			'../../utils/config'
 		);
 		const helpers = await import('../../utils/helpers');
+		const { getApplication } = await import('../../utils/sdk');
+
 		let app: ApplicationWithDeviceType | undefined;
 		let device;
 		let deviceTypeSlug: string;
@@ -199,7 +201,7 @@ export default class OsConfigureCmd extends Command {
 			};
 			deviceTypeSlug = device.is_of__device_type[0].slug;
 		} else {
-			app = (await balena.models.application.get(options.application!, {
+			app = (await getApplication(balena, options.application!, {
 				$expand: {
 					is_for__device_type: { $select: 'slug' },
 				},
@@ -297,11 +299,11 @@ async function validateOptions(options: FlagsDef) {
 		throw new ExpectedError(stripIndent`
 			Unsupported platform error: the 'balena os configure' command currently requires
 			the Windows Subsystem for Linux in order to run on Windows. It was tested with
-			the Ubuntu 18.04 distribution from the Microsoft Store. With WSL, a balenaCLI
+			the Ubuntu 18.04 distribution from the Microsoft Store. With WSL, a balena CLI
 			release for Linux (rather than Windows) should be installed: for example, the
 			standalone zip package for Linux. (It is possible to have both a Windows CLI
 			release and a Linux CLI release installed simultaneously.) For more information
-			on WSL and the balenaCLI installation options, please check:
+			on WSL and the balena CLI installation options, please check:
 			- https://docs.microsoft.com/en-us/windows/wsl/about
 			- https://github.com/balena-io/balena-cli/blob/master/INSTALL.md
 		`);

lib/commands/os/download.ts

@@ -46,11 +46,15 @@ export default class OsDownloadCmd extends Command {
 
 		You can pass \`--version menu\` to pick the OS version from the interactive menu
 		of all available versions.
+
+		To download a development image append \`.dev\` to the version or select from
+		the interactive menu.
 `;
 	public static examples = [
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img',
-		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 1.24.1',
-		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^1.20.0',
+		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1',
+		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version 2.60.1+rev1.dev',
+		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version ^2.60.0',
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version latest',
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version default',
 		'$ balena os download raspberrypi3 -o ../foo/bar/raspberry-pi.img --version menu',

lib/commands/preload.ts

@@ -43,7 +43,7 @@ interface FlagsDef extends DockerConnectionCliFlags {
 	'splash-image'?: string;
 	'dont-check-arch': boolean;
 	'pin-device-to-release': boolean;
-	'add-certificate'?: string;
+	'add-certificate'?: string[];
 	help: void;
 }
 
@@ -84,7 +84,7 @@ export default class PreloadCmd extends Command {
 
 	public static flags: flags.Input<FlagsDef> = {
 		app: flags.string({
-			description: 'name, slug or numeric ID of the application to preload',
+			description: 'name of the application to preload',
 			char: 'a',
 		}),
 		commit: flags.string({
@@ -115,6 +115,7 @@ The file name must end with '.crt' and must not be already contained in the prel
 /etc/ssl/certs folder.
 Can be repeated to add multiple certificates.\
 `,
+			multiple: true,
 		}),
 		...dockerConnectionCliFlags,
 		// Redefining --dockerPort here (defined already in dockerConnectionCliFlags)
@@ -201,14 +202,7 @@ Can be repeated to add multiple certificates.\
 			);
 		}
 
-		let certificates: string[];
-		if (Array.isArray(options['add-certificate'])) {
-			certificates = options['add-certificate'];
-		} else if (options['add-certificate'] === undefined) {
-			certificates = [];
-		} else {
-			certificates = [options['add-certificate']];
-		}
+		const certificates: string[] = options['add-certificate'] || [];
 		for (const certificate of certificates) {
 			if (!certificate.endsWith('.crt')) {
 				throw new ExpectedError('Certificate file name must end with ".crt"');
@@ -463,10 +457,12 @@ Would you like to disable automatic updates for this application now?\
 		});
 	}
 
-	getAppWithReleases(balenaSdk: BalenaSDK, appId: string | number) {
-		return balenaSdk.models.application.get(appId, {
+	async getAppWithReleases(balenaSdk: BalenaSDK, appId: string | number) {
+		const { getApplication } = await import('../utils/sdk');
+
+		return (await getApplication(balenaSdk, appId, {
 			$expand: this.applicationExpandOptions,
-		}) as Promise<Application & { should_be_running__release: [Release?] }>;
+		})) as Application & { should_be_running__release: [Release?] };
 	}
 
 	async prepareAndPreload(

lib/commands/push.ts

@@ -197,7 +197,7 @@ export default class PushCmd extends Command {
 			multiple: true,
 		}),
 		'convert-eol': flags.boolean({
-			description: 'No-op and deprecated since balenaCLI v12.0.0',
+			description: 'No-op and deprecated since balena CLI v12.0.0',
 			char: 'l',
 			hidden: true,
 		}),
@@ -212,7 +212,7 @@ export default class PushCmd extends Command {
 		}),
 		nogitignore: flags.boolean({
 			description:
-				'No-op (default behavior) since balenaCLI v12.0.0. See "balena help push".',
+				'No-op (default behavior) since balena CLI v12.0.0. See "balena help push".',
 			char: 'G',
 			hidden: true,
 		}),

lib/commands/scan.ts

@@ -19,9 +19,10 @@ import { flags } from '@oclif/command';
 import type { LocalBalenaOsDevice } from 'balena-sync';
 import Command from '../command';
 import * as cf from '../utils/common-flags';
-import { getVisuals, stripIndent } from '../utils/lazy';
+import { getCliUx, stripIndent } from '../utils/lazy';
 
 interface FlagsDef {
+	json?: boolean;
 	verbose: boolean;
 	timeout?: number;
 	help: void;
@@ -32,6 +33,11 @@ export default class ScanCmd extends Command {
 		Scan for balenaOS devices on your local network.
 
 		Scan for balenaOS devices on your local network.
+
+		The output includes device information collected through balenaEngine for
+		devices running a development image of balenaOS. Devices running a production
+		image do not expose balenaEngine (on TCP port 2375), which is why less
+		information is printed about them.
 `;
 
 	public static examples = [
@@ -53,6 +59,10 @@ export default class ScanCmd extends Command {
 			description: 'scan timeout in seconds',
 		}),
 		help: cf.help,
+		json: flags.boolean({
+			char: 'j',
+			description: 'produce JSON output instead of tabular output',
+		}),
 	};
 
 	public static primary = true;
@@ -60,10 +70,8 @@ export default class ScanCmd extends Command {
 
 	public async run() {
 		const _ = await import('lodash');
-		const { SpinnerPromise } = getVisuals();
 		const { discover } = await import('balena-sync');
 		const prettyjson = await import('prettyjson');
-		const { ExpectedError } = await import('../errors');
 		const dockerUtils = await import('../utils/docker');
 
 		const dockerPort = 2375;
@@ -75,37 +83,54 @@ export default class ScanCmd extends Command {
 			options.timeout != null ? options.timeout * 1000 : undefined;
 
 		// Find active local devices
-		const activeLocalDevices: LocalBalenaOsDevice[] = await new SpinnerPromise({
-			promise: discover.discoverLocalBalenaOsDevices(discoverTimeout),
-			startMessage: 'Scanning for local balenaOS devices..',
-			stopMessage: 'Reporting scan results',
-		}).filter(async ({ address }: { address: string }) => {
-			const docker = dockerUtils.createClient({
-				host: address,
-				port: dockerPort,
-				timeout: dockerTimeout,
-			}) as any;
-			try {
-				await docker.pingAsync();
-				return true;
-			} catch (err) {
-				return false;
-			}
-		});
+		const ux = getCliUx();
+		ux.action.start('Scanning for local balenaOS devices');
 
-		// Exit with message if no devices found
-		if (_.isEmpty(activeLocalDevices)) {
-			// TODO: Consider whether this should really be an error
-			throw new ExpectedError(
-				process.platform === 'win32'
-					? ScanCmd.noDevicesFoundMessage + ScanCmd.windowsTipMessage
-					: ScanCmd.noDevicesFoundMessage,
-			);
-		}
+		const localDevices: LocalBalenaOsDevice[] = await discover.discoverLocalBalenaOsDevices(
+			discoverTimeout,
+		);
+		const engineReachableDevices: boolean[] = await Promise.all(
+			localDevices.map(async ({ address }: { address: string }) => {
+				const docker = dockerUtils.createClient({
+					host: address,
+					port: dockerPort,
+					timeout: dockerTimeout,
+				}) as any;
+				try {
+					await docker.pingAsync();
+					return true;
+				} catch (err) {
+					return false;
+				}
+			}),
+		);
+
+		const developmentDevices: LocalBalenaOsDevice[] = localDevices.filter(
+			(_localDevice, index) => engineReachableDevices[index],
+		);
+
+		const productionDevices = _.differenceWith(
+			localDevices,
+			developmentDevices,
+			_.isEqual,
+		);
+
+		const productionDevicesInfo = _.map(
+			productionDevices,
+			(device: LocalBalenaOsDevice) => {
+				return {
+					host: device.host,
+					address: device.address,
+					osVariant: 'production',
+					dockerInfo: undefined,
+					dockerVersion: undefined,
+				};
+			},
+		);
 
 		// Query devices for info
 		const devicesInfo = await Promise.all(
-			activeLocalDevices.map(async ({ host, address }) => {
+			developmentDevices.map(async ({ host, address }) => {
 				const docker = dockerUtils.createClient({
 					host: address,
 					port: dockerPort,
@@ -118,12 +143,15 @@ export default class ScanCmd extends Command {
 				return {
 					host,
 					address,
+					osVariant: 'development',
 					dockerInfo,
 					dockerVersion,
 				};
 			}),
 		);
 
+		ux.action.stop('Reporting scan results');
+
 		// Reduce properties if not --verbose
 		if (!options.verbose) {
 			devicesInfo.forEach((d: any) => {
@@ -136,8 +164,22 @@ export default class ScanCmd extends Command {
 			});
 		}
 
+		const cmdOutput = productionDevicesInfo.concat(devicesInfo);
+
 		// Output results
-		console.log(prettyjson.render(devicesInfo, { noColor: true }));
+		if (!options.json && cmdOutput.length === 0) {
+			console.error(
+				process.platform === 'win32'
+					? ScanCmd.noDevicesFoundMessage + ScanCmd.windowsTipMessage
+					: ScanCmd.noDevicesFoundMessage,
+			);
+			return;
+		}
+		console.log(
+			options.json
+				? JSON.stringify(cmdOutput, null, 4)
+				: prettyjson.render(cmdOutput, { noColor: true }),
+		);
 	}
 
 	protected static dockerInfoProperties = [

lib/commands/settings.ts

@@ -28,7 +28,7 @@ export default class SettingsCmd extends Command {
 	public static description = stripIndent`
 		Print current settings.
 
-		Use this command to display current balenaCLI settings.
+		Use this command to display the current balena CLI settings.
 `;
 	public static examples = ['$ balena settings'];
 

lib/commands/version.ts

@@ -32,9 +32,13 @@ export interface JsonVersions {
 
 export default class VersionCmd extends Command {
 	public static description = stripIndent`
-		Display version information for balenaCLI and/or Node.js.
+		Display version information for the balena CLI and/or Node.js.
 
-		Display version information for balenaCLI and/or Node.js.
+		Display version information for the balena CLI and/or Node.js. Note that the
+		balena CLI executable installers for Windows and macOS, and the standalone
+		zip packages, ship with a built-in copy of Node.js.  In this case, the
+		reported version of Node.js regards this built-in copy, rather than any
+		other \`node\` engine that may also be available on the command prompt.
 
 		The --json option is recommended when scripting the output of this command,
 		because the JSON format is less likely to change and it better represents

lib/errors.ts

@@ -148,6 +148,7 @@ const EXPECTED_ERROR_REGEXES = [
 	/^BalenaDeviceNotFound/, // balena-sdk
 	/^BalenaExpiredToken/, // balena-sdk
 	/^BalenaInvalidDeviceType/, // balena-sdk
+	/Request error: Unauthorized$/, // balena-sdk
 	/^Missing \d+ required arg/, // oclif parser: RequiredArgsError
 	/Missing required flag/, // oclif parser: RequiredFlagError
 	/^Unexpected argument/, // oclif parser: UnexpectedArgsError
@@ -155,6 +156,7 @@ const EXPECTED_ERROR_REGEXES = [
 	/must also be provided when using/, // oclif parser (depends-on)
 	/^Expected an integer/, // oclif parser (flags.integer)
 	/^Flag .* expects a value/, // oclif parser
+	/^Error parsing config file.*balenarc\.yml/,
 ];
 
 // Support unit testing of handleError
@@ -162,6 +164,18 @@ export const getSentry = async function () {
 	return await import('@sentry/node');
 };
 
+async function sentryCaptureException(error: Error) {
+	const Sentry = await getSentry();
+	Sentry.captureException(error);
+	try {
+		await Sentry.close(1000);
+	} catch (e) {
+		if (process.env.DEBUG) {
+			console.error('Timeout reporting error to sentry.io');
+		}
+	}
+}
+
 export async function handleError(error: Error) {
 	// Set appropriate exitCode
 	process.exitCode =
@@ -189,15 +203,10 @@ export async function handleError(error: Error) {
 		printErrorMessage(message.join('\n'));
 
 		// Report "unexpected" errors via Sentry.io
-		const Sentry = await getSentry();
-		Sentry.captureException(error);
-		try {
-			await Sentry.close(1000);
-		} catch (e) {
-			if (process.env.DEBUG) {
-				console.error('Timeout reporting error to sentry.io');
-			}
+		if (!process.env.BALENARC_NO_SENTRY) {
+			await sentryCaptureException(error);
 		}
+
 		// Unhandled/unexpected error: ensure that the process terminates.
 		// The exit error code was set above through `process.exitCode`.
 		process.exit();
@@ -215,7 +224,7 @@ export const printErrorMessage = function (message: string) {
 		console.error(line);
 	});
 
-	console.error(`\n${getHelp}\n`);
+	console.error(`\n${getHelp()}\n`);
 };
 
 export const printExpectedErrorMessage = function (message: string) {

lib/events.ts

@@ -34,7 +34,7 @@ interface CachedUsername {
 }
 
 /**
- * Mixpanel.com analytics tracking (information on balenaCLI usage).
+ * Mixpanel.com analytics tracking (information on balena CLI usage).
  *
  * @param commandSignature A string like, for example:
  *      "push <applicationOrDevice>"
@@ -46,13 +46,19 @@ interface CachedUsername {
  * The username and command signature are also added as extra context
  * information in Sentry.io error reporting, for CLI debugging purposes
  * (mainly unexpected/unhandled exceptions -- see also `lib/errors.ts`).
+ *
+ * For more details on the data collected by balena generally, check this page:
+ * https://www.balena.io/docs/learn/more/collected-data/
  */
 export async function trackCommand(commandSignature: string) {
 	try {
-		const Sentry = await import('@sentry/node');
-		Sentry.configureScope((scope) => {
-			scope.setExtra('command', commandSignature);
-		});
+		let Sentry: typeof import('@sentry/node');
+		if (!process.env.BALENARC_NO_SENTRY) {
+			Sentry = await import('@sentry/node');
+			Sentry.configureScope((scope) => {
+				scope.setExtra('command', commandSignature);
+			});
+		}
 		const settings = await import('balena-settings-client');
 		const balenaUrl = settings.get('balenaUrl') as string;
 
@@ -78,7 +84,7 @@ export async function trackCommand(commandSignature: string) {
 			try {
 				const balena = getBalenaSdk();
 				const $username = await balena.auth.whoami();
-				storage.set('cachedUsername', {
+				await storage.set('cachedUsername', {
 					token,
 					username: $username,
 				} as CachedUsername);
@@ -90,14 +96,19 @@ export async function trackCommand(commandSignature: string) {
 
 		const mixpanel = getMixpanel(balenaUrl);
 
-		Sentry.configureScope((scope) => {
-			scope.setUser({
-				id: username,
-				username,
+		if (!process.env.BALENARC_NO_SENTRY) {
+			Sentry!.configureScope((scope) => {
+				scope.setUser({
+					id: username,
+					username,
+				});
 			});
-		});
-		// Don't actually call mixpanel.track() while running test cases
-		if (!process.env.BALENA_CLI_TEST_TYPE) {
+		}
+		// Don't actually call mixpanel.track() while running test cases, or if suppressed
+		if (
+			!process.env.BALENA_CLI_TEST_TYPE &&
+			!process.env.BALENARC_NO_ANALYTICS
+		) {
 			await mixpanel.track(`[CLI] ${commandSignature}`, {
 				distinct_id: username,
 				version: packageJSON.version,

lib/help.ts

@@ -1,3 +1,19 @@
+/**
+ * @license
+ * Copyright 2017-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 import Help from '@oclif/plugin-help';
 import * as indent from 'indent-string';
 import { getChalk } from './utils/lazy';
@@ -123,16 +139,10 @@ export default class BalenaHelp extends Help {
 		console.log('  --help, -h');
 		console.log('  --debug\n');
 
-		console.log(
-			`For help, visit our support forums: ${chalk.grey(
-				'https://forums.balena.io',
-			)}`,
-		);
-		console.log(
-			`For bug reports or feature requests, see: ${chalk.grey(
-				'https://github.com/balena-io/balena-cli/issues/',
-			)}\n`,
-		);
+		const {
+			reachingOut,
+		} = require('./utils/messages') as typeof import('./utils/messages');
+		console.log(reachingOut);
 	}
 
 	protected formatCommands(commands: any[]): string {

lib/preparser.ts

@@ -70,7 +70,7 @@ export async function preparseArgs(argv: string[]): Promise<string[]> {
 	let args = cmdSlice;
 
 	// Convert space separated subcommands (e.g. `end add`), to colon-separated format (e.g. `env:add`)
-	if (isSubcommand(cmdSlice)) {
+	if (await isSubcommand(cmdSlice)) {
 		// convert space-separated commands to oclif's topic:command syntax
 		args = [cmdSlice[0] + ':' + cmdSlice[1], ...cmdSlice.slice(2)];
 		Logger.command = `${cmdSlice[0]} ${cmdSlice[1]}`;
@@ -136,75 +136,8 @@ export function checkDeletedCommand(argvSlice: string[]): void {
 
 // Check if this is a space separated 'topic command' style command subcommand (e.g. `end add`)
 // by comparing with oclif style colon-separated subcommand list (e.g. `env:add`)
-// TODO: Need to find a way of doing this that does not require maintaining list of IDs
-export function isSubcommand(args: string[]) {
-	return oclifCommandIds.includes(`${args[0] || ''}:${args[1] || ''}`);
+export async function isSubcommand(args: string[]) {
+	const { getCommandIdsFromManifest } = await import('./utils/oclif-utils');
+	const commandIds = await getCommandIdsFromManifest();
+	return commandIds.includes(`${args[0] || ''}:${args[1] || ''}`);
 }
-
-export const oclifCommandIds = [
-	'api-key:generate',
-	'app',
-	'app:create',
-	'app:rename',
-	'app:restart',
-	'app:rm',
-	'apps',
-	'build',
-	'config:generate',
-	'config:inject',
-	'config:read',
-	'config:reconfigure',
-	'config:write',
-	'deploy',
-	'device',
-	'device:identify',
-	'device:init',
-	'device:move',
-	'device:os-update',
-	'device:public-url',
-	'device:reboot',
-	'device:register',
-	'device:rename',
-	'device:restart',
-	'device:rm',
-	'device:shutdown',
-	'devices',
-	'devices:supported',
-	'envs',
-	'env:add',
-	'env:rename',
-	'env:rm',
-	'help',
-	'internal:scandevices',
-	'internal:osinit',
-	'join',
-	'keys',
-	'key',
-	'key:add',
-	'key:rm',
-	'leave',
-	'local:configure',
-	'local:flash',
-	'login',
-	'logout',
-	'logs',
-	'note',
-	'os:build-config',
-	'os:configure',
-	'os:versions',
-	'os:download',
-	'os:initialize',
-	'preload',
-	'push',
-	'scan',
-	'settings',
-	'ssh',
-	'support',
-	'tags',
-	'tag:rm',
-	'tag:set',
-	'tunnel',
-	'util:available-drives',
-	'version',
-	'whoami',
-];

lib/utils/bootstrap.ts

@@ -0,0 +1,103 @@
+/**
+ * @license
+ * Copyright 2019-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * THIS MODULE SHOULD NOT IMPORT / REQUIRE ANYTHING AT THE GLOBAL LEVEL.
+ * It is meant to contain elementary helper functions or classes that
+ * can be used very early on during CLI startup, before anything else
+ * like Sentry error reporting, preparser, oclif parser and the like.
+ */
+
+export class CliSettings {
+	public readonly settings: any;
+	constructor() {
+		this.settings = require('balena-settings-client') as typeof import('balena-settings-client');
+	}
+
+	public get<T>(name: string): T {
+		return this.settings.get(name);
+	}
+
+	/**
+	 * Like settings.get(), but return `undefined` instead of throwing an
+	 * error if the setting is not found / not defined.
+	 */
+	public getCatch<T>(name: string): T | undefined {
+		try {
+			return this.settings.get(name);
+		} catch (err) {
+			if (!/Setting not found/i.test(err.message)) {
+				throw err;
+			}
+		}
+	}
+}
+
+export function parseBoolEnvVar(varName: string): boolean {
+	return !['0', 'no', 'false', '', undefined].includes(
+		process.env[varName]?.toLowerCase(),
+	);
+}
+
+export function normalizeEnvVar(varName: string) {
+	process.env[varName] = parseBoolEnvVar(varName) ? '1' : '';
+}
+
+const bootstrapVars = ['DEBUG', 'BALENARC_NO_SENTRY'];
+
+export function normalizeEnvVars(varNames: string[] = bootstrapVars) {
+	for (const varName of varNames) {
+		normalizeEnvVar(varName);
+	}
+}
+
+/**
+ * Implements the 'pkgExec' command, used as a way to provide a Node.js
+ * interpreter for child_process.spawn()-like operations when the CLI is
+ * executing as a standalone zip package (built-in Node interpreter) and
+ * the system may not have a separate Node.js installation. A present use
+ * case is a patched version of the 'windosu' package that requires a
+ * Node.js interpreter to spawn a privileged child process.
+ *
+ * @param modFunc Path to a JS module that will be executed via require().
+ * The modFunc argument may optionally contain a function name separated
+ * by '::', for example '::main' in:
+ * 'C:\\snapshot\\balena-cli\\node_modules\\windosu\\lib\\pipe.js::main'
+ * in which case that function is executed in the require'd module.
+ * @param args Optional arguments to passed through process.argv and as
+ * arguments to the function specified via modFunc.
+ */
+export async function pkgExec(modFunc: string, args: string[]) {
+	const [modPath, funcName] = modFunc.split('::');
+	let replacedModPath = modPath;
+	const match = modPath
+		.replace(/\\/g, '/')
+		.match(/\/snapshot\/balena-cli\/(.+)/);
+	if (match) {
+		replacedModPath = `../${match[1]}`;
+	}
+	process.argv = [process.argv[0], process.argv[1], ...args];
+	try {
+		const mod: any = await import(replacedModPath);
+		if (funcName) {
+			await mod[funcName](...args);
+		}
+	} catch (err) {
+		console.error(`Error executing pkgExec "${modFunc}" [${args.join()}]`);
+		console.error(err);
+	}
+}

lib/utils/compose-types.d.ts

@@ -30,6 +30,8 @@ export interface BuiltImage {
 		dockerfile?: string;
 		projectType?: string;
 		size?: number;
+		startTime?: Date;
+		endTime?: Date;
 	};
 	serviceName: string;
 }
@@ -64,7 +66,7 @@ export interface ComposeCliFlags {
 	'multi-dockerignore': boolean;
 	nogitignore: boolean;
 	'noparent-check': boolean;
-	'registry-secrets'?: string | RegistrySecrets;
+	'registry-secrets'?: RegistrySecrets;
 	'convert-eol': boolean;
 	'noconvert-eol': boolean;
 	projectName?: string;

lib/utils/compose.js

@@ -85,23 +85,6 @@ export function createProject(composePath, composeStr, projectName = null) {
 	};
 }
 
-/**
- * Create a tar stream out of the local filesystem at the given directory,
- * while optionally applying file filters such as '.dockerignore' and
- * optionally converting text file line endings (CRLF to LF).
- * @param {string} dir Source directory
- * @param {import('./compose-types').TarDirectoryOptions} param
- * @returns {Promise<import('stream').Readable>}
- */
-export function tarDirectory(dir, param) {
-	let { nogitignore = false } = param;
-	if (nogitignore) {
-		return require('./compose_ts').tarDirectory(dir, param);
-	} else {
-		return originalTarDirectory(dir, param);
-	}
-}
-
 /**
  * This is the CLI v10 / v11 "original" tarDirectory function. It is still
  * around for the benefit of the `--gitignore` option, but is expected to be
@@ -110,7 +93,7 @@ export function tarDirectory(dir, param) {
  * @param {import('./compose-types').TarDirectoryOptions} param
  * @returns {Promise<import('stream').Readable>}
  */
-function originalTarDirectory(dir, param) {
+export async function originalTarDirectory(dir, param) {
 	let {
 		preFinalizeCallback = null,
 		convertEol = false,
@@ -185,265 +168,6 @@ function originalTarDirectory(dir, param) {
 		});
 }
 
-/**
- * @param {string} str
- * @param {number} len
- * @returns {string}
- */
-const truncateString = function (str, len) {
-	if (str.length < len) {
-		return str;
-	}
-	str = str.slice(0, len);
-	// return everything up to the last line. this is a cheeky way to avoid
-	// having to deal with splitting the string midway through some special
-	// character sequence.
-	return str.slice(0, str.lastIndexOf('\n'));
-};
-
-const LOG_LENGTH_MAX = 512 * 1024; // 512KB
-
-export function buildProject(
-	docker,
-	logger,
-	projectPath,
-	projectName,
-	composition,
-	arch,
-	deviceType,
-	emulated,
-	buildOpts,
-	inlineLogs,
-	convertEol,
-	dockerfilePath,
-	nogitignore,
-	multiDockerignore,
-) {
-	const Bluebird = require('bluebird');
-	const _ = require('lodash');
-	const humanize = require('humanize');
-	const compose = require('resin-compose-parse');
-	const builder = require('resin-multibuild');
-	const transpose = require('docker-qemu-transpose');
-	const { BALENA_ENGINE_TMP_PATH } = require('../config');
-	const {
-		checkBuildSecretsRequirements,
-		makeBuildTasks,
-	} = require('./compose_ts');
-	const qemu = require('./qemu');
-	const { toPosixPath } = builder.PathUtils;
-
-	logger.logInfo(`Building for ${arch}/${deviceType}`);
-
-	const imageDescriptors = compose.parse(composition);
-	const imageDescriptorsByServiceName = _.keyBy(
-		imageDescriptors,
-		'serviceName',
-	);
-
-	let renderer;
-	if (inlineLogs) {
-		renderer = new BuildProgressInline(
-			logger.streams['build'],
-			imageDescriptors,
-		);
-	} else {
-		const tty = require('./tty')(process.stdout);
-		renderer = new BuildProgressUI(tty, imageDescriptors);
-	}
-	renderer.start();
-
-	return Bluebird.resolve(checkBuildSecretsRequirements(docker, projectPath))
-		.then(() => qemu.installQemuIfNeeded(emulated, logger, arch, docker))
-		.tap(function (needsQemu) {
-			if (!needsQemu) {
-				return;
-			}
-			logger.logInfo('Emulation is enabled');
-			// Copy qemu into all build contexts
-			return Promise.all(
-				imageDescriptors.map(function (d) {
-					if (typeof d.image === 'string' || d.image.context == null) {
-						return;
-					}
-					// external image
-					return qemu.copyQemu(path.join(projectPath, d.image.context), arch);
-				}),
-			);
-		})
-		.then((
-			needsQemu, // Tar up the directory, ready for the build stream
-		) =>
-			Bluebird.resolve(
-				tarDirectory(projectPath, {
-					composition,
-					convertEol,
-					multiDockerignore,
-					nogitignore,
-				}),
-			)
-				.then((tarStream) =>
-					makeBuildTasks(
-						composition,
-						tarStream,
-						{ arch, deviceType },
-						logger,
-						projectName,
-					),
-				)
-				.map(function (/** @type {any} */ task) {
-					const d = imageDescriptorsByServiceName[task.serviceName];
-
-					// multibuild parses the composition internally so any tags we've
-					// set before are lost; re-assign them here
-					task.tag ??= [projectName, task.serviceName].join('_').toLowerCase();
-
-					if (typeof d.image !== 'string' && d.image.context != null) {
-						d.image.tag = task.tag;
-					}
-
-					// configure build opts appropriately
-					task.dockerOpts ??= {};
-
-					_.merge(task.dockerOpts, buildOpts, { t: task.tag });
-					if (typeof d.image !== 'string') {
-						/** @type {any} */
-						const context = d.image.context;
-						if (context?.args != null) {
-							task.dockerOpts.buildargs ??= {};
-							_.merge(task.dockerOpts.buildargs, context.args);
-						}
-					}
-
-					// Get the service-specific log stream
-					// Caveat: `multibuild.BuildTask` defines no `logStream` property
-					// but it's convenient to store it there; it's JS ultimately.
-					task.logStream = renderer.streams[task.serviceName];
-					task.logBuffer = [];
-
-					// Setup emulation if needed
-					if (task.external || !needsQemu) {
-						return [task, null];
-					}
-					const binPath = qemu.qemuPathInContext(
-						path.join(projectPath, task.context ?? ''),
-					);
-					if (task.buildStream == null) {
-						throw new Error(`No buildStream for task '${task.tag}'`);
-					}
-					return transpose
-						.transposeTarStream(
-							task.buildStream,
-							{
-								hostQemuPath: toPosixPath(binPath),
-								containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`,
-								qemuFileMode: 0o555,
-							},
-							dockerfilePath || undefined,
-						)
-						.then((/** @type {any} */ stream) => {
-							task.buildStream = stream;
-						})
-						.return([task, binPath]);
-				}),
-		)
-		.map(function ([task, qemuPath]) {
-			const captureStream = buildLogCapture(task.external, task.logBuffer);
-
-			if (task.external) {
-				// External image -- there's no build to be performed,
-				// just follow pull progress.
-				captureStream.pipe(task.logStream);
-				task.progressHook = pullProgressAdapter(captureStream);
-			} else {
-				task.streamHook = function (stream) {
-					let rawStream;
-					stream = createLogStream(stream);
-					if (qemuPath != null) {
-						const buildThroughStream = transpose.getBuildThroughStream({
-							hostQemuPath: toPosixPath(qemuPath),
-							containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`,
-						});
-						rawStream = stream.pipe(buildThroughStream);
-					} else {
-						rawStream = stream;
-					}
-					// `stream` sends out raw strings in contrast to `task.progressHook`
-					// where we're given objects. capture these strings as they come
-					// before we parse them.
-					return rawStream
-						.pipe(dropEmptyLinesStream())
-						.pipe(captureStream)
-						.pipe(buildProgressAdapter(inlineLogs))
-						.pipe(task.logStream);
-				};
-			}
-			return task;
-		})
-		.then(function (tasks) {
-			logger.logDebug('Prepared tasks; building...');
-			return builder
-				.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH)
-				.then(function (builtImages) {
-					return Promise.all(
-						builtImages.map(function (builtImage) {
-							if (!builtImage.successful) {
-								/** @type {Error & {serviceName?: string}} */
-								const error = builtImage.error ?? new Error();
-								error.serviceName = builtImage.serviceName;
-								throw error;
-							}
-
-							const d = imageDescriptorsByServiceName[builtImage.serviceName];
-							const task = _.find(tasks, {
-								serviceName: builtImage.serviceName,
-							});
-
-							const image = {
-								serviceName: d.serviceName,
-								name: typeof d.image === 'string' ? d.image : d.image.tag,
-								logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX),
-								props: {
-									dockerfile: builtImage.dockerfile,
-									projectType: builtImage.projectType,
-								},
-							};
-
-							// Times here are timestamps, so test whether they're null
-							// before creating a date out of them, as `new Date(null)`
-							// creates a date representing UNIX time 0.
-							if (builtImage.startTime) {
-								image.props.startTime = new Date(builtImage.startTime);
-							}
-							if (builtImage.endTime) {
-								image.props.endTime = new Date(builtImage.endTime);
-							}
-							return docker
-								.getImage(image.name)
-								.inspect()
-								.get('Size')
-								.then((size) => {
-									image.props.size = size;
-								})
-								.return(image);
-						}),
-					);
-				})
-				.then(function (images) {
-					const summary = _(images)
-						.map(({ serviceName, props }) => [
-							serviceName,
-							`Image size: ${humanize.filesize(props.size)}`,
-						])
-						.fromPairs()
-						.value();
-					renderer.end(summary);
-					return images;
-				});
-		})
-		.finally(renderer.end);
-}
-
 /**
  * @param {string} apiEndpoint
  * @param {string} auth
@@ -641,8 +365,7 @@ export const pushAndUpdateServiceImages = function (
 		Promise.all(
 			images.map(({ serviceImage, localImage, props, logs }, index) =>
 				Promise.all([
-					// @ts-ignore
-					localImage.inspect().get('Size'),
+					localImage.inspect().then((img) => img.Size),
 					retry(
 						// @ts-ignore
 						() => progress.push(localImage.name, reporters[index], opts),
@@ -708,102 +431,7 @@ var pushProgressRenderer = function (tty, prefix) {
 	return fn;
 };
 
-var createLogStream = function (input) {
-	const split = require('split');
-	const stripAnsi = require('strip-ansi-stream');
-	return input.pipe(stripAnsi()).pipe(split());
-};
-
-var dropEmptyLinesStream = function () {
-	const through = require('through2');
-	return through(function (data, _enc, cb) {
-		const str = data.toString('utf-8');
-		if (str.trim()) {
-			this.push(str);
-		}
-		return cb();
-	});
-};
-
-var buildLogCapture = function (objectMode, buffer) {
-	const through = require('through2');
-
-	return through({ objectMode }, function (data, _enc, cb) {
-		// data from pull stream
-		if (data.error) {
-			buffer.push(`${data.error}`);
-		} else if (data.progress && data.status) {
-			buffer.push(`${data.progress}% ${data.status}`);
-		} else if (data.status) {
-			buffer.push(`${data.status}`);
-
-			// data from build stream
-		} else {
-			buffer.push(data);
-		}
-
-		return cb(null, data);
-	});
-};
-
-var buildProgressAdapter = function (inline) {
-	const through = require('through2');
-
-	const stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/;
-
-	let step = null;
-	let numSteps = null;
-	let progress;
-
-	return through({ objectMode: true }, function (str, _enc, cb) {
-		if (str == null) {
-			return cb(null, str);
-		}
-
-		if (inline) {
-			return cb(null, { status: str });
-		}
-
-		if (/^Successfully tagged /.test(str)) {
-			progress = undefined;
-		} else {
-			const match = stepRegex.exec(str);
-			if (match) {
-				step = match[1];
-				numSteps ??= match[2];
-				str = match[3];
-			}
-			if (step != null) {
-				str = `Step ${step}/${numSteps}: ${str}`;
-				progress = Math.floor(
-					(parseInt(step, 10) * 100) / parseInt(numSteps, 10),
-				);
-			}
-		}
-
-		return cb(null, { status: str, progress });
-	});
-};
-
-var pullProgressAdapter = (outStream) =>
-	function ({ status, id, percentage, error, errorDetail }) {
-		if (status != null) {
-			status = status.replace(/^Status: /, '');
-		}
-		if (id != null) {
-			status = `${id}: ${status}`;
-		}
-		if (percentage === 100) {
-			percentage = undefined;
-		}
-		return outStream.write({
-			status,
-			progress: percentage,
-			error: errorDetail?.message ?? error,
-		});
-	};
-
-class BuildProgressUI {
+export class BuildProgressUI {
 	constructor(tty, descriptors) {
 		this._handleEvent = this._handleEvent.bind(this);
 		this._handleInterrupt = this._handleInterrupt.bind(this);
@@ -978,7 +606,7 @@ class BuildProgressUI {
 	}
 }
 
-class BuildProgressInline {
+export class BuildProgressInline {
 	constructor(outStream, descriptors) {
 		this.start = this.start.bind(this);
 		this.end = this.end.bind(this);
@@ -1029,7 +657,7 @@ class BuildProgressInline {
 
 		if (summary != null) {
 			this._services.forEach((service) => {
-				this._renderEvent(service, summary[service]);
+				this._renderEvent(service, { status: summary[service] });
 			});
 		}
 

lib/utils/compose_ts.ts

@@ -14,17 +14,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { flags } from '@oclif/command';
 import { BalenaSDK } from 'balena-sdk';
+import type { TransposeOptions } from 'docker-qemu-transpose';
 import type * as Dockerode from 'dockerode';
 import * as _ from 'lodash';
 import { promises as fs } from 'fs';
 import * as path from 'path';
-import type { Composition } from 'resin-compose-parse';
+import type {
+	BuildConfig,
+	Composition,
+	ImageDescriptor,
+} from 'resin-compose-parse';
 import type * as MultiBuild from 'resin-multibuild';
-import type { Readable } from 'stream';
+import type { Duplex, Readable } from 'stream';
 import type { Pack } from 'tar-stream';
+
 import { ExpectedError } from '../errors';
-import { getBalenaSdk, getChalk, stripIndent } from './lazy';
 import {
 	BuiltImage,
 	ComposeCliFlags,
@@ -34,16 +40,9 @@ import {
 	TaggedImage,
 	TarDirectoryOptions,
 } from './compose-types';
-import { DeviceInfo } from './device/api';
+import type { DeviceInfo } from './device/api';
+import { getBalenaSdk, getChalk, stripIndent } from './lazy';
 import Logger = require('./logger');
-import { flags } from '@oclif/command';
-
-export interface RegistrySecrets {
-	[registryAddress: string]: {
-		username: string;
-		password: string;
-	};
-}
 
 const exists = async (filename: string) => {
 	try {
@@ -54,8 +53,8 @@ const exists = async (filename: string) => {
 	}
 };
 
+const LOG_LENGTH_MAX = 512 * 1024; // 512KB
 const compositionFileNames = ['docker-compose.yml', 'docker-compose.yaml'];
-
 const hr =
 	'----------------------------------------------------------------------';
 
@@ -131,6 +130,372 @@ async function resolveProject(
 	return [composeFileName, composeFileContents];
 }
 
+interface BuildTaskPlus extends MultiBuild.BuildTask {
+	logBuffer?: string[];
+}
+
+interface Renderer {
+	start: () => void;
+	end: (buildSummaryByService?: Dictionary<string>) => void;
+	streams: Dictionary<NodeJS.ReadWriteStream>;
+}
+
+export async function buildProject(opts: {
+	docker: Dockerode;
+	logger: Logger;
+	projectPath: string;
+	projectName: string;
+	composition: Composition;
+	arch: string;
+	deviceType: string;
+	emulated: boolean;
+	buildOpts: import('./docker').BuildOpts;
+	inlineLogs?: boolean;
+	convertEol: boolean;
+	dockerfilePath?: string;
+	nogitignore: boolean;
+	multiDockerignore: boolean;
+}): Promise<BuiltImage[]> {
+	const { logger, projectName } = opts;
+	logger.logInfo(`Building for ${opts.arch}/${opts.deviceType}`);
+
+	let buildSummaryByService: Dictionary<string> | undefined;
+	const compose = await import('resin-compose-parse');
+	const imageDescriptors = compose.parse(opts.composition);
+	const imageDescriptorsByServiceName = _.keyBy(
+		imageDescriptors,
+		'serviceName',
+	);
+	const renderer = await startRenderer({ imageDescriptors, ...opts });
+	try {
+		await checkBuildSecretsRequirements(opts.docker, opts.projectPath);
+
+		const needsQemu = await installQemuIfNeeded({ ...opts, imageDescriptors });
+
+		const tarStream = await tarDirectory(opts.projectPath, opts);
+
+		const tasks: BuildTaskPlus[] = await makeBuildTasks(
+			opts.composition,
+			tarStream,
+			opts,
+			logger,
+			projectName,
+		);
+
+		setTaskAttributes({ tasks, imageDescriptorsByServiceName, ...opts });
+
+		const transposeOptArray: Array<
+			TransposeOptions | undefined
+		> = await Promise.all(
+			tasks.map((task) => {
+				// Setup emulation if needed
+				if (needsQemu && !task.external) {
+					return qemuTransposeBuildStream({ task, ...opts });
+				}
+			}),
+		);
+
+		await Promise.all(
+			// transposeOptions may be undefined. That's OK.
+			transposeOptArray.map((transposeOptions, index) =>
+				setTaskProgressHooks({
+					task: tasks[index],
+					renderer,
+					transposeOptions,
+					...opts,
+				}),
+			),
+		);
+
+		logger.logDebug('Prepared tasks; building...');
+
+		const { BALENA_ENGINE_TMP_PATH } = await import('../config');
+		const builder = await import('resin-multibuild');
+
+		const builtImages = await builder.performBuilds(
+			tasks,
+			opts.docker,
+			BALENA_ENGINE_TMP_PATH,
+		);
+
+		const [images, summaryMsgByService] = await inspectBuiltImages({
+			builtImages,
+			imageDescriptorsByServiceName,
+			tasks,
+			...opts,
+		});
+		buildSummaryByService = summaryMsgByService;
+
+		return images;
+	} finally {
+		renderer.end(buildSummaryByService);
+	}
+}
+
+async function startRenderer({
+	imageDescriptors,
+	inlineLogs,
+	logger,
+}: {
+	imageDescriptors: ImageDescriptor[];
+	inlineLogs?: boolean;
+	logger: Logger;
+}): Promise<Renderer> {
+	let renderer: Renderer;
+	if (inlineLogs) {
+		renderer = new (await import('./compose')).BuildProgressInline(
+			logger.streams['build'],
+			imageDescriptors,
+		);
+	} else {
+		const tty = (await import('./tty'))(process.stdout);
+		renderer = new (await import('./compose')).BuildProgressUI(
+			tty,
+			imageDescriptors,
+		);
+	}
+	renderer.start();
+	return renderer;
+}
+
+async function installQemuIfNeeded({
+	arch,
+	docker,
+	emulated,
+	imageDescriptors,
+	logger,
+	projectPath,
+}: {
+	arch: string;
+	docker: Dockerode;
+	emulated: boolean;
+	imageDescriptors: ImageDescriptor[];
+	logger: Logger;
+	projectPath: string;
+}): Promise<boolean> {
+	const qemu = await import('./qemu');
+	const needsQemu = await qemu.installQemuIfNeeded(
+		emulated,
+		logger,
+		arch,
+		docker,
+	);
+	if (needsQemu) {
+		logger.logInfo('Emulation is enabled');
+		// Copy qemu into all build contexts
+		await Promise.all(
+			imageDescriptors.map(function (d) {
+				if (isBuildConfig(d.image)) {
+					return qemu.copyQemu(
+						path.join(projectPath, d.image.context || '.'),
+						arch,
+					);
+				}
+			}),
+		);
+	}
+	return needsQemu;
+}
+
+function setTaskAttributes({
+	tasks,
+	buildOpts,
+	imageDescriptorsByServiceName,
+	projectName,
+}: {
+	tasks: BuildTaskPlus[];
+	buildOpts: import('./docker').BuildOpts;
+	imageDescriptorsByServiceName: Dictionary<ImageDescriptor>;
+	projectName: string;
+}) {
+	for (const task of tasks) {
+		const d = imageDescriptorsByServiceName[task.serviceName];
+		// multibuild (splitBuildStream) parses the composition internally so
+		// any tags we've set before are lost; re-assign them here
+		task.tag ??= [projectName, task.serviceName].join('_').toLowerCase();
+		if (isBuildConfig(d.image)) {
+			d.image.tag = task.tag;
+		}
+		// reassign task.args so that the `--buildArg` flag takes precedence
+		// over assignments in the docker-compose.yml file (service.build.args)
+		task.args = {
+			...task.args,
+			...buildOpts.buildargs,
+		};
+
+		// Docker image build options
+		task.dockerOpts ??= {};
+		if (task.args && Object.keys(task.args).length) {
+			task.dockerOpts.buildargs = {
+				...task.dockerOpts.buildargs,
+				...task.args,
+			};
+		}
+		_.merge(task.dockerOpts, buildOpts, { t: task.tag });
+	}
+}
+
+async function qemuTransposeBuildStream({
+	task,
+	dockerfilePath,
+	projectPath,
+}: {
+	task: BuildTaskPlus;
+	dockerfilePath?: string;
+	projectPath: string;
+}): Promise<TransposeOptions> {
+	const qemu = await import('./qemu');
+	const binPath = qemu.qemuPathInContext(
+		path.join(projectPath, task.context ?? ''),
+	);
+	if (task.buildStream == null) {
+		throw new Error(`No buildStream for task '${task.tag}'`);
+	}
+
+	const transpose = await import('docker-qemu-transpose');
+	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
+
+	const transposeOptions: TransposeOptions = {
+		hostQemuPath: toPosixPath(binPath),
+		containerQemuPath: `/tmp/${qemu.QEMU_BIN_NAME}`,
+		qemuFileMode: 0o555,
+	};
+
+	task.buildStream = (await transpose.transposeTarStream(
+		task.buildStream,
+		transposeOptions,
+		dockerfilePath || undefined,
+	)) as Pack;
+
+	return transposeOptions;
+}
+
+async function setTaskProgressHooks({
+	inlineLogs,
+	renderer,
+	task,
+	transposeOptions,
+}: {
+	inlineLogs?: boolean;
+	renderer: Renderer;
+	task: BuildTaskPlus;
+	transposeOptions?: import('docker-qemu-transpose').TransposeOptions;
+}) {
+	const transpose = await import('docker-qemu-transpose');
+	// Get the service-specific log stream
+	const logStream = renderer.streams[task.serviceName];
+	task.logBuffer = [];
+	const captureStream = buildLogCapture(task.external, task.logBuffer);
+
+	if (task.external) {
+		// External image -- there's no build to be performed,
+		// just follow pull progress.
+		captureStream.pipe(logStream);
+		task.progressHook = pullProgressAdapter(captureStream);
+	} else {
+		task.streamHook = function (stream) {
+			let rawStream;
+			stream = createLogStream(stream);
+			if (transposeOptions) {
+				const buildThroughStream = transpose.getBuildThroughStream(
+					transposeOptions,
+				);
+				rawStream = stream.pipe(buildThroughStream);
+			} else {
+				rawStream = stream;
+			}
+			// `stream` sends out raw strings in contrast to `task.progressHook`
+			// where we're given objects. capture these strings as they come
+			// before we parse them.
+			return rawStream
+				.pipe(dropEmptyLinesStream())
+				.pipe(captureStream)
+				.pipe(buildProgressAdapter(!!inlineLogs))
+				.pipe(logStream);
+		};
+	}
+}
+
+async function inspectBuiltImages({
+	builtImages,
+	docker,
+	imageDescriptorsByServiceName,
+	tasks,
+}: {
+	builtImages: MultiBuild.LocalImage[];
+	docker: Dockerode;
+	imageDescriptorsByServiceName: Dictionary<ImageDescriptor>;
+	tasks: BuildTaskPlus[];
+}): Promise<[BuiltImage[], Dictionary<string>]> {
+	const images: BuiltImage[] = await Promise.all(
+		builtImages.map((builtImage: MultiBuild.LocalImage) =>
+			inspectBuiltImage({
+				builtImage,
+				docker,
+				imageDescriptorsByServiceName,
+				tasks,
+			}),
+		),
+	);
+
+	const humanize = require('humanize');
+	const summaryMsgByService: { [serviceName: string]: string } = {};
+	for (const image of images) {
+		summaryMsgByService[image.serviceName] = `Image size: ${humanize.filesize(
+			image.props.size,
+		)}`;
+	}
+
+	return [images, summaryMsgByService];
+}
+
+async function inspectBuiltImage({
+	builtImage,
+	docker,
+	imageDescriptorsByServiceName,
+	tasks,
+}: {
+	builtImage: MultiBuild.LocalImage;
+	docker: Dockerode;
+	imageDescriptorsByServiceName: Dictionary<ImageDescriptor>;
+	tasks: BuildTaskPlus[];
+}): Promise<BuiltImage> {
+	if (!builtImage.successful) {
+		const error: Error & { serviceName?: string } =
+			builtImage.error ?? new Error();
+		error.serviceName = builtImage.serviceName;
+		throw error;
+	}
+
+	const d = imageDescriptorsByServiceName[builtImage.serviceName];
+	const task = _.find(tasks, {
+		serviceName: builtImage.serviceName,
+	});
+
+	const image: BuiltImage = {
+		serviceName: d.serviceName,
+		name: (isBuildConfig(d.image) ? d.image.tag : d.image) || '',
+		logs: truncateString(task?.logBuffer?.join('\n') || '', LOG_LENGTH_MAX),
+		props: {
+			dockerfile: builtImage.dockerfile,
+			projectType: builtImage.projectType,
+		},
+	};
+
+	// Times here are timestamps, so test whether they're null
+	// before creating a date out of them, as `new Date(null)`
+	// creates a date representing UNIX time 0.
+	if (builtImage.startTime) {
+		image.props.startTime = new Date(builtImage.startTime);
+	}
+	if (builtImage.endTime) {
+		image.props.endTime = new Date(builtImage.endTime);
+	}
+	image.props.size = (await docker.getImage(image.name).inspect()).Size;
+
+	return image;
+}
+
 /**
  * Load the ".balena/balena.yml" file (or resin.yml, or yaml or json),
  * which contains "build metadata" for features like "build secrets" and
@@ -178,16 +543,14 @@ async function loadBuildMetatada(
 }
 
 /**
- * Return a map of service name to service subdirectory, obtained from the given
- * composition object. If a composition object is not provided, an attempt will
- * be made to parse a 'docker-compose.yml' file at the given sourceDir.
- * Entries will be NOT be returned for subdirectories equal to '.' (e.g. the
- * 'main' "service" of a single-container application).
- *
+ * Return a map of service name to service subdirectory (relative to sourceDir),
+ * obtained from the given composition object. If a composition object is not
+ * provided, an attempt will be made to parse a 'docker-compose.yml' file at
+ * the given sourceDir.
  * @param sourceDir Project source directory (project root)
  * @param composition Optional previously parsed composition object
  */
-async function getServiceDirsFromComposition(
+export async function getServiceDirsFromComposition(
 	sourceDir: string,
 	composition?: Composition,
 ): Promise<Dictionary<string>> {
@@ -207,9 +570,9 @@ async function getServiceDirsFromComposition(
 		const relPrefix = '.' + path.sep;
 		for (const [serviceName, service] of Object.entries(composition.services)) {
 			let dir =
-				typeof service.build === 'string'
+				(typeof service.build === 'string'
 					? service.build
-					: service.build?.context || '.';
+					: service.build?.context) || '.';
 			// Convert forward slashes to backslashes on Windows
 			dir = path.normalize(dir);
 			// Make sure the path is relative to the project directory
@@ -220,25 +583,65 @@ async function getServiceDirsFromComposition(
 			dir = dir.endsWith(path.sep) ? dir.slice(0, -1) : dir;
 			// remove './' prefix (or '.\\' on Windows)
 			dir = dir.startsWith(relPrefix) ? dir.slice(2) : dir;
-			// filter out a '.' service directory (e.g. for the 'main' service
-			// of a single-container application)
-			if (dir && dir !== '.') {
-				serviceDirs[serviceName] = dir;
-			}
+
+			serviceDirs[serviceName] = dir || '.';
 		}
 	}
 	return serviceDirs;
 }
 
+/**
+ * Return true if `image` is actually a docker-compose.yml `services.service.build`
+ * configuration object, rather than an "external image" (`services.service.image`).
+ *
+ * The `image` argument may therefore refere to either a `build` or `image` property
+ * of a service in a docker-compose.yml file, which is a bit confusing but it matches
+ * the `ImageDescriptor.image` property as defined by `resin-compose-parse`.
+ *
+ * Note that `resin-compose-parse` "normalizes" the docker-compose.yml file such
+ * that, if `services.service.build` is a string, it is converted to a BuildConfig
+ * object with the string value assigned to `services.service.build.context`:
+ * https://github.com/balena-io-modules/resin-compose-parse/blob/v2.1.3/src/compose.ts#L166-L167
+ * This is why this implementation works when `services.service.build` is defined
+ * as a string in the docker-compose.yml file.
+ *
+ * @param image The `ImageDescriptor.image` attribute parsed with `resin-compose-parse`
+ */
+export function isBuildConfig(
+	image: string | BuildConfig,
+): image is BuildConfig {
+	return image != null && typeof image !== 'string';
+}
+
 /**
  * Create a tar stream out of the local filesystem at the given directory,
  * while optionally applying file filters such as '.dockerignore' and
  * optionally converting text file line endings (CRLF to LF).
  * @param dir Source directory
  * @param param Options
- * @returns {Promise<import('stream').Readable>}
+ * @returns Readable stream
  */
 export async function tarDirectory(
+	dir: string,
+	param: TarDirectoryOptions,
+): Promise<import('stream').Readable> {
+	const { nogitignore = false } = param;
+	if (nogitignore) {
+		return newTarDirectory(dir, param);
+	} else {
+		return (await import('./compose')).originalTarDirectory(dir, param);
+	}
+}
+
+/**
+ * Create a tar stream out of the local filesystem at the given directory,
+ * while optionally applying file filters such as '.dockerignore' and
+ * optionally converting text file line endings (CRLF to LF).
+ * @param dir Source directory
+ * @param param Options
+ * @returns Readable stream
+ */
+async function newTarDirectory(
 	dir: string,
 	{
 		composition,
@@ -252,10 +655,6 @@ export async function tarDirectory(
 	const { filterFilesWithDockerignore } = await import('./ignore');
 	const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
 
-	const serviceDirs = multiDockerignore
-		? await getServiceDirsFromComposition(dir, composition)
-		: {};
-
 	let readFile: (file: string) => Promise<Buffer>;
 	if (process.platform === 'win32') {
 		const { readFileWithEolConversion } = require('./eol-conversion');
@@ -265,10 +664,11 @@ export async function tarDirectory(
 	}
 	const tar = await import('tar-stream');
 	const pack = tar.pack();
+	const serviceDirs = await getServiceDirsFromComposition(dir, composition);
 	const {
 		filteredFileList,
 		dockerignoreFiles,
-	} = await filterFilesWithDockerignore(dir, serviceDirs);
+	} = await filterFilesWithDockerignore(dir, multiDockerignore, serviceDirs);
 	printDockerignoreWarn(dockerignoreFiles, serviceDirs, multiDockerignore);
 	for (const fileStats of filteredFileList) {
 		pack.entry(
@@ -295,7 +695,7 @@ export async function tarDirectory(
  * @param serviceDirsByService Map of service names to service subdirectories
  * @param multiDockerignore Whether --multi-dockerignore (-m) was provided
  */
-export function printDockerignoreWarn(
+function printDockerignoreWarn(
 	dockerignoreFiles: Array<import('./ignore').FileStats>,
 	serviceDirsByService: Dictionary<string>,
 	multiDockerignore: boolean,
@@ -441,7 +841,7 @@ export async function checkBuildSecretsRequirements(
 export async function getRegistrySecrets(
 	sdk: BalenaSDK,
 	inputFilename?: string,
-): Promise<RegistrySecrets> {
+): Promise<MultiBuild.RegistrySecrets> {
 	if (inputFilename != null) {
 		return await parseRegistrySecrets(inputFilename);
 	}
@@ -464,7 +864,7 @@ export async function getRegistrySecrets(
 
 async function parseRegistrySecrets(
 	secretsFilename: string,
-): Promise<RegistrySecrets> {
+): Promise<MultiBuild.RegistrySecrets> {
 	try {
 		let isYaml = false;
 		if (/.+\.ya?ml$/i.test(secretsFilename)) {
@@ -661,7 +1061,7 @@ async function validateSpecifiedDockerfile(
 
 export interface ProjectValidationResult {
 	dockerfilePath: string;
-	registrySecrets: RegistrySecrets;
+	registrySecrets: MultiBuild.RegistrySecrets;
 }
 
 /**
@@ -797,7 +1197,7 @@ async function pushServiceImages(
 export async function deployProject(
 	docker: import('docker-toolbelt'),
 	logger: Logger,
-	composition: import('resin-compose-parse').Composition,
+	composition: Composition,
 	images: BuiltImage[],
 	appId: number,
 	userId: number,
@@ -907,6 +1307,123 @@ export function createRunLoop(tick: (...args: any[]) => void) {
 	return runloop;
 }
 
+function createLogStream(input: Readable) {
+	const split = require('split') as typeof import('split');
+	const stripAnsi = require('strip-ansi-stream');
+	return input.pipe<Duplex>(stripAnsi()).pipe(split());
+}
+
+function dropEmptyLinesStream() {
+	const through = require('through2') as typeof import('through2');
+	return through(function (data, _enc, cb) {
+		const str = data.toString('utf-8');
+		if (str.trim()) {
+			this.push(str);
+		}
+		return cb();
+	});
+}
+
+function buildLogCapture(objectMode: boolean, buffer: string[]) {
+	const through = require('through2') as typeof import('through2');
+
+	return through({ objectMode }, function (data, _enc, cb) {
+		// data from pull stream
+		if (data.error) {
+			buffer.push(`${data.error}`);
+		} else if (data.progress && data.status) {
+			buffer.push(`${data.progress}% ${data.status}`);
+		} else if (data.status) {
+			buffer.push(`${data.status}`);
+
+			// data from build stream
+		} else {
+			buffer.push(data);
+		}
+
+		return cb(null, data);
+	});
+}
+
+function buildProgressAdapter(inline: boolean) {
+	const through = require('through2') as typeof import('through2');
+
+	const stepRegex = /^\s*Step\s+(\d+)\/(\d+)\s*: (.+)$/;
+
+	let step = '';
+	let numSteps = '';
+	let progress: number | undefined;
+
+	return through({ objectMode: true }, function (str, _enc, cb) {
+		if (str == null) {
+			return cb(null, str);
+		}
+
+		if (inline) {
+			return cb(null, { status: str });
+		}
+
+		if (!/^Successfully tagged /.test(str)) {
+			const match = stepRegex.exec(str);
+			if (match) {
+				step = match[1];
+				numSteps ??= match[2];
+				str = match[3];
+			}
+			if (step) {
+				str = `Step ${step}/${numSteps}: ${str}`;
+				progress = Math.floor(
+					(parseInt(step, 10) * 100) / parseInt(numSteps, 10),
+				);
+			}
+		}
+
+		return cb(null, { status: str, progress });
+	});
+}
+
+function pullProgressAdapter(outStream: Duplex) {
+	return function ({
+		status,
+		id,
+		percentage,
+		error,
+		errorDetail,
+	}: {
+		status: string;
+		id: string;
+		percentage: number | undefined;
+		error: Error;
+		errorDetail: Error;
+	}) {
+		if (status != null) {
+			status = status.replace(/^Status: /, '');
+		}
+		if (id != null) {
+			status = `${id}: ${status}`;
+		}
+		if (percentage === 100) {
+			percentage = undefined;
+		}
+		return outStream.write({
+			status,
+			progress: percentage,
+			error: errorDetail?.message ?? error,
+		});
+	};
+}
+
+function truncateString(str: string, len: number): string {
+	if (str.length < len) {
+		return str;
+	}
+	str = str.slice(0, len);
+	// return everything up to the last line. this is a cheeky way to avoid
+	// having to deal with splitting the string midway through some special
+	// character sequence.
+	return str.slice(0, str.lastIndexOf('\n'));
+}
+
 export const composeCliFlags: flags.Input<ComposeCliFlags> = {
 	emulated: flags.boolean({
 		description:
@@ -919,7 +1436,7 @@ export const composeCliFlags: flags.Input<ComposeCliFlags> = {
 	}),
 	logs: flags.boolean({
 		description:
-			'No-op and deprecated since balenaCLI v12.0.0. Build logs are now shown by default.',
+			'No-op and deprecated since balena CLI v12.0.0. Build logs are now shown by default.',
 	}),
 	nologs: flags.boolean({
 		description:
@@ -938,7 +1455,7 @@ export const composeCliFlags: flags.Input<ComposeCliFlags> = {
 		char: 'm',
 	}),
 	nogitignore: flags.boolean({
-		description: `No-op (default behavior) since balenaCLI v12.0.0. See "balena help build".`,
+		description: `No-op (default behavior) since balena CLI v12.0.0. See "balena help build".`,
 		char: 'G',
 	}),
 	'noparent-check': flags.boolean({
@@ -951,7 +1468,7 @@ export const composeCliFlags: flags.Input<ComposeCliFlags> = {
 		char: 'R',
 	}),
 	'convert-eol': flags.boolean({
-		description: 'No-op and deprecated since balenaCLI v12.0.0',
+		description: 'No-op and deprecated since balena CLI v12.0.0',
 		char: 'l',
 	}),
 	'noconvert-eol': flags.boolean({

lib/utils/config.ts

@@ -68,7 +68,7 @@ export async function generateBaseConfig(
 	};
 
 	const config = (await getBalenaSdk().models.os.getConfig(
-		application.app_name,
+		application.slug,
 		options,
 	)) as ImgConfig & { apiKey?: string };
 	// os.getConfig always returns a config for an app

lib/utils/deploy-legacy.js

@@ -30,13 +30,18 @@ const getBuilderLogPushEndpoint = function (baseUrl, buildId, owner, app) {
 	return `https://builder.${baseUrl}/v1/pushLogs?${args}`;
 };
 
+/**
+ * @param {import('docker-toolbelt')} docker
+ * @param {string} imageId
+ * @param {string} bufferFile
+ */
 const bufferImage = function (docker, imageId, bufferFile) {
 	const streamUtils = require('./streams');
 
 	const image = docker.getImage(imageId);
-	const imageMetadata = image.inspect();
+	const sizePromise = image.inspect().then((img) => img.Size);
 
-	return Promise.all([image.get(), imageMetadata.get('Size')]).then(
+	return Promise.all([image.get(), sizePromise]).then(
 		([imageStream, imageSize]) =>
 			streamUtils.buffer(imageStream, bufferFile).then((bufferedStream) => {
 				// @ts-ignore adding an extra property
@@ -150,14 +155,17 @@ const uploadLogs = function (logs, token, url, buildId, username, appName) {
 	});
 };
 
-/*
-opts must be a hash with the following keys:
-
-- appName: the name of the app to deploy to
-- imageName: the name of the image to deploy
-- buildLogs: a string with build output
-- shouldUploadLogs
-*/
+/**
+ * @param {import('docker-toolbelt')} docker
+ * @param {import('./logger')} logger
+ * @param {string} token
+ * @param {string} username
+ * @param {string} url
+ * @param {{appName: string; imageName: string; buildLogs: string; shouldUploadLogs: boolean}} opts
+ * - appName: the name of the app to deploy to
+ * - imageName: the name of the image to deploy
+ * - buildLogs: a string with build output
+ */
 export const deployLegacy = async function (
 	docker,
 	logger,

lib/utils/device/deploy.ts

@@ -32,6 +32,7 @@ import {
 	checkBuildSecretsRequirements,
 	loadProject,
 	makeBuildTasks,
+	tarDirectory,
 } from '../compose_ts';
 import Logger = require('../logger');
 import { DeviceAPI, DeviceInfo } from './api';
@@ -121,7 +122,6 @@ async function environmentFromInput(
 }
 
 export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
-	const { tarDirectory } = await import('../compose');
 	const { exitWithExpectedError } = await import('../../errors');
 	const { displayDeviceLogs } = await import('./logs');
 
@@ -400,7 +400,6 @@ export async function rebuildSingleTask(
 	// this should provide the following callback
 	containerIdCb?: (id: string) => void,
 ): Promise<string> {
-	const { tarDirectory } = await import('../compose');
 	const multibuild = await import('resin-multibuild');
 	// First we run the build task, to get the new image id
 	let buildLogs = '';

lib/utils/device/live.ts

@@ -1,5 +1,23 @@
+/**
+ * @license
+ * Copyright 2019-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 import * as chokidar from 'chokidar';
 import type * as Dockerode from 'dockerode';
+import * as fs from 'fs';
 import Livepush, { ContainerNotRunningError } from 'livepush';
 import * as _ from 'lodash';
 import * as path from 'path';
@@ -92,8 +110,22 @@ export class LivepushManager {
 		// Split the composition into a load of differents paths
 		// which we can
 		this.logger.logLivepush('Device state settled');
-		// create livepush instances for
 
+		// Prepare dockerignore data for file watcher
+		const { getDockerignoreByService } = await import('../ignore');
+		const { getServiceDirsFromComposition } = await import('../compose_ts');
+		const rootContext = path.resolve(this.buildContext);
+		const serviceDirsByService = await getServiceDirsFromComposition(
+			this.deployOpts.source,
+			this.composition,
+		);
+		const dockerignoreByService = await getDockerignoreByService(
+			this.deployOpts.source,
+			this.deployOpts.multiDockerignore,
+			serviceDirsByService,
+		);
+
+		// create livepush instances for each service
 		for (const serviceName of _.keys(this.composition.services)) {
 			const service = this.composition.services[serviceName];
 			const buildTask = _.find(this.buildTasks, { serviceName });
@@ -106,7 +138,6 @@ export class LivepushManager {
 
 			// We only care about builds
 			if (service.build != null) {
-				const context = path.join(this.buildContext, service.build.context);
 				if (buildTask.dockerfile == null) {
 					throw new Error(
 						`Could not detect dockerfile for service: ${serviceName}`,
@@ -137,6 +168,10 @@ export class LivepushManager {
 					return;
 				}
 
+				// path.resolve() converts to an absolute path, removes trailing slashes,
+				// and also converts forward slashes to backslashes on Windows.
+				const context = path.resolve(rootContext, service.build.context);
+
 				const livepush = await Livepush.init({
 					dockerfile,
 					context,
@@ -155,27 +190,22 @@ export class LivepushManager {
 
 				this.updateEventsWaiting[serviceName] = [];
 				this.deleteEventsWaiting[serviceName] = [];
-				const addEvent = (eventQueue: string[], changedPath: string) => {
+				const addEvent = ($serviceName: string, changedPath: string) => {
 					this.logger.logDebug(
-						`Got an add filesystem event for service: ${serviceName}. File: ${changedPath}`,
+						`Got an add filesystem event for service: ${$serviceName}. File: ${changedPath}`,
 					);
+					const eventQueue = this.updateEventsWaiting[$serviceName];
 					eventQueue.push(changedPath);
-					this.getDebouncedEventHandler(serviceName)();
+					this.getDebouncedEventHandler($serviceName)();
 				};
-				// TODO: Memoize this for containers which share a context
-				const monitor = chokidar.watch('.', {
-					cwd: context,
-					ignoreInitial: true,
-					ignored: '.git',
-				});
-				monitor.on('add', (changedPath: string) =>
-					addEvent(this.updateEventsWaiting[serviceName], changedPath),
-				);
-				monitor.on('change', (changedPath: string) =>
-					addEvent(this.updateEventsWaiting[serviceName], changedPath),
-				);
-				monitor.on('unlink', (changedPath: string) =>
-					addEvent(this.deleteEventsWaiting[serviceName], changedPath),
+
+				const monitor = this.setupFilesystemWatcher(
+					serviceName,
+					rootContext,
+					context,
+					addEvent,
+					dockerignoreByService,
+					this.deployOpts.multiDockerignore,
 				);
 
 				this.containers[serviceName] = {
@@ -209,6 +239,57 @@ export class LivepushManager {
 		});
 	}
 
+	protected setupFilesystemWatcher(
+		serviceName: string,
+		rootContext: string,
+		serviceContext: string,
+		changedPathHandler: (serviceName: string, changedPath: string) => void,
+		dockerignoreByService: {
+			[serviceName: string]: import('@balena/dockerignore').Ignore;
+		},
+		multiDockerignore: boolean,
+	): chokidar.FSWatcher {
+		const contextForDockerignore = multiDockerignore
+			? serviceContext
+			: rootContext;
+		const dockerignore = dockerignoreByService[serviceName];
+		// TODO: Memoize this for services that share a context
+		const monitor = chokidar.watch('.', {
+			cwd: serviceContext,
+			followSymlinks: true,
+			ignoreInitial: true,
+			ignored: (filePath: string, stats: fs.Stats | undefined) => {
+				if (!stats) {
+					try {
+						// sync because chokidar defines a sync interface
+						stats = fs.lstatSync(filePath);
+					} catch (err) {
+						// OK: the file may have been deleted. See also:
+						// https://github.com/paulmillr/chokidar/blob/3.4.3/lib/fsevents-handler.js#L326-L328
+						// https://github.com/paulmillr/chokidar/blob/3.4.3/lib/nodefs-handler.js#L364
+					}
+				}
+				if (stats && !stats.isFile() && !stats.isSymbolicLink()) {
+					// never ignore directories for compatibility with
+					// dockerignore exclusion patterns
+					return !stats.isDirectory();
+				}
+				const relPath = path.relative(contextForDockerignore, filePath);
+				return dockerignore.ignores(relPath);
+			},
+		});
+		monitor.on('add', (changedPath: string) =>
+			changedPathHandler(serviceName, changedPath),
+		);
+		monitor.on('change', (changedPath: string) =>
+			changedPathHandler(serviceName, changedPath),
+		);
+		monitor.on('unlink', (changedPath: string) =>
+			changedPathHandler(serviceName, changedPath),
+		);
+		return monitor;
+	}
+
 	public static preprocessDockerfile(content: string): string {
 		return new Dockerfile(content).generateLiveDockerfile();
 	}

lib/utils/device/ssh.ts

@@ -49,7 +49,7 @@ export async function performLocalDeviceSSH(
 			port: 2375,
 		});
 
-		const regex = new RegExp(`\\/?${escapeRegExp(opts.service)}_\\d+_\\d+`);
+		const regex = new RegExp(`(^|\\/)${escapeRegExp(opts.service)}_\\d+_\\d+`);
 		const nameRegex = /\/?([a-zA-Z0-9_]+)_\d+_\d+/;
 		let allContainers: ContainerInfo[];
 		try {
@@ -61,21 +61,19 @@ export async function performLocalDeviceSSH(
 		}
 
 		const serviceNames: string[] = [];
-		const containers = allContainers
-			.map((container) => {
-				for (const name of container.Names) {
-					if (regex.test(name)) {
-						return { id: container.Id, name };
-					}
-					const match = name.match(nameRegex);
-					if (match) {
-						serviceNames.push(match[1]);
-					}
+		const containers: Array<{ id: string; name: string }> = [];
+		for (const container of allContainers) {
+			for (const name of container.Names) {
+				if (regex.test(name)) {
+					containers.push({ id: container.Id, name });
+					break;
 				}
-				return;
-			})
-			.filter((c) => c != null);
-
+				const match = name.match(nameRegex);
+				if (match) {
+					serviceNames.push(match[1]);
+				}
+			}
+		}
 		if (containers.length === 0) {
 			throw new ExpectedError(
 				`Could not find a service on device with name ${opts.service}. ${
@@ -91,12 +89,13 @@ export async function performLocalDeviceSSH(
 		}
 		if (containers.length > 1) {
 			throw new ExpectedError(stripIndent`
-				Found more than one container with a service name ${opts.service}.
-				This state is not supported, please contact support.
+				Found more than one container matching service name "${opts.service}":
+				${containers.map((container) => container.name).join(', ')}
+				Use different service names to avoid ambiguity.
 			`);
 		}
 
-		const containerId = containers[0]!.id;
+		const containerId = containers[0].id;
 		const shellCmd = `/bin/sh -c "if [ -e /bin/bash ]; then exec /bin/bash; else exec /bin/sh; fi"`;
 		// stdin (fd=0) is not a tty when data is piped in, for example
 		// echo 'ls -la; exit;' | balena ssh 192.168.0.20 service1

lib/utils/docker-js.js

@@ -91,48 +91,6 @@ const generateConnectOpts = async function (opts) {
 	return connectOpts;
 };
 
-const parseBuildArgs = function (args) {
-	if (!Array.isArray(args)) {
-		args = [args];
-	}
-	const buildArgs = {};
-	args.forEach(function (arg) {
-		// note: [^] matches any character, including line breaks
-		const pair = /^([^\s]+?)=([^]*)$/.exec(arg);
-		if (pair != null) {
-			buildArgs[pair[1]] = pair[2] ?? '';
-		} else {
-			throw new ExpectedError(`Could not parse build argument: '${arg}'`);
-		}
-	});
-	return buildArgs;
-};
-
-export function generateBuildOpts(options) {
-	const opts = {};
-	if (options.tag != null) {
-		opts.t = options.tag;
-	}
-	if (options.nocache != null) {
-		opts.nocache = true;
-	}
-	if (options['cache-from']?.trim()) {
-		opts.cachefrom = options['cache-from'].split(',').filter((i) => !!i.trim());
-	}
-	if (options.pull != null) {
-		opts.pull = true;
-	}
-	if (options.squash != null) {
-		opts.squash = true;
-	}
-	if (options.buildArg != null) {
-		opts.buildargs = parseBuildArgs(options.buildArg);
-	}
-	if (!_.isEmpty(options['registry-secrets'])) {
-		opts.registryconfig = options['registry-secrets'];
-	}
-	return opts;
-}
 /**
  * @param {{
  * 	ca?: string; // path to ca (Certificate Authority) file (TLS)

lib/utils/docker.ts

@@ -17,6 +17,8 @@
 
 import type * as dockerode from 'dockerode';
 import { flags } from '@oclif/command';
+
+import { ExpectedError } from '../errors';
 import { parseAsInteger } from './validation';
 
 export * from './docker-js';
@@ -98,6 +100,70 @@ Implements the same feature as the "docker build --cache-from" option.`,
 	...dockerConnectionCliFlags,
 };
 
+export interface BuildOpts {
+	buildargs?: Dictionary<string>;
+	cachefrom?: string[];
+	nocache?: boolean;
+	pull?: boolean;
+	registryconfig?: import('resin-multibuild').RegistrySecrets;
+	squash?: boolean;
+	t?: string;
+}
+
+function parseBuildArgs(args: string[]): Dictionary<string> {
+	if (!Array.isArray(args)) {
+		args = [args];
+	}
+	const buildArgs: Dictionary<string> = {};
+	args.forEach(function (arg) {
+		// note: [^] matches any character, including line breaks
+		const pair = /^([^\s]+?)=([^]*)$/.exec(arg);
+		if (pair != null) {
+			buildArgs[pair[1]] = pair[2] ?? '';
+		} else {
+			throw new ExpectedError(`Could not parse build argument: '${arg}'`);
+		}
+	});
+	return buildArgs;
+}
+
+export function generateBuildOpts(options: {
+	buildArg?: string[];
+	'cache-from'?: string;
+	nocache: boolean;
+	pull?: boolean;
+	'registry-secrets'?: import('resin-multibuild').RegistrySecrets;
+	squash: boolean;
+	tag?: string;
+}): BuildOpts {
+	const opts: BuildOpts = {};
+	if (options.buildArg != null) {
+		opts.buildargs = parseBuildArgs(options.buildArg);
+	}
+	if (options['cache-from']?.trim()) {
+		opts.cachefrom = options['cache-from'].split(',').filter((i) => !!i.trim());
+	}
+	if (options.nocache != null) {
+		opts.nocache = true;
+	}
+	if (options.pull != null) {
+		opts.pull = true;
+	}
+	if (
+		options['registry-secrets'] &&
+		Object.keys(options['registry-secrets']).length
+	) {
+		opts.registryconfig = options['registry-secrets'];
+	}
+	if (options.squash != null) {
+		opts.squash = true;
+	}
+	if (options.tag != null) {
+		opts.t = options.tag;
+	}
+	return opts;
+}
+
 export async function isBalenaEngine(docker: dockerode): Promise<boolean> {
 	// dockerVersion.Engine should equal 'balena-engine' for the current/latest
 	// version of balenaEngine, but it was at one point (mis)spelt 'balaena':

lib/utils/helpers.ts

@@ -69,7 +69,7 @@ export function stateToString(state: OperationState) {
  * bash/sh and the Windows cmd.exe in relation to escape characters.
  * @param msg Optional message for the user, before the password prompt
  * @param stderr Optional stream to which stderr should be piped
- * @param isCLIcmd (default: true) Whether the command array is a balenaCLI command
+ * @param isCLIcmd (default: true) Whether the command array is a balena CLI command
  * (e.g. ['internal', 'osinit', ...]), in which case process.argv[0] and argv[1] are
  * added as necessary, depending on whether the CLI is running as a standalone zip
  * package (with Node built in).

lib/utils/ignore.ts

@@ -21,6 +21,7 @@ import * as MultiBuild from 'resin-multibuild';
 
 import dockerIgnore = require('@zeit/dockerignore');
 import ignore from 'ignore';
+import type { Ignore } from '@balena/dockerignore';
 
 import { ExpectedError } from '../errors';
 
@@ -196,37 +197,6 @@ export interface FileStats {
 	stats: Stats;
 }
 
-/**
- * Create a list of files (FileStats[]) for the filesystem subtree rooted at
- * projectDir, listing each file with both a full path and a relative path,
- * but excluding entries for directories themselves.
- * @param projectDir Source directory (root of subtree to be listed)
- * @param dir Used for recursive calls only (omit on first function call)
- */
-async function listFiles(
-	projectDir: string,
-	dir: string = projectDir,
-): Promise<FileStats[]> {
-	const files: FileStats[] = [];
-	const dirEntries = await fs.readdir(dir);
-	await Promise.all(
-		dirEntries.map(async (entry) => {
-			const filePath = path.join(dir, entry);
-			const stats = await fs.stat(filePath);
-			if (stats.isDirectory()) {
-				files.push(...(await listFiles(projectDir, filePath)));
-			} else if (stats.isFile()) {
-				files.push({
-					filePath,
-					relPath: path.relative(projectDir, filePath),
-					stats,
-				});
-			}
-		}),
-	);
-	return files;
-}
-
 /**
  * Return the contents of a .dockerignore file at projectDir, as a string.
  * Return an empty string if a .dockerignore file does not exist.
@@ -254,9 +224,9 @@ async function readDockerIgnoreFile(projectDir: string): Promise<string> {
  * a set of default/hardcoded patterns.
  * @param directory Directory where to look for a .dockerignore file
  */
-async function getDockerIgnoreInstance(
+export async function getDockerIgnoreInstance(
 	directory: string,
-): Promise<import('@balena/dockerignore').Ignore> {
+): Promise<Ignore> {
 	const dockerIgnoreStr = await readDockerIgnoreFile(directory);
 	const $dockerIgnore = (await import('@balena/dockerignore')).default;
 	const ig = $dockerIgnore({ ignorecase: false });
@@ -283,7 +253,8 @@ export interface ServiceDirs {
  * Create a list of files (FileStats[]) for the filesystem subtree rooted at
  * projectDir, filtered against the applicable .dockerignore files, including
  * a few default/hardcoded dockerignore patterns.
- * @param projectDir Source directory to
+ * @param projectDir Source directory
+ * @param multiDockerignore The --multi-dockerignore (-m) option
  * @param serviceDirsByService Map of service names to their subdirectories.
  * The service directory names/paths must be relative to the project root dir
  * and be "normalized" (path.normalize()) before the call to this function:
@@ -293,39 +264,106 @@ export interface ServiceDirs {
  */
 export async function filterFilesWithDockerignore(
 	projectDir: string,
-	serviceDirsByService?: ServiceDirs,
+	multiDockerignore: boolean,
+	serviceDirsByService: ServiceDirs,
 ): Promise<{ filteredFileList: FileStats[]; dockerignoreFiles: FileStats[] }> {
 	// path.resolve() also converts forward slashes to backslashes on Windows
 	projectDir = path.resolve(projectDir);
-	// ignoreByDir stores an instance of the dockerignore filter for each service dir
-	const ignoreByDir: {
-		[serviceDir: string]: import('@balena/dockerignore').Ignore;
-	} = {
-		'.': await getDockerIgnoreInstance(projectDir),
-	};
-	const serviceDirs: string[] = Object.values(serviceDirsByService || {})
-		// filter out the project source/root dir
-		.filter((dir) => dir && dir !== '.')
+	const root = '.' + path.sep;
+	const ignoreByService = await getDockerignoreByService(
+		projectDir,
+		multiDockerignore,
+		serviceDirsByService,
+	);
+	// Sample contents of ignoreByDir:
+	// { './': (dockerignore instance), 'foo/': (dockerignore instance) }
+	const ignoreByDir: { [serviceDir: string]: Ignore } = {};
+	for (let [serviceName, dir] of Object.entries(serviceDirsByService)) {
+		// convert slashes to backslashes on Windows, resolve '..' segments
+		dir = path.normalize(dir);
 		// add a trailing '/' (or '\' on Windows) to the path
-		.map((dir) => (dir.endsWith(path.sep) ? dir : dir + path.sep));
-
-	for (const serviceDir of serviceDirs) {
-		ignoreByDir[serviceDir] = await getDockerIgnoreInstance(
-			path.join(projectDir, serviceDir),
-		);
+		dir = dir.endsWith(path.sep) ? dir : dir + path.sep;
+		ignoreByDir[dir] = ignoreByService[serviceName];
 	}
-	const files = await listFiles(projectDir);
+	if (!ignoreByDir[root]) {
+		ignoreByDir[root] = await getDockerIgnoreInstance(projectDir);
+	}
+	const dockerignoreServiceDirs: string[] = multiDockerignore
+		? Object.keys(ignoreByDir).filter((dir) => dir && dir !== root)
+		: [];
 	const dockerignoreFiles: FileStats[] = [];
-	const filteredFileList = files.filter((file: FileStats) => {
-		if (path.basename(file.relPath) === '.dockerignore') {
-			dockerignoreFiles.push(file);
-		}
-		for (const dir of serviceDirs) {
-			if (file.relPath.startsWith(dir)) {
-				return !ignoreByDir[dir].ignores(file.relPath.substring(dir.length));
-			}
-		}
-		return !ignoreByDir['.'].ignores(file.relPath);
+	const filteredFileList: FileStats[] = [];
+	const klaw = await import('klaw');
+
+	await new Promise((resolve, reject) => {
+		// Looking at klaw's source code, `preserveSymlinks` appears to only
+		// afect the `stats` argument to the `data` event handler
+		klaw(projectDir, { preserveSymlinks: false })
+			.on('error', reject)
+			.on('end', resolve)
+			.on('data', (item: { path: string; stats: Stats }) => {
+				const { path: filePath, stats } = item;
+				// With `preserveSymlinks: false`, filePath cannot be a symlink.
+				// filePath may be a directory or a regular or special file
+				if (!stats.isFile()) {
+					return;
+				}
+				const relPath = path.relative(projectDir, filePath);
+				const fileInfo = {
+					filePath,
+					relPath,
+					stats,
+				};
+				if (path.basename(relPath) === '.dockerignore') {
+					dockerignoreFiles.push(fileInfo);
+				}
+				for (const dir of dockerignoreServiceDirs) {
+					if (relPath.startsWith(dir)) {
+						if (!ignoreByDir[dir].ignores(relPath.substring(dir.length))) {
+							filteredFileList.push(fileInfo);
+						}
+						return;
+					}
+				}
+				if (!ignoreByDir[root].ignores(relPath)) {
+					filteredFileList.push(fileInfo);
+				}
+			});
 	});
 	return { filteredFileList, dockerignoreFiles };
 }
+
+let dockerignoreByService: { [serviceName: string]: Ignore } | null = null;
+
+/**
+ * Get dockerignore instances for each service in serviceDirsByService.
+ * Dockerignore instances are cached and may be shared between services.
+ * @param projectDir Source directory
+ * @param multiDockerignore The --multi-dockerignore (-m) option
+ * @param serviceDirsByService Map of service names to their subdirectories
+ */
+export async function getDockerignoreByService(
+	projectDir: string,
+	multiDockerignore: boolean,
+	serviceDirsByService: ServiceDirs,
+): Promise<{ [serviceName: string]: Ignore }> {
+	if (dockerignoreByService) {
+		return dockerignoreByService;
+	}
+	const cachedDirs: { [dir: string]: Ignore } = {};
+	// path.resolve() converts to an absolute path, removes trailing slashes,
+	// and also converts forward slashes to backslashes on Windows.
+	projectDir = path.resolve(projectDir);
+	dockerignoreByService = {};
+
+	for (let [serviceName, dir] of Object.entries(serviceDirsByService)) {
+		dir = multiDockerignore ? dir : '.';
+		const absDir = path.resolve(projectDir, dir);
+		if (!cachedDirs[absDir]) {
+			cachedDirs[absDir] = await getDockerIgnoreInstance(absDir);
+		}
+		dockerignoreByService[serviceName] = cachedDirs[absDir];
+	}
+
+	return dockerignoreByService;
+}

lib/utils/messages.ts

@@ -15,26 +15,20 @@
  * limitations under the License.
  */
 
-const DEBUG_MODE = !!process.env.DEBUG;
-
 export const reachingOut = `\
-If you need help, or just want to say hi, don't hesitate in reaching out
-through our discussion and support forums at https://forums.balena.io
-
-For bug reports or feature requests, have a look at the GitHub issues or
-create a new one at: https://github.com/balena-io/balena-cli/issues/\
+For further help or support, visit:
+https://www.balena.io/docs/reference/balena-cli/#support-faq-and-troubleshooting
 `;
 
 const debugHint = `\
 Additional information may be available with the \`--debug\` flag.
-`;
+\n`;
 
-export const help = `\
-For help, visit our support forums: https://forums.balena.io
-For bug reports or feature requests, see: https://github.com/balena-io/balena-cli/issues/
-`;
+export const help = reachingOut;
 
-export const getHelp = (DEBUG_MODE ? '' : debugHint) + help;
+// Note that the value of process.env.DEBUG may change after the --debug flag
+// is parsed, so its evaluation cannot happen at module loading time.
+export const getHelp = () => (process.env.DEBUG ? '' : debugHint) + help;
 
 export const balenaAsciiArt = `\
  _            _
@@ -69,7 +63,7 @@ this file will be used instead.`;
 
 export const dockerignoreHelp = `\
 DOCKERIGNORE AND GITIGNORE FILES  
-By default, balenaCLI will use a single ".dockerignore" file (if any) at
+By default, the balena CLI will use a single ".dockerignore" file (if any) at
 the project root (--source directory) in order to decide which source files to
 exclude from the "build context" (tar stream) sent to balenaCloud, Docker
 daemon or balenaEngine. In a microservices (multicontainer) application, the
@@ -94,7 +88,7 @@ compatibility with the standard docker-compose tool, while still allowing a
 root .dockerignore file (at the overall project root) to filter files and
 folders that are outside service subdirectories.
 
-balenaCLI releases older than v12.0.0 also took .gitignore files into account.
+balena CLI releases older than v12.0.0 also took .gitignore files into account.
 This behavior is deprecated, but may still be enabled with the --gitignore (-g)
 option if compatibility is required. This option is mutually exclusive with
 --multi-dockerignore (-m) and will be removed in the CLI's next major version

lib/utils/normalization.ts

@@ -1,18 +1,19 @@
-/*
-Copyright 2016-2018 Balena
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+/**
+ * @license
+ * Copyright 2016-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 import type { BalenaSDK } from 'balena-sdk';
 import _ = require('lodash');

lib/utils/oclif-utils.ts

@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+import { Main } from '@oclif/command';
 import type * as Config from '@oclif/config';
 
 /**
@@ -48,6 +49,17 @@ export class CommandHelp {
 	}
 }
 
+export class CustomMain extends Main {
+	protected _helpOverride(): boolean {
+		// Disable oclif's default handler for the 'version' command
+		if (['-v', '--version', 'version'].includes(this.argv[0])) {
+			return false;
+		} else {
+			return super._helpOverride();
+		}
+	}
+}
+
 /** Convert e.g. 'env add NAME [VALUE]' to 'env add <name> [value]' */
 export function capitanoizeOclifUsage(
 	oclifUsage: string | string[] | undefined,
@@ -57,3 +69,17 @@ export function capitanoizeOclifUsage(
 		.replace(/(?<=\s)[A-Z]+(?=(\s|$))/g, (match) => `<${match}>`)
 		.toLowerCase();
 }
+
+export async function getCommandsFromManifest() {
+	const manifest = require('../../oclif.manifest.json');
+
+	if (manifest.commands == null) {
+		throw new Error('Commands section not found in manifest.');
+	}
+	return manifest.commands;
+}
+
+export async function getCommandIdsFromManifest() {
+	const commands = await getCommandsFromManifest();
+	return Object.keys(commands);
+}

lib/utils/proxy.ts

@@ -15,71 +15,11 @@
  * limitations under the License.
  */
 
-import * as packageJSON from '../package.json';
-import { onceAsync, stripIndent } from './utils/lazy';
-
-class CliSettings {
-	public readonly settings: any;
-	constructor() {
-		this.settings = require('balena-settings-client') as typeof import('balena-settings-client');
-	}
-
-	public get<T>(name: string): T {
-		return this.settings.get(name);
-	}
-
-	/**
-	 * Like settings.get(), but return `undefined` instead of throwing an
-	 * error if the setting is not found / not defined.
-	 */
-	public getCatch<T>(name: string): T | undefined {
-		try {
-			return this.settings.get(name);
-		} catch (err) {
-			if (!/Setting not found/i.test(err.message)) {
-				throw err;
-			}
-		}
-	}
-}
-
-/**
- * Sentry.io setup
- * @see https://docs.sentry.io/error-reporting/quickstart/?platform=node
- */
-export const setupSentry = onceAsync(async () => {
-	const config = await import('./config');
-	const Sentry = await import('@sentry/node');
-	Sentry.init({
-		dsn: config.sentryDsn,
-		release: packageJSON.version,
-	});
-	Sentry.configureScope((scope) => {
-		scope.setExtras({
-			is_pkg: !!(process as any).pkg,
-			node_version: process.version,
-			platform: process.platform,
-		});
-	});
-	return Sentry.getCurrentHub();
-});
-
-async function checkNodeVersion() {
-	const validNodeVersions = packageJSON.engines.node;
-	if (!(await import('semver')).satisfies(process.version, validNodeVersions)) {
-		console.warn(stripIndent`
-			------------------------------------------------------------------------------
-			Warning: Node version "${process.version}" does not match required versions "${validNodeVersions}".
-			This may cause unexpected behavior. To upgrade Node, visit:
-			https://nodejs.org/en/download/
-			------------------------------------------------------------------------------
-			`);
-	}
-}
-
 import type { Options as GlobalTunnelNgConfig } from 'global-tunnel-ng';
 export type { GlobalTunnelNgConfig };
 
+import { CliSettings } from './bootstrap';
+
 type ProxyConfig = string | GlobalTunnelNgConfig;
 
 /**
@@ -104,7 +44,7 @@ type ProxyConfig = string | GlobalTunnelNgConfig;
  * 'localhost' and '127.0.0.1' are always excluded. If NO_PROXY is not defined,
  * default exclusion patterns are added for all private IPv4 address ranges.
  */
-async function setupGlobalHttpProxy(settings: CliSettings) {
+export async function setupGlobalHttpProxy(settings: CliSettings) {
 	// `global-tunnel-ng` accepts lowercase variables with higher precedence
 	// than uppercase variables, but `global-agent` does not accept lowercase.
 	// Set uppercase versions for backwards compatibility.
@@ -207,37 +147,3 @@ export function makeUrlFromTunnelNgConfig(cfg: GlobalTunnelNgConfig): string {
 	}
 	return url;
 }
-
-function setupBalenaSdkSharedOptions(settings: CliSettings) {
-	// We don't yet use balena-sdk directly everywhere, but we set up shared
-	// options correctly so we can do safely in submodules
-	const BalenaSdk = require('balena-sdk') as typeof import('balena-sdk');
-	BalenaSdk.setSharedOptions({
-		apiUrl: settings.get<string>('apiUrl'),
-		dataDirectory: settings.get<string>('dataDirectory'),
-	});
-}
-
-/**
- * Addresses the console warning:
- * (node:49500) MaxListenersExceededWarning: Possible EventEmitter memory
- * leak detected. 11 error listeners added. Use emitter.setMaxListeners() to
- * increase limit
- */
-export function setMaxListeners(maxListeners: number) {
-	require('events').EventEmitter.defaultMaxListeners = maxListeners;
-}
-
-export async function globalInit() {
-	await setupSentry();
-	checkNodeVersion();
-
-	const settings = new CliSettings();
-
-	// Proxy setup should be done early on, before loading balena-sdk
-	await setupGlobalHttpProxy(settings);
-	setupBalenaSdkSharedOptions(settings);
-
-	// check for CLI updates once a day
-	(await import('./utils/update')).notify();
-}

lib/utils/remote-build.ts

@@ -24,7 +24,7 @@ import type { Pack } from 'tar-stream';
 
 import { ExpectedError } from '../errors';
 import { exitWithExpectedError } from '../errors';
-import { tarDirectory } from './compose';
+import { tarDirectory } from './compose_ts';
 import { getVisuals, stripIndent } from './lazy';
 import Logger = require('./logger');
 

lib/utils/sdk.ts

@@ -0,0 +1,49 @@
+/**
+ * @license
+ * Copyright 2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Application, BalenaSDK, PineOptions } from 'balena-sdk';
+
+/**
+ * Wraps the sdk application.get method,
+ * adding disambiguation in cases where the provided
+ * identifier could be interpreted in multiple valid ways.
+ */
+export async function getApplication(
+	sdk: BalenaSDK,
+	nameOrSlugOrId: string | number,
+	options?: PineOptions<Application>,
+): Promise<Application> {
+	// TODO: Consider whether it would be useful to generally include interactive selection of application here,
+	//       when nameOrSlugOrId not provided.
+	//       e.g. nameOrSlugOrId || (await (await import('../../utils/patterns')).selectApplication()),
+	//       See commands/device/init.ts ~ln100 for example
+	const { looksLikeInteger } = await import('./validation');
+	if (looksLikeInteger(nameOrSlugOrId as string)) {
+		try {
+			// Test for existence of app with this numerical ID
+			return await sdk.models.application.get(Number(nameOrSlugOrId), options);
+		} catch (e) {
+			const { instanceOf } = await import('../errors');
+			const { BalenaApplicationNotFound } = await import('balena-errors');
+			if (!instanceOf(e, BalenaApplicationNotFound)) {
+				throw e;
+			}
+			// App with this numerical ID not found, but there may be an app with this numerical name.
+		}
+	}
+	return sdk.models.application.get(nameOrSlugOrId, options);
+}

lib/utils/sudo.ts

@@ -31,7 +31,7 @@ import { stripIndent } from './lazy';
  * differences between bash/sh and the Windows cmd.exe in relation to escape
  * characters.
  * @param stderr Optional stream to which stderr should be piped
- * @param isCLIcmd (default: true) Whether the command array is a balenaCLI command
+ * @param isCLIcmd (default: true) Whether the command array is a balena CLI command
  * (e.g. ['internal', 'osinit', ...]), in which case process.argv[0] and argv[1] are
  * added as necessary, depending on whether the CLI is running as a standalone zip
  * package (with Node built in).

package.json

@@ -1,6 +1,6 @@
 {
   "name": "balena-cli",
-  "version": "12.23.3",
+  "version": "12.29.0",
   "description": "The official balena Command Line Interface",
   "main": "./build/app.js",
   "homepage": "https://github.com/balena-io/balena-cli",
@@ -61,13 +61,13 @@
     "test:shrinkwrap": "ts-node --transpile-only automation/run.ts test-shrinkwrap",
     "test:source": "cross-env BALENA_CLI_TEST_TYPE=source mocha",
     "test:standalone": "npm run build:standalone && npm run test:standalone:fast",
-    "test:standalone:fast": "cross-env BALENA_CLI_TEST_TYPE=standalone mocha",
+    "test:standalone:fast": "cross-env BALENA_CLI_TEST_TYPE=standalone mocha --config .mocharc-standalone.js",
     "test:fast": "npm run build:fast && npm run test:source",
     "test:only": "npm run build:fast && cross-env BALENA_CLI_TEST_TYPE=source mocha \"tests/**/${npm_config_test}.spec.ts\"",
     "catch-uncommitted": "ts-node --transpile-only automation/run.ts catch-uncommitted",
     "ci": "npm run test && npm run catch-uncommitted",
     "watch": "gulp watch",
-    "lint": "balena-lint -e ts -e js --typescript --fix automation/ lib/ typings/ tests/ gulpfile.js",
+    "lint": "balena-lint -e ts -e js --typescript --fix automation/ lib/ typings/ tests/ bin/balena bin/balena-dev gulpfile.js .mocharc.js .mocharc-standalone.js",
     "update": "ts-node --transpile-only ./automation/update-module.ts",
     "prepare": "echo {} > bin/.fast-boot.json",
     "prepublishOnly": "npm run build"
@@ -91,13 +91,6 @@
       "pre-commit": "node automation/check-npm-version.js && node automation/check-doc.js"
     }
   },
-  "mocha": {
-    "reporter": "spec",
-    "require": "ts-node/register/transpile-only",
-    "file": "./tests/config-tests",
-    "timeout": 12000,
-    "_": "tests/**/*.spec.ts"
-  },
   "oclif": {
     "bin": "balena",
     "commands": "./build/commands",
@@ -142,7 +135,7 @@
     "@types/klaw": "^3.0.1",
     "@types/lodash": "^4.14.159",
     "@types/mixpanel": "^2.14.2",
-    "@types/mocha": "^5.2.7",
+    "@types/mocha": "^8.0.4",
     "@types/mock-require": "^2.0.0",
     "@types/moment-duration-format": "^2.2.2",
     "@types/net-keepalive": "^0.4.1",
@@ -156,7 +149,7 @@
     "@types/rewire": "^2.5.28",
     "@types/rimraf": "^3.0.0",
     "@types/shell-escape": "^0.2.0",
-    "@types/sinon": "^9.0.4",
+    "@types/sinon": "^9.0.8",
     "@types/split": "^1.0.0",
     "@types/stream-to-promise": "2.2.0",
     "@types/tar-stream": "^2.1.0",
@@ -180,15 +173,15 @@
     "intercept-stdout": "^0.1.2",
     "jsonwebtoken": "^8.5.1",
     "mkdirp": "^0.5.5",
-    "mocha": "^6.2.3",
+    "mocha": "^8.2.1",
     "mock-require": "^3.0.3",
     "nock": "^12.0.3",
     "parse-link-header": "~1.0.1",
     "pkg": "^4.4.9",
     "publish-release": "^1.6.1",
-    "rewire": "^4.0.1",
+    "rewire": "^5.0.0",
     "simple-git": "^1.132.0",
-    "sinon": "^9.0.3",
+    "sinon": "^9.2.1",
     "ts-node": "^8.10.2",
     "typescript": "^4.0.2"
   },
@@ -197,7 +190,7 @@
     "@balena/es-version": "^1.0.0",
     "@oclif/command": "^1.8.0",
     "@resin.io/valid-email": "^0.1.0",
-    "@sentry/node": "^5.21.1",
+    "@sentry/node": "^5.25.0",
     "@types/fast-levenshtein": "0.0.1",
     "@types/update-notifier": "^4.1.1",
     "@zeit/dockerignore": "0.0.3",
@@ -216,7 +209,7 @@
     "bluebird": "^3.7.2",
     "body-parser": "^1.19.0",
     "chalk": "^3.0.0",
-    "chokidar": "^3.3.1",
+    "chokidar": "^3.4.3",
     "cli-truncate": "^2.1.0",
     "color-hash": "^1.0.3",
     "columnify": "^1.5.2",

tests/commands/build.spec.ts

@@ -16,6 +16,7 @@
  */
 
 import { expect } from 'chai';
+import * as _ from 'lodash';
 import mock = require('mock-require');
 import { promises as fs } from 'fs';
 import * as path from 'path';
@@ -45,13 +46,16 @@ const commonResponseLines: { [key: string]: string[] } = {
 
 const commonQueryParams = {
 	t: '${tag}',
-	buildargs: '{}',
+	buildargs: {},
 	labels: '',
 };
 
 const commonComposeQueryParams = {
 	t: '${tag}',
-	buildargs: '{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable"}',
+	buildargs: {
+		MY_VAR_1: 'This is a variable',
+		MY_VAR_2: 'Also a variable',
+	},
 	labels: '',
 };
 
@@ -375,19 +379,26 @@ describe('balena build', function () {
 			'utf8',
 		);
 		const expectedQueryParamsByService = {
-			service1: Object.entries({
-				...commonComposeQueryParams,
-				buildargs:
-					'{"BARG1":"b1","barg2":"B2","MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable","SERVICE1_VAR":"This is a service specific variable"}',
-				cachefrom: '["my/img1","my/img2"]',
-			}),
-			service2: Object.entries({
-				...commonComposeQueryParams,
-				buildargs:
-					'{"BARG1":"b1","barg2":"B2","MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable"}',
-				cachefrom: '["my/img1","my/img2"]',
-				dockerfile: 'Dockerfile-alt',
-			}),
+			service1: Object.entries(
+				_.merge({}, commonComposeQueryParams, {
+					buildargs: {
+						COMPOSE_ARG: 'A',
+						barg: 'b',
+						SERVICE1_VAR: 'This is a service specific variable',
+					},
+					cachefrom: ['my/img1', 'my/img2'],
+				}),
+			),
+			service2: Object.entries(
+				_.merge({}, commonComposeQueryParams, {
+					buildargs: {
+						COMPOSE_ARG: 'A',
+						barg: 'b',
+					},
+					cachefrom: ['my/img1', 'my/img2'],
+					dockerfile: 'Dockerfile-alt',
+				}),
+			),
 		};
 		const expectedResponseLines: string[] = [
 			...commonResponseLines[responseFilename],
@@ -417,7 +428,7 @@ describe('balena build', function () {
 		}
 		docker.expectGetInfo({});
 		await testDockerBuildStream({
-			commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -G -B BARG1=b1 -B barg2=B2 --cache-from my/img1,my/img2`,
+			commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -G -B COMPOSE_ARG=A -B barg=b --cache-from my/img1,my/img2`,
 			dockerMock: docker,
 			expectedFilesByService,
 			expectedQueryParamsByService,
@@ -464,15 +475,19 @@ describe('balena build', function () {
 			'utf8',
 		);
 		const expectedQueryParamsByService = {
-			service1: Object.entries({
-				...commonComposeQueryParams,
-				buildargs:
-					'{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable","SERVICE1_VAR":"This is a service specific variable"}',
-			}),
-			service2: Object.entries({
-				...commonComposeQueryParams,
-				dockerfile: 'Dockerfile-alt',
-			}),
+			service1: Object.entries(
+				_.merge({}, commonComposeQueryParams, {
+					buildargs: { SERVICE1_VAR: 'This is a service specific variable' },
+				}),
+			),
+			service2: Object.entries(
+				_.merge({}, commonComposeQueryParams, {
+					buildargs: {
+						COMPOSE_ARG: 'an argument defined in the docker-compose.yml file',
+					},
+					dockerfile: 'Dockerfile-alt',
+				}),
+			),
 		};
 		const expectedResponseLines: string[] = [
 			...commonResponseLines[responseFilename],

tests/commands/deploy.spec.ts

@@ -17,6 +17,7 @@
 
 import { expect } from 'chai';
 import { promises as fs } from 'fs';
+import * as _ from 'lodash';
 import * as path from 'path';
 import * as sinon from 'sinon';
 
@@ -53,14 +54,14 @@ const commonQueryParams = [
 	['labels', ''],
 ];
 
-const commonComposeQueryParams = [
-	['t', '${tag}'],
-	[
-		'buildargs',
-		'{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable"}',
-	],
-	['labels', ''],
-];
+const commonComposeQueryParams = {
+	t: '${tag}',
+	buildargs: {
+		MY_VAR_1: 'This is a variable',
+		MY_VAR_2: 'Also a variable',
+	},
+	labels: '',
+};
 
 const hr =
 	'----------------------------------------------------------------------';
@@ -268,15 +269,19 @@ describe('balena deploy', function () {
 			'utf8',
 		);
 		const expectedQueryParamsByService = {
-			service1: [
-				['t', '${tag}'],
-				[
-					'buildargs',
-					'{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable","SERVICE1_VAR":"This is a service specific variable"}',
-				],
-				['labels', ''],
-			],
-			service2: [...commonComposeQueryParams, ['dockerfile', 'Dockerfile-alt']],
+			service1: Object.entries(
+				_.merge({}, commonComposeQueryParams, {
+					buildargs: { SERVICE1_VAR: 'This is a service specific variable' },
+				}),
+			),
+			service2: Object.entries(
+				_.merge({}, commonComposeQueryParams, {
+					buildargs: {
+						COMPOSE_ARG: 'an argument defined in the docker-compose.yml file',
+					},
+					dockerfile: 'Dockerfile-alt',
+				}),
+			),
 		};
 		const expectedResponseLines: string[] = [
 			...commonResponseLines[responseFilename],

tests/commands/device/devices.spec.ts

@@ -40,8 +40,9 @@ DESCRIPTION
 
   The --json option is recommended when scripting the output of this command,
   because field names are less likely to change in JSON format and because it
-  better represents data types like arrays and empty strings. The 'jq' utility
-  may also be helpful in shell scripts (https://stedolan.github.io/jq/manual/).
+  better represents data types like arrays, empty strings and null values.
+  The 'jq' utility may be helpful for querying JSON fields in shell scripts
+  (https://stedolan.github.io/jq/manual/).
 
 EXAMPLES
   $ balena devices

tests/commands/help.spec.ts

@@ -19,6 +19,7 @@ import { expect } from 'chai';
 
 import { BalenaAPIMock } from '../balena-api-mock';
 import { cleanOutput, runCommand } from '../helpers';
+import * as messages from '../../build/utils/messages';
 
 const SIMPLE_HELP = `
 USAGE
@@ -87,7 +88,7 @@ ADDITIONAL COMMANDS
   tag set <tagKey> [value]               set a tag on an application, device or release
   tags                                   list all tags for an application, device or release
   util available-drives                  list available drives
-  version                                display version information for balenaCLI and/or Node.js
+  version                                display version information for the balena CLI and/or Node.js
   whoami                                 display account information for current user
 
 `;
@@ -103,10 +104,7 @@ GLOBAL OPTIONS
 
 `;
 
-const ONLINE_RESOURCES = `
-For help, visit our support forums: https://forums.balena.io
-For bug reports or feature requests, see: https://github.com/balena-io/balena-cli/issues/
-`;
+const ONLINE_RESOURCES = messages.reachingOut;
 
 describe.skip('balena help', function () {
 	let api: BalenaAPIMock;

tests/commands/push.spec.ts

@@ -465,7 +465,7 @@ describe('balena push', function () {
 		const expectedFiles: ExpectedTarStreamFiles = {
 			'.balena/balena.yml': { fileSize: 197, type: 'file' },
 			'.dockerignore': { fileSize: 22, type: 'file' },
-			'docker-compose.yml': { fileSize: 245, type: 'file' },
+			'docker-compose.yml': { fileSize: 332, type: 'file' },
 			'service1/Dockerfile.template': { fileSize: 144, type: 'file' },
 			'service1/file1.sh': { fileSize: 12, type: 'file' },
 			'service2/Dockerfile-alt': { fileSize: 40, type: 'file' },
@@ -523,7 +523,7 @@ describe('balena push', function () {
 		const expectedFiles: ExpectedTarStreamFiles = {
 			'.balena/balena.yml': { fileSize: 197, type: 'file' },
 			'.dockerignore': { fileSize: 22, type: 'file' },
-			'docker-compose.yml': { fileSize: 245, type: 'file' },
+			'docker-compose.yml': { fileSize: 332, type: 'file' },
 			'service1/Dockerfile.template': { fileSize: 144, type: 'file' },
 			'service1/file1.sh': { fileSize: 12, type: 'file' },
 			'service1/test-ignore.txt': { fileSize: 12, type: 'file' },

tests/commands/ssh.spec.ts

@@ -120,6 +120,13 @@ describe('balena ssh', function () {
 async function checkSsh(): Promise<boolean> {
 	const { which } = await import('../../build/utils/helpers');
 	const sshPath = await which('ssh', false);
+	if ((sshPath || '').includes('\\Windows\\System32\\OpenSSH\\ssh')) {
+		// don't use Windows' built-in ssh tool for these test cases
+		// because it messes up with the terminal window such that
+		// "line breaks stop working" (and not even '\033c' fixes it)
+		// and all mocha output gets printed on a single very long line...
+		return false;
+	}
 	return !!sshPath;
 }
 
@@ -127,11 +134,13 @@ async function checkSsh(): Promise<boolean> {
 async function startMockSshServer(): Promise<[Server, number]> {
 	const server = createServer((c) => {
 		// 'connection' listener
-		c.on('end', () => {
+		const handler = (msg: string) => {
 			if (process.env.DEBUG) {
-				console.error('[debug] mock ssh server: client disconnected');
+				console.error(`[debug] mock ssh server: ${msg}`);
 			}
-		});
+		};
+		c.on('error', (err) => handler(err.message));
+		c.on('end', () => handler('client disconnected'));
 		c.end();
 	});
 	server.on('error', (err) => {

tests/docker-build.ts

@@ -20,6 +20,7 @@ import * as _ from 'lodash';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 import { PathUtils } from 'resin-multibuild';
+import rewire = require('rewire');
 import * as sinon from 'sinon';
 import { Readable } from 'stream';
 import * as tar from 'tar-stream';
@@ -29,7 +30,12 @@ import { URL } from 'url';
 import { stripIndent } from '../lib/utils/lazy';
 import { BuilderMock } from './builder-mock';
 import { DockerMock } from './docker-mock';
-import { cleanOutput, fillTemplateArray, runCommand } from './helpers';
+import {
+	cleanOutput,
+	deepJsonParse,
+	deepTemplateReplace,
+	runCommand,
+} from './helpers';
 import {
 	ExpectedTarStreamFile,
 	ExpectedTarStreamFiles,
@@ -152,7 +158,7 @@ export async function testDockerBuildStream(o: {
 	commandLine: string;
 	dockerMock: DockerMock;
 	expectedFilesByService: ExpectedTarStreamFilesByService;
-	expectedQueryParamsByService: { [service: string]: string[][] };
+	expectedQueryParamsByService: { [service: string]: any[][] };
 	expectedErrorLines?: string[];
 	expectedExitCode?: number;
 	expectedResponseLines: string[];
@@ -161,15 +167,15 @@ export async function testDockerBuildStream(o: {
 	responseBody: string;
 	services: string[]; // e.g. ['main'] or ['service1', 'service2']
 }) {
-	const expectedErrorLines = fillTemplateArray(o.expectedErrorLines || [], o);
-	const expectedResponseLines = fillTemplateArray(o.expectedResponseLines, o);
+	const expectedErrorLines = deepTemplateReplace(o.expectedErrorLines || [], o);
+	const expectedResponseLines = deepTemplateReplace(o.expectedResponseLines, o);
 
 	for (const service of o.services) {
 		// tagPrefix is, for example, 'myApp' if the path is 'path/to/myApp'
 		const tagPrefix = o.projectPath.split(path.sep).pop();
 		const tag = `${tagPrefix}_${service}`;
 		const expectedFiles = o.expectedFilesByService[service];
-		const expectedQueryParams = fillTemplateArray(
+		const expectedQueryParams = deepTemplateReplace(
 			o.expectedQueryParamsByService[service],
 			{ tag, ...o },
 		);
@@ -181,7 +187,9 @@ export async function testDockerBuildStream(o: {
 			checkURI: async (uri: string) => {
 				const url = new URL(uri, 'http://test.net/');
 				const queryParams = Array.from(url.searchParams.entries());
-				expect(queryParams).to.have.deep.members(expectedQueryParams);
+				expect(deepJsonParse(queryParams)).to.have.deep.members(
+					deepJsonParse(expectedQueryParams),
+				);
 			},
 			checkBuildRequestBody: (buildRequestBody: string) =>
 				inspectTarStream(buildRequestBody, expectedFiles, projectPath),
@@ -192,6 +200,8 @@ export async function testDockerBuildStream(o: {
 		}
 	}
 
+	resetDockerignoreCache();
+
 	const { exitCode, out, err } = await runCommand(o.commandLine);
 
 	if (expectedErrorLines.length) {
@@ -226,22 +236,36 @@ export async function testPushBuildStream(o: {
 	responseCode: number;
 	responseBody: string;
 }) {
-	const expectedQueryParams = fillTemplateArray(o.expectedQueryParams, o);
-	const expectedResponseLines = fillTemplateArray(o.expectedResponseLines, o);
+	const expectedQueryParams = deepTemplateReplace(o.expectedQueryParams, o);
+	const expectedResponseLines = deepTemplateReplace(o.expectedResponseLines, o);
 
 	o.builderMock.expectPostBuild({
 		...o,
 		checkURI: async (uri: string) => {
 			const url = new URL(uri, 'http://test.net/');
 			const queryParams = Array.from(url.searchParams.entries());
-			expect(queryParams).to.have.deep.members(expectedQueryParams);
+			expect(deepJsonParse(queryParams)).to.have.deep.members(
+				deepJsonParse(expectedQueryParams),
+			);
 		},
 		checkBuildRequestBody: (buildRequestBody) =>
 			inspectTarStream(buildRequestBody, o.expectedFiles, o.projectPath),
 	});
 
+	resetDockerignoreCache();
+
 	const { out, err } = await runCommand(o.commandLine);
 
 	expect(err).to.be.empty;
 	expect(cleanOutput(out, true)).to.include.members(expectedResponseLines);
 }
+
+export function resetDockerignoreCache() {
+	if (process.env.BALENA_CLI_TEST_TYPE !== 'source') {
+		return;
+	}
+	const ignorePath = '../build/utils/ignore';
+	delete require.cache[require.resolve(ignorePath)];
+	const ignoreMod = rewire(ignorePath);
+	ignoreMod.__set__('dockerignoreByService', null);
+}

tests/errors.spec.ts

@@ -126,6 +126,7 @@ describe('handleError() function', () => {
 		'must also be provided when using', // oclif
 		'Expected an integer', // oclif
 		'Flag --foo expects a value', // oclif
+		'BalenaRequestError: Request error: Unauthorized', // sdk
 	];
 
 	messagesToMatch.forEach((message) => {
@@ -186,7 +187,7 @@ describe('printErrorMessage() function', () => {
 		expect(consoleError.getCall(0).args[0]).to.equal(expectedOutputMessages[0]);
 		expect(consoleError.getCall(1).args[0]).to.equal(expectedOutputMessages[1]);
 		expect(consoleError.getCall(2).args[0]).to.equal(expectedOutputMessages[2]);
-		expect(consoleError.getCall(3).args[0]).to.equal(`\n${getHelp}\n`);
+		expect(consoleError.getCall(3).args[0]).to.equal(`\n${getHelp()}\n`);
 
 		consoleError.restore();
 	});

tests/helpers.ts

@@ -23,7 +23,6 @@ import * as nock from 'nock';
 import * as path from 'path';
 
 import * as balenaCLI from '../build/app';
-import { setupSentry } from '../build/app-common';
 
 const balenaExe = process.platform === 'win32' ? 'balena.exe' : 'balena';
 const standalonePath = path.resolve(__dirname, '..', 'build-bin', balenaExe);
@@ -48,6 +47,7 @@ function filterCliOutputForTests(testOutput: TestOutput): TestOutput {
 				// TODO stop this warning message from appearing when running
 				// sdk.setSharedOptions multiple times in the same process
 				!line.startsWith('Shared SDK options') &&
+				!line.startsWith('WARN: disabling Sentry.io error reporting') &&
 				// Node 12: '[DEP0066] DeprecationWarning: OutgoingMessage.prototype._headers is deprecated'
 				!line.includes('[DEP0066]'),
 		),
@@ -265,29 +265,61 @@ export function fillTemplate(
 	return unescaped;
 }
 
-export function fillTemplateArray(
-	templateStringArray: string[],
-	templateVars: object,
-): string[];
-export function fillTemplateArray(
-	templateStringArray: Array<string | string[]>,
-	templateVars: object,
-): Array<string | string[]>;
-export function fillTemplateArray(
-	templateStringArray: Array<string | string[]>,
-	templateVars: object,
-): Array<string | string[]> {
-	return templateStringArray.map((i) =>
-		Array.isArray(i)
-			? fillTemplateArray(i, templateVars)
-			: fillTemplate(i, templateVars),
-	);
+/**
+ * Recursively navigate the `data` argument (if it is an array or object),
+ * finding and replacing "template strings" such as 'hello ${name}!' with
+ * the variable values given in `templateVars` such as { name: 'world' }.
+ *
+ * @param data Any data type (array, object, string) containing template
+ * strings to be replaced
+ * @param templateVars Map of template variable names to values
+ */
+export function deepTemplateReplace(
+	data: any,
+	templateVars: { [key: string]: any },
+): any {
+	switch (typeof data) {
+		case 'string':
+			return fillTemplate(data, templateVars);
+		case 'object':
+			if (Array.isArray(data)) {
+				return data.map((i) => deepTemplateReplace(i, templateVars));
+			}
+			return _.mapValues(data, (value) =>
+				deepTemplateReplace(value, templateVars),
+			);
+		default:
+			// number, undefined, null, or something else
+			return data;
+	}
+}
+
+export const fillTemplateArray = deepTemplateReplace;
+
+/**
+ * Recursively navigate the `data` argument (if it is an array or object),
+ * looking for strings that start with `[` or `{` which are assumed to contain
+ * JSON arrays or objects that are then parsed with JSON.parse().
+ * @param data
+ */
+export function deepJsonParse(data: any): any {
+	if (typeof data === 'string') {
+		const maybeJson = data.trim();
+		if (maybeJson.startsWith('{') || maybeJson.startsWith('[')) {
+			return JSON.parse(maybeJson);
+		}
+	} else if (Array.isArray(data)) {
+		return data.map((i) => deepJsonParse(i));
+	} else if (typeof data === 'object') {
+		return _.mapValues(data, (value) => deepJsonParse(value));
+	}
+	return data;
 }
 
 export async function switchSentry(
 	enabled: boolean | undefined,
 ): Promise<boolean | undefined> {
-	const sentryOpts = (await setupSentry()).getClient()?.getOptions();
+	const sentryOpts = (await balenaCLI.setupSentry()).getClient()?.getOptions();
 	if (sentryOpts) {
 		const sentryStatus = sentryOpts.enabled;
 		sentryOpts.enabled = enabled;

tests/proxy.spec.ts

@@ -20,7 +20,7 @@ import { expect } from 'chai';
 import {
 	GlobalTunnelNgConfig,
 	makeUrlFromTunnelNgConfig,
-} from '../build/app-common';
+} from '../build/utils/proxy';
 
 describe('makeUrlFromTunnelNgConfig() function', function () {
 	it('should return a URL given a GlobalTunnelNgConfig object', () => {

tests/test-data/projects/docker-compose/basic/docker-compose.yml

@@ -12,3 +12,5 @@ services:
     build:
       context: ./service2
       dockerfile: Dockerfile-alt
+      args:
+        - 'COMPOSE_ARG=an argument defined in the docker-compose.yml file'

tests/utils/device/live.spec.ts

@@ -0,0 +1,374 @@
+/**
+ * @license
+ * Copyright 2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect } from 'chai';
+import * as chokidar from 'chokidar';
+import { promises as fs } from 'fs';
+import * as _ from 'lodash';
+import * as path from 'path';
+import { promisify } from 'util';
+
+import { LivepushManager } from '../../../lib/utils/device/live';
+import { resetDockerignoreCache } from '../../docker-build';
+import { setupDockerignoreTestData } from '../../projects';
+
+const delay = promisify(setTimeout);
+const FS_WATCH_DURATION_MS = 500;
+
+const repoPath = path.normalize(path.join(__dirname, '..', '..', '..'));
+const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects');
+
+interface ByService<T> {
+	[serviceName: string]: T;
+}
+
+class MockLivepushManager extends LivepushManager {
+	public constructor() {
+		super({
+			buildContext: '',
+			composition: { version: '2.1', services: {} },
+			buildTasks: [],
+			docker: {} as import('dockerode'),
+			api: {} as import('../../../lib/utils/device/api').DeviceAPI,
+			logger: {} as import('../../../lib/utils/logger'),
+			buildLogs: {},
+			deployOpts: {} as import('../../../lib/utils/device/deploy').DeviceDeployOptions,
+		});
+	}
+
+	public testSetupFilesystemWatcher(
+		serviceName: string,
+		rootContext: string,
+		serviceContext: string,
+		changedPathHandler: (serviceName: string, changedPath: string) => void,
+		dockerignoreByService: ByService<import('@balena/dockerignore').Ignore>,
+		multiDockerignore: boolean,
+	): import('chokidar').FSWatcher {
+		return super.setupFilesystemWatcher(
+			serviceName,
+			rootContext,
+			serviceContext,
+			changedPathHandler,
+			dockerignoreByService,
+			multiDockerignore,
+		);
+	}
+}
+
+// "describeSS" stands for "describe Skip Standalone"
+const describeSS =
+	process.env.BALENA_CLI_TEST_TYPE === 'standalone' ? describe.skip : describe;
+
+describeSS('LivepushManager::setupFilesystemWatcher', function () {
+	const manager = new MockLivepushManager();
+
+	async function createMonitors(
+		projectPath: string,
+		composition: import('resin-compose-parse').Composition,
+		multiDockerignore: boolean,
+		changedPathHandler: (serviceName: string, changedPath: string) => void,
+	): Promise<ByService<chokidar.FSWatcher>> {
+		const { getServiceDirsFromComposition } = await import(
+			'../../../build/utils/compose_ts'
+		);
+		const { getDockerignoreByService } = await import(
+			'../../../build/utils/ignore'
+		);
+		const rootContext = path.resolve(projectPath);
+
+		const monitors: ByService<chokidar.FSWatcher> = {};
+
+		const serviceDirsByService = await getServiceDirsFromComposition(
+			projectPath,
+			composition,
+		);
+		const dockerignoreByService = await getDockerignoreByService(
+			projectPath,
+			multiDockerignore,
+			serviceDirsByService,
+		);
+
+		for (const serviceName of Object.keys(composition.services)) {
+			const service = composition.services[serviceName];
+			const serviceContext = path.resolve(rootContext, service.build!.context);
+
+			const monitor = manager.testSetupFilesystemWatcher(
+				serviceName,
+				rootContext,
+				serviceContext,
+				changedPathHandler,
+				dockerignoreByService,
+				multiDockerignore,
+			);
+			monitors[serviceName] = monitor;
+
+			await new Promise((resolve, reject) => {
+				monitor.on('error', reject);
+				monitor.on('ready', resolve);
+			});
+		}
+		return monitors;
+	}
+
+	this.beforeAll(async () => {
+		await setupDockerignoreTestData();
+	});
+
+	this.afterAll(async () => {
+		await setupDockerignoreTestData({ cleanup: true });
+	});
+
+	this.beforeEach(() => {
+		resetDockerignoreCache();
+	});
+
+	describe('for project no-docker-compose/basic', function () {
+		const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic');
+		const composition = {
+			version: '2.1',
+			services: {
+				main: { build: { context: '.' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored', async () => {
+			const changedPaths: ByService<string[]> = { main: [] };
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'Dockerfile')),
+				touch(path.join(projectPath, 'src', 'start.sh')),
+				touch(path.join(projectPath, 'src', 'windows-crlf.sh')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['main']).to.have.members([
+				'Dockerfile',
+				path.join('src', 'start.sh'),
+				path.join('src', 'windows-crlf.sh'),
+			]);
+		});
+	});
+
+	describe('for project no-docker-compose/dockerignore1', function () {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore1',
+		);
+		const composition = {
+			version: '2.1',
+			services: {
+				main: { build: { context: '.' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored', async () => {
+			const changedPaths: ByService<string[]> = { main: [] };
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'a.txt')),
+				touch(path.join(projectPath, 'b.txt')),
+				touch(path.join(projectPath, 'vendor', '.git', 'vendor-git-contents')),
+				touch(path.join(projectPath, 'src', 'src-a.txt')),
+				touch(path.join(projectPath, 'src', 'src-b.txt')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['main']).to.have.members([
+				'a.txt',
+				path.join('src', 'src-a.txt'),
+				path.join('vendor', '.git', 'vendor-git-contents'),
+			]);
+		});
+	});
+
+	describe('for project no-docker-compose/dockerignore2', function () {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'dockerignore2',
+		);
+		const composition = {
+			version: '2.1',
+			services: {
+				main: { build: { context: '.' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored', async () => {
+			const changedPaths: ByService<string[]> = { main: [] };
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'a.txt')),
+				touch(path.join(projectPath, 'b.txt')),
+				touch(path.join(projectPath, 'lib', 'src-a.txt')),
+				touch(path.join(projectPath, 'lib', 'src-b.txt')),
+				touch(path.join(projectPath, 'src', 'src-a.txt')),
+				touch(path.join(projectPath, 'src', 'src-b.txt')),
+				touch(path.join(projectPath, 'symlink-a.txt')),
+				touch(path.join(projectPath, 'symlink-b.txt')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			// chokidar appears to treat symbolic links differently on different
+			// platforms like Linux and macOS. On Linux only, change events are
+			// reported for symlinks when the target file they point to is changed.
+			// We tolerate this difference in this test case.
+			const expectedNoSymlink = [
+				'b.txt',
+				path.join('lib', 'src-b.txt'),
+				path.join('src', 'src-b.txt'),
+			];
+			const expectedWithSymlink = [...expectedNoSymlink, 'symlink-a.txt'];
+			expect(changedPaths['main']).to.include.members(expectedNoSymlink);
+			expect(expectedWithSymlink).to.include.members(changedPaths['main']);
+		});
+	});
+
+	describe('for project docker-compose/basic', function () {
+		const projectPath = path.join(projectsPath, 'docker-compose', 'basic');
+		const composition = {
+			version: '2.1',
+			services: {
+				service1: { build: { context: 'service1' } },
+				service2: { build: { context: 'service2' } },
+			},
+		};
+
+		it('should trigger change events for paths that are not ignored (docker-compose)', async () => {
+			const changedPaths: ByService<string[]> = {
+				service1: [],
+				service2: [],
+			};
+			const multiDockerignore = false;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'service1', 'test-ignore.txt')),
+				touch(path.join(projectPath, 'service1', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'src', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'file2-crlf.sh')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['service1']).to.have.members(['file1.sh']);
+			expect(changedPaths['service2']).to.have.members([
+				path.join('src', 'file1.sh'),
+				'file2-crlf.sh',
+			]);
+		});
+
+		it('should trigger change events for paths that are not ignored (docker-compose, multi-dockerignore)', async () => {
+			const changedPaths: ByService<string[]> = {
+				service1: [],
+				service2: [],
+			};
+			const multiDockerignore = true;
+			const monitors = await createMonitors(
+				projectPath,
+				composition,
+				multiDockerignore,
+				(serviceName: string, changedPath: string) => {
+					changedPaths[serviceName].push(changedPath);
+				},
+			);
+
+			await Promise.all([
+				touch(path.join(projectPath, 'service1', 'test-ignore.txt')),
+				touch(path.join(projectPath, 'service1', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'src', 'file1.sh')),
+				touch(path.join(projectPath, 'service2', 'file2-crlf.sh')),
+			]);
+
+			// wait a bit so that filesystem modifications are notified
+			await delay(FS_WATCH_DURATION_MS);
+
+			await Promise.all(
+				Object.values(monitors).map((monitor) => monitor.close()),
+			);
+
+			expect(changedPaths['service1']).to.have.members([
+				'file1.sh',
+				'test-ignore.txt',
+			]);
+			expect(changedPaths['service2']).to.have.members(['file2-crlf.sh']);
+		});
+	});
+});
+
+async function touch(filePath: string) {
+	const time = new Date();
+	return fs.utimes(filePath, time, time);
+}

tests/utils/tarDirectory.spec.ts

@@ -20,7 +20,7 @@ import * as _ from 'lodash';
 import * as path from 'path';
 import * as tar from 'tar-stream';
 
-import { tarDirectory } from '../../build/utils/compose';
+import { tarDirectory } from '../../build/utils/compose_ts';
 import { setupDockerignoreTestData } from '../projects';
 
 const repoPath = path.normalize(path.join(__dirname, '..', '..'));

typings/balena-sync/index.d.ts

@@ -23,6 +23,7 @@ declare module 'balena-sync' {
 	export interface LocalBalenaOsDevice {
 		address: string;
 		host: string;
+		osVariant: string;
 		port: number;
 	}