Merge pull request #2309 from balena-io/warn-deprecation-policy

Add deprecation policy checker and --unsupported global flag

README.md

@@ -156,14 +156,18 @@ of major, minor and patch version releases.
 
 The latest release of a major version of the balena CLI will remain compatible with
 the balenaCloud backend services for at least one year from the date when the
-following major version is released. For example, balena CLI v10.17.5, as the
+following major version is released. For example, balena CLI v11.36.0, as the
-latest v10 release, would remain compatible with the balenaCloud backend for one
+latest v11 release, would remain compatible with the balenaCloud backend for one
-year from the date when v11.0.0 is released.
+year from the date when v12.0.0 was released.
 
-At the end of this period, the older major version is considered deprecated and
+Half way through to that period (6 months after the release of the next major
-some of the functionality that depends on balenaCloud services may stop working
+version), older major versions of the balena CLI will start printing a deprecation
-at any time.
+warning message when it is used interactively (when `stderr` is attached to a TTY
-Users are encouraged to regularly update the balena CLI to the latest version.
+device file). At the end of that period, older major versions will exit with an
+error message unless the `--unsupported` flag is used.  This behavior was
+introduced in CLI version 12.47.0 and is also documented by `balena help`.
+To take advantage of the latest backend features and ensure compatibility, users
+are encouraged to regularly update the balena CLI to the latest version.
 
 ## Contributing (including editing documentation files)
 

automation/build-bin.ts

@@ -22,25 +22,26 @@ import * as archiver from 'archiver';
 import * as Bluebird from 'bluebird';
 import { execFile } from 'child_process';
 import * as filehound from 'filehound';
+import { Stats } from 'fs';
 import * as fs from 'fs-extra';
+import * as klaw from 'klaw';
 import * as _ from 'lodash';
 import * as path from 'path';
 import * as rimraf from 'rimraf';
 import * as semver from 'semver';
-import * as util from 'util';
+import { promisify } from 'util';
-import * as klaw from 'klaw';
-import { Stats } from 'fs';
 
 import { stripIndent } from '../lib/utils/lazy';
 import {
 	diffLines,
-	getSubprocessStdout,
 	loadPackageJson,
 	ROOT,
 	StdOutTap,
 	whichSpawn,
 } from './utils';
 
+const execFileAsync = promisify(execFile);
+
 export const packageJSON = loadPackageJson();
 export const version = 'v' + packageJSON.version;
 const arch = process.arch;
@@ -246,7 +247,17 @@ async function testPkg() {
 	console.log(`Testing standalone package "${pkgBalenaPath}"...`);
 	// Run `balena version -j`, parse its stdout as JSON, and check that the
 	// reported Node.js major version matches semver.major(process.version)
-	const stdout = await getSubprocessStdout(pkgBalenaPath, ['version', '-j']);
+	let { stdout, stderr } = await execFileAsync(pkgBalenaPath, [
+		'version',
+		'-j',
+	]);
+	const { filterCliOutputForTests } = await import('../tests/helpers');
+	const filtered = filterCliOutputForTests({
+		err: stderr.split(/\r?\n/),
+		out: stdout.split(/\r?\n/),
+	});
+	stdout = filtered.out.join('\n');
+	stderr = filtered.err.join('\n');
 	let pkgNodeVersion = '';
 	let pkgNodeMajorVersion = 0;
 	try {
@@ -263,6 +274,10 @@ async function testPkg() {
 			`Mismatched major version: built-in pkg Node version="${pkgNodeVersion}" vs process.version="${process.version}"`,
 		);
 	}
+	if (filtered.err.length > 0) {
+		const err = filtered.err.join('\n');
+		throw new Error(`"${pkgBalenaPath}": non-empty stderr "${err}"`);
+	}
 	console.log('Success! (standalone package test successful)');
 }
 
@@ -411,8 +426,6 @@ async function renameInstallerFiles() {
 async function signWindowsInstaller() {
 	if (process.env.CSC_LINK && process.env.CSC_KEY_PASSWORD) {
 		const exeName = renamedOclifInstallers[process.platform];
-		const execFileAsync = util.promisify<string, string[], void>(execFile);
-
 		console.log(`Signing installer "${exeName}"`);
 		await execFileAsync(MSYS2_BASH, [
 			'sign-exe.sh',

automation/utils.ts

@@ -21,22 +21,6 @@ import * as path from 'path';
 
 export const ROOT = path.join(__dirname, '..');
 
-const nodeEngineWarn = `\
-------------------------------------------------------------------------------
-Warning: Node version "v14.x.x" does not match required versions ">=10.20.0 <13.0.0".
-This may cause unexpected behavior. To upgrade Node, visit:
-https://nodejs.org/en/download/
-------------------------------------------------------------------------------
-`;
-const nodeEngineWarnArray = nodeEngineWarn.split('\n').filter((l) => l);
-
-export function matchesNodeEngineVersionWarn(line: string) {
-	line = line.replace(/"v14\.\d{1,3}\.\d{1,3}"/, '"v14.x.x"');
-	return (
-		line === nodeEngineWarn || nodeEngineWarnArray.includes(line.trimEnd())
-	);
-}
-
 /** Tap and buffer this process' stdout and stderr */
 export class StdOutTap {
 	public stdoutBuf: string[] = [];
@@ -104,60 +88,6 @@ export function loadPackageJson() {
 	return require(path.join(ROOT, 'package.json'));
 }
 
-/**
- * Run the executable at execPath as a child process, and resolve a promise
- * to the executable's stdout output as a string. Reject the promise if
- * anything is printed to stderr, or if the child process exits with a
- * non-zero exit code.
- * @param execPath Executable path
- * @param args Command-line argument for the executable
- */
-export async function getSubprocessStdout(
-	execPath: string,
-	args: string[],
-): Promise<string> {
-	const child = spawn(execPath, args);
-	return new Promise((resolve, reject) => {
-		let stdout = '';
-		child.stdout.on('error', reject);
-		child.stderr.on('error', reject);
-		child.stdout.on('data', (data: Buffer) => {
-			try {
-				stdout = data.toString();
-			} catch (err) {
-				reject(err);
-			}
-		});
-		child.stderr.on('data', (data: Buffer) => {
-			try {
-				const stderr = data.toString();
-
-				// ignore any debug lines, but ensure that we parse
-				// every line provided to the stderr stream
-				const lines = _.filter(
-					stderr.trim().split(/\r?\n/),
-					(line) =>
-						!line.startsWith('[debug]') && !matchesNodeEngineVersionWarn(line),
-				);
-				if (lines.length > 0) {
-					reject(
-						new Error(`"${execPath}": non-empty stderr "${lines.join('\n')}"`),
-					);
-				}
-			} catch (err) {
-				reject(err);
-			}
-		});
-		child.on('exit', (code: number) => {
-			if (code) {
-				reject(new Error(`"${execPath}": non-zero exit code "${code}"`));
-			} else {
-				resolve(stdout);
-			}
-		});
-	});
-}
-
 /**
  * Error handling wrapper around the npm `which` package:
  * "Like the unix which utility. Finds the first instance of a specified

doc/cli.markdown

@@ -144,14 +144,18 @@ of major, minor and patch version releases.
 
 The latest release of a major version of the balena CLI will remain compatible with
 the balenaCloud backend services for at least one year from the date when the
-following major version is released. For example, balena CLI v10.17.5, as the
+following major version is released. For example, balena CLI v11.36.0, as the
-latest v10 release, would remain compatible with the balenaCloud backend for one
+latest v11 release, would remain compatible with the balenaCloud backend for one
-year from the date when v11.0.0 is released.
+year from the date when v12.0.0 was released.
 
-At the end of this period, the older major version is considered deprecated and
+Half way through to that period (6 months after the release of the next major
-some of the functionality that depends on balenaCloud services may stop working
+version), older major versions of the balena CLI will start printing a deprecation
-at any time.
+warning message when it is used interactively (when `stderr` is attached to a TTY
-Users are encouraged to regularly update the balena CLI to the latest version.
+device file). At the end of that period, older major versions will exit with an
+error message unless the `--unsupported` flag is used.  This behavior was
+introduced in CLI version 12.47.0 and is also documented by `balena help`.
+To take advantage of the latest backend features and ensure compatibility, users
+are encouraged to regularly update the balena CLI to the latest version.
 
 
 # CLI Command Reference

lib/app.ts

@@ -16,8 +16,14 @@
  */
 
 import * as packageJSON from '../package.json';
+import {
+	AppOptions,
+	checkDeletedCommand,
+	preparseArgs,
+	unsupportedFlag,
+} from './preparser';
 import { CliSettings } from './utils/bootstrap';
-import { onceAsync, stripIndent } from './utils/lazy';
+import { onceAsync } from './utils/lazy';
 
 /**
  * Sentry.io setup
@@ -43,13 +49,8 @@ export const setupSentry = onceAsync(async () => {
 async function checkNodeVersion() {
 	const validNodeVersions = packageJSON.engines.node;
 	if (!(await import('semver')).satisfies(process.version, validNodeVersions)) {
-		console.warn(stripIndent`
+		const { getNodeEngineVersionWarn } = await import('./utils/messages');
-			------------------------------------------------------------------------------
+		console.warn(getNodeEngineVersionWarn(process.version, validNodeVersions));
-			Warning: Node version "${process.version}" does not match required versions "${validNodeVersions}".
-			This may cause unexpected behavior. To upgrade Node, visit:
-			https://nodejs.org/en/download/
-			------------------------------------------------------------------------------
-			`);
 	}
 }
 
@@ -93,10 +94,20 @@ async function init() {
 }
 
 /** Execute the oclif parser and the CLI command. */
-async function oclifRun(
+async function oclifRun(command: string[], options: AppOptions) {
-	command: string[],
+	let deprecationPromise: Promise<void>;
-	options: import('./preparser').AppOptions,
+	// check and enforce the CLI's deprecation policy
-) {
+	if (unsupportedFlag) {
+		deprecationPromise = Promise.resolve();
+	} else {
+		const { DeprecationChecker } = await import('./deprecation');
+		const deprecationChecker = new DeprecationChecker(packageJSON.version);
+		// warnAndAbortIfDeprecated uses previously cached data only
+		await deprecationChecker.warnAndAbortIfDeprecated();
+		// checkForNewReleasesIfNeeded may query the npm registry
+		deprecationPromise = deprecationChecker.checkForNewReleasesIfNeeded();
+	}
+
 	const runPromise = (async function (shouldFlush: boolean) {
 		const { CustomMain } = await import('./utils/oclif-utils');
 		let isEEXIT = false;
@@ -130,14 +141,12 @@ async function oclifRun(
 	})(!options.noFlush);
 
 	const { trackPromise } = await import('./hooks/prerun/track');
-	await Promise.all([trackPromise, runPromise]);
+
+	await Promise.all([trackPromise, deprecationPromise, runPromise]);
 }
 
 /** CLI entrypoint. Called by the `bin/balena` and `bin/balena-dev` scripts. */
-export async function run(
+export async function run(cliArgs = process.argv, options: AppOptions = {}) {
-	cliArgs = process.argv,
-	options: import('./preparser').AppOptions = {},
-) {
 	try {
 		const { normalizeEnvVars, pkgExec } = await import('./utils/bootstrap');
 		normalizeEnvVars();
@@ -150,8 +159,6 @@ export async function run(
 
 		await init();
 
-		const { preparseArgs, checkDeletedCommand } = await import('./preparser');
-
 		// Look for commands that have been removed and if so, exit with a notice
 		checkDeletedCommand(cliArgs.slice(2));
 

lib/deprecation.ts

@@ -0,0 +1,241 @@
+/**
+ * @license
+ * Copyright 2021 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface ReleaseTimestampsByVersion {
+	[version: string]: string; // e.g. { '12.0.0': '2021-06-16T12:54:52.000Z' }
+	lastFetched: string; // ISO 8601 timestamp, e.g. '2021-06-27T16:46:10.000Z'
+}
+
+/**
+ * Warn about and enforce the CLI deprecation policy stated in the README
+ * file. In particular:
+ *   The latest release of a major version will remain compatible with
+ *   the backend services for at least one year from the date when the
+ *   following major version is released. [...]
+ *   Half way through to that period (6 months), old major versions of the
+ *   balena CLI will start printing a deprecation warning message.
+ *   At the end of that period, older major versions will abort with an error
+ *   message unless the `--unsupported` flag is used.
+ *
+ * - Check for new balena-cli releases by querying the npm registry.
+ * - Cache results for a number of days to improve performance.
+ *
+ * For this feature's specification and planning, see (restricted access):
+ * https://jel.ly.fish/ed8d2395-9323-418c-bb67-d11d32a17d00
+ */
+export class DeprecationChecker {
+	protected static disabled = false; // for the benefit of testing
+	readonly majorVersionFetchIntervalDays = 7;
+	readonly expiryDays = 365;
+	readonly deprecationDays = Math.ceil(this.expiryDays / 2);
+	readonly msInDay = 24 * 60 * 60 * 1000; // milliseconds in a day
+	readonly debugPrefix = 'Deprecation check';
+	readonly cacheFile = 'cachedReleaseTimestamps';
+	readonly now = new Date().getTime();
+	private initialized = false;
+	storage: ReturnType<typeof import('balena-settings-storage')>;
+	cachedTimestamps: ReleaseTimestampsByVersion;
+	nextMajorVersion: string; // semver without the 'v' prefix
+
+	constructor(protected currentVersion: string) {
+		const semver = require('semver') as typeof import('semver');
+		const major = semver.major(this.currentVersion, { loose: true });
+		this.nextMajorVersion = `${major + 1}.0.0`;
+	}
+
+	public async init() {
+		if (this.initialized) {
+			return;
+		}
+		this.initialized = true;
+
+		const settings = await import('balena-settings-client');
+		const getStorage = await import('balena-settings-storage');
+		const dataDirectory = settings.get<string>('dataDirectory');
+		this.storage = getStorage({ dataDirectory });
+		let stored: ReleaseTimestampsByVersion | undefined;
+		try {
+			stored = (await this.storage.get(
+				this.cacheFile,
+			)) as ReleaseTimestampsByVersion;
+		} catch {
+			// ignore
+		}
+		this.cachedTimestamps = {
+			...stored,
+			// '1970-01-01T00:00:00.000Z' is new Date(0).toISOString()
+			lastFetched: stored?.lastFetched || '1970-01-01T00:00:00.000Z',
+		};
+	}
+
+	/**
+	 * Get NPM registry URL to retrieve the package.json file for a given version.
+	 * @param version Semver without 'v' prefix, e.g. '12.0.0.'
+	 */
+	protected getNpmUrl(version: string) {
+		return `http://registry.npmjs.org/balena-cli/${version}`;
+	}
+
+	/**
+	 * Query the npm registry (HTTP request) for a given balena-cli version.
+	 *
+	 * @param version semver version without the 'v' prefix, e.g. '13.0.0'
+	 * @returns `undefined` if the request status code is 404 (version not
+	 *   published), otherwise a publishedAt date in ISO 8601 format, e.g.
+	 *   '2021-06-27T16:46:10.000Z'.
+	 */
+	protected async fetchPublishedTimestampForVersion(
+		version: string,
+	): Promise<string | undefined> {
+		const { default: got } = await import('got');
+		const url = this.getNpmUrl(version);
+		let response: import('got').Response<Dictionary<any>> | undefined;
+		try {
+			response = await got(url, { responseType: 'json', retry: 0 });
+		} catch (e) {
+			// 404 is expected if `version` hasn't been published yet
+			if (e.response?.statusCode !== 404) {
+				throw new Error(`Failed to query "${url}":\n${e}`);
+			}
+		}
+		// response.body looks like a package.json file, plus possibly a
+		// `versionist.publishedAt` field added by `github.com/product-os/versionist`
+		const publishedAt: string | undefined =
+			response?.body?.versionist?.publishedAt;
+		if (!publishedAt && process.env.DEBUG) {
+			console.error(`\
+[debug] ${this.debugPrefix}: balena CLI next major version "${this.nextMajorVersion}" not released, \
+or release date not available`);
+		}
+		return publishedAt; // ISO 8601, e.g. '2021-06-27T16:46:10.000Z'
+	}
+
+	/**
+	 * Check if we already know (cached value) when the next major version
+	 * was released. If we don't know, check how long ago the npm registry
+	 * was last fetched, and fetch again if it has been longer than
+	 * `majorVersionFetchIntervalDays`.
+	 */
+	public async checkForNewReleasesIfNeeded() {
+		if (DeprecationChecker.disabled) {
+			return; // for the benefit of code testing
+		}
+		await this.init();
+		if (this.cachedTimestamps[this.nextMajorVersion]) {
+			// A cached value exists: no need to check the npm registry
+			return;
+		}
+		const lastFetched = new Date(this.cachedTimestamps.lastFetched).getTime();
+		const daysSinceLastFetch = (this.now - lastFetched) / this.msInDay;
+		if (daysSinceLastFetch < this.majorVersionFetchIntervalDays) {
+			if (process.env.DEBUG) {
+				// toFixed(5) results in a precision of ~1 second
+				const days = daysSinceLastFetch.toFixed(5);
+				console.error(`\
+[debug] ${this.debugPrefix}: ${days} days since last npm registry query for next major version release date.
+[debug] Will not query the registry again until at least ${this.majorVersionFetchIntervalDays} days have passed.`);
+			}
+			return;
+		}
+		if (process.env.DEBUG) {
+			console.error(`\
+[debug] ${
+				this.debugPrefix
+			}: Cache miss for the balena CLI next major version release date.
+[debug] Will query ${this.getNpmUrl(this.nextMajorVersion)}`);
+		}
+		try {
+			const publishedAt = await this.fetchPublishedTimestampForVersion(
+				this.nextMajorVersion,
+			);
+			if (publishedAt) {
+				this.cachedTimestamps[this.nextMajorVersion] = publishedAt;
+			}
+		} catch (e) {
+			if (process.env.DEBUG) {
+				console.error(`[debug] ${this.debugPrefix}: ${e}`);
+			}
+		}
+		// Refresh `lastFetched` regardless of whether or not the request to the npm
+		// registry was successful. Will try again after `majorVersionFetchIntervalDays`.
+		this.cachedTimestamps.lastFetched = new Date(this.now).toISOString();
+		await this.storage.set(this.cacheFile, this.cachedTimestamps);
+	}
+
+	/**
+	 * Use previously cached data (local cache only, fast execution) to check
+	 * whether this version of the CLI is deprecated as per deprecation policy,
+	 * in which case warn about it and conditionally throw an error.
+	 */
+	public async warnAndAbortIfDeprecated() {
+		if (DeprecationChecker.disabled) {
+			return; // for the benefit of code testing
+		}
+		await this.init();
+		const nextMajorDateStr = this.cachedTimestamps[this.nextMajorVersion];
+		if (!nextMajorDateStr) {
+			return;
+		}
+		const nextMajorDate = new Date(nextMajorDateStr).getTime();
+		const daysElapsed = Math.trunc((this.now - nextMajorDate) / this.msInDay);
+		if (daysElapsed > this.expiryDays) {
+			const { ExpectedError } = await import('./errors');
+			throw new ExpectedError(this.getExpiryMsg(daysElapsed));
+		} else if (daysElapsed > this.deprecationDays && process.stderr.isTTY) {
+			console.error(this.getDeprecationMsg(daysElapsed));
+		}
+	}
+
+	/** Separate function for the benefit of code testing */
+	getDeprecationMsg(daysElapsed: number) {
+		const { warnify } =
+			require('./utils/messages') as typeof import('./utils/messages');
+		return warnify(`\
+CLI version ${this.nextMajorVersion} was released ${daysElapsed} days ago: please upgrade.
+This version of the balena CLI (${this.currentVersion}) will exit with an error
+message after ${this.expiryDays} days from the release of version ${this.nextMajorVersion},
+as per deprecation policy: https://git.io/JRHUW#deprecation-policy
+
+The --unsupported flag may be used to bypass this deprecation check and
+allow the CLI to keep working beyond the deprecation period.  However,
+note that the balenaCloud or openBalena backends may be updated in a way
+that is no longer compatible with this version.`);
+	}
+
+	/** Separate function the benefit of code testing */
+	getExpiryMsg(daysElapsed: number) {
+		return `
+This version of the balena CLI (${this.currentVersion}) has expired: please upgrade.
+${daysElapsed} days have passed since the release of CLI version ${this.nextMajorVersion}.
+See deprecation policy at: https://git.io/JRHUW#deprecation-policy
+
+The --unsupported flag may be used to bypass this deprecation check and
+continue using this version of the CLI. However, note that the balenaCloud
+or openBalena backends may be updated in a way that is no longer compatible
+with this CLI version.`;
+	}
+
+	/** Disable deprecation checks (for the benefit of code testing). */
+	public static disable() {
+		DeprecationChecker.disabled = true;
+	}
+
+	/** Re-enable deprecation checks (for the benefit of code testing). */
+	public static enable() {
+		DeprecationChecker.disabled = false;
+	}
+}

lib/events.ts

@@ -60,11 +60,11 @@ export async function trackCommand(commandSignature: string) {
 			});
 		}
 		const settings = await import('balena-settings-client');
-		const balenaUrl = settings.get('balenaUrl') as string;
+		const balenaUrl = settings.get<string>('balenaUrl');
 
 		const username = await (async () => {
 			const getStorage = await import('balena-settings-storage');
-			const dataDirectory = settings.get('dataDirectory') as string;
+			const dataDirectory = settings.get<string>('dataDirectory');
 			const storage = getStorage({ dataDirectory });
 			let token;
 			try {

lib/help.ts

@@ -46,7 +46,7 @@ export default class BalenaHelp extends Help {
 		const subject = getHelpSubject(argv);
 		if (!subject) {
 			const verbose = argv.includes('-v') || argv.includes('--verbose');
-			this.showCustomRootHelp(verbose);
+			console.log(this.getCustomRootHelp(verbose));
 			return;
 		}
 
@@ -80,67 +80,106 @@ export default class BalenaHelp extends Help {
 		throw new ExpectedError(`command ${chalk.cyan.bold(subject)} not found`);
 	}
 
-	showCustomRootHelp(showAllCommands: boolean): void {
+	getCustomRootHelp(showAllCommands: boolean): string {
-		const chalk = getChalk();
+		const { bold, cyan } = getChalk();
-		const bold = chalk.bold;
-		const cmd = chalk.cyan.bold;
 
 		let commands = this.config.commands;
 		commands = commands.filter((c) => this.opts.all || !c.hidden);
 
 		// Get Primary Commands, sorted as in manual list
-		const primaryCommands = this.manuallySortedPrimaryCommands.map((pc) => {
+		const primaryCommands = this.manuallySortedPrimaryCommands
-			return commands.find((c) => c.id === pc.replace(' ', ':'));
+			.map((pc) => {
-		});
+				return commands.find((c) => c.id === pc.replace(' ', ':'));
+			})
+			.filter((c): c is typeof commands[0] => !!c);
 
-		// Get the rest as Additional Commands
+		let usageLength = 0;
-		const additionalCommands = commands.filter(
+		for (const cmd of primaryCommands) {
-			(c) =>
+			usageLength = Math.max(usageLength, cmd.usage?.length || 0);
-				!this.manuallySortedPrimaryCommands.includes(c.id.replace(':', ' ')),
-		);
-
-		// Find longest usage, and pad usage of first command in each category
-		// This is to ensure that both categories align visually
-		const usageLength = commands
-			.map((c) => c.usage?.length || 0)
-			.reduce((longest, l) => {
-				return l > longest ? l : longest;
-			});
-
-		if (
-			typeof primaryCommands[0]?.usage === 'string' &&
-			typeof additionalCommands[0]?.usage === 'string'
-		) {
-			primaryCommands[0].usage = primaryCommands[0].usage.padEnd(usageLength);
-			additionalCommands[0].usage =
-				additionalCommands[0].usage.padEnd(usageLength);
 		}
 
-		// Output help
+		let additionalCmdSection: string[];
-		console.log(bold('USAGE'));
-		console.log('$ balena [COMMAND] [OPTIONS]');
-
-		console.log(bold('\nPRIMARY COMMANDS'));
-		console.log(this.formatCommands(primaryCommands));
-
 		if (showAllCommands) {
-			console.log(bold('\nADDITIONAL COMMANDS'));
+			// Get the rest as Additional Commands
-			console.log(this.formatCommands(additionalCommands));
+			const additionalCommands = commands.filter(
+				(c) =>
+					!this.manuallySortedPrimaryCommands.includes(c.id.replace(':', ' ')),
+			);
+
+			// Find longest usage, and pad usage of first command in each category
+			// This is to ensure that both categories align visually
+			for (const cmd of additionalCommands) {
+				usageLength = Math.max(usageLength, cmd.usage?.length || 0);
+			}
+
+			if (
+				typeof primaryCommands[0].usage === 'string' &&
+				typeof additionalCommands[0].usage === 'string'
+			) {
+				primaryCommands[0].usage = primaryCommands[0].usage.padEnd(usageLength);
+				additionalCommands[0].usage =
+					additionalCommands[0].usage.padEnd(usageLength);
+			}
+
+			additionalCmdSection = [
+				bold('\nADDITIONAL COMMANDS'),
+				this.formatCommands(additionalCommands),
+			];
 		} else {
-			console.log(
+			const cmd = cyan.bold('balena help --verbose');
-				`\n${bold('...MORE')} run ${cmd(
+			additionalCmdSection = [
-					'balena help --verbose',
+				`\n${bold('...MORE')} run ${cmd} to list additional commands.`,
-				)} to list additional commands.`,
+			];
+		}
+
+		const globalOps = [
+			['--help, -h', 'display command help'],
+			['--debug', 'enable debug output'],
+			[
+				'--unsupported',
+				`\
+prevent exit with an error as per Deprecation Policy
+See: https://git.io/JRHUW#deprecation-policy`,
+			],
+		];
+		globalOps[0][0] = globalOps[0][0].padEnd(usageLength);
+
+		const { deprecationPolicyNote, reachingOut } =
+			require('./utils/messages') as typeof import('./utils/messages');
+
+		return [
+			bold('USAGE'),
+			'$ balena [COMMAND] [OPTIONS]',
+			bold('\nPRIMARY COMMANDS'),
+			this.formatCommands(primaryCommands),
+			...additionalCmdSection,
+			bold('\nGLOBAL OPTIONS'),
+			this.formatGlobalOpts(globalOps),
+			bold('\nDeprecation Policy Reminder'),
+			deprecationPolicyNote,
+			reachingOut,
+		].join('\n');
+	}
+
+	protected formatGlobalOpts(opts: string[][]) {
+		const { dim } = getChalk();
+		const outLines: string[] = [];
+		let flagWidth = 0;
+		for (const opt of opts) {
+			flagWidth = Math.max(flagWidth, opt[0].length);
+		}
+		for (const opt of opts) {
+			const descriptionLines = opt[1].split('\n');
+			outLines.push(
+				`  ${opt[0].padEnd(flagWidth + 2)}${dim(descriptionLines[0])}`,
+			);
+			outLines.push(
+				...descriptionLines
+					.slice(1)
+					.map((line) => `  ${' '.repeat(flagWidth + 2)}${dim(line)}`),
 			);
 		}
-
+		return outLines.join('\n');
-		console.log(bold('\nGLOBAL OPTIONS'));
-		console.log('  --help, -h');
-		console.log('  --debug\n');
-
-		const { reachingOut } =
-			require('./utils/messages') as typeof import('./utils/messages');
-		console.log(reachingOut);
 	}
 
 	protected formatCommands(commands: any[]): string {

lib/preparser.ts

@@ -14,8 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { stripIndent } from './utils/lazy';
+
-import { exitWithExpectedError } from './errors';
+export let unsupportedFlag = false;
 
 export interface AppOptions {
 	// Prevent the default behavior of flushing stdout after running a command
@@ -50,11 +50,10 @@ export async function preparseArgs(argv: string[]): Promise<string[]> {
 		}
 
 		// support global --debug flag
-		const debugIndex = cmdSlice.indexOf('--debug');
+		if (extractBooleanFlag(cmdSlice, '--debug')) {
-		if (debugIndex > -1) {
 			process.env.DEBUG = '1';
-			cmdSlice.splice(debugIndex, 1);
 		}
+		unsupportedFlag = extractBooleanFlag(cmdSlice, '--unsupported');
 	}
 
 	// Enable bluebird long stack traces when in debug mode, must be set
@@ -87,11 +86,22 @@ export async function preparseArgs(argv: string[]): Promise<string[]> {
 	return args;
 }
 
+function extractBooleanFlag(argv: string[], flag: string): boolean {
+	const index = argv.indexOf(flag);
+	if (index >= 0) {
+		argv.splice(index, 1);
+		return true;
+	}
+	return false;
+}
+
 /**
  * Check whether the command line refers to a command that has been deprecated
  * and removed and, if so, exit with an informative error message.
  */
 export function checkDeletedCommand(argvSlice: string[]): void {
+	const { ExpectedError } = require('./errors') as typeof import('./errors');
+
 	if (argvSlice[0] === 'help') {
 		argvSlice = argvSlice.slice(1);
 	}
@@ -101,17 +111,16 @@ export function checkDeletedCommand(argvSlice: string[]): void {
 		version: string,
 		verb = 'replaced',
 	) {
-		exitWithExpectedError(stripIndent`
+		throw new ExpectedError(`\
-			Note: the command "balena ${oldCmd}" was ${verb} in CLI version ${version}.
+Note: the command "balena ${oldCmd}" was ${verb} in CLI version ${version}.
-			Please use "balena ${alternative}" instead.
+Please use "balena ${alternative}" instead.`);
-		`);
 	}
 	function removed(oldCmd: string, alternative: string, version: string) {
 		let msg = `Note: the command "balena ${oldCmd}" was removed in CLI version ${version}.`;
 		if (alternative) {
 			msg = [msg, alternative].join('\n');
 		}
-		exitWithExpectedError(msg);
+		throw new ExpectedError(msg);
 	}
 	const stopAlternative =
 		'Please use "balena ssh -s" to access the host OS, then use `balena-engine stop`.';

lib/utils/messages.ts

@@ -30,6 +30,12 @@ export const help = reachingOut;
 // is parsed, so its evaluation cannot happen at module loading time.
 export const getHelp = () => (process.env.DEBUG ? '' : debugHint) + help;
 
+export const deprecationPolicyNote = `\
+The balena CLI enforces its deprecation policy by exiting with an error a year
+after the release of the next major version, unless the --unsupported option is
+used. Find out more at: https://git.io/JRHUW#deprecation-policy
+`;
+
 /**
  * Take a multiline string like:
  *     Line One
@@ -41,8 +47,8 @@ export const getHelp = () => (process.env.DEBUG ? '' : debugHint) + help;
  *     ---------------
  * where the length of the dash rows matches the length of the longest line.
  */
-export function warnify(msg: string) {
+export function warnify(msg: string, prefix = '[Warn] ') {
-	const lines = msg.split('\n').map((l) => `[Warn] ${l}`);
+	const lines = msg.split('\n').map((l) => `${prefix}${l}`);
 	const maxLength = Math.max(...lines.map((l) => l.length));
 	const hr = '-'.repeat(maxLength);
 	return [hr, ...lines, hr].join('\n');
@@ -184,3 +190,13 @@ the next major version of the CLI (v13). The --v13 option may be used
 to enable the new names already now, and suppress a warning message.
 (The --v13 option will be silently ignored in CLI v13.)
 Find out more at: https://git.io/JRuZr`;
+
+export function getNodeEngineVersionWarn(
+	version: string,
+	validVersions: string,
+) {
+	version = version.startsWith('v') ? version.substring(1) : version;
+	return warnify(`\
+Node.js version "${version}" does not satisfy requirement "${validVersions}"
+This may cause unexpected behavior.`);
+}

npm-shrinkwrap.json

@@ -2385,6 +2385,17 @@
         "@types/node": "*"
       }
     },
+    "@types/cacheable-request": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz",
+      "integrity": "sha512-B3xVo+dlKM6nnKTcmm5ZtY/OL8bOAOd2Olee9M1zft65ox50OzjEHW91sDiU9j6cvW8Ejg1/Qkf4xd2kugApUA==",
+      "requires": {
+        "@types/http-cache-semantics": "*",
+        "@types/keyv": "*",
+        "@types/node": "*",
+        "@types/responselike": "*"
+      }
+    },
     "@types/caseless": {
       "version": "0.12.2",
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz",
@@ -2534,6 +2545,11 @@
       "integrity": "sha512-TikApqV8CkUsI1GGUgVydkJFrq9sYCBWv4fc/r3zvl6Oqe2YU1ASeWBrG5bw1D2XvS07YS3s05hCor/lEtIoYw==",
       "dev": true
     },
+    "@types/http-cache-semantics": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz",
+      "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ=="
+    },
     "@types/http-proxy": {
       "version": "1.17.7",
       "resolved": "https://registry.npmjs.org/@types/http-proxy/-/http-proxy-1.17.7.tgz",
@@ -2600,6 +2616,14 @@
       "resolved": "https://registry.npmjs.org/@types/jwt-decode/-/jwt-decode-2.2.1.tgz",
       "integrity": "sha512-aWw2YTtAdT7CskFyxEX2K21/zSDStuf/ikI3yBqmwpwJF0pS+/IX5DWv+1UFffZIbruP6cnT9/LAJV1gFwAT1A=="
     },
+    "@types/keyv": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.2.tgz",
+      "integrity": "sha512-/FvAK2p4jQOaJ6CGDHJTqZcUtbZe820qIeTg7o0Shg7drB4JHeL+V/dhSaly7NXx6u8eSee+r7coT+yuJEvDLg==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/klaw": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/@types/klaw/-/klaw-3.0.2.tgz",
@@ -2791,6 +2815,14 @@
         }
       }
     },
+    "@types/responselike": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
+      "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/rewire": {
       "version": "2.5.28",
       "resolved": "https://registry.npmjs.org/@types/rewire/-/rewire-2.5.28.tgz",
@@ -4561,6 +4593,11 @@
         "unset-value": "^1.0.0"
       }
     },
+    "cacheable-lookup": {
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
+      "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="
+    },
     "cacheable-request": {
       "version": "6.1.0",
       "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz",
@@ -8429,6 +8466,31 @@
       "requires": {
         "got": "^6.2.0",
         "is-plain-obj": "^1.1.0"
+      },
+      "dependencies": {
+        "get-stream": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+          "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ="
+        },
+        "got": {
+          "version": "6.7.1",
+          "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
+          "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=",
+          "requires": {
+            "create-error-class": "^3.0.0",
+            "duplexer3": "^0.1.4",
+            "get-stream": "^3.0.0",
+            "is-redirect": "^1.0.0",
+            "is-retry-allowed": "^1.0.0",
+            "is-stream": "^1.0.0",
+            "lowercase-keys": "^1.0.0",
+            "safe-buffer": "^5.0.1",
+            "timed-out": "^4.0.0",
+            "unzip-response": "^2.0.1",
+            "url-parse-lax": "^1.0.0"
+          }
+        }
       }
     },
     "ghauth": {
@@ -8905,27 +8967,111 @@
       }
     },
     "got": {
-      "version": "6.7.1",
+      "version": "11.8.2",
-      "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
+      "resolved": "https://registry.npmjs.org/got/-/got-11.8.2.tgz",
-      "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=",
+      "integrity": "sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==",
       "requires": {
-        "create-error-class": "^3.0.0",
+        "@sindresorhus/is": "^4.0.0",
-        "duplexer3": "^0.1.4",
+        "@szmarczak/http-timer": "^4.0.5",
-        "get-stream": "^3.0.0",
+        "@types/cacheable-request": "^6.0.1",
-        "is-redirect": "^1.0.0",
+        "@types/responselike": "^1.0.0",
-        "is-retry-allowed": "^1.0.0",
+        "cacheable-lookup": "^5.0.3",
-        "is-stream": "^1.0.0",
+        "cacheable-request": "^7.0.1",
-        "lowercase-keys": "^1.0.0",
+        "decompress-response": "^6.0.0",
-        "safe-buffer": "^5.0.1",
+        "http2-wrapper": "^1.0.0-beta.5.2",
-        "timed-out": "^4.0.0",
+        "lowercase-keys": "^2.0.0",
-        "unzip-response": "^2.0.1",
+        "p-cancelable": "^2.0.0",
-        "url-parse-lax": "^1.0.0"
+        "responselike": "^2.0.0"
       },
       "dependencies": {
+        "@sindresorhus/is": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.1.tgz",
+          "integrity": "sha512-Qm9hBEBu18wt1PO2flE7LPb30BHMQt1eQgbV76YntdNk73XZGpn3izvGTYxbGgzXKgbCjiia0uxTd3aTNQrY/g=="
+        },
+        "@szmarczak/http-timer": {
+          "version": "4.0.6",
+          "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz",
+          "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==",
+          "requires": {
+            "defer-to-connect": "^2.0.0"
+          }
+        },
+        "cacheable-request": {
+          "version": "7.0.2",
+          "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz",
+          "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==",
+          "requires": {
+            "clone-response": "^1.0.2",
+            "get-stream": "^5.1.0",
+            "http-cache-semantics": "^4.0.0",
+            "keyv": "^4.0.0",
+            "lowercase-keys": "^2.0.0",
+            "normalize-url": "^6.0.1",
+            "responselike": "^2.0.0"
+          }
+        },
+        "decompress-response": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
+          "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
+          "requires": {
+            "mimic-response": "^3.1.0"
+          }
+        },
+        "defer-to-connect": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
+          "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="
+        },
         "get-stream": {
-          "version": "3.0.0",
+          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
-          "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ="
+          "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "json-buffer": {
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+          "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
+        },
+        "keyv": {
+          "version": "4.0.3",
+          "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz",
+          "integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==",
+          "requires": {
+            "json-buffer": "3.0.1"
+          }
+        },
+        "lowercase-keys": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
+          "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
+        },
+        "mimic-response": {
+          "version": "3.1.0",
+          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
+          "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
+        },
+        "normalize-url": {
+          "version": "6.1.0",
+          "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz",
+          "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A=="
+        },
+        "p-cancelable": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz",
+          "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg=="
+        },
+        "responselike": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+          "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
+          "requires": {
+            "lowercase-keys": "^2.0.0"
+          }
         }
       }
     },
@@ -9388,6 +9534,15 @@
         "sshpk": "^1.7.0"
       }
     },
+    "http2-wrapper": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz",
+      "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==",
+      "requires": {
+        "quick-lru": "^5.1.1",
+        "resolve-alpn": "^1.0.0"
+      }
+    },
     "https-proxy-agent": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
@@ -14676,6 +14831,11 @@
       "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
       "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
     },
+    "quick-lru": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
+      "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="
+    },
     "randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -15851,6 +16011,11 @@
         "path-parse": "^1.0.6"
       }
     },
+    "resolve-alpn": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.0.tgz",
+      "integrity": "sha512-e4FNQs+9cINYMO5NMFc6kOUCdohjqFPSgMuwuZAOUWqrfWsen+Yjy5qZFkV5K7VO7tFSLKcUL97olkED7sCBHA=="
+    },
     "resolve-dir": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz",
@@ -16347,13 +16512,13 @@
       "dev": true
     },
     "sinon": {
-      "version": "11.1.1",
+      "version": "11.1.2",
-      "resolved": "https://registry.npmjs.org/sinon/-/sinon-11.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/sinon/-/sinon-11.1.2.tgz",
-      "integrity": "sha512-ZSSmlkSyhUWbkF01Z9tEbxZLF/5tRC9eojCdFh33gtQaP7ITQVaMWQHGuFM7Cuf/KEfihuh1tTl3/ABju3AQMg==",
+      "integrity": "sha512-59237HChms4kg7/sXhiRcUzdSkKuydDeTiamT/jesUVHshBgL8XAmhgFo0GfK6RruMDM/iRSij1EybmMog9cJw==",
       "dev": true,
       "requires": {
         "@sinonjs/commons": "^1.8.3",
-        "@sinonjs/fake-timers": "^7.1.0",
+        "@sinonjs/fake-timers": "^7.1.2",
         "@sinonjs/samsam": "^6.0.2",
         "diff": "^5.0.0",
         "nise": "^5.1.0",

package.json

@@ -49,6 +49,7 @@
     "postinstall": "node patches/apply-patches.js",
     "prebuild": "rimraf build/ build-bin/",
     "build": "npm run build:src && npm run catch-uncommitted",
+    "build:t": "npm run lint && npm run build:fast && npm run build:test",
     "build:src": "npm run lint && npm run build:fast && npm run build:test && npm run build:doc && npm run build:completion",
     "build:fast": "gulp pages && tsc && npx oclif-dev manifest",
     "build:test": "tsc -P ./tsconfig.dev.json --noEmit && tsc -P ./tsconfig.js.json --noEmit",
@@ -185,7 +186,7 @@
     "publish-release": "^1.6.1",
     "rewire": "^5.0.0",
     "simple-git": "^2.40.0",
-    "sinon": "^11.1.1",
+    "sinon": "^11.1.2",
     "ts-node": "^10.0.0",
     "typescript": "^4.3.5"
   },
@@ -236,6 +237,7 @@
     "glob": "^7.1.7",
     "global-agent": "^2.1.12",
     "global-tunnel-ng": "^2.1.1",
+    "got": "^11.8.2",
     "humanize": "0.0.9",
     "ignore": "^5.1.8",
     "inquirer": "^7.3.3",

tests/commands/app/create.spec.ts

@@ -16,7 +16,7 @@
  */
 
 import { expect } from 'chai';
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../../helpers';
 
 const HELP_MESSAGE = '';

tests/commands/build.spec.ts

@@ -22,9 +22,9 @@ import { promises as fs } from 'fs';
 import * as path from 'path';
 
 import { stripIndent } from '../../lib/utils/lazy';
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
 import { expectStreamNoCRLF, testDockerBuildStream } from '../docker-build';
-import { DockerMock, dockerResponsePath } from '../docker-mock';
+import { DockerMock, dockerResponsePath } from '../nock/docker-mock';
 import { cleanOutput, runCommand } from '../helpers';
 import {
 	ExpectedTarStreamFiles,

tests/commands/deploy.spec.ts

@@ -21,9 +21,9 @@ import * as _ from 'lodash';
 import * as path from 'path';
 import * as sinon from 'sinon';
 
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
 import { expectStreamNoCRLF, testDockerBuildStream } from '../docker-build';
-import { DockerMock, dockerResponsePath } from '../docker-mock';
+import { DockerMock, dockerResponsePath } from '../nock/docker-mock';
 import { cleanOutput, runCommand, switchSentry } from '../helpers';
 import {
 	ExpectedTarStreamFiles,

tests/commands/device/device-move.spec.ts

@@ -16,7 +16,7 @@
  */
 
 import { expect } from 'chai';
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../../helpers';
 
 describe('balena device move', function () {

tests/commands/device/device.spec.ts

@@ -18,7 +18,7 @@
 import { expect } from 'chai';
 import * as path from 'path';
 
-import { apiResponsePath, BalenaAPIMock } from '../../balena-api-mock';
+import { apiResponsePath, BalenaAPIMock } from '../../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../../helpers';
 
 import { appToFleetOutputMsg, warnify } from '../../../build/utils/messages';

tests/commands/device/devices.spec.ts

@@ -18,7 +18,7 @@
 import { expect } from 'chai';
 import * as path from 'path';
 
-import { apiResponsePath, BalenaAPIMock } from '../../balena-api-mock';
+import { apiResponsePath, BalenaAPIMock } from '../../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../../helpers';
 
 import { appToFleetOutputMsg, warnify } from '../../../build/utils/messages';

tests/commands/device/supported.spec.ts

@@ -17,7 +17,7 @@
 
 import { expect } from 'chai';
 
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../../helpers';
 
 import { isV13 } from '../../../lib/utils/version';

tests/commands/env/add.spec.ts

@@ -17,7 +17,7 @@
 
 import { expect } from 'chai';
 
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { runCommand } from '../../helpers';
 
 describe('balena env add', function () {

tests/commands/env/envs.spec.ts

@@ -18,7 +18,7 @@
 import { expect } from 'chai';
 import { stripIndent } from '../../../lib/utils/lazy';
 
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { runCommand } from '../../helpers';
 
 import {

tests/commands/env/rename.spec.ts

@@ -17,7 +17,7 @@
 
 import { expect } from 'chai';
 
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { runCommand } from '../../helpers';
 
 describe('balena env rename', function () {

tests/commands/env/rm.spec.ts

@@ -17,7 +17,7 @@
 
 import { expect } from 'chai';
 
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 import { runCommand } from '../../helpers';
 
 describe('balena env rm', function () {

tests/commands/help.spec.ts

@@ -17,7 +17,7 @@
 
 import { expect } from 'chai';
 
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../helpers';
 import * as messages from '../../build/utils/messages';
 

tests/commands/logs.spec.ts

@@ -17,9 +17,9 @@
 
 import { expect } from 'chai';
 
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../helpers';
-import { SupervisorMock } from '../supervisor-mock';
+import { SupervisorMock } from '../nock/supervisor-mock';
 
 const itS = process.env.BALENA_CLI_TEST_TYPE === 'standalone' ? it : it.skip;
 

tests/commands/os/configure.spec.ts

@@ -25,7 +25,7 @@ import * as tmp from 'tmp';
 tmp.setGracefulCleanup();
 const tmpNameAsync = promisify(tmp.tmpName);
 
-import { BalenaAPIMock } from '../../balena-api-mock';
+import { BalenaAPIMock } from '../../nock/balena-api-mock';
 
 if (process.platform !== 'win32') {
 	describe('balena os configure', function () {

tests/commands/push.spec.ts

@@ -19,8 +19,8 @@ import { expect } from 'chai';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
-import { BuilderMock, builderResponsePath } from '../builder-mock';
+import { BuilderMock, builderResponsePath } from '../nock/builder-mock';
 import { expectStreamNoCRLF, testPushBuildStream } from '../docker-build';
 import { cleanOutput, runCommand } from '../helpers';
 import {

tests/commands/ssh.spec.ts

@@ -19,7 +19,7 @@ import { expect } from 'chai';
 import mock = require('mock-require');
 import { createServer, Server } from 'net';
 
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
 import { cleanOutput, runCommand } from '../helpers';
 
 // "itSS" means "it() Skip Standalone"

tests/commands/version.spec.ts

@@ -17,7 +17,7 @@
 import { expect } from 'chai';
 import * as fs from 'fs';
 
-import { BalenaAPIMock } from '../balena-api-mock';
+import { BalenaAPIMock } from '../nock/balena-api-mock';
 import { runCommand } from '../helpers';
 
 const packageJSON = JSON.parse(fs.readFileSync('./package.json', 'utf8'));

tests/config-tests.ts

@@ -19,10 +19,20 @@ import { set as setEsVersion } from '@balena/es-version';
 // Set the desired es version for downstream modules that support it
 setEsVersion('es2018');
 
+// Disable Sentry.io error reporting while running test code
+process.env.BALENARC_NO_SENTRY = '1';
+
+// Disable deprecation checks while running test code
+import { DeprecationChecker } from '../build/deprecation';
+DeprecationChecker.disable();
+
 import * as tmp from 'tmp';
 tmp.setGracefulCleanup();
 // Use a temporary dir for tests data
 process.env.BALENARC_DATA_DIRECTORY = tmp.dirSync().name;
+console.error(
+	`[debug] tests/config-tests.ts: BALENARC_DATA_DIRECTORY="${process.env.BALENARC_DATA_DIRECTORY}"`,
+);
 
 import { EventEmitter } from 'events';
 EventEmitter.defaultMaxListeners = 35; // it appears that 'nock' adds a bunch of listeners - bug?

tests/deprecation.spec.ts

@@ -0,0 +1,320 @@
+/**
+ * @license
+ * Copyright 2021 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as settings from 'balena-settings-client';
+import * as getStorage from 'balena-settings-storage';
+import { expect } from 'chai';
+import mock = require('mock-require');
+import * as semver from 'semver';
+import * as sinon from 'sinon';
+
+import * as packageJSON from '../package.json';
+import {
+	DeprecationChecker,
+	ReleaseTimestampsByVersion,
+} from '../build/deprecation';
+import { BalenaAPIMock } from './nock/balena-api-mock';
+import { NpmMock } from './nock/npm-mock';
+import { runCommand, TestOutput } from './helpers';
+
+// "itSS" means "it() Skip Standalone"
+const itSS = process.env.BALENA_CLI_TEST_TYPE === 'standalone' ? it.skip : it;
+
+describe('DeprecationChecker', function () {
+	const sandbox = sinon.createSandbox();
+	const now = new Date().getTime();
+	const anHourAgo = now - 3600000;
+	const currentMajor = semver.major(packageJSON.version, { loose: true });
+	const nextMajorVersion = `${currentMajor + 1}.0.0`;
+	const dataDirectory = settings.get<string>('dataDirectory');
+	const storageModPath = 'balena-settings-storage';
+	const mockStorage = getStorage({ dataDirectory });
+	let api: BalenaAPIMock;
+	let npm: NpmMock;
+	let checker: DeprecationChecker;
+	let getStub: sinon.SinonStub<
+		Parameters<typeof mockStorage.get>,
+		ReturnType<typeof mockStorage.get>
+	>;
+	let setStub: sinon.SinonStub<
+		Parameters<typeof mockStorage.set>,
+		ReturnType<typeof mockStorage.set>
+	>;
+
+	// see also DeprecationChecker.disable() in tests/config-tests.ts
+	this.beforeAll(() => DeprecationChecker.enable());
+	this.afterAll(() => DeprecationChecker.disable());
+
+	this.beforeEach(() => {
+		npm = new NpmMock();
+		api = new BalenaAPIMock();
+		api.expectGetWhoAmI({ optional: true, persist: true });
+		api.expectGetMixpanel({ optional: true });
+		checker = new DeprecationChecker(packageJSON.version);
+
+		getStub = sandbox.stub(mockStorage, 'get').withArgs(checker.cacheFile);
+
+		setStub = sandbox
+			.stub(mockStorage, 'set')
+			.withArgs(checker.cacheFile, sinon.match.any);
+
+		mock(storageModPath, () => mockStorage);
+	});
+
+	this.afterEach(() => {
+		// Check all expected api calls have been made and clean up.
+		(mockStorage.get as sinon.SinonStub).restore();
+		(mockStorage.set as sinon.SinonStub).restore();
+
+		// originalStorage.set.restore();
+		api.done();
+		npm.done();
+		mock.stop(storageModPath);
+	});
+
+	itSS(
+		'should warn if this version of the CLI is deprecated (isTTY is true)',
+		async () => {
+			const mockCache: ReleaseTimestampsByVersion = {
+				lastFetched: '1970-01-01T00:00:00.000Z',
+			};
+			// pretend the next major was released just over half a year ago
+			mockCache[nextMajorVersion] = new Date(
+				checker.now - (checker.deprecationDays + 1) * checker.msInDay,
+			).toISOString();
+
+			getStub.resolves(mockCache);
+
+			// Force isTTY to be true. It happens to be false (undefined) when
+			// the tests run on balenaCI on Windows.
+			const originalIsTTY = process.stderr.isTTY;
+			process.stderr.isTTY = true;
+			let result: TestOutput;
+			try {
+				result = await runCommand('version');
+			} finally {
+				process.stderr.isTTY = originalIsTTY;
+			}
+			const { out, err } = result;
+
+			expect(setStub.callCount).to.equal(0);
+			expect(getStub.callCount).to.equal(1);
+			expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+			expect(out.join('')).to.equal(packageJSON.version + '\n');
+			expect(err.join('')).to.equal(
+				checker.getDeprecationMsg(checker.deprecationDays + 1) + '\n',
+			);
+		},
+	);
+
+	itSS(
+		'should NOT warn if this version of the CLI is deprecated (isTTY is false)',
+		async () => {
+			const mockCache: ReleaseTimestampsByVersion = {
+				lastFetched: '1970-01-01T00:00:00.000Z',
+			};
+			// pretend the next major was released just over half a year ago
+			mockCache[nextMajorVersion] = new Date(
+				checker.now - (checker.deprecationDays + 1) * checker.msInDay,
+			).toISOString();
+
+			getStub.resolves(mockCache);
+
+			// Force isTTY to be false (undefined). It happens to be true when
+			// the tests run on balenaCI on macOS and Linux.
+			const originalIsTTY = process.stderr.isTTY;
+			process.stderr.isTTY = undefined;
+			let result: TestOutput;
+			try {
+				result = await runCommand('version');
+			} finally {
+				process.stderr.isTTY = originalIsTTY;
+			}
+			const { out, err } = result;
+
+			expect(setStub.callCount).to.equal(0);
+			expect(getStub.callCount).to.equal(1);
+			expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+			expect(out.join('')).to.equal(packageJSON.version + '\n');
+			expect(err.join('')).to.equal('');
+		},
+	);
+
+	itSS(
+		'should NOT warn with --unsupported (deprecated but not expired)',
+		async () => {
+			const mockCache: ReleaseTimestampsByVersion = {
+				lastFetched: '1970-01-01T00:00:00.000Z',
+			};
+			// pretend the next major was released just over half a year ago
+			mockCache[nextMajorVersion] = new Date(
+				checker.now - (checker.deprecationDays + 1) * checker.msInDay,
+			).toISOString();
+
+			getStub.resolves(mockCache);
+
+			const { out, err } = await runCommand('version --unsupported');
+
+			expect(setStub.callCount).to.equal(0);
+			expect(getStub.callCount).to.equal(0);
+
+			expect(out.join('')).to.equal(packageJSON.version + '\n');
+			expect(err.join('')).to.be.empty;
+		},
+	);
+
+	itSS('should exit if this version of the CLI has expired', async () => {
+		const mockCache: ReleaseTimestampsByVersion = {
+			lastFetched: '1970-01-01T00:00:00.000Z',
+		};
+		// pretend the next major was released just over a year ago
+		mockCache[nextMajorVersion] = new Date(
+			checker.now - (checker.expiryDays + 1) * checker.msInDay,
+		).toISOString();
+
+		getStub.resolves(mockCache);
+
+		const { out, err } = await runCommand('version');
+
+		expect(setStub.callCount).to.equal(0);
+		expect(getStub.callCount).to.equal(1);
+		expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+		expect(out.join('')).to.equal('');
+		expect(err.join('')).to.include(
+			checker.getExpiryMsg(checker.expiryDays + 1) + '\n\n',
+		);
+	});
+
+	itSS('should NOT exit with --unsupported (expired version)', async () => {
+		const mockCache: ReleaseTimestampsByVersion = {
+			lastFetched: '1970-01-01T00:00:00.000Z',
+		};
+		// pretend the next major was released just over a year ago
+		mockCache[nextMajorVersion] = new Date(
+			checker.now - (checker.expiryDays + 1) * checker.msInDay,
+		).toISOString();
+
+		getStub.resolves(mockCache);
+
+		const { out, err } = await runCommand('--unsupported version');
+
+		expect(setStub.callCount).to.equal(0);
+		expect(getStub.callCount).to.equal(0);
+
+		expect(out.join('')).to.equal(packageJSON.version + '\n');
+		expect(err.join('')).to.be.empty;
+	});
+
+	it('should query the npm registry (empty cache file)', async () => {
+		npm.expectGetBalenaCli({
+			version: nextMajorVersion,
+			publishedAt: new Date().toISOString(),
+		});
+
+		getStub.resolves(undefined);
+
+		const { out, err } = await runCommand('version');
+
+		expect(setStub.callCount).to.equal(1);
+		expect(setStub.firstCall.args.length).to.equal(2);
+		const [name, obj] = setStub.firstCall.args;
+		expect(name).to.equal(checker.cacheFile);
+		expect(obj).to.have.property(nextMajorVersion);
+		const lastFetched = new Date(obj[nextMajorVersion]).getTime();
+		expect(lastFetched).to.be.greaterThan(anHourAgo);
+
+		expect(getStub.callCount).to.equal(1);
+		expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+		expect(out.join('')).to.equal(packageJSON.version + '\n');
+		expect(err.join('')).to.equal('');
+	});
+
+	it('should query the npm registry (not recently fetched)', async () => {
+		npm.expectGetBalenaCli({
+			version: nextMajorVersion,
+			publishedAt: new Date().toISOString(),
+		});
+
+		const mockCache: ReleaseTimestampsByVersion = {
+			lastFetched: new Date(
+				checker.now -
+					(checker.majorVersionFetchIntervalDays + 1) * checker.msInDay,
+			).toISOString(),
+		};
+		getStub.resolves(mockCache);
+
+		const { out, err } = await runCommand('version');
+
+		expect(setStub.callCount).to.equal(1);
+		expect(setStub.firstCall.args.length).to.equal(2);
+		const [name, obj] = setStub.firstCall.args;
+		expect(name).to.equal(checker.cacheFile);
+		expect(obj).to.have.property(nextMajorVersion);
+		const lastFetched = new Date(obj[nextMajorVersion]).getTime();
+		expect(lastFetched).to.be.greaterThan(anHourAgo);
+
+		expect(getStub.callCount).to.equal(1);
+		expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+		expect(out.join('')).to.equal(packageJSON.version + '\n');
+		expect(err.join('')).to.equal('');
+	});
+
+	itSS('should NOT query the npm registry (recently fetched)', async () => {
+		const mockCache: ReleaseTimestampsByVersion = {
+			lastFetched: new Date(
+				checker.now -
+					(checker.majorVersionFetchIntervalDays - 1) * checker.msInDay,
+			).toISOString(),
+		};
+		getStub.resolves(mockCache);
+
+		const { out, err } = await runCommand('version');
+
+		expect(setStub.callCount).to.equal(0);
+		expect(getStub.callCount).to.equal(1);
+		expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+		expect(out.join('')).to.equal(packageJSON.version + '\n');
+		expect(err.join('')).to.equal('');
+	});
+
+	itSS('should NOT query the npm registry (cached value)', async () => {
+		const mockCache: ReleaseTimestampsByVersion = {
+			lastFetched: '1970-01-01T00:00:00.000Z',
+		};
+		// pretend the next major was released just under half a year ago
+		mockCache[nextMajorVersion] = new Date(
+			checker.now - (checker.deprecationDays - 1) * checker.msInDay,
+		).toISOString();
+
+		getStub.resolves(mockCache);
+
+		const { out, err } = await runCommand('version');
+
+		expect(setStub.callCount).to.equal(0);
+		expect(getStub.callCount).to.equal(1);
+		expect(getStub.firstCall.args).to.deep.equal([checker.cacheFile]);
+
+		expect(out.join('')).to.equal(packageJSON.version + '\n');
+		expect(err.join('')).to.equal('');
+	});
+});

tests/docker-build.ts

@@ -28,8 +28,8 @@ import { streamToBuffer } from 'tar-utils';
 import { URL } from 'url';
 
 import { stripIndent } from '../lib/utils/lazy';
-import { BuilderMock } from './builder-mock';
+import { BuilderMock } from './nock/builder-mock';
-import { DockerMock } from './docker-mock';
+import { DockerMock } from './nock/docker-mock';
 import {
 	cleanOutput,
 	deepJsonParse,

tests/helpers.ts

@@ -1,6 +1,6 @@
 /**
  * @license
- * Copyright 2019-2020 Balena Ltd.
+ * Copyright 2019-2021 Balena Ltd.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,36 +15,59 @@
  * limitations under the License.
  */
 
-import { execFile } from 'child_process';
-import intercept = require('intercept-stdout');
 import * as _ from 'lodash';
-import { promises as fs } from 'fs';
-import * as nock from 'nock';
 import * as path from 'path';
 
-import * as balenaCLI from '../build/app';
+import * as packageJSON from '../package.json';
 
 const balenaExe = process.platform === 'win32' ? 'balena.exe' : 'balena';
 const standalonePath = path.resolve(__dirname, '..', 'build-bin', balenaExe);
 
-interface TestOutput {
+export interface TestOutput {
 	err: string[]; // stderr
 	out: string[]; // stdout
 	exitCode?: number; // process.exitCode
 }
 
+function matchesNodeEngineVersionWarn(msg: string) {
+	if (/^-----+\r?\n?$/.test(msg)) {
+		return true;
+	}
+	const cleanup = (line: string): string[] =>
+		line
+			.replace(/-----+/g, '')
+			.replace(/"\d+\.\d+\.\d+"/, '"x.y.z"')
+			.split(/\r?\n/)
+			.map((l) => l.trim())
+			.filter((l) => l);
+
+	const { getNodeEngineVersionWarn } = require('../build/utils/messages');
+	let nodeEngineWarn: string = getNodeEngineVersionWarn(
+		'x.y.z',
+		packageJSON.engines.node,
+	);
+	const nodeEngineWarnArray = cleanup(nodeEngineWarn);
+	nodeEngineWarn = nodeEngineWarnArray.join('\n');
+	msg = cleanup(msg).join('\n');
+	return msg === nodeEngineWarn || nodeEngineWarnArray.includes(msg);
+}
+
 /**
  * Filter stdout / stderr lines to remove lines that start with `[debug]` and
  * other lines that can be ignored for testing purposes.
  * @param testOutput
  */
-function filterCliOutputForTests(testOutput: TestOutput): TestOutput {
+export function filterCliOutputForTests({
-	const { matchesNodeEngineVersionWarn } =
+	err,
-		require('../automation/utils') as typeof import('../automation/utils');
+	out,
+}: {
+	err: string[];
+	out: string[];
+}): { err: string[]; out: string[] } {
 	return {
-		exitCode: testOutput.exitCode,
+		err: err.filter(
-		err: testOutput.err.filter(
 			(line: string) =>
+				line &&
 				!line.match(/\[debug\]/i) &&
 				// TODO stop this warning message from appearing when running
 				// sdk.setSharedOptions multiple times in the same process
@@ -52,7 +75,7 @@ function filterCliOutputForTests(testOutput: TestOutput): TestOutput {
 				!line.startsWith('WARN: disabling Sentry.io error reporting') &&
 				!matchesNodeEngineVersionWarn(line),
 		),
-		out: testOutput.out.filter((line: string) => !line.match(/\[debug\]/i)),
+		out: out.filter((line: string) => line && !line.match(/\[debug\]/i)),
 	};
 }
 
@@ -61,6 +84,9 @@ function filterCliOutputForTests(testOutput: TestOutput): TestOutput {
  * @param cmd Command to execute, e.g. `push myApp` (without 'balena' prefix)
  */
 async function runCommandInProcess(cmd: string): Promise<TestOutput> {
+	const balenaCLI = await import('../build/app');
+	const intercept = await import('intercept-stdout');
+
 	const preArgs = [process.argv[0], path.join(process.cwd(), 'bin', 'balena')];
 
 	const err: string[] = [];
@@ -85,12 +111,13 @@ async function runCommandInProcess(cmd: string): Promise<TestOutput> {
 	} finally {
 		unhookIntercept();
 	}
-	return filterCliOutputForTests({
+	const filtered = filterCliOutputForTests({ err, out });
-		err,
+	return {
-		out,
+		err: filtered.err,
+		out: filtered.out,
 		// this makes sense if `process.exit()` was stubbed with sinon
 		exitCode: process.exitCode,
-	});
+	};
 }
 
 /**
@@ -129,6 +156,7 @@ async function runCommandInSubprocess(
 		// override default proxy exclusion to allow proxying of requests to 127.0.0.1
 		BALENARC_DO_PROXY: '127.0.0.1,localhost',
 	};
+	const { execFile } = await import('child_process');
 	await new Promise<void>((resolve) => {
 		const child = execFile(
 			standalonePath,
@@ -141,11 +169,12 @@ async function runCommandInSubprocess(
 				// non-zero exit code. Usually this is harmless/expected, as
 				// the CLI child process is tested for error conditions.
 				if ($error && process.env.DEBUG) {
-					console.error(`
+					const msg = `
-[debug] Error (possibly expected) executing child CLI process "${standalonePath}"
+Error (possibly expected) executing child CLI process "${standalonePath}"
-------------------------------------------------------------------
+${$error}`;
-${$error}
+					const { warnify } =
-------------------------------------------------------------------`);
+						require('../build/utils/messages') as typeof import('../build/utils/messages');
+					console.error(warnify(msg, '[debug] '));
 				}
 				resolve();
 			},
@@ -166,11 +195,16 @@ ${$error}
 			.filter((l) => l)
 			.map((l) => l + '\n');
 
-	return filterCliOutputForTests({
+	const filtered = filterCliOutputForTests({
-		exitCode,
 		err: splitLines(stderr),
 		out: splitLines(stdout),
 	});
+	return {
+		err: filtered.err,
+		out: filtered.out,
+		// this makes sense if `process.exit()` was stubbed with sinon
+		exitCode,
+	};
 }
 
 /**
@@ -190,11 +224,12 @@ export async function runCommand(cmd: string): Promise<TestOutput> {
 			);
 		}
 		try {
+			const { promises: fs } = await import('fs');
 			await fs.access(standalonePath);
 		} catch {
 			throw new Error(`Standalone executable not found: "${standalonePath}"`);
 		}
-		const proxy = await import('./proxy-server');
+		const proxy = await import('./nock/proxy-server');
 		const [proxyPort] = await proxy.createProxyServerOnce();
 		return runCommandInSubprocess(cmd, proxyPort);
 	} else {
@@ -202,22 +237,6 @@ export async function runCommand(cmd: string): Promise<TestOutput> {
 	}
 }
 
-export const balenaAPIMock = () => {
-	if (!nock.isActive()) {
-		nock.activate();
-	}
-
-	return nock(/./).get('/config/vars').reply(200, {
-		reservedNames: [],
-		reservedNamespaces: [],
-		invalidRegex: '/^d|W/',
-		whiteListedNames: [],
-		whiteListedNamespaces: [],
-		blackListedNames: [],
-		configVarSchema: [],
-	});
-};
-
 export function cleanOutput(
 	output: string[] | string,
 	collapseBlank = false,
@@ -226,11 +245,17 @@ export function cleanOutput(
 		? (line: string) => monochrome(line.trim()).replace(/\s{2,}/g, ' ')
 		: (line: string) => monochrome(line.trim());
 
-	return _(_.castArray(output))
+	const result: string[] = [];
-		.map((log: string) => log.split('\n').map(cleanLine))
+	output = typeof output === 'string' ? [output] : output;
-		.flatten()
+	for (const lines of output) {
-		.compact()
+		for (let line of lines.split('\n')) {
-		.value();
+			line = cleanLine(line);
+			if (line) {
+				result.push(line);
+			}
+		}
+	}
+	return result;
 }
 
 /**
@@ -320,6 +345,7 @@ export function deepJsonParse(data: any): any {
 export async function switchSentry(
 	enabled: boolean | undefined,
 ): Promise<boolean | undefined> {
+	const balenaCLI = await import('../build/app');
 	const sentryOpts = (await balenaCLI.setupSentry()).getClient()?.getOptions();
 	if (sentryOpts) {
 		const sentryStatus = sentryOpts.enabled;

tests/nock/balena-api-mock.ts

@@ -21,7 +21,7 @@ import * as path from 'path';
 import { NockMock, ScopeOpts } from './nock-mock';
 
 export const apiResponsePath = path.normalize(
-	path.join(__dirname, 'test-data', 'api-response'),
+	path.join(__dirname, '..', 'test-data', 'api-response'),
 );
 
 const jHeader = { 'Content-Type': 'application/json' };

tests/nock/builder-mock.ts

@@ -22,7 +22,7 @@ import * as zlib from 'zlib';
 import { NockMock } from './nock-mock';
 
 export const builderResponsePath = path.normalize(
-	path.join(__dirname, 'test-data', 'builder-response'),
+	path.join(__dirname, '..', 'test-data', 'builder-response'),
 );
 
 export class BuilderMock extends NockMock {

tests/nock/docker-mock.ts

@@ -21,7 +21,7 @@ import * as path from 'path';
 import { NockMock, ScopeOpts } from './nock-mock';
 
 export const dockerResponsePath = path.normalize(
-	path.join(__dirname, 'test-data', 'docker-response'),
+	path.join(__dirname, '..', 'test-data', 'docker-response'),
 );
 
 export class DockerMock extends NockMock {

tests/nock/npm-mock.ts

@@ -0,0 +1,50 @@
+/**
+ * @license
+ * Copyright 2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { NockMock } from './nock-mock';
+
+const jHeader = { 'Content-Type': 'application/json' };
+
+export class NpmMock extends NockMock {
+	constructor() {
+		super(/registry\.npmjs\.org/);
+	}
+
+	public expectGetBalenaCli({
+		version,
+		publishedAt,
+		notFound = false,
+		optional = false,
+		persist = false,
+	}: {
+		version: string;
+		publishedAt: string;
+		notFound?: boolean;
+		optional?: boolean;
+		persist?: boolean;
+	}) {
+		const interceptor = this.optGet(`/balena-cli/${version}`, {
+			optional,
+			persist,
+		});
+		if (notFound) {
+			interceptor.reply(404, `version not found: ${version}`, jHeader);
+		} else {
+			interceptor.reply(200, { versionist: { publishedAt } }, jHeader);
+		}
+	}
+}

tests/nock/supervisor-mock.ts

@@ -22,7 +22,7 @@ import { Readable } from 'stream';
 import { NockMock, ScopeOpts } from './nock-mock';
 
 export const dockerResponsePath = path.normalize(
-	path.join(__dirname, 'test-data', 'docker-response'),
+	path.join(__dirname, '..', 'test-data', 'docker-response'),
 );
 
 export class SupervisorMock extends NockMock {