Tests: add verification of 'pkg' warnings against saved output

Change-type: patch

automation/build-bin.ts

@@ -25,17 +25,18 @@ import * as filehound from 'filehound';
 import * as fs from 'fs-extra';
 import * as _ from 'lodash';
 import * as path from 'path';
-import { exec as execPkg } from 'pkg';
 import * as rimraf from 'rimraf';
 import * as semver from 'semver';
 import * as util from 'util';
 
 import { stripIndent } from '../lib/utils/lazy';
 import {
+	diffLines,
 	getSubprocessStdout,
 	loadPackageJson,
 	MSYS2_BASH,
 	ROOT,
+	StdOutTap,
 	whichSpawn,
 } from './utils';
 
@@ -73,6 +74,99 @@ export const finalReleaseAssets: { [platform: string]: string[] } = {
 	linux: [standaloneZips['linux']],
 };
 
+/**
+ * Given the output of `pkg` as a string (containing warning messages),
+ * diff it against previously saved output of known "safe" warnings.
+ * Throw an error if the diff is not empty.
+ */
+async function diffPkgOutput(pkgOut: string) {
+	const { monochrome } = await import('../tests/helpers');
+	const relSavedPath = path.join(
+		'tests',
+		'test-data',
+		'pkg',
+		`expected-warnings-${process.platform}.txt`,
+	);
+	const absSavedPath = path.join(ROOT, relSavedPath);
+	const ignoreStartsWith = [
+		'> pkg@',
+		'> Fetching base Node.js binaries',
+		'  fetched-',
+	];
+	const modulesRE =
+		process.platform === 'win32'
+			? /(?<=[ '])([A-Z]:)?\\.+?\\node_modules(?=\\)/
+			: /(?<=[ '])\/.+?\/node_modules(?=\/)/;
+	const buildRE =
+		process.platform === 'win32'
+			? /(?<=[ '])([A-Z]:)?\\.+\\build(?=\\)/
+			: /(?<=[ '])\/.+\/build(?=\/)/;
+
+	const cleanLines = (chunks: string | string[]) => {
+		const lines = typeof chunks === 'string' ? chunks.split('\n') : chunks;
+		return lines
+			.map((line: string) => monochrome(line)) // remove ASCII colors
+			.filter((line: string) => !/^\s*$/.test(line)) // blank lines
+			.filter((line: string) =>
+				ignoreStartsWith.every((i) => !line.startsWith(i)),
+			)
+			.map((line: string) => {
+				// replace absolute paths with relative paths
+				let replaced = line.replace(modulesRE, 'node_modules');
+				if (replaced === line) {
+					replaced = line.replace(buildRE, 'build');
+				}
+				return replaced;
+			});
+	};
+
+	pkgOut = cleanLines(pkgOut).join('\n');
+	const { readFile } = (await import('fs')).promises;
+	const expectedOut = cleanLines(await readFile(absSavedPath, 'utf8')).join(
+		'\n',
+	);
+	if (expectedOut !== pkgOut) {
+		const sep =
+			'================================================================================';
+		const diff = diffLines(expectedOut, pkgOut);
+		const msg = `pkg output does not match expected output from "${relSavedPath}"
+Diff:
+${sep}
+${diff}
+${sep}
+Check whether the new or changed pkg warnings are safe to ignore, then update
+"${relSavedPath}"
+and share the result of your investigation as comments on the pull request.
+Hint: the fix is often a matter of updating the 'pkg.scripts' or 'pkg.assets'
+sections in the CLI's 'package.json' file, or a matter of updating the
+'buildPkg' function in 'automation/build-bin.ts'.  Sometimes it requires
+patching dependencies: See for example 'patches/all/open+7.0.2.patch'.
+${sep}
+`;
+		throw new Error(msg);
+	}
+}
+
+/**
+ * Call `pkg.exec` to generate the standalone zip file, capturing its warning
+ * messages (stdout and stderr) in order to call diffPkgOutput().
+ */
+async function execPkg(...args: any[]) {
+	const { exec: pkgExec } = await import('pkg');
+	const outTap = new StdOutTap(true);
+	try {
+		outTap.tap();
+		await (pkgExec as any)(...args);
+	} catch (err) {
+		outTap.untap();
+		console.log(outTap.stdoutBuf.join(''));
+		console.error(outTap.stderrBuf.join(''));
+		throw err;
+	}
+	outTap.untap();
+	await diffPkgOutput(outTap.allBuf.join(''));
+}
+
 /**
  * Use the 'pkg' module to create a single large executable file with
  * the contents of 'node_modules' and the CLI's javascript code.
@@ -208,11 +302,11 @@ export async function buildStandaloneZip() {
 		await buildPkg();
 		await testPkg();
 		await zipPkg();
+		console.log(`Standalone zip package build completed`);
 	} catch (error) {
-		console.log(`Error creating or testing standalone zip package:\n ${error}`);
-		process.exit(1);
+		console.error(`Error creating or testing standalone zip package`);
+		throw error;
 	}
-	console.log(`Standalone zip package build completed`);
 }
 
 async function renameInstallerFiles() {

automation/utils.ts

@@ -23,6 +23,69 @@ import * as shellEscape from 'shell-escape';
 export const MSYS2_BASH = 'C:\\msys64\\usr\\bin\\bash.exe';
 export const ROOT = path.join(__dirname, '..');
 
+/** Tap and buffer this process' stdout and stderr */
+export class StdOutTap {
+	public stdoutBuf: string[] = [];
+	public stderrBuf: string[] = [];
+	public allBuf: string[] = []; // both stdout and stderr
+
+	protected origStdoutWrite: typeof process.stdout.write;
+	protected origStderrWrite: typeof process.stdout.write;
+
+	constructor(protected printDots = false) {}
+
+	tap() {
+		this.origStdoutWrite = process.stdout.write;
+		this.origStderrWrite = process.stderr.write;
+
+		process.stdout.write = (chunk: string, ...args: any[]): boolean => {
+			this.stdoutBuf.push(chunk);
+			this.allBuf.push(chunk);
+			const str = this.printDots ? '.' : chunk;
+			return this.origStdoutWrite.call(process.stdout, str, ...args);
+		};
+
+		process.stderr.write = (chunk: string, ...args: any[]): boolean => {
+			this.stderrBuf.push(chunk);
+			this.allBuf.push(chunk);
+			const str = this.printDots ? '.' : chunk;
+			return this.origStderrWrite.call(process.stderr, str, ...args);
+		};
+	}
+
+	untap() {
+		process.stdout.write = this.origStdoutWrite;
+		process.stderr.write = this.origStderrWrite;
+		if (this.printDots) {
+			console.error('');
+		}
+	}
+}
+
+/**
+ * Diff strings by line, using the 'diff' npm package:
+ * https://www.npmjs.com/package/diff
+ */
+export function diffLines(str1: string, str2: string): string {
+	const { diffTrimmedLines } = require('diff');
+	const diffObjs = diffTrimmedLines(str1, str2);
+	const prefix = (chunk: string, char: string) =>
+		chunk
+			.split('\n')
+			.map((line: string) => `${char} ${line}`)
+			.join('\n');
+	const diffStr = diffObjs
+		.map((part: any) => {
+			return part.added
+				? prefix(part.value, '+')
+				: part.removed
+				? prefix(part.value, '-')
+				: prefix(part.value, ' ');
+		})
+		.join('\n');
+	return diffStr;
+}
+
 export function loadPackageJson() {
 	return require(path.join(ROOT, 'package.json'));
 }

npm-shrinkwrap.json

@@ -5021,9 +5021,9 @@
       "integrity": "sha1-oqLie025mSjW2NRNXF9++9TLQ3I="
     },
     "diff": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz",
-      "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q=="
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A=="
     },
     "dir-glob": {
       "version": "2.0.0",

package.json

@@ -156,6 +156,7 @@
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.2",
     "deep-object-diff": "^1.1.0",
+    "diff": "^4.0.2",
     "ent": "^2.2.0",
     "filehound": "^1.17.4",
     "fs-extra": "^8.0.1",

tests/test-data/pkg/expected-warnings-darwin.txt

@@ -0,0 +1,78 @@
+> Warning Cannot resolve 'path.join(...pathComponents)'
+  build/actions/help_ts.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot resolve ''./' + command'
+  node_modules/balena-sync/build/capitano/index.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot resolve ''./' + target'
+  node_modules/balena-sync/build/sync/index.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/open/xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/open/xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/build/Release/drivelist.node
+  %2: path-to-executable/drivelist.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/darwin.sh
+  %2: path-to-executable/drivelist/darwin.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/linux.sh
+  %2: path-to-executable/drivelist/linux.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/win32.bat
+  %2: path-to-executable/drivelist/win32.bat
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/build/Release/drivelist.node
+  %2: path-to-executable/drivelist.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/darwin.sh
+  %2: path-to-executable/drivelist/darwin.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/linux.sh
+  %2: path-to-executable/drivelist/linux.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/win32.bat
+  %2: path-to-executable/drivelist/win32.bat
+> Warning Cannot include addon %1 into executable.
+  The addon must be distributed with executable as %2.
+  %1: node_modules/xxhash/build/Release/hash.node
+  %2: path-to-executable/hash.node
+> Warning Cannot include addon %1 into executable.
+  The addon must be distributed with executable as %2.
+  %1: node_modules/fsevents/fsevents.node
+  %2: path-to-executable/fsevents.node
+> Warning Cannot include addon %1 into executable.
+  The addon must be distributed with executable as %2.
+  %1: node_modules/fsevents/fsevents.node
+  %2: path-to-executable/fsevents.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/opn/xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/opn/xdg-open
+  %2: path-to-executable/xdg-open

tests/test-data/pkg/expected-warnings-linux.txt

@@ -0,0 +1,70 @@
+> Warning Cannot resolve 'path.join(...pathComponents)'
+  build/actions/help_ts.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot resolve ''./' + command'
+  node_modules/balena-sync/build/capitano/index.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot resolve ''./' + target'
+  node_modules/balena-sync/build/sync/index.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/open/xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/open/xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/build/Release/drivelist.node
+  %2: path-to-executable/drivelist.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/darwin.sh
+  %2: path-to-executable/drivelist/darwin.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/linux.sh
+  %2: path-to-executable/drivelist/linux.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/win32.bat
+  %2: path-to-executable/drivelist/win32.bat
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/build/Release/drivelist.node
+  %2: path-to-executable/drivelist.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/darwin.sh
+  %2: path-to-executable/drivelist/darwin.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/linux.sh
+  %2: path-to-executable/drivelist/linux.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/drivelist/scripts/win32.bat
+  %2: path-to-executable/drivelist/win32.bat
+> Warning Cannot include addon %1 into executable.
+  The addon must be distributed with executable as %2.
+  %1: node_modules/xxhash/build/Release/hash.node
+  %2: path-to-executable/hash.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/opn/xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules/opn/xdg-open
+  %2: path-to-executable/xdg-open

tests/test-data/pkg/expected-warnings-win32.txt

@@ -0,0 +1,72 @@
+> Warning Cannot resolve 'path.join(...pathComponents)'
+  build\actions\help_ts.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot find module 'net-keepalive' from 'build\utils\device'
+  %1: build\utils\device\api.js
+> Warning Cannot resolve ''./' + command'
+  node_modules\balena-sync\build\capitano\index.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot resolve ''./' + target'
+  node_modules\balena-sync\build\sync\index.js
+  Dynamic require may fail at run time, because the requested file
+  is unknown at compilation time and not included into executable.
+  Use a string literal as an argument for 'require', or leave it
+  as is and specify the resolved file name in 'scripts' option.
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\open\xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\open\xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\build\Release\drivelist.node
+  %2: path-to-executable/drivelist.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\scripts\darwin.sh
+  %2: path-to-executable/drivelist/darwin.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\scripts\linux.sh
+  %2: path-to-executable/drivelist/linux.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\scripts\win32.bat
+  %2: path-to-executable/drivelist/win32.bat
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\build\Release\drivelist.node
+  %2: path-to-executable/drivelist.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\scripts\darwin.sh
+  %2: path-to-executable/drivelist/darwin.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\scripts\linux.sh
+  %2: path-to-executable/drivelist/linux.sh
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\drivelist\scripts\win32.bat
+  %2: path-to-executable/drivelist/win32.bat
+> Warning Cannot include addon %1 into executable.
+  The addon must be distributed with executable as %2.
+  %1: node_modules\xxhash\build\Release\hash.node
+  %2: path-to-executable/hash.node
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\opn\xdg-open
+  %2: path-to-executable/xdg-open
+> Warning Cannot include file %1 into executable.
+  The file must be distributed with executable as %2.
+  %1: node_modules\opn\xdg-open
+  %2: path-to-executable/xdg-open