From 9db6961a7eb3905f37abe55e33989029128d2612 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Mon, 20 Jan 2020 14:01:58 +0000 Subject: [PATCH] Add `catch-uncommitted` to balena CI build Change-type: patch Signed-off-by: Paulo Castro --- automation/build-bin.ts | 122 +++++++--------------------- automation/run.ts | 8 +- automation/utils.ts | 174 ++++++++++++++++++++++++++++++++++++++++ npm-shrinkwrap.json | 6 +- package.json | 5 +- 5 files changed, 213 insertions(+), 102 deletions(-) create mode 100644 automation/utils.ts diff --git a/automation/build-bin.ts b/automation/build-bin.ts index 73527cb4..a412b236 100644 --- a/automation/build-bin.ts +++ b/automation/build-bin.ts @@ -18,7 +18,7 @@ import { run as oclifRun } from '@oclif/dev-cli'; import * as archiver from 'archiver'; import * as Bluebird from 'bluebird'; -import { execFile, spawn } from 'child_process'; +import { execFile } from 'child_process'; import { stripIndent } from 'common-tags'; import * as filehound from 'filehound'; import * as fs from 'fs-extra'; @@ -27,15 +27,17 @@ import * as path from 'path'; import { exec as execPkg } from 'pkg'; import * as rimraf from 'rimraf'; import * as semver from 'semver'; -import * as shellEscape from 'shell-escape'; import * as util from 'util'; -export const ROOT = path.join(__dirname, '..'); -// Note: the following 'tslint disable' line was only required to -// satisfy ts-node under Appveyor's MSYS2 on Windows -- oddly specific. -// Maybe something to do with '/' vs '\' in paths in some tslint file. -// tslint:disable-next-line:no-var-requires -export const packageJSON = require(path.join(ROOT, 'package.json')); +import { + getSubprocessStdout, + loadPackageJson, + MSYS2_BASH, + ROOT, + whichSpawn, +} from './utils'; + +export const packageJSON = loadPackageJson(); export const version = 'v' + packageJSON.version; const arch = process.arch; @@ -69,34 +71,6 @@ export const finalReleaseAssets: { [platform: string]: string[] } = { linux: [standaloneZips['linux']], }; -const MSYS2_BASH = 'C:\\msys64\\usr\\bin\\bash.exe'; - -/** - * Run the MSYS2 bash.exe shell in a child process (child_process.spawn()). - * The given argv arguments are escaped using the 'shell-escape' package, - * so that backslashes in Windows paths, and other bash-special characters, - * are preserved. If argv is not provided, defaults to process.argv, to the - * effect that this current (parent) process is re-executed under MSYS2 bash. - * This is useful to change the default shell from cmd.exe to MSYS2 bash on - * Windows. - * @param argv Arguments to be shell-escaped and given to MSYS2 bash.exe. - */ -export async function runUnderMsys(argv?: string[]) { - const newArgv = argv || process.argv; - await new Promise((resolve, reject) => { - const args = ['-lc', shellEscape(newArgv)]; - const child = spawn(MSYS2_BASH, args, { stdio: 'inherit' }); - child.on('close', code => { - if (code) { - console.log(`runUnderMsys: child process exited with code ${code}`); - reject(code); - } else { - resolve(); - } - }); - }); -} - /** * Use the 'pkg' module to create a single large executable file with * the contents of 'node_modules' and the CLI's javascript code. @@ -313,62 +287,24 @@ export async function buildOclifInstaller() { } /** - * Convert e.g. 'C:\myfolder' -> '/C/myfolder' so that the path can be given - * as argument to "unix tools" like 'tar' under MSYS or MSYS2 on Windows. + * Wrapper around the npm `catch-uncommitted` package in order to run it + * conditionally, only when: + * - A CI env var is set (CI=true), and + * - The OS is not Windows. (`catch-uncommitted` fails on Windows) */ -export function fixPathForMsys(p: string): string { - return p.replace(/\\/g, '/').replace(/^([a-zA-Z]):/, '/$1'); -} - -/** - * Run the executable at execPath as a child process, and resolve a promise - * to the executable's stdout output as a string. Reject the promise if - * anything is printed to stderr, or if the child process exits with a - * non-zero exit code. - * @param execPath Executable path - * @param args Command-line argument for the executable - */ -async function getSubprocessStdout( - execPath: string, - args: string[], -): Promise { - const child = spawn(execPath, args); - return new Promise((resolve, reject) => { - let stdout = ''; - child.stdout.on('error', reject); - child.stderr.on('error', reject); - child.stdout.on('data', (data: Buffer) => { - try { - stdout = data.toString(); - } catch (err) { - reject(err); - } - }); - child.stderr.on('data', (data: Buffer) => { - try { - const stderr = data.toString(); - - // ignore any debug lines, but ensure that we parse - // every line provided to the stderr stream - const lines = _.filter( - stderr.trim().split(/\r?\n/), - line => !line.startsWith('[debug]'), - ); - if (lines.length > 0) { - reject( - new Error(`"${execPath}": non-empty stderr "${lines.join('\n')}"`), - ); - } - } catch (err) { - reject(err); - } - }); - child.on('exit', (code: number) => { - if (code) { - reject(new Error(`"${execPath}": non-zero exit code "${code}"`)); - } else { - resolve(stdout); - } - }); - }); +export async function catchUncommitted(): Promise { + if (process.env.DEBUG) { + console.error(`[debug] CI=${process.env.CI} platform=${process.platform}`); + } + if ( + process.env.CI && + ['true', 'yes', '1'].includes(process.env.CI.toLowerCase()) && + process.platform !== 'win32' + ) { + await whichSpawn('npx', [ + 'catch-uncommitted', + '--catch-no-git', + '--skip-node-versionbot-changes', + ]); + } } diff --git a/automation/run.ts b/automation/run.ts index c31e557c..45921a2b 100644 --- a/automation/run.ts +++ b/automation/run.ts @@ -20,14 +20,13 @@ import * as _ from 'lodash'; import { buildOclifInstaller, buildStandaloneZip, - fixPathForMsys, - ROOT, - runUnderMsys, + catchUncommitted, } from './build-bin'; import { release, updateDescriptionOfReleasesAffectedByIssue1359, } from './deploy-bin'; +import { fixPathForMsys, ROOT, runUnderMsys } from './utils'; function exitWithError(error: Error | string): never { console.error(`Error: ${error}`); @@ -54,9 +53,10 @@ export async function run(args?: string[]) { if (_.isEmpty(args)) { return exitWithError('missing command-line arguments'); } - const commands: { [cmd: string]: () => void } = { + const commands: { [cmd: string]: () => void | Promise } = { 'build:installer': buildOclifInstaller, 'build:standalone': buildStandaloneZip, + 'catch-uncommitted': catchUncommitted, fix1359: updateDescriptionOfReleasesAffectedByIssue1359, release, }; diff --git a/automation/utils.ts b/automation/utils.ts new file mode 100644 index 00000000..cfdd0302 --- /dev/null +++ b/automation/utils.ts @@ -0,0 +1,174 @@ +/** + * @license + * Copyright 2019-2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { spawn } from 'child_process'; +import * as _ from 'lodash'; +import * as path from 'path'; +import * as shellEscape from 'shell-escape'; + +export const MSYS2_BASH = 'C:\\msys64\\usr\\bin\\bash.exe'; +export const ROOT = path.join(__dirname, '..'); + +export function loadPackageJson() { + return require(path.join(ROOT, 'package.json')); +} + +/** + * Convert e.g. 'C:\myfolder' -> '/C/myfolder' so that the path can be given + * as argument to "unix tools" like 'tar' under MSYS or MSYS2 on Windows. + */ +export function fixPathForMsys(p: string): string { + return p.replace(/\\/g, '/').replace(/^([a-zA-Z]):/, '/$1'); +} + +/** + * Run the MSYS2 bash.exe shell in a child process (child_process.spawn()). + * The given argv arguments are escaped using the 'shell-escape' package, + * so that backslashes in Windows paths, and other bash-special characters, + * are preserved. If argv is not provided, defaults to process.argv, to the + * effect that this current (parent) process is re-executed under MSYS2 bash. + * This is useful to change the default shell from cmd.exe to MSYS2 bash on + * Windows. + * @param argv Arguments to be shell-escaped and given to MSYS2 bash.exe. + */ +export async function runUnderMsys(argv?: string[]) { + const newArgv = argv || process.argv; + await new Promise((resolve, reject) => { + const args = ['-lc', shellEscape(newArgv)]; + const child = spawn(MSYS2_BASH, args, { stdio: 'inherit' }); + child.on('close', code => { + if (code) { + console.log(`runUnderMsys: child process exited with code ${code}`); + reject(code); + } else { + resolve(); + } + }); + }); +} + +/** + * Run the executable at execPath as a child process, and resolve a promise + * to the executable's stdout output as a string. Reject the promise if + * anything is printed to stderr, or if the child process exits with a + * non-zero exit code. + * @param execPath Executable path + * @param args Command-line argument for the executable + */ +export async function getSubprocessStdout( + execPath: string, + args: string[], +): Promise { + const child = spawn(execPath, args); + return new Promise((resolve, reject) => { + let stdout = ''; + child.stdout.on('error', reject); + child.stderr.on('error', reject); + child.stdout.on('data', (data: Buffer) => { + try { + stdout = data.toString(); + } catch (err) { + reject(err); + } + }); + child.stderr.on('data', (data: Buffer) => { + try { + const stderr = data.toString(); + + // ignore any debug lines, but ensure that we parse + // every line provided to the stderr stream + const lines = _.filter( + stderr.trim().split(/\r?\n/), + line => !line.startsWith('[debug]'), + ); + if (lines.length > 0) { + reject( + new Error(`"${execPath}": non-empty stderr "${lines.join('\n')}"`), + ); + } + } catch (err) { + reject(err); + } + }); + child.on('exit', (code: number) => { + if (code) { + reject(new Error(`"${execPath}": non-zero exit code "${code}"`)); + } else { + resolve(stdout); + } + }); + }); +} + +/** + * Error handling wrapper around the npm `which` package: + * "Like the unix which utility. Finds the first instance of a specified + * executable in the PATH environment variable. Does not cache the results, + * so hash -r is not needed when the PATH changes." + * + * @param program Basename of a program, for example 'ssh' + * @returns The program's full path, e.g. 'C:\WINDOWS\System32\OpenSSH\ssh.EXE' + */ +export async function which(program: string): Promise { + const whichMod = await import('which'); + let programPath: string; + try { + programPath = await whichMod(program); + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error(`'${program}' program not found. Is it installed?`); + } + throw err; + } + return programPath; +} + +/** + * Call which(programName) and spawn() with the given arguments. Throw an error + * if the process exit code is not zero. + */ +export async function whichSpawn( + programName: string, + args: string[], +): Promise { + const program = await which(programName); + let error: Error | undefined; + let exitCode: number | undefined; + try { + exitCode = await new Promise((resolve, reject) => { + try { + spawn(program, args, { stdio: 'inherit' }) + .on('error', reject) + .on('close', resolve); + } catch (err) { + reject(err); + } + }); + } catch (err) { + error = err; + } + if (error || exitCode) { + const msg = [ + `${programName} failed with exit code ${exitCode}:`, + `"${program}" [${args}]`, + ]; + if (error) { + msg.push(`${error}`); + } + throw new Error(msg.join('\n')); + } +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 68a8c2ba..72a3fed3 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -2702,9 +2702,9 @@ "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "catch-uncommitted": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/catch-uncommitted/-/catch-uncommitted-1.3.0.tgz", - "integrity": "sha512-JJrlxvOX8mLEmQ7zk/w+su70FQeuTkRH9OYqWg8df3YLjz+rEkHKlWx0+C3/jjWZxRSrB1JBVhS5MhXJ3VhU1A==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/catch-uncommitted/-/catch-uncommitted-1.4.0.tgz", + "integrity": "sha512-xrLMj7iYrMc3TXSLsRO9tTxfcWEUICGCDZm+WI40WznxLp/+mVE8v4RxipC/ufL5TDfAYAe1ppu5VURBN990SQ==", "dev": true }, "chai": { diff --git a/package.json b/package.json index 1d83e397..87d9806e 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "scripts": { "postinstall": "patch-package", "prebuild": "rimraf build/ build-bin/", - "build": "npm run build:src", + "build": "npm run build:src && npm run catch-uncommitted", "build:src": "npm run prettify && npm run lint && npm run build:fast && npm run build:doc", "build:fast": "gulp build && tsc", "build:doc": "mkdirp doc/ && ts-node --type-check -P automation/tsconfig.json automation/capitanodoc/index.ts > doc/cli.markdown", @@ -52,6 +52,7 @@ "pretest": "npm run build", "test": "mocha --timeout 6000 -r ts-node/register \"tests/**/*.spec.ts\"", "test:fast": "npm run build:fast && npm run test", + "catch-uncommitted": "ts-node --type-check -P automation/tsconfig.json automation/run.ts catch-uncommitted", "ci": "npm run test && catch-uncommitted", "watch": "gulp watch", "prettify": "prettier --write \"{lib,tests,automation,typings}/**/*.[tj]s\" --config ./node_modules/resin-lint/config/.prettierrc", @@ -124,7 +125,7 @@ "@types/tar-stream": "1.6.0", "@types/through2": "2.0.33", "@types/which": "^1.3.2", - "catch-uncommitted": "^1.3.0", + "catch-uncommitted": "^1.4.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "ent": "^2.2.0",