From 66b997d98c1895df827bca4b3c343660a27ce893 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Tue, 4 Jun 2019 01:45:54 +0100 Subject: [PATCH 1/6] balena CI integration: Use C:\tmp to avoid 260-char path length limit Change-type: patch Signed-off-by: Paulo Castro --- automation/run.ts | 9 +++ patches/@oclif+dev-cli+1.22.0.patch | 88 ++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/automation/run.ts b/automation/run.ts index 08beb0e2..41a34e41 100644 --- a/automation/run.ts +++ b/automation/run.ts @@ -89,6 +89,15 @@ export async function run(args?: string[]) { // the current working dir becomes the MSYS2 homedir, so we change back. process.chdir(ROOT); + if (process.platform === 'win32' && !process.env.BUILD_TMP) { + const randID = require('crypto') + .randomBytes(6) + .toString('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_'); // base64url (RFC 4648) + process.env.BUILD_TMP = `C:\\tmp\\${randID}`; + } + for (const arg of args) { if (arg === 'build:installer' && process.platform === 'win32') { // ensure running under MSYS2 diff --git a/patches/@oclif+dev-cli+1.22.0.patch b/patches/@oclif+dev-cli+1.22.0.patch index 84bcc952..61ce7fec 100644 --- a/patches/@oclif+dev-cli+1.22.0.patch +++ b/patches/@oclif+dev-cli+1.22.0.patch @@ -1,3 +1,15 @@ +diff --git a/node_modules/@oclif/dev-cli/lib/commands/pack/macos.js b/node_modules/@oclif/dev-cli/lib/commands/pack/macos.js +index cd771cd..4a66939 100644 +--- a/node_modules/@oclif/dev-cli/lib/commands/pack/macos.js ++++ b/node_modules/@oclif/dev-cli/lib/commands/pack/macos.js +@@ -37,6 +37,7 @@ class PackMacos extends command_1.Command { + if (process.env.OSX_KEYCHAIN) + args.push('--keychain', process.env.OSX_KEYCHAIN); + args.push(dist); ++ console.log(`pkgbuild "${args.join('" "')}"`); + await qq.x('pkgbuild', args); + } + } diff --git a/node_modules/@oclif/dev-cli/lib/commands/pack/win.js b/node_modules/@oclif/dev-cli/lib/commands/pack/win.js index a9d4276..75c2f8b 100644 --- a/node_modules/@oclif/dev-cli/lib/commands/pack/win.js @@ -36,12 +48,15 @@ index a9d4276..75c2f8b 100644 exports.default = PackWin; const scripts = { diff --git a/node_modules/@oclif/dev-cli/lib/tarballs/build.js b/node_modules/@oclif/dev-cli/lib/tarballs/build.js -index 3e613e0..dd53603 100644 +index 3e613e0..c73a04c 100644 --- a/node_modules/@oclif/dev-cli/lib/tarballs/build.js +++ b/node_modules/@oclif/dev-cli/lib/tarballs/build.js -@@ -19,6 +19,9 @@ const pack = async (from, to) => { +@@ -17,8 +17,11 @@ const pack = async (from, to) => { + qq.cd(prevCwd); + }; async function build(c, options = {}) { - const { xz, config } = c; +- const { xz, config } = c; ++ const { xz, config, tmp } = c; const prevCwd = qq.cwd(); + + console.log(`[patched @oclif/dev-cli] cwd="${prevCwd}"\n c.root="${c.root}" c.workspace()="${c.workspace()}"`); @@ -49,7 +64,7 @@ index 3e613e0..dd53603 100644 const packCLI = async () => { const stdout = await qq.x.stdout('npm', ['pack', '--unsafe-perm'], { cwd: c.root }); return path.join(c.root, stdout.split('\n').pop()); -@@ -34,6 +37,23 @@ async function build(c, options = {}) { +@@ -34,6 +37,28 @@ async function build(c, options = {}) { await qq.mv(f, '.'); await qq.rm('package', tarball, 'bin/run.cmd'); }; @@ -69,11 +84,16 @@ index 3e613e0..dd53603 100644 + // rename the original balena-cli ./bin/balena entry point for oclif compatibility + await qq.mv('bin/balena', 'bin/run'); + await qq.rm('bin/run.cmd'); ++ // The oclif installers are produced with `npm i --production`, while the ++ // source `bin` folder may contain a `.fast-boot.json` produced with `npm i`. ++ // This has previously led to issues preventing the CLI from starting, so ++ // delete `.fast-boot.json` (if any) from the destination folder. ++ await qq.rm('bin/.fast-boot.json'); + } const updatePJSON = async () => { qq.cd(c.workspace()); const pjson = await qq.readJSON('package.json'); -@@ -55,7 +75,11 @@ async function build(c, options = {}) { +@@ -55,7 +80,11 @@ async function build(c, options = {}) { if (!await qq.exists(lockpath)) { lockpath = qq.join(c.root, 'npm-shrinkwrap.json'); } @@ -86,7 +106,17 @@ index 3e613e0..dd53603 100644 await qq.x('npm install --production'); } }; -@@ -124,7 +148,8 @@ async function build(c, options = {}) { +@@ -71,7 +100,8 @@ async function build(c, options = {}) { + output: path.join(workspace, 'bin', 'node'), + platform: target.platform, + arch: target.arch, +- tmp: qq.join(config.root, 'tmp'), ++ tmp, ++ projectRootPath: c.root, + }); + if (options.pack === false) + return; +@@ -124,7 +154,8 @@ async function build(c, options = {}) { await qq.writeJSON(c.dist(config.s3Key('manifest')), manifest); }; log_1.log(`gathering workspace for ${config.bin} to ${c.workspace()}`); @@ -96,11 +126,39 @@ index 3e613e0..dd53603 100644 await updatePJSON(); await addDependencies(); await bin_1.writeBinScripts({ config, baseWorkspace: c.workspace(), nodeVersion: c.nodeVersion }); +diff --git a/node_modules/@oclif/dev-cli/lib/tarballs/config.js b/node_modules/@oclif/dev-cli/lib/tarballs/config.js +index 320fc52..efe3f2f 100644 +--- a/node_modules/@oclif/dev-cli/lib/tarballs/config.js ++++ b/node_modules/@oclif/dev-cli/lib/tarballs/config.js +@@ -10,7 +10,13 @@ function gitSha(cwd, options = {}) { + } + exports.gitSha = gitSha; + async function Tmp(config) { +- const tmp = path.join(config.root, 'tmp'); ++ let tmp; ++ if (process.env.BUILD_TMP) { ++ tmp = path.join(process.env.BUILD_TMP, 'oclif'); ++ } else { ++ tmp = path.join(config.root, 'tmp'); ++ } ++ console.log(`@oclif/dev-cli tmp="${tmp}"`); + await qq.mkdirp(tmp); + return tmp; + } +@@ -36,7 +42,7 @@ async function buildConfig(root, options = {}) { + s3Config: updateConfig.s3, + nodeVersion: updateConfig.node.version || process.versions.node, + workspace(target) { +- const base = qq.join(config.root, 'tmp'); ++ const base = tmp; + if (target && target.platform) + return qq.join(base, [target.platform, target.arch].join('-'), config.s3Key('baseDir', target)); + return qq.join(base, config.s3Key('baseDir', target)); diff --git a/node_modules/@oclif/dev-cli/lib/tarballs/node.js b/node_modules/@oclif/dev-cli/lib/tarballs/node.js -index 343eb00..865d5a5 100644 +index 343eb00..5521e2d 100644 --- a/node_modules/@oclif/dev-cli/lib/tarballs/node.js +++ b/node_modules/@oclif/dev-cli/lib/tarballs/node.js -@@ -1,19 +1,45 @@ +@@ -1,28 +1,58 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); const errors_1 = require("@oclif/errors"); @@ -152,9 +210,13 @@ index 343eb00..865d5a5 100644 } + return foundPath; } - async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { +-async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { ++async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp, projectRootPath }) { ++ ++ console.log(`fetchNodeBinary using tmp="${tmp}`); ++ if (arch === 'arm') -@@ -21,8 +47,9 @@ async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { + arch = 'armv6l'; let nodeBase = `node-v${nodeVersion}-${platform}-${arch}`; let tarball = path.join(tmp, 'node', `${nodeBase}.tar.xz`); let url = `https://nodejs.org/dist/v${nodeVersion}/${nodeBase}.tar.xz`; @@ -162,11 +224,11 @@ index 343eb00..865d5a5 100644 - await checkFor7Zip(); + let zPath = ''; + if (platform === 'win32') { -+ zPath = await checkFor7Zip(path.join(tmp, '..')); ++ zPath = await checkFor7Zip(projectRootPath); nodeBase = `node-v${nodeVersion}-win-${arch}`; tarball = path.join(tmp, 'node', `${nodeBase}.7z`); url = `https://nodejs.org/dist/v${nodeVersion}/${nodeBase}.7z`; -@@ -40,7 +67,8 @@ async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { +@@ -40,7 +70,8 @@ async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { const basedir = path.dirname(tarball); await qq.mkdirp(basedir); await qq.download(url, tarball); @@ -176,7 +238,7 @@ index 343eb00..865d5a5 100644 }; const extract = async () => { log_1.log(`extracting ${nodeBase}`); -@@ -50,7 +78,7 @@ async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { +@@ -50,7 +81,7 @@ async function fetchNodeBinary({ nodeVersion, output, platform, arch, tmp }) { await qq.mkdirp(path.dirname(cache)); if (platform === 'win32') { qq.pushd(nodeTmp); From 0afbd6f17a2c7857006ca46386330b9c01a2c3c6 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Sun, 2 Jun 2019 15:23:37 +0100 Subject: [PATCH 2/6] Refactor build:standalone / build:installer / run release So that: - Standalone zip files are created in the standalone step, - oclif installers are renamed in the installer step, and - npm run release (which is skipped by balena CI) is reduced to uploading the files to the GitHub releases page. Change-type: patch Signed-off-by: Paulo Castro --- automation/build-bin.ts | 116 ++++++++++++++++++++++++++++++++++++--- automation/deploy-bin.ts | 92 +------------------------------ automation/run.ts | 40 ++++++++------ package.json | 4 +- 4 files changed, 136 insertions(+), 116 deletions(-) diff --git a/automation/build-bin.ts b/automation/build-bin.ts index fef58004..c71c7c78 100644 --- a/automation/build-bin.ts +++ b/automation/build-bin.ts @@ -16,6 +16,7 @@ */ import { run as oclifRun } from '@oclif/dev-cli'; +import * as archiver from 'archiver'; import * as Bluebird from 'bluebird'; import * as filehound from 'filehound'; import * as fs from 'fs-extra'; @@ -24,6 +25,43 @@ import { exec as execPkg } from 'pkg'; import * as rimraf from 'rimraf'; export const ROOT = path.join(__dirname, '..'); +// Note: the following 'tslint disable' line was only required to +// satisfy ts-node under Appveyor's MSYS2 on Windows -- oddly specific. +// Maybe something to do with '/' vs '\' in paths in some tslint file. +// tslint:disable-next-line:no-var-requires +export const packageJSON = require(path.join(ROOT, 'package.json')); +export const version = 'v' + packageJSON.version; +const arch = process.arch; + +function dPath(...paths: string[]) { + return path.join(ROOT, 'dist', ...paths); +} + +interface PathByPlatform { + [platform: string]: string; +} + +const standaloneZips: PathByPlatform = { + linux: dPath(`balena-cli-${version}-linux-${arch}-standalone.zip`), + darwin: dPath(`balena-cli-${version}-macOS-${arch}-standalone.zip`), + win32: dPath(`balena-cli-${version}-windows-${arch}-standalone.zip`), +}; + +const oclifInstallers: PathByPlatform = { + darwin: dPath('macos', `balena-${version}.pkg`), + win32: dPath('win', `balena-${version}-${arch}.exe`), +}; + +const renamedOclifInstallers: PathByPlatform = { + darwin: dPath(`balena-cli-${version}-macOS-${arch}-installer-BETA.pkg`), + win32: dPath(`balena-cli-${version}-windows-${arch}-installer-BETA.exe`), +}; + +export const finalReleaseAssets: { [platform: string]: string[] } = { + win32: [standaloneZips['win32'], renamedOclifInstallers['win32']], + darwin: [standaloneZips['darwin'], renamedOclifInstallers['darwin']], + linux: [standaloneZips['linux']], +}; /** * Use the 'pkg' module to create a single large executable file with @@ -34,16 +72,21 @@ export const ROOT = path.join(__dirname, '..'); * because of a pkg limitation that does not allow binary executables * to be directly executed from inside another binary executable.) */ -export async function buildPkg() { - console.log('Building package...\n'); - - await execPkg([ +async function buildPkg() { + const args = [ '--target', 'node10', '--output', 'build-bin/balena', 'package.json', - ]); + ]; + console.log('======================================================='); + console.log(`execPkg ${args.join(' ')}`); + console.log(`cwd="${process.cwd()}" ROOT="${ROOT}"`); + console.log('======================================================='); + + await execPkg(args); + const xpaths: Array<[string, string[]]> = [ // [platform, [path, to, file]] ['*', ['opn', 'xdg-open']], @@ -77,6 +120,60 @@ export async function buildPkg() { ); } +/** + * Create the zip file for the standalone 'pkg' bundle previously created + * by the buildPkg() function in 'build-bin.ts'. + */ +async function zipPkg() { + const outputFile = standaloneZips[process.platform]; + if (!outputFile) { + throw new Error( + `Standalone installer unavailable for platform "${process.platform}"`, + ); + } + await fs.mkdirp(path.dirname(outputFile)); + await new Promise((resolve, reject) => { + console.log(`Zipping standalone package to "${outputFile}"...`); + + const archive = archiver('zip', { + zlib: { level: 7 }, + }); + archive.directory(path.join(ROOT, 'build-bin'), 'balena-cli'); + + const outputStream = fs.createWriteStream(outputFile); + + outputStream.on('close', resolve); + outputStream.on('error', reject); + + archive.on('error', reject); + archive.on('warning', console.warn); + + archive.pipe(outputStream); + archive.finalize(); + }); +} + +export async function buildStandaloneZip() { + console.log(`Building standalone zip package for CLI ${version}`); + try { + await buildPkg(); + await zipPkg(); + } catch (error) { + console.log(`Error creating standalone zip package: ${error}`); + process.exit(1); + } + console.log(`Standalone zip package build completed`); +} + +async function renameInstallerFiles() { + if (await fs.pathExists(oclifInstallers[process.platform])) { + await fs.rename( + oclifInstallers[process.platform], + renamedOclifInstallers[process.platform], + ); + } +} + /** * Run the `oclif-dev pack:win` or `pack:macos` command (depending on the value * of process.platform) to generate the native installers (which end up under @@ -84,7 +181,6 @@ export async function buildPkg() { * 64-bit binaries under Windows. */ export async function buildOclifInstaller() { - console.log(`buildOclifInstaller cwd="${process.cwd()}" ROOT="${ROOT}"`); let packOS = ''; let packOpts = ['-r', ROOT]; if (process.platform === 'darwin') { @@ -94,6 +190,7 @@ export async function buildOclifInstaller() { packOpts = packOpts.concat('-t', 'win32-x64'); } if (packOS) { + console.log(`Building oclif installer for CLI ${version}`); const packCmd = `pack:${packOS}`; const dirs = [path.join(ROOT, 'dist', packOS)]; if (packOS === 'win') { @@ -104,9 +201,12 @@ export async function buildOclifInstaller() { await Bluebird.fromCallback(cb => rimraf(dir, cb)); } console.log('======================================================='); - console.log(`oclif-dev "${packCmd}" [${packOpts}]`); + console.log(`oclif-dev "${packCmd}" "${packOpts.join('" "')}"`); + console.log(`cwd="${process.cwd()}" ROOT="${ROOT}"`); console.log('======================================================='); - oclifRun([packCmd].concat(...packOpts)); + await oclifRun([packCmd].concat(...packOpts)); + await renameInstallerFiles(); + console.log(`oclif installer build completed`); } } diff --git a/automation/deploy-bin.ts b/automation/deploy-bin.ts index 15852ed7..4038eeb7 100644 --- a/automation/deploy-bin.ts +++ b/automation/deploy-bin.ts @@ -14,86 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import * as archiver from 'archiver'; import * as Bluebird from 'bluebird'; -import * as fs from 'fs-extra'; import * as _ from 'lodash'; -import * as path from 'path'; import * as publishRelease from 'publish-release'; +import { finalReleaseAssets, version } from './build-bin'; + const { GITHUB_TOKEN } = process.env; -const ROOT = path.join(__dirname, '..'); -// Note: the following 'tslint disable' line was only required to -// satisfy ts-node under Appveyor's MSYS2 on Windows -- oddly specific. -// Maybe something to do with '/' vs '\' in paths in some tslint file. -// tslint:disable-next-line:no-var-requires -const packageJSON = require(path.join(ROOT, 'package.json')); -const version = 'v' + packageJSON.version; -const arch = process.arch; - -function dPath(...paths: string[]) { - return path.join(ROOT, 'dist', ...paths); -} - -interface PathByPlatform { - [platform: string]: string; -} - -const standaloneZips: PathByPlatform = { - linux: dPath(`balena-cli-${version}-linux-${arch}-standalone.zip`), - darwin: dPath(`balena-cli-${version}-macOS-${arch}-standalone.zip`), - win32: dPath(`balena-cli-${version}-windows-${arch}-standalone.zip`), -}; - -const oclifInstallers: PathByPlatform = { - darwin: dPath('macos', `balena-${version}.pkg`), - win32: dPath('win', `balena-${version}-${arch}.exe`), -}; - -const renamedOclifInstallers: PathByPlatform = { - darwin: dPath(`balena-cli-${version}-macOS-${arch}-installer-BETA.pkg`), - win32: dPath(`balena-cli-${version}-windows-${arch}-installer-BETA.exe`), -}; - -const finalReleaseAssets: { [platform: string]: string[] } = { - win32: [standaloneZips['win32'], renamedOclifInstallers['win32']], - darwin: [standaloneZips['darwin'], renamedOclifInstallers['darwin']], - linux: [standaloneZips['linux']], -}; - -/** - * Create the zip file for the standalone 'pkg' bundle previously created - * by the buildPkg() function in 'build-bin.ts'. - */ -export async function zipStandaloneInstaller() { - const outputFile = standaloneZips[process.platform]; - if (!outputFile) { - throw new Error( - `Standalone installer unavailable for platform "${process.platform}"`, - ); - } - await fs.mkdirp(path.dirname(outputFile)); - await new Bluebird((resolve, reject) => { - console.log(`Zipping build to "${outputFile}"...`); - - const archive = archiver('zip', { - zlib: { level: 7 }, - }); - archive.directory(path.join(ROOT, 'build-bin'), 'balena-cli'); - - const outputStream = fs.createWriteStream(outputFile); - - outputStream.on('close', resolve); - outputStream.on('error', reject); - - archive.on('error', reject); - archive.on('warning', console.warn); - - archive.pipe(outputStream); - archive.finalize(); - }); - console.log('Build zipped'); -} /** * Create or update a release in GitHub's releases page, uploading the @@ -122,21 +49,6 @@ export async function createGitHubRelease() { * the files. */ export async function release() { - console.log(`Creating release assets for CLI ${version}`); - try { - await zipStandaloneInstaller(); - } catch (error) { - console.log(`Error creating standalone installer zip file: ${error}`); - process.exit(1); - } - if (process.platform === 'win32' || process.platform === 'darwin') { - if (await fs.pathExists(oclifInstallers[process.platform])) { - await fs.rename( - oclifInstallers[process.platform], - renamedOclifInstallers[process.platform], - ); - } - } try { await createGitHubRelease(); } catch (err) { diff --git a/automation/run.ts b/automation/run.ts index 41a34e41..c106094a 100644 --- a/automation/run.ts +++ b/automation/run.ts @@ -21,7 +21,7 @@ import * as shellEscape from 'shell-escape'; import { buildOclifInstaller, - buildPkg, + buildStandaloneZip, fixPathForMsys, ROOT, } from './build-bin'; @@ -76,7 +76,7 @@ export async function run(args?: string[]) { } const commands: { [cmd: string]: () => void } = { 'build:installer': buildOclifInstaller, - 'build:standalone': buildPkg, + 'build:standalone': buildStandaloneZip, release, }; for (const arg of args) { @@ -99,22 +99,30 @@ export async function run(args?: string[]) { } for (const arg of args) { - if (arg === 'build:installer' && process.platform === 'win32') { - // ensure running under MSYS2 - if (!process.env.MSYSTEM) { - process.env.MSYS2_PATH_TYPE = 'inherit'; - await runUnderMsys([ - fixPathForMsys(process.argv[0]), - fixPathForMsys(process.argv[1]), - arg, - ]); - continue; - } - if (process.env.MSYS2_PATH_TYPE !== 'inherit') { - throw new Error('the MSYS2_PATH_TYPE env var must be set to "inherit"'); + try { + if (arg === 'build:installer' && process.platform === 'win32') { + // ensure running under MSYS2 + if (!process.env.MSYSTEM) { + process.env.MSYS2_PATH_TYPE = 'inherit'; + await runUnderMsys([ + fixPathForMsys(process.argv[0]), + fixPathForMsys(process.argv[1]), + arg, + ]); + continue; + } + if (process.env.MSYS2_PATH_TYPE !== 'inherit') { + throw new Error( + 'the MSYS2_PATH_TYPE env var must be set to "inherit"', + ); + } } + const cmdFunc = commands[arg]; + await cmdFunc(); + } catch (err) { + console.log(`Error running command "${arg}": ${err}`); + process.exit(1); } - await commands[arg](); } } diff --git a/package.json b/package.json index fa84d17c..358a1ddf 100644 --- a/package.json +++ b/package.json @@ -41,8 +41,8 @@ "build:src": "npm run prettify && npm run lint && npm run build:fast && npm run build:doc", "build:fast": "gulp build && tsc", "build:doc": "mkdirp doc/ && ts-node --type-check -P automation/tsconfig.json automation/capitanodoc/index.ts > doc/cli.markdown", - "build:standalone": "ts-node --type-check -P automation/tsconfig.json automation/run.ts build:standalone", - "build:installer": "ts-node --type-check -P automation/tsconfig.json automation/run.ts build:installer", + "build:standalone": "npm run build:fast && ts-node --type-check -P automation/tsconfig.json automation/run.ts build:standalone", + "build:installer": "npm run build:fast && ts-node --type-check -P automation/tsconfig.json automation/run.ts build:installer", "release": "ts-node --type-check -P automation/tsconfig.json automation/run.ts release", "pretest": "npm run build", "test": "gulp test", From 55bf4dc0f0efd99841d28ed054383d3061ff93b9 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Thu, 6 Jun 2019 15:19:35 +0100 Subject: [PATCH 3/6] Add 'npm run package' command Change-type: patch Signed-off-by: Paulo Castro --- .travis.yml | 7 +++++-- appveyor.yml | 15 +++++++++------ package.json | 5 +++-- repo.yml | 1 + 4 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 repo.yml diff --git a/.travis.yml b/.travis.yml index 183aff5b..7864b37c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,15 @@ os: - osx node_js: - "10" +matrix: + exclude: + node_js: "10" script: - node --version - npm --version - npm run ci - - npm run build:standalone - - npm run build:installer + # - npm run build:standalone + # - npm run build:installer notifications: email: false deploy: diff --git a/appveyor.yml b/appveyor.yml index 8eb60841..84a36088 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,19 +22,22 @@ install: - ps: Install-Product node $env:nodejs_version x64 - set PATH=%APPDATA%\npm;%PATH% - npm config set python 'C:\Python27\python.exe' - - npm install + - npm --version + # - npm install build: off +test: off +deploy: off test_script: - node --version - npm --version - - npm test + # - npm test deploy_script: - node --version - npm --version - - npm run build:standalone - - npm run build:installer - - IF "%APPVEYOR_REPO_TAG%" == "true" (npm run release) - - IF NOT "%APPVEYOR_REPO_TAG%" == "true" (echo 'Not tagged, skipping deploy') + # - npm run build:standalone + # - npm run build:installer + # - IF "%APPVEYOR_REPO_TAG%" == "true" (npm run release) + # - IF NOT "%APPVEYOR_REPO_TAG%" == "true" (echo 'Not tagged, skipping deploy') diff --git a/package.json b/package.json index 358a1ddf..4b62456e 100644 --- a/package.json +++ b/package.json @@ -41,8 +41,9 @@ "build:src": "npm run prettify && npm run lint && npm run build:fast && npm run build:doc", "build:fast": "gulp build && tsc", "build:doc": "mkdirp doc/ && ts-node --type-check -P automation/tsconfig.json automation/capitanodoc/index.ts > doc/cli.markdown", - "build:standalone": "npm run build:fast && ts-node --type-check -P automation/tsconfig.json automation/run.ts build:standalone", - "build:installer": "npm run build:fast && ts-node --type-check -P automation/tsconfig.json automation/run.ts build:installer", + "build:standalone": "ts-node --type-check -P automation/tsconfig.json automation/run.ts build:standalone", + "build:installer": "ts-node --type-check -P automation/tsconfig.json automation/run.ts build:installer", + "package": "npm run build:fast && npm run build:standalone && npm run build:installer", "release": "ts-node --type-check -P automation/tsconfig.json automation/run.ts release", "pretest": "npm run build", "test": "gulp test", diff --git a/repo.yml b/repo.yml new file mode 100644 index 00000000..f66c634c --- /dev/null +++ b/repo.yml @@ -0,0 +1 @@ +type: node-cli From 7c750f9e4330ff3ac135dfff109e015ea4e14f7d Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Wed, 19 Jun 2019 00:59:56 +0100 Subject: [PATCH 4/6] balena CI: Add balena-cli executable signing step Change-type: minor Signed-off-by: Paulo Castro --- automation/build-bin.ts | 63 +++++++++++++++++++++++++++++++++++++++++ automation/run.ts | 36 +++++------------------ package.json | 3 +- 3 files changed, 72 insertions(+), 30 deletions(-) diff --git a/automation/build-bin.ts b/automation/build-bin.ts index c71c7c78..a662cf67 100644 --- a/automation/build-bin.ts +++ b/automation/build-bin.ts @@ -18,11 +18,14 @@ import { run as oclifRun } from '@oclif/dev-cli'; import * as archiver from 'archiver'; import * as Bluebird from 'bluebird'; +import { execFile, spawn } from 'child_process'; import * as filehound from 'filehound'; import * as fs from 'fs-extra'; import * as path from 'path'; import { exec as execPkg } from 'pkg'; import * as rimraf from 'rimraf'; +import * as shellEscape from 'shell-escape'; +import * as util from 'util'; export const ROOT = path.join(__dirname, '..'); // Note: the following 'tslint disable' line was only required to @@ -63,6 +66,34 @@ export const finalReleaseAssets: { [platform: string]: string[] } = { linux: [standaloneZips['linux']], }; +const MSYS2_BASH = 'C:\\msys64\\usr\\bin\\bash.exe'; + +/** + * Run the MSYS2 bash.exe shell in a child process (child_process.spawn()). + * The given argv arguments are escaped using the 'shell-escape' package, + * so that backslashes in Windows paths, and other bash-special characters, + * are preserved. If argv is not provided, defaults to process.argv, to the + * effect that this current (parent) process is re-executed under MSYS2 bash. + * This is useful to change the default shell from cmd.exe to MSYS2 bash on + * Windows. + * @param argv Arguments to be shell-escaped and given to MSYS2 bash.exe. + */ +export async function runUnderMsys(argv?: string[]) { + const newArgv = argv || process.argv; + await new Promise((resolve, reject) => { + const args = ['-lc', shellEscape(newArgv)]; + const child = spawn(MSYS2_BASH, args, { stdio: 'inherit' }); + child.on('close', code => { + if (code) { + console.log(`runUnderMsys: child process exited with code ${code}`); + reject(code); + } else { + resolve(); + } + }); + }); +} + /** * Use the 'pkg' module to create a single large executable file with * the contents of 'node_modules' and the CLI's javascript code. @@ -174,6 +205,31 @@ async function renameInstallerFiles() { } } +/** + * If the CSC_LINK and CSC_KEY_PASSWORD env vars are set, digitally sign the + * executable installer by running the balena-io/scripts/shared/sign-exe.sh + * script (which must be in the PATH) using a MSYS2 bash shell. + */ +async function signWindowsInstaller() { + if (process.env.CSC_LINK && process.env.CSC_KEY_PASSWORD) { + const exeName = renamedOclifInstallers[process.platform]; + const execFileAsync = util.promisify(execFile); + + console.log(`Signing installer "${exeName}"`); + await execFileAsync(MSYS2_BASH, [ + 'sign-exe.sh', + '-f', + exeName, + '-d', + `balena-cli ${version}`, + ]); + } else { + console.log( + 'Skipping installer signing step because CSC_* env vars are not set', + ); + } +} + /** * Run the `oclif-dev pack:win` or `pack:macos` command (depending on the value * of process.platform) to generate the native installers (which end up under @@ -206,6 +262,13 @@ export async function buildOclifInstaller() { console.log('======================================================='); await oclifRun([packCmd].concat(...packOpts)); await renameInstallerFiles(); + // The Windows installer is explicitly signed here (oclif doesn't do it). + // The macOS installer is automatically signed by oclif (which runs the + // `pkgbuild` tool), using the certificate name given in package.json + // (`oclif.macos.sign` section). + if (process.platform === 'win32') { + await signWindowsInstaller(); + } console.log(`oclif installer build completed`); } } diff --git a/automation/run.ts b/automation/run.ts index c106094a..e298b6e6 100644 --- a/automation/run.ts +++ b/automation/run.ts @@ -15,45 +15,17 @@ * limitations under the License. */ -import { spawn } from 'child_process'; import * as _ from 'lodash'; -import * as shellEscape from 'shell-escape'; import { buildOclifInstaller, buildStandaloneZip, fixPathForMsys, ROOT, + runUnderMsys, } from './build-bin'; import { release } from './deploy-bin'; -/** - * Run the MSYS2 bash.exe shell in a child process (child_process.spawn()). - * The given argv arguments are escaped using the 'shell-escape' package, - * so that backslashes in Windows paths, and other bash-special characters, - * are preserved. If argv is not provided, defaults to process.argv, to the - * effect that this current (parent) process is re-executed under MSYS2 bash. - * This is useful to change the default shell from cmd.exe to MSYS2 bash on - * Windows. - * @param argv Arguments to be shell-escaped and given to MSYS2 bash.exe. - */ -export async function runUnderMsys(argv?: string[]) { - const newArgv = argv || process.argv; - await new Promise((resolve, reject) => { - const cmd = 'C:\\msys64\\usr\\bin\\bash.exe'; - const args = ['-lc', shellEscape(newArgv)]; - const child = spawn(cmd, args, { stdio: 'inherit' }); - child.on('close', code => { - if (code) { - console.log(`runUnderMsys: child process exited with code ${code}`); - reject(code); - } else { - resolve(); - } - }); - }); -} - /** * Trivial command-line parser. Check whether the command-line argument is one * of the following strings, then call the appropriate functions: @@ -89,6 +61,12 @@ export async function run(args?: string[]) { // the current working dir becomes the MSYS2 homedir, so we change back. process.chdir(ROOT); + // The BUILD_TMP env var is used as an alternative location for oclif + // (patched) to copy/extract the CLI files, run npm install and then + // create the NSIS executable installer for Windows. This was necessary + // to avoid issues with a 260-char limit on Windows paths (possibly a + // limitation of some library used by NSIS), as the "current working dir" + // provided by balena CI is a rather long path to start with. if (process.platform === 'win32' && !process.env.BUILD_TMP) { const randID = require('crypto') .randomBytes(6) diff --git a/package.json b/package.json index 4b62456e..bee58969 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,8 @@ "bin": "balena", "commands": "./build/actions-oclif", "macos": { - "identifier": "io.balena.cli" + "identifier": "io.balena.cli", + "sign": "Developer ID Installer: Rulemotion Ltd (66H43P8FRG)" } }, "devDependencies": { From 05aaed07b2aad9c983db2cbf217c3471b404186f Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Wed, 3 Jul 2019 22:50:53 +0100 Subject: [PATCH 5/6] Patch oclif to use "npx npm@6.9.0 install" if npm is older than 6.9.0 Change-type: patch Signed-off-by: Paulo Castro --- patches/@oclif+dev-cli+1.22.0.patch | 24 +++++++++++++----------- patches/qqjs++execa+0.10.0.patch | 19 +++++++++++++++---- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/patches/@oclif+dev-cli+1.22.0.patch b/patches/@oclif+dev-cli+1.22.0.patch index 61ce7fec..6c84a29b 100644 --- a/patches/@oclif+dev-cli+1.22.0.patch +++ b/patches/@oclif+dev-cli+1.22.0.patch @@ -48,7 +48,7 @@ index a9d4276..75c2f8b 100644 exports.default = PackWin; const scripts = { diff --git a/node_modules/@oclif/dev-cli/lib/tarballs/build.js b/node_modules/@oclif/dev-cli/lib/tarballs/build.js -index 3e613e0..c73a04c 100644 +index 3e613e0..129d041 100644 --- a/node_modules/@oclif/dev-cli/lib/tarballs/build.js +++ b/node_modules/@oclif/dev-cli/lib/tarballs/build.js @@ -17,8 +17,11 @@ const pack = async (from, to) => { @@ -93,20 +93,22 @@ index 3e613e0..c73a04c 100644 const updatePJSON = async () => { qq.cd(c.workspace()); const pjson = await qq.readJSON('package.json'); -@@ -55,7 +80,11 @@ async function build(c, options = {}) { - if (!await qq.exists(lockpath)) { +@@ -56,7 +81,13 @@ async function build(c, options = {}) { lockpath = qq.join(c.root, 'npm-shrinkwrap.json'); } -- await qq.cp(lockpath, '.'); -+ try { -+ await qq.cp(lockpath, '.'); -+ } catch (err) { -+ console.log('WARNING: found neiter package-lock.json nor npm-shrinkwrap.json') + await qq.cp(lockpath, '.'); +- await qq.x('npm install --production'); ++ ++ const npmVersion = await qq.x.stdout('npm', ['--version']); ++ if (require('semver').lt(npmVersion, '6.9.0')) { ++ await qq.x('npx npm@6.9.0 install --production'); ++ } else { ++ await qq.x('npm install --production'); + } - await qq.x('npm install --production'); } }; -@@ -71,7 +100,8 @@ async function build(c, options = {}) { + const buildTarget = async (target) => { +@@ -71,7 +102,8 @@ async function build(c, options = {}) { output: path.join(workspace, 'bin', 'node'), platform: target.platform, arch: target.arch, @@ -116,7 +118,7 @@ index 3e613e0..c73a04c 100644 }); if (options.pack === false) return; -@@ -124,7 +154,8 @@ async function build(c, options = {}) { +@@ -124,7 +156,8 @@ async function build(c, options = {}) { await qq.writeJSON(c.dist(config.s3Key('manifest')), manifest); }; log_1.log(`gathering workspace for ${config.bin} to ${c.workspace()}`); diff --git a/patches/qqjs++execa+0.10.0.patch b/patches/qqjs++execa+0.10.0.patch index 226ea0f1..dddfc7e6 100644 --- a/patches/qqjs++execa+0.10.0.patch +++ b/patches/qqjs++execa+0.10.0.patch @@ -1,5 +1,5 @@ diff --git a/node_modules/qqjs/node_modules/execa/index.js b/node_modules/qqjs/node_modules/execa/index.js -index 06f3969..7ab1b66 100644 +index 06f3969..8bca191 100644 --- a/node_modules/qqjs/node_modules/execa/index.js +++ b/node_modules/qqjs/node_modules/execa/index.js @@ -14,6 +14,17 @@ const stdio = require('./lib/stdio'); @@ -20,7 +20,7 @@ index 06f3969..7ab1b66 100644 function handleArgs(cmd, args, opts) { let parsed; -@@ -104,13 +115,21 @@ function handleShell(fn, cmd, opts) { +@@ -104,13 +115,22 @@ function handleShell(fn, cmd, opts) { opts = Object.assign({}, opts); @@ -38,12 +38,23 @@ index 06f3969..7ab1b66 100644 opts.windowsVerbatimArguments = true; } -+ console.log(`[patched execa] handleShell file="${file}" args="[${args}]"`); ++ const argStr = (args && args.length) ? `["${args.join('", "')}"]` : args; ++ console.log(`[patched execa] handleShell file="${file}" args=${argStr}`); + if (opts.shell) { file = opts.shell; delete opts.shell; -@@ -364,3 +383,7 @@ module.exports.sync = (cmd, args, opts) => { +@@ -199,6 +219,9 @@ module.exports = (cmd, args, opts) => { + const maxBuffer = parsed.opts.maxBuffer; + const joinedCmd = joinCmd(cmd, args); + ++ const argStr = (args && args.length) ? `["${args.join('", "')}"]` : args; ++ console.log(`[patched execa] parsed.cmd="${parsed.cmd}" parsed.args=${argStr}`); ++ + let spawned; + try { + spawned = childProcess.spawn(parsed.cmd, parsed.args, parsed.opts); +@@ -364,3 +387,7 @@ module.exports.sync = (cmd, args, opts) => { module.exports.shellSync = (cmd, opts) => handleShell(module.exports.sync, cmd, opts); module.exports.spawn = util.deprecate(module.exports, 'execa.spawn() is deprecated. Use execa() instead.'); From 32e72c832f10f96048fda9f2d532bfbb49c632c0 Mon Sep 17 00:00:00 2001 From: Giovanni Garufi Date: Fri, 5 Jul 2019 15:16:59 +0200 Subject: [PATCH 6/6] Add release target in repo.yml Change-type: patch Signed-off-by: Giovanni Garufi --- repo.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/repo.yml b/repo.yml index f66c634c..a49a6d89 100644 --- a/repo.yml +++ b/repo.yml @@ -1 +1,2 @@ type: node-cli +release: github