From 913f09924a3cf5efda79b2f27da705e29f48ed00 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Sat, 25 Jan 2020 22:49:29 +0000 Subject: [PATCH 1/2] Update Github's templates for new CLI pull requests and issues Change-type: patch Signed-off-by: Paulo Castro --- .github/ISSUE_TEMPLATE.md | 8 ++++---- .github/PULL_REQUEST_TEMPLATE.md | 29 +++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 14f3a92b..e4069fe6 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -6,10 +6,10 @@ --- -*Please keep in mind that we try to use the issue tracker of this repository for specific bug -reports & CLI feature requests. General & troubleshooting questions are encouraged to be posted to -the [balena forums](https://forums.balena.io), which are monitored by balena's support team and -where the community can both contribute and benefit from the answers.* +*Please note that this issue tracker is used for specific bug reports and feature requests. +General and troubleshooting questions are encouraged to be posted to the [balena +forums](https://forums.balena.io), which are monitored by balena's support team and where +the community can both contribute and benefit from the answers.* *Before submitting this issue please check that this issue is not a duplicate. If there is another issue describing the same problem or feature please add your information to the existing issue's diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 122bae35..aeedc290 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,24 @@ -Resolves: # -See: -Depends-on: -Change-type: major|minor|patch +Resolves: # +Change-type: major|minor|patch +Depends-on: +See: --- -##### Contributor checklist - -- [ ] Introduces security considerations -- [ ] Affects the development, build or deployment processes of the component +Please check the CONTRIBUTING.md file for relevant information and some +guidance. Keep in mind that the CLI is a cross-platform application that runs +on Windows, macOS and Linux. Tests will be automatically run by balena CI on +all three operating systems, but this will only help if you have added test +code that exercises the modified or added feature code. + +Note that each commit's message (currently only the first line) will be +automatically copied to the CHANGELOG.md file, so try writing it in a way +that describes the feature or fix for CLI users. + +If there isn't a linked issue or if the linked issue doesn't quite match the +PR, please add a description to explain the PR's motivation and the features +that it adds. Adding comments to blocks of code that aren't self explanatory +usually helps with the review process. + +If he PR introduces security considerations or affects the development, build +or release process, please be sure to add a description and highlight this. From 1e37c97ffbff1816fd50eaa2991f38e8c04bb810 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Fri, 24 Jan 2020 18:43:04 +0000 Subject: [PATCH 2/2] Fix proxy support and add proxy exclusion feature (Node.js >= 10.16.0 only) See README for more details on proxy configuration and Node.js compatibility. Resolves: #1579 Resolves: #1335 Connects-to: #1580 Change-type: minor Signed-off-by: Paulo Castro --- .github/ISSUE_TEMPLATE.md | 4 +- .github/PULL_REQUEST_TEMPLATE.md | 10 +- INSTALL.md | 12 +- README.md | 68 ++++++++++-- doc/cli.markdown | 68 ++++++++++-- lib/actions/ssh.ts | 35 +++--- lib/app-common.ts | 181 +++++++++++++++++++++++++++---- lib/app.ts | 2 +- lib/utils/helpers.ts | 75 ++++++++++++- npm-shrinkwrap.json | 117 +++++++++++++++++--- package.json | 2 +- tests/app-common.spec.ts | 38 +++++++ tests/utils/helpers.spec.ts | 87 +++++++++++++++ 13 files changed, 598 insertions(+), 101 deletions(-) create mode 100644 tests/app-common.spec.ts create mode 100644 tests/utils/helpers.spec.ts diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index e4069fe6..7f0d66cb 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -8,8 +8,8 @@ *Please note that this issue tracker is used for specific bug reports and feature requests. General and troubleshooting questions are encouraged to be posted to the [balena -forums](https://forums.balena.io), which are monitored by balena's support team and where -the community can both contribute and benefit from the answers.* +forums](https://forums.balena.io), which are monitored by balena's support team and +where the community can both contribute and benefit from the answers.* *Before submitting this issue please check that this issue is not a duplicate. If there is another issue describing the same problem or feature please add your information to the existing issue's diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index aeedc290..53839365 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,14 +11,14 @@ on Windows, macOS and Linux. Tests will be automatically run by balena CI on all three operating systems, but this will only help if you have added test code that exercises the modified or added feature code. -Note that each commit's message (currently only the first line) will be +Note that each commit message (currently only the first line) will be automatically copied to the CHANGELOG.md file, so try writing it in a way that describes the feature or fix for CLI users. If there isn't a linked issue or if the linked issue doesn't quite match the -PR, please add a description to explain the PR's motivation and the features -that it adds. Adding comments to blocks of code that aren't self explanatory +PR, please add a PR description to explain its purpose or the features that it +implements. Adding PR comments to blocks of code that aren't self explanatory usually helps with the review process. -If he PR introduces security considerations or affects the development, build -or release process, please be sure to add a description and highlight this. +If the PR introduces security considerations or affects the development, build +or release process, please be sure to highlight this in the PR description. diff --git a/INSTALL.md b/INSTALL.md index fe791059..82263138 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -141,11 +141,13 @@ especially if you're using a user-managed node install such as [nvm](https://git ([more information](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse)). For other versions of Windows, there are several ssh/OpenSSH clients provided by 3rd parties. - * If you need SSH to work behind a proxy, you will also need to install - [`proxytunnel`](http://proxytunnel.sourceforge.net/) (available as a `proxytunnel` package - for Ubuntu, for example). - Check the [README](https://github.com/balena-io/balena-cli/blob/master/README.md) file - for proxy configuration instructions. + * The [`proxytunnel`](http://proxytunnel.sourceforge.net/) package (command-line tool) is needed + for the `balena ssh` command to work behind a proxy. It is available for Linux distributions + like Ubuntu/Debian (`apt install proxytunnel`), and for macOS through + [Homebrew](https://brew.sh/). Windows support is limited to the Windows Subsystem for Linux + (e.g., by installing Ubuntu through the Microsoft App Store). Check the + [README](https://github.com/balena-io/balena-cli/blob/master/README.md) file for proxy + configuration instructions. * The `balena preload`, `balena build` and `balena deploy --build` commands require [Docker](https://docs.docker.com/install/overview/) or [balenaEngine](https://www.balena.io/engine/) diff --git a/README.md b/README.md index f31ff0d6..26f8fb73 100644 --- a/README.md +++ b/README.md @@ -64,19 +64,65 @@ $ balena login ### Proxy support -HTTP(S) proxies can be configured through any of the following methods, in order of preference: +HTTP(S) proxies can be configured through any of the following methods, in precedence order +(from higher to lower): -* Set the `BALENARC_PROXY` environment variable in URL format (with protocol, host, port, and - optionally basic auth). -* Alternatively, use the [balena config file](https://www.npmjs.com/package/balena-settings-client#documentation) - (project-specific or user-level) and set the `proxy` setting. It can be: - * A string in URL format, or - * An object in the [global-tunnel-ng options format](https://www.npmjs.com/package/global-tunnel-ng#options) (which allows more control). -* Alternatively, set the conventional `https_proxy` / `HTTPS_PROXY` / `http_proxy` / `HTTP_PROXY` - environment variable (in the same standard URL format). +* The `BALENARC_PROXY` environment variable in URL format, with protocol (`http` or `https`), + host, port and optionally basic auth. Examples: + * `export BALENARC_PROXY='https://bob:secret@proxy.company.com:12345'` + * `export BALENARC_PROXY='http://localhost:8000'` -To get a proxy to work with the `balena ssh` command, check the -[installation instructions](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md). +* The `proxy` setting in the [CLI config + file](https://www.npmjs.com/package/balena-settings-client#documentation). It may be: + * A string in URL format, e.g. `proxy: 'http://localhost:8000'` + * An object in the format: + + ```yaml + proxy: + protocol: 'http' + host: 'proxy.company.com' + port: 12345 + proxyAuth: 'bob:secret' + ``` + +* The `HTTPS_PROXY` and/or `HTTP_PROXY` environment variables, in the same URL format as + `BALENARC_PROXY`. + +> Note: The `balena ssh` command has additional setup requirements to work behind a proxy. +> Check the [installation instructions](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md). + +Some installations of the balena CLI also include support for the `BALENARC_NO_PROXY` environment +variable, which allows proxy exclusion patterns to be defined. The current support status is listed +below. Eventually, all installation types will have support for it. + +OS | Installation type | BALENARC_NO_PROXY environment variable support +-- | ----------------- | ---------------------------------------------- +Windows | standalone zip | Supported with CLI v11.24.0 and later +Windows | native/GUI | Not supported +macOS | standalone zip | Not supported +macOS | native/GUI | Supported with CLI v11.24.0 and later +Linux | standalone zip | Not supported +Any | npm | Supported with Node.js >= v10.16.0 and CLI >= v11.24.0 + +The format of the `BALENARC_NO_PROXY` environment variable is a comma-separated list of patterns +that are matched against hostnames or IP addresses. For example: + +``` +export BALENARC_NO_PROXY='*.local,dev*.mycompany.com,192.168.*' +``` + +Matched patterns are excluded from proxying. Matching takes place _before_ name resolution, so a +pattern like `'192.168.*'` will **not** match a hostname like `proxy.company.com` even if the +hostname resolves to an IP address like `192.168.1.2`. Pattern matching expressions are documented +at [matcher](https://www.npmjs.com/package/matcher#usage). + +By default, if BALENARC_NO_PROXY is not defined, all [private IPv4 +addresses](https://en.wikipedia.org/wiki/Private_network) and `'*.local'` are excluded from +proxying. Other hostnames that may resolve to private IPv4 addresses are **not** excluded by +default, as matching takes place _before_ name resolution. In addition, `localhost` and `127.0.0.1` +are always excluded from proxying, regardless of the value of BALENARC_NO_PROXY. These default +exclusions only apply to the CLI installations where BALENARC_NO_PROXY is supported, as listed in +the table above. ## Command reference documentation diff --git a/doc/cli.markdown b/doc/cli.markdown index 49cea935..3739ebd3 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -57,19 +57,65 @@ $ balena login ### Proxy support -HTTP(S) proxies can be configured through any of the following methods, in order of preference: +HTTP(S) proxies can be configured through any of the following methods, in precedence order +(from higher to lower): -* Set the `BALENARC_PROXY` environment variable in URL format (with protocol, host, port, and - optionally basic auth). -* Alternatively, use the [balena config file](https://www.npmjs.com/package/balena-settings-client#documentation) - (project-specific or user-level) and set the `proxy` setting. It can be: - * A string in URL format, or - * An object in the [global-tunnel-ng options format](https://www.npmjs.com/package/global-tunnel-ng#options) (which allows more control). -* Alternatively, set the conventional `https_proxy` / `HTTPS_PROXY` / `http_proxy` / `HTTP_PROXY` - environment variable (in the same standard URL format). +* The `BALENARC_PROXY` environment variable in URL format, with protocol (`http` or `https`), + host, port and optionally basic auth. Examples: + * `export BALENARC_PROXY='https://bob:secret@proxy.company.com:12345'` + * `export BALENARC_PROXY='http://localhost:8000'` -To get a proxy to work with the `balena ssh` command, check the -[installation instructions](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md). +* The `proxy` setting in the [CLI config + file](https://www.npmjs.com/package/balena-settings-client#documentation). It may be: + * A string in URL format, e.g. `proxy: 'http://localhost:8000'` + * An object in the format: + + ```yaml + proxy: + protocol: 'http' + host: 'proxy.company.com' + port: 12345 + proxyAuth: 'bob:secret' + ``` + +* The `HTTPS_PROXY` and/or `HTTP_PROXY` environment variables, in the same URL format as + `BALENARC_PROXY`. + +> Note: The `balena ssh` command has additional setup requirements to work behind a proxy. +> Check the [installation instructions](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md). + +Some installations of the balena CLI also include support for the `BALENARC_NO_PROXY` environment +variable, which allows proxy exclusion patterns to be defined. The current support status is listed +below. Eventually, all installation types will have support for it. + +OS | Installation type | BALENARC_NO_PROXY environment variable support +-- | ----------------- | ---------------------------------------------- +Windows | standalone zip | Supported with CLI v11.24.0 and later +Windows | native/GUI | Not supported +macOS | standalone zip | Not supported +macOS | native/GUI | Supported with CLI v11.24.0 and later +Linux | standalone zip | Not supported +Any | npm | Supported with Node.js >= v10.16.0 and CLI >= v11.24.0 + +The format of the `BALENARC_NO_PROXY` environment variable is a comma-separated list of patterns +that are matched against hostnames or IP addresses. For example: + +``` +export BALENARC_NO_PROXY='*.local,dev*.mycompany.com,192.168.*' +``` + +Matched patterns are excluded from proxying. Matching takes place _before_ name resolution, so a +pattern like `'192.168.*'` will **not** match a hostname like `proxy.company.com` even if the +hostname resolves to an IP address like `192.168.1.2`. Pattern matching expressions are documented +at [matcher](https://www.npmjs.com/package/matcher#usage). + +By default, if BALENARC_NO_PROXY is not defined, all [private IPv4 +addresses](https://en.wikipedia.org/wiki/Private_network) and `'*.local'` are excluded from +proxying. Other hostnames that may resolve to private IPv4 addresses are **not** excluded by +default, as matching takes place _before_ name resolution. In addition, `localhost` and `127.0.0.1` +are always excluded from proxying, regardless of the value of BALENARC_NO_PROXY. These default +exclusions only apply to the CLI installations where BALENARC_NO_PROXY is supported, as listed in +the table above. ## Support, FAQ and troubleshooting diff --git a/lib/actions/ssh.ts b/lib/actions/ssh.ts index a1688e4b..c4bf5dde 100644 --- a/lib/actions/ssh.ts +++ b/lib/actions/ssh.ts @@ -227,9 +227,9 @@ export const ssh: CommandDefinition< const applicationOrDevice = params.applicationOrDevice_raw || params.applicationOrDevice; const bash = await import('bash'); - // TODO: Make this typed - const hasbin = require('hasbin'); - const { getSubShellCommand } = await import('../utils/helpers'); + const { getProxyConfig, getSubShellCommand, which } = await import( + '../utils/helpers' + ); const { child_process } = await import('mz'); const { exitIfNotLoggedIn, @@ -239,8 +239,7 @@ export const ssh: CommandDefinition< const sdk = BalenaSdk.fromSharedOptions(); const verbose = options.verbose === true; - // ugh TODO: Fix this - const proxyConfig = (global as any).PROXY_CONFIG; + const proxyConfig = getProxyConfig(); const useProxy = !!proxyConfig && !options.noProxy; const port = options.port != null ? parseInt(options.port, 10) : undefined; @@ -276,8 +275,8 @@ export const ssh: CommandDefinition< } } - const [hasTunnelBin, username, proxyUrl] = await Promise.all([ - useProxy ? await hasbin('proxytunnel') : undefined, + const [whichProxytunnel, username, proxyUrl] = await Promise.all([ + useProxy ? which('proxytunnel', false) : undefined, sdk.auth.whoami(), sdk.settings.get('proxyUrl'), ]); @@ -287,7 +286,7 @@ export const ssh: CommandDefinition< return ''; } - if (!hasTunnelBin) { + if (!whichProxytunnel) { console.warn(stripIndent` Proxy is enabled but the \`proxytunnel\` binary cannot be found. Please install it if you want to route the \`balena ssh\` requests through the proxy. @@ -298,18 +297,14 @@ export const ssh: CommandDefinition< return ''; } - let tunnelOptions: Dictionary = { - proxy: `${proxyConfig.host}:${proxyConfig.port}`, + const p = proxyConfig!; + const tunnelOptions: Dictionary = { + proxy: `${p.host}:${p.port}`, dest: '%h:%p', }; - const { proxyAuth } = proxyConfig; - if (proxyAuth) { - const i = proxyAuth.indexOf(':'); - tunnelOptions = { - user: proxyAuth.substring(0, i), - pass: proxyAuth.substring(i + 1), - ...tunnelOptions, - }; + if (p.username && p.password) { + tunnelOptions.user = p.username; + tunnelOptions.pass = p.password; } const ProxyCommand = `proxytunnel ${bash.args(tunnelOptions, '--', '=')}`; @@ -335,7 +330,7 @@ export const ssh: CommandDefinition< { port, proxyCommand, - proxyUrl, + proxyUrl: proxyUrl || '', username: username!, }, version, @@ -356,7 +351,7 @@ export const ssh: CommandDefinition< verbose, port, proxyCommand, - proxyUrl, + proxyUrl: proxyUrl || '', username: username!, }); diff --git a/lib/app-common.ts b/lib/app-common.ts index 67fd8141..a8ccdbe8 100644 --- a/lib/app-common.ts +++ b/lib/app-common.ts @@ -15,6 +15,36 @@ * limitations under the License. */ +class CliSettings { + public readonly settings: any; + constructor() { + // TODO figure out why the typescript compiler attempts to type-check + // the `balena-settings-client` module (and then fails with errors) if + // a straighforward `require('balena-settings-client')` statement is + // used here. It may even be a compiler bug, because `tsconfig.json` + // has a `"skipLibCheck": true` setting. + this.settings = require(['balena', 'settings', 'client'].join('-')); + } + + public get(name: string): T { + return this.settings.get(name); + } + + /** + * Like settings.get(), but return `undefined` instead of throwing an + * error if the setting is not found / not defined. + */ + public getCatch(name: string): T | undefined { + try { + return this.settings.get(name); + } catch (err) { + if (!/Setting not found/i.test(err.message)) { + throw err; + } + } + } +} + /** * Sentry.io setup * @see https://docs.sentry.io/clients/node/ @@ -53,37 +83,136 @@ function checkNodeVersion() { } } -function setupGlobalHttpProxy() { - // Doing this before requiring any other modules, - // including the 'balena-sdk', to prevent any module from reading the http proxy config - // before us - const globalTunnel = require('global-tunnel-ng'); - const settings = require('balena-settings-client'); - let proxy; - try { - proxy = settings.get('proxy') || null; - } catch (error1) { - proxy = null; +export interface GlobalTunnelNgConfig { + host: string; + port: number; + protocol: string; + proxyAuth?: string; + connect?: string; + sockets?: number; +} + +/** + * Global proxy setup. Originally, `global-tunnel-ng` was used, but it only + * supports Node.js versions older than 10.16.0. For v10.16.0 and later, + * we use `global-agent` (which only supports Node.js v10.0.0 and later). + * + * For backwards compatibility reasons, in either case we still accept a + * 'proxy' setting in `.balenarc.yml` that follows the + * `global-tunnel-ng` object configuration format: + * https://www.npmjs.com/package/global-tunnel-ng#options + * + * The proxy may also be configured with the environment variables: + * BALENARC_PROXY, HTTP_PROXY, HTTPS_PROXY, http_proxy, and https_proxy, + * any of which should contain a URL in the usual format (authentication + * details are optional): http://username:password@domain.com:1234 + * + * A proxy exclusion list in the NO_PROXY variable is only supported when + * `global-agent` is used, i.e. with Node.js v10.16.0 or later. The format + * is specified at: https://www.npmjs.com/package/global-agent#exclude-urls + * Patterns are matched with matcher: https://www.npmjs.com/package/matcher + * 'localhost' and '127.0.0.1' are always excluded. If NO_PROXY is not defined, + * default exclusion patterns are added for all private IPv4 address ranges. + */ +async function setupGlobalHttpProxy(settings: CliSettings) { + const semver = await import('semver'); + if (semver.lt(process.version, '10.16.0')) { + setupGlobalTunnelNgProxy(settings); + } else { + // use global-agent instead of global-tunnel-ng + await setupGlobalAgentProxy(settings); } +} - // Init the tunnel even if the proxy is not configured - // because it can also get the proxy from the http(s)_proxy env var - // If that is not set as well the initialize will do nothing +/** + * `global-tunnel-ng` proxy setup. + * See docs for setupGlobalHttpProxy() above. + */ +function setupGlobalTunnelNgProxy(settings: CliSettings) { + const proxy = settings.getCatch('proxy'); + const globalTunnel = require('global-tunnel-ng'); + // Init the tunnel even if BALENARC_PROXY is not defined, because + // other env vars may be defined. If no proxy configuration exists, + // initialize() does nothing. globalTunnel.initialize(proxy); - - // TODO: make this a feature of capitano https://github.com/balena-io/capitano/issues/48 (global as any).PROXY_CONFIG = globalTunnel.proxyConfig; } -function setupBalenaSdkSharedOptions() { +/** + * `global-agent` proxy setup. + * See docs for setupGlobalHttpProxy() above, and also the README file + * (Proxy Support section). + */ +async function setupGlobalAgentProxy(settings: CliSettings) { + const proxy = settings.getCatch('proxy'); + const noProxy = settings.getCatch('noProxy'); + // Always exclude localhost, even if NO_PROXY is set + const requiredNoProxy = ['localhost', '127.0.0.1']; + // Private IPv4 address patterns in `matcher` format: https://www.npmjs.com/package/matcher + const privateNoProxy = ['*.local', '10.*', '192.168.*']; + for (let i = 16; i <= 31; i++) { + privateNoProxy.push(`172.${i}.*`); + } + + const env = process.env; + env.GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE = ''; + + if (proxy) { + const _ = await import('lodash'); + const proxyUrl: string = + typeof proxy === 'string' ? proxy : makeUrlFromTunnelNgConfig(proxy); + + env.HTTPS_PROXY = env.HTTP_PROXY = proxyUrl; + delete env.http_proxy; + delete env.https_proxy; + + env.NO_PROXY = [ + ...requiredNoProxy, + ...(noProxy ? _.filter((noProxy || '').split(',')) : privateNoProxy), + ].join(','); + } else { + // `global-tunnel-ng` accepts lowercase variables with higher precedence + // than uppercase variables, but `global-agent` does not accept lowercase. + // Set uppercase versions for backwards compatibility. + if (env.http_proxy) { + env.HTTP_PROXY = env.http_proxy; + } + if (env.https_proxy) { + env.HTTPS_PROXY = env.https_proxy; + } + } + + const { bootstrap } = require('global-agent'); + bootstrap(); +} + +/** Make a URL in the format 'http://bob:secret@proxy.company.com:12345' */ +export function makeUrlFromTunnelNgConfig(cfg: GlobalTunnelNgConfig): string { + let url: string = cfg.host; + if (cfg.proxyAuth) { + url = `${cfg.proxyAuth}@${url}`; + } + if (cfg.protocol) { + // accept 'http', 'http:', 'http://' and the like + const match = cfg.protocol.match(/^[^:/]+/); + if (match) { + url = `${match[0].toLowerCase()}://${url}`; + } + } + if (cfg.port) { + url = `${url}:${cfg.port}`; + } + return url; +} + +function setupBalenaSdkSharedOptions(settings: CliSettings) { // We don't yet use balena-sdk directly everywhere, but we set up shared // options correctly so we can do safely in submodules const BalenaSdk = require('balena-sdk'); - const settings = require('balena-settings-client'); BalenaSdk.setSharedOptions({ - apiUrl: settings.get('apiUrl'), - imageMakerUrl: settings.get('imageMakerUrl'), - dataDirectory: settings.get('dataDirectory'), + apiUrl: settings.get('apiUrl'), + imageMakerUrl: settings.get('imageMakerUrl'), + dataDirectory: settings.get('dataDirectory'), retries: 2, }); } @@ -120,12 +249,16 @@ export function setMaxListeners(maxListeners: number) { require('events').EventEmitter.defaultMaxListeners = maxListeners; } -export function globalInit() { +export async function globalInit() { setupRaven(); checkNodeVersion(); configureBluebird(); - setupGlobalHttpProxy(); - setupBalenaSdkSharedOptions(); + + const settings = new CliSettings(); + + // Proxy setup should be done early on, before loading balena-sdk + await setupGlobalHttpProxy(settings); + setupBalenaSdkSharedOptions(settings); // check for CLI updates once a day require('./utils/update').notify(); diff --git a/lib/app.ts b/lib/app.ts index 8ae67afb..7e96af83 100644 --- a/lib/app.ts +++ b/lib/app.ts @@ -35,7 +35,7 @@ export async function run( // globalInit() must be called very early on (before other imports) because // it sets up Sentry error reporting, global HTTP proxy settings, balena-sdk // shared options, and performs node version requirement checks. - globalInit(); + await globalInit(); await routeCliFramework(cliArgs, options); // Windows fix: reading from stdin prevents the process from exiting diff --git a/lib/utils/helpers.ts b/lib/utils/helpers.ts index e9797fef..92e458cb 100644 --- a/lib/utils/helpers.ts +++ b/lib/utils/helpers.ts @@ -379,20 +379,87 @@ export async function workaroundWindowsDnsIssue(ipOrHostname: string) { * so hash -r is not needed when the PATH changes." * * @param program Basename of a program, for example 'ssh' + * @param rejectOnMissing If the program cannot be found, reject the promise + * with an ExpectedError instead of fulfilling it with an empty string. * @returns The program's full path, e.g. 'C:\WINDOWS\System32\OpenSSH\ssh.EXE' */ -export async function which(program: string): Promise { +export async function which( + program: string, + rejectOnMissing = true, +): Promise { const whichMod = await import('which'); let programPath: string; try { programPath = await whichMod(program); } catch (err) { if (err.code === 'ENOENT') { - throw new ExpectedError( - `'${program}' program not found. Is it installed?`, - ); + if (rejectOnMissing) { + throw new ExpectedError( + `'${program}' program not found. Is it installed?`, + ); + } else { + return ''; + } } throw err; } return programPath; } + +export interface ProxyConfig { + host: string; + port: string; + username?: string; + password?: string; + proxyAuth?: string; +} + +/** + * Check whether a proxy has been configured (whether global-tunnel-ng or + * global-agent) and if so, return a ProxyConfig object. + */ +export function getProxyConfig(): ProxyConfig | undefined { + const tunnelNgConfig: any = (global as any).PROXY_CONFIG; + // global-tunnel-ng + if (tunnelNgConfig) { + let username: string | undefined; + let password: string | undefined; + const proxyAuth: string = tunnelNgConfig.proxyAuth; + if (proxyAuth) { + const i = proxyAuth.lastIndexOf(':'); + if (i > 0) { + username = proxyAuth.substring(0, i); + password = proxyAuth.substring(i + 1); + } + } + return { + host: tunnelNgConfig.host, + port: `${tunnelNgConfig.port}`, + username, + password, + proxyAuth: tunnelNgConfig.proxyAuth, + }; + // global-agent, or no proxy config + } else { + const proxyUrl = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; + if (proxyUrl) { + const { URL } = require('url') as typeof import('url'); + let url: URL; + try { + url = new URL(proxyUrl); + } catch (_e) { + return; + } + return { + host: url.hostname, + port: url.port, + username: url.username, + password: url.password, + proxyAuth: + url.username && url.password + ? `${url.username}:${url.password}` + : undefined, + }; + } + } +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 37334b33..ce8f45a9 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -2452,6 +2452,11 @@ "multicast-dns-service-types": "^1.1.0" } }, + "boolean": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.0.0.tgz", + "integrity": "sha512-OElxJ1lUSinuoUnkpOgLmxp0DC4ytEhODEL6QJU0NpxE/mI4rUSh8h1P1Wkvfi3xQEBcxXR2gBIPNYNuaFcAbQ==" + }, "bottleneck": { "version": "2.19.5", "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", @@ -4038,6 +4043,11 @@ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" }, + "detect-node": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.0.4.tgz", + "integrity": "sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw==" + }, "dev-null-stream": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/dev-null-stream/-/dev-null-stream-0.0.1.tgz", @@ -4650,6 +4660,11 @@ "next-tick": "^1.0.0" } }, + "es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==" + }, "es6-iterator": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", @@ -6482,6 +6497,32 @@ "object.defaults": "^1.1.0" } }, + "global-agent": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-2.1.7.tgz", + "integrity": "sha512-ooK7eqGYZku+LgnbfH/Iv0RJ74XfhrBZDlke1QSzcBt0bw1PmJcnRADPAQuFE+R45pKKDTynAr25SBasY2kvow==", + "requires": { + "boolean": "^3.0.0", + "core-js": "^3.4.1", + "es6-error": "^4.1.1", + "matcher": "^2.0.0", + "roarr": "^2.14.5", + "semver": "^6.3.0", + "serialize-error": "^5.0.0" + }, + "dependencies": { + "core-js": { + "version": "3.6.4", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.4.tgz", + "integrity": "sha512-4paDGScNgZP2IXXilaffL9X7968RuvwlkK3xWtZRVqgd8SYNiVKRJvkFd1aqqEuPfN7E68ZHEp9hDj6lHj4Hyw==" + }, + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, "global-dirs": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz", @@ -6542,6 +6583,14 @@ "integrity": "sha512-uNUtxIZpGyuaq+5BqGGQHsL4wUlJAXRqOm6g3Y48/CWNGTLONgBibI0lh6lGxjR2HljFYUfszb+mk4WkgMntsA==", "dev": true }, + "globalthis": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.1.tgz", + "integrity": "sha512-mJPRTc/P39NH/iNG4mXa9aIhNymaQikTrnspeCa2ZuJ+mH2QN/rXwtX3XwKrHqWgUQFbNZKtHM105aHzJalElw==", + "requires": { + "define-properties": "^1.1.3" + } + }, "globby": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/globby/-/globby-4.1.0.tgz", @@ -6957,21 +7006,6 @@ } } }, - "hasbin": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/hasbin/-/hasbin-1.2.3.tgz", - "integrity": "sha1-eMWSaJPIAhXCtWiuH9P8q3omlrA=", - "requires": { - "async": "~1.5" - }, - "dependencies": { - "async": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", - "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" - } - } - }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -8576,6 +8610,21 @@ } } }, + "matcher": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-2.1.0.tgz", + "integrity": "sha512-o+nZr+vtJtgPNklyeUKkkH42OsK8WAfdgaJE2FNxcjLPg+5QbeEoT6vRj8Xq/iv18JlQ9cmKsEu0b94ixWf1YQ==", + "requires": { + "escape-string-regexp": "^2.0.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==" + } + } + }, "mbr": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/mbr/-/mbr-1.1.3.tgz", @@ -16044,6 +16093,26 @@ "string-to-stream": "^1.0.1" } }, + "roarr": { + "version": "2.14.6", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.14.6.tgz", + "integrity": "sha512-qjbw0BEesKA+3XFBPt+KVe1PC/Z6ShfJ4wPlx2XifqH5h2Lj8/KQT5XJTsy3n1Es5kai+BwKALaECW3F70B1cg==", + "requires": { + "boolean": "^3.0.0", + "detect-node": "^2.0.4", + "globalthis": "^1.0.0", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "dependencies": { + "sprintf-js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==" + } + } + }, "rsync": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/rsync/-/rsync-0.4.0.tgz", @@ -16139,8 +16208,7 @@ "semver-compare": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", - "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", - "dev": true + "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=" }, "semver-diff": { "version": "2.1.0", @@ -16208,6 +16276,21 @@ } } }, + "serialize-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-5.0.0.tgz", + "integrity": "sha512-/VtpuyzYf82mHYTtI4QKtwHa79vAdU5OQpNPAmE/0UDdlGT0ZxHwC+J6gXkw29wwoVI8fMPsfcVHOwXtUQYYQA==", + "requires": { + "type-fest": "^0.8.0" + }, + "dependencies": { + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==" + } + } + }, "serve-static": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", diff --git a/package.json b/package.json index b8ca5f64..03145e34 100644 --- a/package.json +++ b/package.json @@ -189,8 +189,8 @@ "event-stream": "3.3.4", "express": "^4.13.3", "fast-boot2": "^1.0.9", + "global-agent": "^2.1.7", "global-tunnel-ng": "^2.1.1", - "hasbin": "^1.2.3", "humanize": "0.0.9", "ignore": "^5.1.4", "inquirer": "^3.1.1", diff --git a/tests/app-common.spec.ts b/tests/app-common.spec.ts new file mode 100644 index 00000000..72510c03 --- /dev/null +++ b/tests/app-common.spec.ts @@ -0,0 +1,38 @@ +/** + * @license + * Copyright 2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; + +import { + GlobalTunnelNgConfig, + makeUrlFromTunnelNgConfig, +} from '../build/app-common'; + +describe('makeUrlFromTunnelNgConfig() function', function() { + it('should return a URL given a GlobalTunnelNgConfig object', () => { + const tunnelNgConfig: GlobalTunnelNgConfig = { + host: 'proxy.company.com', + port: 8080, + proxyAuth: 'bob:secret', + protocol: 'http:', + connect: 'https', + }; + const expectedUrl = 'http://bob:secret@proxy.company.com:8080'; + const url = makeUrlFromTunnelNgConfig(tunnelNgConfig); + expect(url).to.deep.equal(expectedUrl); + }); +}); diff --git a/tests/utils/helpers.spec.ts b/tests/utils/helpers.spec.ts new file mode 100644 index 00000000..7913c8db --- /dev/null +++ b/tests/utils/helpers.spec.ts @@ -0,0 +1,87 @@ +/** + * @license + * Copyright 2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; + +import { getProxyConfig } from '../../build/utils/helpers'; + +describe('getProxyConfig() function', function() { + let originalProxyConfig: [boolean, object | undefined]; + let originalHttpProxy: [boolean, string | undefined]; + let originalHttpsProxy: [boolean, string | undefined]; + + this.beforeEach(() => { + originalProxyConfig = [ + global.hasOwnProperty('PROXY_CONFIG'), + (global as any).PROXY_CONFIG, + ]; + originalHttpProxy = [ + process.env.hasOwnProperty('HTTP_PROXY'), + process.env.HTTP_PROXY, + ]; + originalHttpsProxy = [ + process.env.hasOwnProperty('HTTPS_PROXY'), + process.env.HTTPS_PROXY, + ]; + delete (global as any).PROXY_CONFIG; + delete process.env.HTTP_PROXY; + delete process.env.HTTPS_PROXY; + }); + + this.afterEach(() => { + if (originalProxyConfig[0]) { + (global as any).PROXY_CONFIG = originalProxyConfig[1]; + } + if (originalHttpProxy[0]) { + process.env.HTTP_PROXY = originalHttpProxy[1]; + } + if (originalHttpsProxy[0]) { + process.env.HTTPS_PROXY = originalHttpsProxy[1]; + } + }); + + it('should return a ProxyConfig object when global-tunnel-ng is in use', () => { + (global as any).PROXY_CONFIG = { + host: '127.0.0.1', + port: 8080, + proxyAuth: 'bob:secret', + protocol: 'http:', + connect: 'https', + }; + const expectedProxyConfig = { + host: '127.0.0.1', + port: '8080', + proxyAuth: 'bob:secret', + username: 'bob', + password: 'secret', + }; + expect(getProxyConfig()).to.deep.equal(expectedProxyConfig); + }); + + it('should return a ProxyConfig object when the HTTP(S)_PROXY env vars are defined', () => { + process.env.HTTPS_PROXY = 'http://bob:secret@proxy.company.com:12345'; + process.env.HTTP_PROXY = 'http://my.net:8080'; + const expectedProxyConfig = { + host: 'proxy.company.com', + port: '12345', + proxyAuth: 'bob:secret', + username: 'bob', + password: 'secret', + }; + expect(getProxyConfig()).to.deep.equal(expectedProxyConfig); + }); +});