Merge branch 'master' into typo

CHANGELOG.md

@@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file
 automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## 9.9.2 - 2019-01-11
+
+* Lazy-load etcher-sdk to speed up startup [Pagan Gazzard]
+* Lazy-load resin-cli-form and resin-cli-visuals to speed up startup [Pagan Gazzard]
+
+## 9.9.1 - 2019-01-11
+
+* Update util available-drives action [Alexis Svinartchouk]
+* Update lib/actions/local/flash.coffee [Alexis Svinartchouk]
+
+## 9.9.0 - 2019-01-10
+
+* Request access to previously pushed release via `balena deploy` [Matthew McGinn]
+
+## 9.8.0 - 2019-01-01
+
+* Escape backticks in JS template literal [Trevor Sullivan]
+* Moving docs from PR #1055 [Trevor Sullivan]
+
 ## 9.7.0 - 2018-12-28
 
 * Added documentation about the dependencies required to build balena-cli [Trevor Sullivan]

capitanodoc.ts

@@ -7,6 +7,23 @@ Please make sure your system meets the requirements as specified in the [README]
 
 ## Install the CLI
 
+### Dependencies
+
+Before installing the Balena CLI from npm, make sure you have the following dependencies installed:
+
+* make
+* g++ compiler
+* Python 2.7
+* git
+
+For example, to install these packages on a Debian-based Linux operating systems:
+
+\`\`\`
+$ sudo apt-get install g++ make python git --yes
+\`\`\`
+
+**NOTE**: If you are installing the stand-alone binary CLI, you will not need to install these dependencies.
+
 ### Npm install
 
 The best supported way to install the CLI is from npm:

lib/actions/local/common.coffee

@@ -1,6 +1,5 @@
 Promise = require('bluebird')
 _ = require('lodash')
-form = require('resin-cli-form')
 chalk = require('chalk')
 
 dockerUtils = require('../../utils/docker')
@@ -15,6 +14,7 @@ exports.filterOutSupervisorContainer = filterOutSupervisorContainer = (container
 	return true
 
 exports.selectContainerFromDevice = Promise.method (deviceIp, filterSupervisor = false) ->
+	form = require('resin-cli-form')
 	docker = dockerUtils.createClient(host: deviceIp, port: dockerPort, timeout: dockerTimeout)
 
 	# List all containers, including those not running

lib/actions/local/flash.coffee

@@ -1,120 +0,0 @@
-###
-Copyright 2017 Balena
-
-Licensed under the Apache License, Version 2.0 (the 'License');
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	 http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an 'AS IS' BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-###
-
-module.exports =
-	signature: 'local flash <image>'
-	description: 'Flash an image to a drive'
-	help: '''
-		Use this command to flash a balenaOS image to a drive.
-
-		Examples:
-
-			$ balena local flash path/to/balenaos.img
-			$ balena local flash path/to/balenaos.img --drive /dev/disk2
-			$ balena local flash path/to/balenaos.img --drive /dev/disk2 --yes
-	'''
-	options: [
-			signature: 'yes'
-			boolean: true
-			description: 'confirm non-interactively'
-			alias: 'y'
-	,
-			signature: 'drive'
-			parameter: 'drive'
-			description: 'drive'
-			alias: 'd'
-	]
-	root: true
-	action: (params, options, done) ->
-
-		_ = require('lodash')
-		os = require('os')
-		Promise = require('bluebird')
-		umountAsync = Promise.promisify(require('umount').umount)
-		fs = Promise.promisifyAll(require('fs'))
-		driveListAsync = Promise.promisify(require('drivelist').list)
-		chalk = require('chalk')
-		visuals = require('resin-cli-visuals')
-		form = require('resin-cli-form')
-		imageWrite = require('etcher-image-write')
-
-		form.run [
-			{
-				message: 'Select drive'
-				type: 'drive'
-				name: 'drive'
-			},
-			{
-				message: 'This will erase the selected drive. Are you sure?'
-				type: 'confirm'
-				name: 'yes'
-				default: false
-			}
-		],
-			override:
-				drive: options.drive
-
-				# If `options.yes` is `false`, pass `undefined`,
-				# otherwise the question will not be asked because
-				# `false` is a defined value.
-				yes: options.yes || undefined
-
-		# TODO: dedupe with the resin-device-operations
-		.then (answers) ->
-			if answers.yes isnt true
-				console.log(chalk.red.bold('Aborted image flash'))
-				process.exit(0)
-
-			driveListAsync().then (drives) ->
-				selectedDrive = _.find(drives, device: answers.drive)
-
-				if not selectedDrive?
-					throw new Error("Drive not found: #{answers.drive}")
-
-				return selectedDrive
-		.then (selectedDrive) ->
-			progressBars =
-				write: new visuals.Progress('Flashing')
-				check: new visuals.Progress('Validating')
-
-			umountAsync(selectedDrive.device).then ->
-				Promise.props
-					imageSize: fs.statAsync(params.image).get('size'),
-					imageStream: Promise.resolve(fs.createReadStream(params.image))
-					driveFileDescriptor: fs.openAsync(selectedDrive.raw, 'rs+')
-			.then (results) ->
-				imageWrite.write
-					fd: results.driveFileDescriptor
-					device: selectedDrive.raw
-					size: selectedDrive.size
-				,
-					stream: results.imageStream,
-					size: results.imageSize
-				,
-					check: true
-			.then (writer) ->
-				new Promise (resolve, reject) ->
-					writer.on 'progress', (state) ->
-						progressBars[state.type].update(state)
-					writer.on('error', reject)
-					writer.on('done', resolve)
-			.then ->
-				if (os.platform() is 'win32') and selectedDrive.mountpoint?
-					ejectAsync = Promise.promisify(require('removedrive').eject)
-					return ejectAsync(selectedDrive.mountpoint)
-
-				return umountAsync(selectedDrive.device)
-		.asCallback(done)

lib/actions/local/flash.ts

@@ -0,0 +1,120 @@
+/*
+Copyright 2017 Balena
+
+Licensed under the Apache License, Version 2.0 (the 'License');
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an 'AS IS' BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { CommandDefinition } from 'capitano';
+import chalk from 'chalk';
+import { stripIndent } from 'common-tags';
+import * as sdk from 'etcher-sdk';
+
+async function getDrive(options: {
+	drive?: string;
+}): Promise<sdk.sourceDestination.BlockDevice> {
+	const sdk = await import('etcher-sdk');
+
+	const adapter = new sdk.scanner.adapters.BlockDeviceAdapter(() => false);
+	const scanner = new sdk.scanner.Scanner([adapter]);
+	await scanner.start();
+	let drive: sdk.sourceDestination.BlockDevice;
+	if (options.drive !== undefined) {
+		const d = scanner.getBy('device', options.drive);
+		if (d === undefined || !(d instanceof sdk.sourceDestination.BlockDevice)) {
+			throw new Error(`Drive not found: ${options.drive}`);
+		}
+		drive = d;
+	} else {
+		const { DriveList } = await import('../../utils/visuals/drive-list');
+		const driveList = new DriveList(scanner);
+		drive = await driveList.run();
+	}
+	scanner.stop();
+	return drive;
+}
+
+export const flash: CommandDefinition<
+	{ image: string },
+	{ drive: string; yes: boolean }
+> = {
+	signature: 'local flash <image>',
+	description: 'Flash an image to a drive',
+	//root: true,
+	help: stripIndent`
+		Use this command to flash a balenaOS image to a drive.
+
+		Examples:
+
+			$ balena local flash path/to/balenaos.img
+			$ balena local flash path/to/balenaos.img --drive /dev/disk2
+			$ balena local flash path/to/balenaos.img --drive /dev/disk2 --yes
+	`,
+	options: [
+		{
+			signature: 'yes',
+			boolean: true,
+			description: 'confirm non-interactively',
+			alias: 'y',
+		},
+		{
+			signature: 'drive',
+			parameter: 'drive',
+			description: 'drive',
+			alias: 'd',
+		},
+	],
+	async action(params, options) {
+		const visuals = await import('resin-cli-visuals');
+		const form = await import('resin-cli-form');
+		const { sourceDestination, multiWrite } = await import('etcher-sdk');
+
+		const drive = await getDrive(options);
+
+		const yes =
+			options.yes ||
+			(await form.ask({
+				message: 'This will erase the selected drive. Are you sure?',
+				type: 'confirm',
+				name: 'yes',
+				default: false,
+			}));
+		if (yes !== true) {
+			console.log(chalk.red.bold('Aborted image flash'));
+			process.exit(0);
+		}
+
+		const source = new sourceDestination.File(
+			params.image,
+			sourceDestination.File.OpenFlags.Read,
+		);
+
+		const progressBars: { [key: string]: any } = {
+			flashing: new visuals.Progress('Flashing'),
+			verifying: new visuals.Progress('Validating'),
+		};
+
+		await multiWrite.pipeSourceToDestinations(
+			source,
+			[drive],
+			(_, error) => {
+				// onFail
+				console.log(chalk.red.bold(error.message));
+			},
+			(progress: sdk.multiWrite.MultiDestinationProgress) => {
+				// onProgress
+				progressBars[progress.type].update(progress);
+			},
+			true, // verify
+		);
+	},
+};

lib/actions/local/index.coffee

@@ -15,7 +15,7 @@ limitations under the License.
 ###
 
 exports.configure = require('./configure')
-exports.flash = require('./flash')
+exports.flash = require('./flash').flash
 exports.logs = require('./logs')
 exports.scan = require('./scan')
 exports.ssh = require('./ssh')

lib/actions/util.coffee

@@ -1,56 +0,0 @@
-###
-Copyright 2016-2017 Balena
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-###
-
-_ = require('lodash')
-
-exports.availableDrives =
-	# TODO: dedupe with https://github.com/balena-io-modules/resin-cli-visuals/blob/master/lib/widgets/drive/index.coffee
-	signature: 'util available-drives'
-	description: 'list available drives'
-	help: """
-		Use this command to list your machine's drives usable for writing the OS image to.
-		Skips the system drives.
-	"""
-	action: ->
-		Promise = require('bluebird')
-		drivelist = require('drivelist')
-		driveListAsync = Promise.promisify(drivelist.list)
-		chalk = require('chalk')
-		visuals = require('resin-cli-visuals')
-
-		formatDrive = (drive) ->
-			size = drive.size / 1000000000
-			return {
-				device: drive.device
-				size: "#{size.toFixed(1)} GB"
-				description: drive.description
-			}
-
-		getDrives = ->
-			driveListAsync().then (drives) ->
-				return _.reject(drives, system: true)
-
-		getDrives()
-		.then (drives) ->
-			if not drives.length
-				console.error("#{chalk.red('x')} No available drives were detected, plug one in!")
-				return
-
-			console.log visuals.table.horizontal drives.map(formatDrive), [
-				'device'
-				'size'
-				'description'
-			]

lib/actions/util.ts

@@ -0,0 +1,65 @@
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+/*
+Copyright 2016-2017 Balena
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { CommandDefinition } from 'capitano';
+import chalk from 'chalk';
+import { stripIndent } from 'common-tags';
+
+export const availableDrives: CommandDefinition<{}, {}> = {
+	signature: 'util available-drives',
+	description: 'list available drives',
+	help: stripIndent`
+		Use this command to list your machine's drives usable for writing the OS image to.
+		Skips the system drives.
+	`,
+	async action() {
+		const sdk = await import('etcher-sdk');
+		const visuals = await import('resin-cli-visuals');
+
+		const adapter = new sdk.scanner.adapters.BlockDeviceAdapter(() => false);
+		const scanner = new sdk.scanner.Scanner([adapter]);
+		await scanner.start();
+
+		function formatDrive(drive: any) {
+			const size = drive.size / 1000000000;
+			return {
+				device: drive.device,
+				size: `${size.toFixed(1)} GB`,
+				description: drive.description,
+			};
+		}
+
+		if (scanner.drives.size === 0) {
+			console.error(
+				`${chalk.red('x')} No available drives were detected, plug one in!`,
+			);
+		} else {
+			console.log(
+				visuals.table.horizontal(Array.from(scanner.drives).map(formatDrive), [
+					'device',
+					'size',
+					'description',
+				]),
+			);
+		}
+		scanner.stop();
+	},
+};

lib/utils/compose.coffee

@@ -349,13 +349,54 @@ tagServiceImages = (docker, images, serviceImages) ->
 			logs: d.logs
 			props: d.props
 
-authorizePush = (tokenAuthEndpoint, registry, images) ->
+
+getPreviousRepos = (sdk, docker, logger, appID) ->
+	sdk.pine.get(
+		resource: 'release'
+		options:
+			$filter:
+				belongs_to__application: appID
+				status: 'success'
+			$select:
+				[ 'id' ]
+			$expand:
+				contains__image:
+					$expand: 'image'
+			$orderby: 'id desc'
+			$top: 1
+	)
+	.then (release) ->
+		# grab all images from the latest release, return all image locations in the registry
+		if release?.length > 0
+			images = release[0].contains__image
+			Promise.map images, (d) ->
+				imageName = d.image[0].is_stored_at__image_location
+				docker.getRegistryAndName(imageName)
+				.then ( registry ) ->
+					logger.logDebug("Requesting access to previously pushed image repo (#{registry.imageName})")
+					return registry.imageName
+	.catch (e) ->
+		logger.logDebug("Failed to access previously pushed image repo: #{e}")
+
+authorizePush = (sdk, logger, tokenAuthEndpoint, registry, images, previousRepos) ->
 	_ = require('lodash')
-	sdk = require('balena-sdk').fromSharedOptions()
+
+	# TODO: https://github.com/balena-io/balena-cli/issues/1070
+	maxRepos = 20
 
 	if not _.isArray(images)
 		images = [ images ]
 
+	if images.length > maxRepos
+		throw new Error (
+			"More than #{maxRepos} containers is currently not supported, see " +
+			'https://github.com/balena-io/balena-cli/issues/1070 for more information'
+		)
+	images.push previousRepos...
+	if images.length + previousRepos?.length > maxRepos
+		logger.logDebug("Truncating requested repositories to #{maxRepos} by limiting previously pushed repo access")
+		# at this point, we know we're only truncating access to previously pushed repos
+		images = images[0...maxRepos]
 	sdk.request.send
 		baseUrl: tokenAuthEndpoint
 		url: '/auth/v1/token'
@@ -423,7 +464,10 @@ exports.deployProject = (
 		tagServiceImages(docker, images, serviceImages)
 		.tap (images) ->
 			logger.logDebug('Authorizing push...')
-			authorizePush(apiEndpoint, images[0].registry, _.map(images, 'repo'))
+			sdk = require('balena-sdk').fromSharedOptions()
+			getPreviousRepos(sdk, docker, logger, appId)
+			.then (previousRepos) ->
+				authorizePush(sdk, logger, apiEndpoint, images[0].registry, _.map(images, 'repo'), previousRepos)
 			.then (token) ->
 				logger.logInfo('Pushing images to registry...')
 				pushAndUpdateServiceImages docker, token, images, (serviceImage) ->

lib/utils/patterns.ts

@@ -13,11 +13,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+import _form = require('resin-cli-form');
+import _visuals = require('resin-cli-visuals');
 
 import _ = require('lodash');
 import Promise = require('bluebird');
-import form = require('resin-cli-form');
-import visuals = require('resin-cli-visuals');
 import BalenaSdk = require('balena-sdk');
 import chalk from 'chalk';
 import validation = require('./validation');
@@ -25,8 +25,11 @@ import messages = require('./messages');
 
 const balena = BalenaSdk.fromSharedOptions();
 
+const getForm = _.once((): typeof _form => require('resin-cli-form'));
+const getVisuals = _.once((): typeof _visuals => require('resin-cli-visuals'));
+
 export function authenticate(options: {}): Promise<void> {
-	return form
+	return getForm()
 		.run(
 			[
 				{
@@ -50,7 +53,7 @@ export function authenticate(options: {}): Promise<void> {
 				return;
 			}
 
-			return form
+			return getForm()
 				.ask({
 					message: 'Two factor auth challenge:',
 					name: 'code',
@@ -72,7 +75,7 @@ export function authenticate(options: {}): Promise<void> {
 }
 
 export function askLoginType() {
-	return form.ask({
+	return getForm().ask({
 		message: 'How would you like to login?',
 		name: 'loginType',
 		type: 'list',
@@ -100,7 +103,7 @@ export function askLoginType() {
 export function selectDeviceType() {
 	return balena.models.config.getDeviceTypes().then(deviceTypes => {
 		deviceTypes = _.sortBy(deviceTypes, 'name');
-		return form.ask({
+		return getForm().ask({
 			message: 'Device Type',
 			type: 'list',
 			choices: _.map(deviceTypes, ({ slug: value, name }) => ({
@@ -124,7 +127,7 @@ export function confirm(
 			return true;
 		}
 
-		return form.ask({
+		return getForm().ask({
 			message,
 			type: 'confirm',
 			default: false,
@@ -150,7 +153,7 @@ export function selectApplication(
 		})
 		.filter(filter || _.constant(true))
 		.then(applications => {
-			return form.ask({
+			return getForm().ask({
 				message: 'Select an application',
 				type: 'list',
 				choices: _.map(applications, application => ({
@@ -181,7 +184,7 @@ export function selectOrCreateApplication() {
 					value: null,
 				});
 
-				return form.ask({
+				return getForm().ask({
 					message: 'Select an application',
 					type: 'list',
 					choices: appOptions,
@@ -193,7 +196,7 @@ export function selectOrCreateApplication() {
 				return application;
 			}
 
-			return form.ask({
+			return getForm().ask({
 				message: 'Choose a Name for your new application',
 				type: 'input',
 				validate: validation.validateApplicationName,
@@ -203,6 +206,7 @@ export function selectOrCreateApplication() {
 
 export function awaitDevice(uuid: string) {
 	return balena.models.device.getName(uuid).then(deviceName => {
+		const visuals = getVisuals();
 		const spinner = new visuals.Spinner(
 			`Waiting for ${deviceName} to come online`,
 		);
@@ -243,7 +247,7 @@ export function inferOrSelectDevice(preferredUuid: string) {
 				? preferredUuid
 				: onlineDevices[0].uuid;
 
-			return form.ask({
+			return getForm().ask({
 				message: 'Select a device',
 				type: 'list',
 				default: defaultUuid,
@@ -262,7 +266,7 @@ export function selectFromList<T>(
 	message: string,
 	choices: Array<T & { name: string }>,
 ): Promise<T> {
-	return form.ask({
+	return getForm().ask({
 		message,
 		type: 'list',
 		choices: _.map(choices, s => ({

lib/utils/visuals/custom-dynamic-list.ts

@@ -0,0 +1,25 @@
+import DynamicList = require('inquirer-dynamic-list');
+
+export abstract class CustomDynamicList<T> extends DynamicList {
+	constructor(message: string, emptyMessage: string) {
+		super({ message, emptyMessage, choices: [] });
+	}
+
+	protected abstract getThings(): Iterable<T>;
+
+	protected abstract format(thing: T): string;
+
+	refresh(): void {
+		this.opt.choices.choices = [];
+		this.opt.choices.realChoices = [];
+		for (const thing of this.getThings()) {
+			this.addChoice({ name: this.format(thing), value: thing });
+		}
+		this.render();
+	}
+
+	async run(): Promise<T> {
+		this.refresh();
+		return await super.run();
+	}
+}

lib/utils/visuals/drive-list.ts

@@ -0,0 +1,32 @@
+import chalk from 'chalk';
+import * as _sdk from 'etcher-sdk';
+
+import { CustomDynamicList } from './custom-dynamic-list';
+
+export class DriveList extends CustomDynamicList<
+	_sdk.sourceDestination.BlockDevice
+> {
+	constructor(private scanner: _sdk.scanner.Scanner) {
+		super(
+			'Select a drive',
+			`${chalk.red('x')} No available drives were detected, plug one in!`,
+		);
+		const refresh = this.refresh.bind(this);
+		scanner.on('attach', refresh);
+		scanner.on('detach', refresh);
+	}
+
+	protected *getThings() {
+		const sdk: typeof _sdk = require('etcher-sdk')
+		for (const drive of this.scanner.drives) {
+			if (drive instanceof sdk.sourceDestination.BlockDevice) {
+				yield drive;
+			}
+		}
+	}
+
+	protected format(drive: _sdk.sourceDestination.BlockDevice) {
+		const size = drive.size / 1e9;
+		return `${drive.device} (${size.toFixed(1)} GB) - ${drive.description}`;
+	}
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "balena-cli",
-  "version": "9.7.0",
+  "version": "9.9.2",
   "description": "The official balena CLI tool",
   "main": "./build/actions/index.js",
   "homepage": "https://github.com/balena-io/balena-cli",
@@ -126,9 +126,8 @@
     "docker-toolbelt": "^3.3.5",
     "dockerode": "^2.5.5",
     "dockerode-options": "^0.2.1",
-    "drivelist": "^5.0.22",
     "ejs": "^2.5.7",
-    "etcher-image-write": "^9.0.3",
+    "etcher-sdk": "^0.2.0",
     "event-stream": "3.3.4",
     "express": "^4.13.3",
     "global-tunnel-ng": "^2.1.1",

tsconfig.json

@@ -24,6 +24,7 @@
 	},
 	"include": [
 		"./typings/*.d.ts",
+		"./node_modules/etcher-sdk/typings/**/*.d.ts",
 		"./lib/**/*.ts"
 	]
 }

typings/inquire-dynamic-list.d.ts

@@ -0,0 +1,26 @@
+declare module 'inquirer-dynamic-list' {
+	interface Choice {
+		name: string;
+		value: any;
+	}
+
+	class DynamicList {
+		opt: {
+			choices: {
+				choices: Choice[];
+				realChoices: Choice[];
+			};
+		};
+
+		constructor(options: {
+			message?: string;
+			emptyMessage?: string;
+			choices: Choice[];
+		});
+		addChoice(choice: Choice): void;
+		render(): void;
+		run(): Promise<any>;
+	}
+
+	export = DynamicList;
+}