From 4369a2d1610828082a54fce4fdc572e41fc40f0d Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Tue, 16 Oct 2018 12:53:16 +0100 Subject: [PATCH 1/8] tconfig: Add skipLibCheck to tsconfig Signed-off-by: Cameron Diver --- tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.json b/tsconfig.json index d6f3e9c3..95a2584a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,6 +10,7 @@ "preserveConstEnums": true, "removeComments": true, "sourceMap": true, + "skipLibCheck": true, "lib": [ // es5 defaults: "dom", From 221666f59a582b9daf37eedb328800206fe6de55 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Wed, 29 Aug 2018 10:24:11 -0700 Subject: [PATCH 2/8] Stop accepting resin-compose.yml as a build composition definition These files are not supported by any other part of the resin infrastructure, and it could cause confusion with it not being supported everywhere. The idea was originally added because we thought we might need to make extensions on docker-compose, but that hasn't happened. Change-type: major Signed-off-by: Cameron Diver --- lib/utils/compose.coffee | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index b29d9455..4102a938 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -34,8 +34,6 @@ exports.generateOpts = (options) -> inlineLogs: !!options.logs compositionFileNames = [ - 'resin-compose.yml' - 'resin-compose.yaml' 'docker-compose.yml' 'docker-compose.yaml' ] From bf062124f79b106dbc5fd373563146b8b21628d6 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Mon, 15 Oct 2018 12:31:10 +0100 Subject: [PATCH 3/8] compose: Add compose typings Signed-off-by: Cameron Diver --- lib/utils/compose.d.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/utils/compose.d.ts b/lib/utils/compose.d.ts index 3b251828..39cb23d1 100644 --- a/lib/utils/compose.d.ts +++ b/lib/utils/compose.d.ts @@ -1,3 +1,32 @@ +import * as Bluebird from 'bluebird'; import * as Stream from 'stream'; +import { Composition } from 'resin-compose-parse'; +import Logger = require('./logger'); + +interface Image { + context: string; + tag: string; +} + +interface Descriptor { + image: Image | string; + serviceName: string; +} + +export function resolveProject(projectRoot: string): Bluebird; + +export interface ComposeProject { + path: string; + name: string; + composition: Composition; + descriptors: Descriptor[]; +} + +export function loadProject( + logger: Logger, + projectPath: string, + projectName: string, + image?: string, +): Bluebird; export function tarDirectory(source: string): Promise; From 6bcfb2dd51a591c7f01e6e825daae7dc8e803101 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Mon, 15 Oct 2018 12:32:27 +0100 Subject: [PATCH 4/8] logs: Add log build function to logger Signed-off-by: Cameron Diver --- lib/utils/logger.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/utils/logger.ts b/lib/utils/logger.ts index 31b0e983..eec22b9a 100644 --- a/lib/utils/logger.ts +++ b/lib/utils/logger.ts @@ -61,6 +61,10 @@ class Logger { logError(msg: string) { return this.streams.error.write(msg + eol); } + + logBuild(msg: string) { + return this.streams.build.write(msg + eol); + } } export = Logger; From f560aa75233ed3cd870578a291a82d1aa7e3907c Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Tue, 16 Oct 2018 11:23:20 +0100 Subject: [PATCH 5/8] export resolveProject function from compose module Signed-off-by: Cameron Diver --- lib/utils/compose.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils/compose.coffee b/lib/utils/compose.coffee index 4102a938..3a772f98 100644 --- a/lib/utils/compose.coffee +++ b/lib/utils/compose.coffee @@ -40,7 +40,7 @@ compositionFileNames = [ # look into the given directory for valid compose files and return # the contents of the first one found. -resolveProject = (rootDir) -> +exports.resolveProject = resolveProject = (rootDir) -> fs = require('mz/fs') Promise.any compositionFileNames.map (filename) -> fs.readFile(path.join(rootDir, filename), 'utf-8') From c5d4e30e240840b93daf8a72b818534bb5685897 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Tue, 16 Oct 2018 11:24:28 +0100 Subject: [PATCH 6/8] logger: Add logs logging function Signed-off-by: Cameron Diver --- lib/utils/logger.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/utils/logger.ts b/lib/utils/logger.ts index eec22b9a..d86e0545 100644 --- a/lib/utils/logger.ts +++ b/lib/utils/logger.ts @@ -11,6 +11,7 @@ class Logger { success: NodeJS.ReadWriteStream; warn: NodeJS.ReadWriteStream; error: NodeJS.ReadWriteStream; + logs: NodeJS.ReadWriteStream; }; public formatMessage: (name: string, message: string) => string; @@ -23,6 +24,7 @@ class Logger { logger.addPrefix('success', chalk.green('[Success]')); logger.addPrefix('warn', chalk.yellow('[Warn]')); logger.addPrefix('error', chalk.red('[Error]')); + logger.addPrefix('logs', chalk.green('[Logs]')); this.streams = { build: logger.createLogStream('build'), @@ -31,6 +33,7 @@ class Logger { success: logger.createLogStream('success'), warn: logger.createLogStream('warn'), error: logger.createLogStream('error'), + logs: logger.createLogStream('logs'), }; _.forEach(this.streams, function(stream, key) { @@ -65,6 +68,10 @@ class Logger { logBuild(msg: string) { return this.streams.build.write(msg + eol); } + + logLogs(msg: string) { + return this.streams.logs.write(msg + eol); + } } export = Logger; From 947f91d570a5f25b9e8e671efc6540060eda84a1 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Tue, 16 Oct 2018 11:25:37 +0100 Subject: [PATCH 7/8] Support multicontainer local mode in resin push Change-type: minor Signed-off-by: Cameron Diver --- doc/cli.markdown | 16 +- lib/actions/push.ts | 128 ++++++++++---- lib/utils/device/api.ts | 170 ++++++++++++++++++ lib/utils/device/deploy.ts | 284 +++++++++++++++++++++++++++++++ lib/utils/device/errors.ts | 30 ++++ lib/utils/device/logs.ts | 83 +++++++++ lib/utils/logger.ts | 4 +- lib/utils/patterns.ts | 4 +- package.json | 13 +- typings/color-hash.d.ts | 12 ++ typings/dockerfile-template.d.ts | 13 ++ 11 files changed, 714 insertions(+), 43 deletions(-) create mode 100644 lib/utils/device/api.ts create mode 100644 lib/utils/device/deploy.ts create mode 100644 lib/utils/device/errors.ts create mode 100644 lib/utils/device/logs.ts create mode 100644 typings/color-hash.d.ts create mode 100644 typings/dockerfile-template.d.ts diff --git a/doc/cli.markdown b/doc/cli.markdown index 1feb9512..1bc925eb 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -159,7 +159,7 @@ environment variable (in the same standard URL format). - Push - - [push <application>](#push-application-) + - [push <applicationOrDevice>](#push-applicationordevice-) - Settings @@ -1249,19 +1249,29 @@ Docker host TLS key file # Push -## push <application> +## push <applicationOrDevice> This command can be used to start a build on the remote -resin.io cloud builders. The given source directory will be sent to the +resin.io cloud builders, or a local mode resin device. + +When building on the resin cloud the given source directory will be sent to the resin.io builder, and the build will proceed. This can be used as a drop-in replacement for git push to deploy. +When building on a local mode device, the given source directory will be built on +device, and the resulting contianers will be run on the device. Logs will be +streamed back from the device as part of the same invocation. + Examples: $ resin push myApp $ resin push myApp --source $ resin push myApp -s + $ resin push 10.0.0.1 + $ resin push 10.0.0.1 --source + $ resin push 10.0.0.1 -s + ### Options #### --source, -s <source> diff --git a/lib/actions/push.ts b/lib/actions/push.ts index 53657443..8392135c 100644 --- a/lib/actions/push.ts +++ b/lib/actions/push.ts @@ -15,8 +15,33 @@ limitations under the License. */ import { CommandDefinition } from 'capitano'; -import { ResinSDK } from 'resin-sdk'; import { stripIndent } from 'common-tags'; +import { ResinSDK } from 'resin-sdk'; + +import { BuildError } from '../utils/device/errors'; + +// An regex to detect an IP address, from https://www.regular-expressions.info/ip.html +const IP_REGEX = new RegExp( + /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/, +); + +enum BuildTarget { + Cloud, + Device, +} + +function getBuildTarget(appOrDevice: string): BuildTarget | null { + // First try the application regex from the api + if (/^[a-zA-Z0-9_-]+$/.test(appOrDevice)) { + return BuildTarget.Cloud; + } + + if (IP_REGEX.test(appOrDevice)) { + return BuildTarget.Device; + } + + return null; +} async function getAppOwner(sdk: ResinSDK, appName: string) { const { @@ -75,7 +100,7 @@ async function getAppOwner(sdk: ResinSDK, appName: string) { export const push: CommandDefinition< { - application: string; + applicationOrDevice: string; }, { source: string; @@ -83,19 +108,30 @@ export const push: CommandDefinition< nocache: boolean; } > = { - signature: 'push ', - description: 'Start a remote build on the resin.io cloud build servers', + signature: 'push ', + description: + 'Start a remote build on the resin.io cloud build servers or a local mode device', help: stripIndent` This command can be used to start a build on the remote - resin.io cloud builders. The given source directory will be sent to the + resin.io cloud builders, or a local mode resin device. + + When building on the resin cloud the given source directory will be sent to the resin.io builder, and the build will proceed. This can be used as a drop-in replacement for git push to deploy. + When building on a local mode device, the given source directory will be built on + device, and the resulting containers will be run on the device. Logs will be + streamed back from the device as part of the same invocation. + Examples: $ resin push myApp $ resin push myApp --source $ resin push myApp -s + + $ resin push 10.0.0.1 + $ resin push 10.0.0.1 --source + $ resin push 10.0.0.1 -s `, permission: 'user', options: [ @@ -123,11 +159,12 @@ export const push: CommandDefinition< const sdk = (await import('resin-sdk')).fromSharedOptions(); const Bluebird = await import('bluebird'); const remote = await import('../utils/remote-build'); + const deviceDeploy = await import('../utils/device/deploy'); const { exitWithExpectedError } = await import('../utils/patterns'); - const app: string | null = params.application; - if (app == null) { - exitWithExpectedError('You must specify an application'); + const appOrDevice: string | null = params.applicationOrDevice; + if (appOrDevice == null) { + exitWithExpectedError('You must specify an application or a device'); } const source = options.source || '.'; @@ -135,29 +172,58 @@ export const push: CommandDefinition< console.log(`[debug] Using ${source} as build source`); } - Bluebird.join( - sdk.auth.getToken(), - sdk.settings.get('resinUrl'), - getAppOwner(sdk, app), - (token, baseUrl, owner) => { - const opts = { - emulated: options.emulated, - nocache: options.nocache, - }; - const args = { - app, - owner, - source, - auth: token, - baseUrl, - sdk, - opts, - }; + const buildTarget = getBuildTarget(appOrDevice); + switch (buildTarget) { + case BuildTarget.Cloud: + const app = appOrDevice; + Bluebird.join( + sdk.auth.getToken(), + sdk.settings.get('resinUrl'), + getAppOwner(sdk, app), + (token, baseUrl, owner) => { + const opts = { + emulated: options.emulated, + nocache: options.nocache, + }; + const args = { + app, + owner, + source, + auth: token, + baseUrl, + sdk, + opts, + }; - return remote.startRemoteBuild(args); - }, - ) - .catch(remote.RemoteBuildFailedError, exitWithExpectedError) - .nodeify(done); + return remote.startRemoteBuild(args); + }, + ).nodeify(done); + break; + case BuildTarget.Device: + const device = appOrDevice; + // TODO: Support passing a different port + Bluebird.resolve( + deviceDeploy.deployToDevice({ + source, + deviceHost: device, + }), + ) + .catch(BuildError, e => { + exitWithExpectedError(e.toString()); + }) + .nodeify(done); + break; + default: + exitWithExpectedError( + stripIndent` + Build target not recognised. Please provide either an application name or device address. + + The only supported device addresses currently are IP addresses. + + If you believe your build target should have been detected, and this is an error, please + create an issue.`, + ); + break; + } }, }; diff --git a/lib/utils/device/api.ts b/lib/utils/device/api.ts new file mode 100644 index 00000000..5006e51f --- /dev/null +++ b/lib/utils/device/api.ts @@ -0,0 +1,170 @@ +import * as Bluebird from 'bluebird'; +import * as request from 'request'; +import * as Stream from 'stream'; + +import Logger = require('../logger'); + +import * as ApiErrors from './errors'; + +export interface DeviceResponse { + [key: string]: any; + + status: 'success' | 'failed'; + message?: string; +} + +export interface DeviceInfo { + deviceType: string; + arch: string; +} + +const deviceEndpoints = { + setTargetState: 'v2/local/target-state', + getTargetState: 'v2/local/target-state', + getDeviceInformation: 'v2/local/device-info', + logs: 'v2/local/logs', + ping: 'ping', +}; + +export class DeviceAPI { + private deviceAddress: string; + + public constructor( + private logger: Logger, + addr: string, + port: number = 48484, + ) { + this.deviceAddress = `http://${addr}:${port}/`; + } + + // Either return nothing, or throw an error with the info + public async setTargetState(state: any): Promise { + const url = this.getUrlForAction('setTargetState'); + return DeviceAPI.promisifiedRequest( + request.post, + { + url, + json: true, + body: state, + }, + this.logger, + ); + } + + public async getTargetState(): Promise { + const url = this.getUrlForAction('getTargetState'); + + return DeviceAPI.promisifiedRequest( + request.get, + { + url, + json: true, + }, + this.logger, + ).then(body => { + return body.state; + }); + } + + public async getDeviceInformation(): Promise { + const url = this.getUrlForAction('getDeviceInformation'); + + return DeviceAPI.promisifiedRequest( + request.get, + { + url, + json: true, + }, + this.logger, + ).then(body => { + return body.info; + }); + } + + public async ping(): Promise { + const url = this.getUrlForAction('ping'); + + return DeviceAPI.promisifiedRequest( + request.get, + { + url, + }, + this.logger, + ); + } + + public getLogStream(): Bluebird { + const url = this.getUrlForAction('logs'); + + // Don't use the promisified version here as we want to stream the output + return new Bluebird((resolve, reject) => { + const req = request.get(url); + + req.on('error', reject).on('response', res => { + if (res.statusCode !== 200) { + reject( + new ApiErrors.DeviceAPIError( + 'Non-200 response from log streaming endpoint', + ), + ); + } + resolve(res); + }); + }); + } + + private getUrlForAction(action: keyof typeof deviceEndpoints): string { + return `${this.deviceAddress}${deviceEndpoints[action]}`; + } + + // A helper method for promisifying general (non-streaming) requests. Streaming + // requests should use a seperate setup + private static async promisifiedRequest( + requestMethod: ( + opts: T, + cb: (err?: any, res?: any, body?: any) => void, + ) => void, + opts: T, + logger?: Logger, + ): Promise { + const Bluebird = await import('bluebird'); + const _ = await import('lodash'); + + type ObjectWithUrl = { url?: string }; + + if (logger != null) { + let url: string | null = null; + if (_.isObject(opts) && (opts as ObjectWithUrl).url != null) { + // the `as string` shouldn't be necessary, but the type system + // is getting a little confused + url = (opts as ObjectWithUrl).url as string; + } else if (_.isString(opts)) { + url = opts; + } + + if (url != null) { + logger.logDebug(`Sending request to ${url}`); + } + } + + return Bluebird.fromCallback( + cb => { + return requestMethod(opts, cb); + }, + { multiArgs: true }, + ).then(([response, body]) => { + switch (response.statusCode) { + case 200: + return body; + case 400: + throw new ApiErrors.BadRequestDeviceAPIError(body.message); + case 503: + throw new ApiErrors.ServiceUnavailableAPIError(body.message); + default: + throw new ApiErrors.DeviceAPIError(body.message); + } + }); + } +} + +export default DeviceAPI; diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts new file mode 100644 index 00000000..8301cbe8 --- /dev/null +++ b/lib/utils/device/deploy.ts @@ -0,0 +1,284 @@ +import * as Bluebird from 'bluebird'; +import * as Docker from 'dockerode'; +import * as _ from 'lodash'; +import { Composition } from 'resin-compose-parse'; +import { BuildTask, LocalImage } from 'resin-multibuild'; +import { Readable } from 'stream'; + +import Logger = require('../logger'); +import { displayBuildLog } from './logs'; + +import { DeviceInfo } from './api'; +import * as LocalPushErrors from './errors'; + +// Define the logger here so the debug output +// can be used everywhere +const logger = new Logger(); + +export interface DeviceDeployOptions { + source: string; + deviceHost: string; + devicePort?: number; +} + +async function checkSource(source: string): Promise { + const { fs } = await import('mz'); + return (await fs.exists(source)) && (await fs.stat(source)).isDirectory(); +} + +export async function deployToDevice(opts: DeviceDeployOptions): Promise { + const { loadProject, tarDirectory } = await import('../compose'); + const { exitWithExpectedError } = await import('../patterns'); + + const { DeviceAPI } = await import('./api'); + const { displayDeviceLogs } = await import('./logs'); + + if (!(await checkSource(opts.source))) { + exitWithExpectedError(`Could not access source directory: ${opts.source}`); + } + + const api = new DeviceAPI(logger, opts.deviceHost); + + // TODO: Before merge, replace this with the supervisor version endpoint, to + // ensure we're working with a supervisor version that supports the stuff we need + await api.ping(); + + logger.logInfo(`Starting build on device ${opts.deviceHost}`); + + const project = await loadProject(logger, opts.source, 'local'); + + // Attempt to attach to the device's docker daemon + const docker = connectToDocker( + opts.deviceHost, + opts.devicePort != null ? opts.devicePort : 2375, + ); + + const tarStream = await tarDirectory(opts.source); + + // Try to detect the device information + const deviceInfo = await api.getDeviceInformation(); + + await performBuilds( + project.composition, + tarStream, + docker, + deviceInfo, + logger, + ); + + logger.logDebug('Setting device state...'); + // Now set the target state on the device + + const currentTargetState = await api.getTargetState(); + + const targetState = generateTargetState( + currentTargetState, + project.composition, + ); + logger.logDebug(`Sending target state: ${JSON.stringify(targetState)}`); + + await api.setTargetState(targetState); + + // Print an empty newline to seperate the build output + // from the device output + console.log(); + logger.logInfo('Streaming device logs...'); + // Now all we need to do is stream back the logs + const logStream = await api.getLogStream(); + + await displayDeviceLogs(logStream, logger); +} + +function connectToDocker(host: string, port: number): Docker { + return new Docker({ + host, + port, + Promise: Bluebird as any, + }); +} + +export async function performBuilds( + composition: Composition, + tarStream: Readable, + docker: Docker, + deviceInfo: DeviceInfo, + logger: Logger, +): Promise { + const multibuild = await import('resin-multibuild'); + + const buildTasks = await multibuild.splitBuildStream(composition, tarStream); + + logger.logDebug('Found build tasks:'); + _.each(buildTasks, task => { + let infoStr: string; + if (task.external) { + infoStr = `image pull [${task.imageName}]`; + } else { + infoStr = `build [${task.context}]`; + } + logger.logDebug(` ${task.serviceName}: ${infoStr}`); + }); + + logger.logDebug( + `Resolving services with [${deviceInfo.deviceType}|${deviceInfo.arch}]`, + ); + await multibuild.performResolution( + buildTasks, + deviceInfo.arch, + deviceInfo.deviceType, + ); + + logger.logDebug('Found project types:'); + _.each(buildTasks, task => { + if (!task.external) { + logger.logDebug(` ${task.serviceName}: ${task.projectType}`); + } else { + logger.logDebug(` ${task.serviceName}: External image`); + } + }); + + logger.logDebug('Probing remote daemon for cache images'); + await assignDockerBuildOpts(docker, buildTasks); + + logger.logDebug('Starting builds...'); + await assignOutputHandlers(buildTasks, logger); + const localImages = await multibuild.performBuilds(buildTasks, docker); + + // Check for failures + await inspectBuildResults(localImages); + + // Now tag any external images with the correct name that they should be, + // as this won't be done by resin-multibuild + await Bluebird.map(localImages, async localImage => { + if (localImage.external) { + // We can be sure that localImage.name is set here, because of the failure code above + const image = docker.getImage(localImage.name!); + await image.tag({ + repo: generateImageName(localImage.serviceName), + force: true, + }); + await image.remove({ force: true }); + } + }); +} + +function assignOutputHandlers(buildTasks: BuildTask[], logger: Logger) { + _.each(buildTasks, task => { + if (task.external) { + task.progressHook = progressObj => { + displayBuildLog( + { serviceName: task.serviceName, message: progressObj.progress }, + logger, + ); + }; + } else { + task.streamHook = stream => { + stream.on('data', (buf: Buffer) => { + const str = buf.toString().trimRight(); + if (str !== '') { + displayBuildLog( + { serviceName: task.serviceName, message: str }, + logger, + ); + } + }); + }; + } + }); +} + +async function getDeviceDockerImages(docker: Docker): Promise { + const images = await docker.listImages(); + + return _.map(images, 'Id'); +} + +// Mutates buildTasks +async function assignDockerBuildOpts( + docker: Docker, + buildTasks: BuildTask[], +): Promise { + // Get all of the images on the remote docker daemon, so + // that we can use all of them for cache + const images = await getDeviceDockerImages(docker); + + logger.logDebug(`Using ${images.length} on-device images for cache...`); + + _.each(buildTasks, (task: BuildTask) => { + task.dockerOpts = { + cachefrom: images, + labels: { + 'io.resin.local.image': '1', + 'io.resin.local.service': task.serviceName, + }, + t: generateImageName(task.serviceName), + }; + }); +} + +function generateImageName(serviceName: string): string { + return `local_image_${serviceName}:latest`; +} + +function generateTargetState( + currentTargetState: any, + composition: Composition, +): any { + const services: { [serviceId: string]: any } = {}; + let idx = 1; + _.each(composition.services, (opts, name) => { + // Get rid of any build specific stuff + opts = _.cloneDeep(opts); + delete opts.build; + delete opts.image; + + const defaults = { + environment: {}, + labels: {}, + }; + + services[idx] = _.merge(defaults, opts, { + imageId: idx, + serviceName: name, + serviceId: idx, + image: generateImageName(name), + running: true, + }); + idx += 1; + }); + + const targetState = _.cloneDeep(currentTargetState); + delete targetState.local.apps; + + targetState.local.apps = { + 1: { + name: 'localapp', + commit: 'localcommit', + releaseId: '1', + services, + volumes: composition.volumes || {}, + networks: composition.networks || {}, + }, + }; + + return targetState; +} + +async function inspectBuildResults(images: LocalImage[]): Promise { + const { exitWithExpectedError } = await import('../patterns'); + + const failures: LocalPushErrors.BuildFailure[] = []; + + _.each(images, image => { + if (!image.successful) { + failures.push({ + error: image.error!, + serviceName: image.serviceName, + }); + } + }); + + if (failures.length > 0) { + exitWithExpectedError(new LocalPushErrors.BuildError(failures)); + } +} diff --git a/lib/utils/device/errors.ts b/lib/utils/device/errors.ts new file mode 100644 index 00000000..319dbfec --- /dev/null +++ b/lib/utils/device/errors.ts @@ -0,0 +1,30 @@ +import * as _ from 'lodash'; +import { TypedError } from 'typed-error'; + +export interface BuildFailure { + error: Error; + serviceName: string; +} + +export class BuildError extends TypedError { + private failures: BuildFailure[]; + + public constructor(failures: BuildFailure[]) { + super('Build error'); + + this.failures = failures; + } + + public toString(): string { + let str = 'Some services failed to build:\n'; + _.each(this.failures, failure => { + str += `\t${failure.serviceName}: ${failure.error.message}\n`; + }); + return str; + } +} + +export class DeviceAPIError extends TypedError {} + +export class BadRequestDeviceAPIError extends DeviceAPIError {} +export class ServiceUnavailableAPIError extends DeviceAPIError {} diff --git a/lib/utils/device/logs.ts b/lib/utils/device/logs.ts new file mode 100644 index 00000000..5f38789b --- /dev/null +++ b/lib/utils/device/logs.ts @@ -0,0 +1,83 @@ +import * as Bluebird from 'bluebird'; +import chalk from 'chalk'; +import ColorHash = require('color-hash'); +import * as _ from 'lodash'; +import { Readable } from 'stream'; + +import Logger = require('../logger'); + +interface Log { + message: string; + timestamp?: number; + serviceName?: string; + + // There's also a serviceId and imageId, but they're + // meaningless in local mode +} + +interface BuildLog { + serviceName: string; + message: string; +} + +/** + * Display logs from a device logging stream. This function will return + * when the log stream ends. + * + * @param logs A stream which produces newline seperated log objects + */ +export function displayDeviceLogs( + logs: Readable, + logger: Logger, +): Bluebird { + return new Bluebird((resolve, reject) => { + logs.on('data', log => { + displayLogLine(log, logger); + }); + + logs.on('error', reject); + logs.on('end', resolve); + }); +} + +export function displayBuildLog(log: BuildLog, logger: Logger): void { + const toPrint = `${getServiceColourFn(log.serviceName)( + `[${log.serviceName}]`, + )} ${log.message}`; + logger.logBuild(toPrint); +} + +// mutates serviceColours +function displayLogLine(log: string | Buffer, logger: Logger): void { + try { + const obj: Log = JSON.parse(log.toString()); + + let toPrint: string; + if (obj.timestamp != null) { + toPrint = `[${new Date(obj.timestamp).toLocaleString()}]`; + } else { + toPrint = `[${new Date().toLocaleString()}]`; + } + + if (obj.serviceName != null) { + const colourFn = getServiceColourFn(obj.serviceName); + + toPrint += ` ${colourFn(`[${obj.serviceName}]`)}`; + } + + toPrint += ` ${obj.message}`; + + logger.logLogs(toPrint); + } catch (e) { + logger.logDebug(`Dropping device log due to failed parsing: ${e}`); + } +} + +const getServiceColourFn = _.memoize(_getServiceColourFn); + +const colorHash = new ColorHash(); +function _getServiceColourFn(serviceName: string): (msg: string) => string { + const [r, g, b] = colorHash.rgb(serviceName); + + return chalk.rgb(r, g, b); +} diff --git a/lib/utils/logger.ts b/lib/utils/logger.ts index d86e0545..816e4087 100644 --- a/lib/utils/logger.ts +++ b/lib/utils/logger.ts @@ -1,6 +1,6 @@ -import { EOL as eol } from 'os'; -import _ = require('lodash'); import chalk from 'chalk'; +import _ = require('lodash'); +import { EOL as eol } from 'os'; import { StreamLogger } from 'resin-stream-logger'; class Logger { diff --git a/lib/utils/patterns.ts b/lib/utils/patterns.ts index 5960ae9b..b5fb7aa5 100644 --- a/lib/utils/patterns.ts +++ b/lib/utils/patterns.ts @@ -229,7 +229,9 @@ export function inferOrSelectDevice(preferredUuid: string) { throw new Error("You don't have any devices online"); } - const defaultUuid = _.map(onlineDevices, 'uuid').includes(preferredUuid) + const defaultUuid = _(onlineDevices) + .map('uuid') + .includes(preferredUuid) ? preferredUuid : onlineDevices[0].uuid; diff --git a/package.json b/package.json index d7317b54..2d3d7083 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "test:fast": "npm run build:fast && gulp test", "ci": "npm run test && catch-uncommitted", "watch": "gulp watch", - "prettify": "prettier --write \"{lib,tests,automation,typings}/**/*.ts\"", + "prettify": "prettier --write \"{lib,tests,automation,typings}/**/*.ts\" --config ./node_modules/resin-lint/config/.prettierrc", "lint": "resin-lint lib/ tests/ && resin-lint --typescript automation/ lib/ typings/ tests/", "prepublish": "require-npm4-to-publish", "prepublishOnly": "npm run build" @@ -80,10 +80,10 @@ "gulp-shell": "^0.5.2", "mochainon": "^2.0.0", "pkg": "^4.3.0-beta.1", - "prettier": "1.13.5", + "prettier": "^1.14.2", "publish-release": "^1.3.3", "require-npm4-to-publish": "^1.0.0", - "resin-lint": "^1.5.0", + "resin-lint": "^2.0.0", "rewire": "^3.0.2", "ts-node": "^4.0.1", "typescript": "2.8.1" @@ -104,6 +104,7 @@ "chalk": "^2.3.0", "cli-truncate": "^1.1.0", "coffeescript": "^1.12.6", + "color-hash": "^1.0.3", "columnify": "^1.5.2", "common-tags": "^1.7.2", "denymount": "^2.2.0", @@ -139,17 +140,17 @@ "raven": "^2.5.0", "reconfix": "^0.1.0", "request": "^2.81.0", - "resin-bundle-resolve": "^0.5.3", + "resin-bundle-resolve": "^0.6.0", "resin-cli-form": "^2.0.0", "resin-cli-visuals": "^1.4.0", - "resin-compose-parse": "^1.10.2", + "resin-compose-parse": "^2.0.0", "resin-config-json": "^1.0.0", "resin-device-config": "^4.0.0", "resin-device-init": "^4.0.0", "resin-doodles": "0.0.1", "resin-image-fs": "^5.0.2", "resin-image-manager": "^5.0.0", - "resin-multibuild": "^0.5.1", + "resin-multibuild": "^0.9.0", "resin-preload": "^7.0.0", "resin-release": "^1.2.0", "resin-sdk": "10.0.0-beta2", diff --git a/typings/color-hash.d.ts b/typings/color-hash.d.ts new file mode 100644 index 00000000..8acd5be0 --- /dev/null +++ b/typings/color-hash.d.ts @@ -0,0 +1,12 @@ +declare module 'color-hash' { + interface Hasher { + hex(text: string): string; + } + + class ColorHash { + hex(text: string): string; + rgb(text: string): [number, number, number]; + } + + export = ColorHash; +} diff --git a/typings/dockerfile-template.d.ts b/typings/dockerfile-template.d.ts new file mode 100644 index 00000000..15cc7623 --- /dev/null +++ b/typings/dockerfile-template.d.ts @@ -0,0 +1,13 @@ +declare module 'dockerfile-template' { + /** + * Variables which define what will be replaced, and what they will be replaced with. + */ + export interface TemplateVariables { + [key: string]: string; + } + + export function process( + content: string, + variables: TemplateVariables, + ): string; +} From fe751fdb23554d1a69184c51efa0d06990898a41 Mon Sep 17 00:00:00 2001 From: Cameron Diver Date: Tue, 16 Oct 2018 16:41:38 +0100 Subject: [PATCH 8/8] Check supervisor version before attempting to do a local push Signed-off-by: Cameron Diver --- lib/utils/device/api.ts | 18 ++++++++++++++++++ lib/utils/device/deploy.ts | 28 +++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/lib/utils/device/api.ts b/lib/utils/device/api.ts index 5006e51f..4e1e1429 100644 --- a/lib/utils/device/api.ts +++ b/lib/utils/device/api.ts @@ -24,6 +24,7 @@ const deviceEndpoints = { getDeviceInformation: 'v2/local/device-info', logs: 'v2/local/logs', ping: 'ping', + version: 'v2/version', }; export class DeviceAPI { @@ -93,6 +94,23 @@ export class DeviceAPI { ); } + public getVersion(): Promise { + const url = this.getUrlForAction('version'); + + return DeviceAPI.promisifiedRequest(request.get, { + url, + json: true, + }).then(body => { + if (body.status !== 'success') { + throw new ApiErrors.DeviceAPIError( + 'Non-successful response from supervisor version endpoint', + ); + } + + return body.version; + }); + } + public getLogStream(): Bluebird { const url = this.getUrlForAction('logs'); diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts index 8301cbe8..6aace978 100644 --- a/lib/utils/device/deploy.ts +++ b/lib/utils/device/deploy.ts @@ -3,6 +3,7 @@ import * as Docker from 'dockerode'; import * as _ from 'lodash'; import { Composition } from 'resin-compose-parse'; import { BuildTask, LocalImage } from 'resin-multibuild'; +import * as semver from 'resin-semver'; import { Readable } from 'stream'; import Logger = require('../logger'); @@ -39,9 +40,30 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise { const api = new DeviceAPI(logger, opts.deviceHost); - // TODO: Before merge, replace this with the supervisor version endpoint, to - // ensure we're working with a supervisor version that supports the stuff we need - await api.ping(); + // First check that we can access the device with a ping + try { + await api.ping(); + } catch (e) { + exitWithExpectedError( + `Could not communicate with local mode device at address ${ + opts.deviceHost + }`, + ); + } + + const versionError = new Error( + 'The supervisor version on this remote device does not support multicontainer local mode. ' + + 'Please update your device to resinOS v2.20.0 or greater from the dashboard.', + ); + + try { + const version = await api.getVersion(); + if (!semver.satisfies(version, '>=7.21.4')) { + exitWithExpectedError(versionError); + } + } catch { + exitWithExpectedError(versionError); + } logger.logInfo(`Starting build on device ${opts.deviceHost}`);