ExternalVendorCode/balena-cli
1
0
Fork 0
mirror of https://github.com/balena-io/balena-cli.git synced 2025-04-27 06:20:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request from resin-io/v8-meta-branch

Browse Source 
Release CLI v8
This commit is contained in:
Tim Perry 2018-10-19 17:29:40 +02:00 committed by GitHub
commit 61160fd2f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1115 additions and 73 deletions

20
doc/cli.markdown

@ -159,7 +159,7 @@ environment variable (in the same standard URL format).
- Push
- [push <application>](#push-application-)
- [push <applicationOrDevice>](#push-applicationordevice-)
- Settings
@ -1215,9 +1215,9 @@ the commit hash for a specific application release to preload, use "latest" to s
path to a png image to replace the splash screen
#### --dont-check-device-type
#### --dont-check-arch
Disables check for matching device types in image and application
Disables check for matching architecture in image and application
#### --pin-device-to-release, -p
@ -1249,19 +1249,29 @@ Docker host TLS key file
# Push
## push <application>
## push <applicationOrDevice>
This command can be used to start a build on the remote
resin.io cloud builders. The given source directory will be sent to the
resin.io cloud builders, or a local mode resin device.
When building on the resin cloud the given source directory will be sent to the
resin.io builder, and the build will proceed. This can be used as a drop-in
replacement for git push to deploy.
When building on a local mode device, the given source directory will be built on
device, and the resulting containers will be run on the device. Logs will be
streamed back from the device as part of the same invocation.
Examples:
$ resin push myApp
$ resin push myApp --source <source directory>
$ resin push myApp -s <source directory>
$ resin push 10.0.0.1
$ resin push 10.0.0.1 --source <source directory>
$ resin push 10.0.0.1 -s <source directory>
### Options
#### --source, -s &#60;source&#62;

66
lib/actions/preload.coffee

@ -18,24 +18,46 @@ dockerUtils = require('../utils/docker')
LATEST = 'latest'
allDeviceTypes = undefined
getDeviceTypes = ->
Bluebird = require('bluebird')
if allDeviceTypes != undefined
return Bluebird.resolve(allDeviceTypes)
resin = require('resin-sdk').fromSharedOptions()
resin.models.config.getDeviceTypes()
.tap (dt) ->
allDeviceTypes = dt
getDeviceTypesWithSameArch = (deviceTypeSlug) ->
_ = require('lodash')
getDeviceTypes()
.then (deviceTypes) ->
deviceType = _.find(deviceTypes, slug: deviceTypeSlug)
_(deviceTypes).filter(arch: deviceType.arch).map('slug').value()
getApplicationsWithSuccessfulBuilds = (deviceType) ->
preload = require('resin-preload')
resin = require('resin-sdk').fromSharedOptions()
resin.pine.get
resource: 'my_application'
options:
$filter:
device_type: deviceType
owns__release:
$any:
$alias: 'r'
$expr:
r:
status: 'success'
$expand: preload.applicationExpandOptions
$select: [ 'id', 'app_name', 'device_type', 'commit', 'should_track_latest_release' ]
$orderby: 'app_name asc'
getDeviceTypesWithSameArch(deviceType)
.then (deviceTypes) ->
resin.pine.get
resource: 'my_application'
options:
$filter:
device_type:
$in: deviceTypes
owns__release:
$any:
$alias: 'r'
$expr:
r:
status: 'success'
$expand: preload.applicationExpandOptions
$select: [ 'id', 'app_name', 'device_type', 'commit', 'should_track_latest_release' ]
$orderby: 'app_name asc'
selectApplication = (deviceType) ->
visuals = require('resin-cli-visuals')
@ -144,9 +166,9 @@ module.exports =
alias: 's'
}
{
signature: 'dont-check-device-type'
signature: 'dont-check-arch'
boolean: true
description: 'Disables check for matching device types in image and application'
description: 'Disables check for matching architecture in image and application'
}
{
signature: 'pin-device-to-release'
@ -191,12 +213,12 @@ module.exports =
options.splashImage = options['splash-image']
delete options['splash-image']
options.dontCheckDeviceType = options['dont-check-device-type']
delete options['dont-check-device-type']
if options.dontCheckDeviceType and not options.appId
exitWithExpectedError('You need to specify an app id if you disable the device type check.')
options.dontCheckArch = options['dont-check-arch'] || false
delete options['dont-check-arch']
if options.dontCheckArch and not options.appId
exitWithExpectedError('You need to specify an app id if you disable the architecture check.')
options.pinDevice = options['pin-device-to-release']
options.pinDevice = options['pin-device-to-release'] || false
delete options['pin-device-to-release']
# Get a configured dockerode instance
@ -211,7 +233,7 @@ module.exports =
options.image
options.splashImage
options.proxy
options.dontCheckDeviceType
options.dontCheckArch
options.pinDevice
)

126
lib/actions/push.ts

@ -15,8 +15,33 @@ limitations under the License.
*/
import { CommandDefinition } from 'capitano';
import { ResinSDK } from 'resin-sdk';
import { stripIndent } from 'common-tags';
import { ResinSDK } from 'resin-sdk';
import { BuildError } from '../utils/device/errors';
// An regex to detect an IP address, from https://www.regular-expressions.info/ip.html
const IP_REGEX = new RegExp(
/\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/,
);
enum BuildTarget {
Cloud,
Device,
}
function getBuildTarget(appOrDevice: string): BuildTarget | null {
// First try the application regex from the api
if (/^[a-zA-Z0-9_-]+$/.test(appOrDevice)) {
return BuildTarget.Cloud;
}
if (IP_REGEX.test(appOrDevice)) {
return BuildTarget.Device;
}
return null;
}
async function getAppOwner(sdk: ResinSDK, appName: string) {
const {
@ -75,7 +100,7 @@ async function getAppOwner(sdk: ResinSDK, appName: string) {
export const push: CommandDefinition<
{
application: string;
applicationOrDevice: string;
},
{
source: string;
@ -83,19 +108,30 @@ export const push: CommandDefinition<
nocache: boolean;
}
> = {
signature: 'push <application>',
description: 'Start a remote build on the resin.io cloud build servers',
signature: 'push <applicationOrDevice>',
description:
'Start a remote build on the resin.io cloud build servers or a local mode device',
help: stripIndent`
This command can be used to start a build on the remote
resin.io cloud builders. The given source directory will be sent to the
resin.io cloud builders, or a local mode resin device.
When building on the resin cloud the given source directory will be sent to the
resin.io builder, and the build will proceed. This can be used as a drop-in
replacement for git push to deploy.
When building on a local mode device, the given source directory will be built on
device, and the resulting containers will be run on the device. Logs will be
streamed back from the device as part of the same invocation.
Examples:
$ resin push myApp
$ resin push myApp --source <source directory>
$ resin push myApp -s <source directory>
$ resin push 10.0.0.1
$ resin push 10.0.0.1 --source <source directory>
$ resin push 10.0.0.1 -s <source directory>
`,
permission: 'user',
options: [
@ -123,11 +159,12 @@ export const push: CommandDefinition<
const sdk = (await import('resin-sdk')).fromSharedOptions();
const Bluebird = await import('bluebird');
const remote = await import('../utils/remote-build');
const deviceDeploy = await import('../utils/device/deploy');
const { exitWithExpectedError } = await import('../utils/patterns');
const app: string | null = params.application;
if (app == null) {
exitWithExpectedError('You must specify an application');
const appOrDevice: string | null = params.applicationOrDevice;
if (appOrDevice == null) {
exitWithExpectedError('You must specify an application or a device');
}
const source = options.source || '.';
@ -135,27 +172,58 @@ export const push: CommandDefinition<
console.log(`[debug] Using ${source} as build source`);
}
Bluebird.join(
sdk.auth.getToken(),
sdk.settings.get('resinUrl'),
getAppOwner(sdk, app),
(token, baseUrl, owner) => {
const opts = {
emulated: options.emulated,
nocache: options.nocache,
};
const args = {
app,
owner,
source,
auth: token,
baseUrl,
sdk,
opts,
};
const buildTarget = getBuildTarget(appOrDevice);
switch (buildTarget) {
case BuildTarget.Cloud:
const app = appOrDevice;
Bluebird.join(
sdk.auth.getToken(),
sdk.settings.get('resinUrl'),
getAppOwner(sdk, app),
(token, baseUrl, owner) => {
const opts = {
emulated: options.emulated,
nocache: options.nocache,
};
const args = {
app,
owner,
source,
auth: token,
baseUrl,
sdk,
opts,
};
return remote.startRemoteBuild(args);
},
).nodeify(done);
return remote.startRemoteBuild(args);
},
).nodeify(done);
break;
case BuildTarget.Device:
const device = appOrDevice;
// TODO: Support passing a different port
Bluebird.resolve(
deviceDeploy.deployToDevice({
source,
deviceHost: device,
}),
)
.catch(BuildError, e => {
exitWithExpectedError(e.toString());
})
.nodeify(done);
break;
default:
exitWithExpectedError(
stripIndent`
Build target not recognised. Please provide either an application name or device address.
The only supported device addresses currently are IP addresses.
If you believe your build target should have been detected, and this is an error, please
create an issue.`,
);
break;
}
},
};

11
lib/utils/compose.coffee

@ -34,15 +34,13 @@ exports.generateOpts = (options) ->
inlineLogs: !!options.logs
compositionFileNames = [
'resin-compose.yml'
'resin-compose.yaml'
'docker-compose.yml'
'docker-compose.yaml'
]
# look into the given directory for valid compose files and return
# the contents of the first one found.
resolveProject = (rootDir) ->
exports.resolveProject = resolveProject = (rootDir) ->
fs = require('mz/fs')
Promise.any compositionFileNames.map (filename) ->
fs.readFile(path.join(rootDir, filename), 'utf-8')
@ -108,14 +106,21 @@ exports.tarDirectory = tarDirectory = (dir) ->
path = require('path')
fs = require('mz/fs')
streamToPromise = require('stream-to-promise')
{ FileIgnorer } = require('./ignore')
getFiles = ->
streamToPromise(klaw(dir))
.filter((item) -> not item.stats.isDirectory())
.map((item) -> item.path)
ignore = new FileIgnorer(dir)
pack = tar.pack()
getFiles(dir)
.each (file) ->
type = ignore.getIgnoreFileType(path.relative(dir, file))
if type?
ignore.addIgnoreFile(file, type)
.filter(ignore.filter)
.map (file) ->
relPath = path.relative(path.resolve(dir), file)
Promise.join relPath, fs.stat(file), fs.readFile(file),

29
lib/utils/compose.d.ts vendored

@ -1,3 +1,32 @@
import * as Bluebird from 'bluebird';
import * as Stream from 'stream';
import { Composition } from 'resin-compose-parse';
import Logger = require('./logger');
interface Image {
context: string;
tag: string;
}
interface Descriptor {
image: Image | string;
serviceName: string;
}
export function resolveProject(projectRoot: string): Bluebird<string>;
export interface ComposeProject {
path: string;
name: string;
composition: Composition;
descriptors: Descriptor[];
}
export function loadProject(
logger: Logger,
projectPath: string,
projectName: string,
image?: string,
): Bluebird<ComposeProject>;
export function tarDirectory(source: string): Promise<Stream.Readable>;

188
lib/utils/device/api.ts Normal file

@ -0,0 +1,188 @@
import * as Bluebird from 'bluebird';
import * as request from 'request';
import * as Stream from 'stream';
import Logger = require('../logger');
import * as ApiErrors from './errors';
export interface DeviceResponse {
[key: string]: any;
status: 'success' | 'failed';
message?: string;
}
export interface DeviceInfo {
deviceType: string;
arch: string;
}
const deviceEndpoints = {
setTargetState: 'v2/local/target-state',
getTargetState: 'v2/local/target-state',
getDeviceInformation: 'v2/local/device-info',
logs: 'v2/local/logs',
ping: 'ping',
version: 'v2/version',
};
export class DeviceAPI {
private deviceAddress: string;
public constructor(
private logger: Logger,
addr: string,
port: number = 48484,
) {
this.deviceAddress = `http://${addr}:${port}/`;
}
// Either return nothing, or throw an error with the info
public async setTargetState(state: any): Promise<void> {
const url = this.getUrlForAction('setTargetState');
return DeviceAPI.promisifiedRequest(
request.post,
{
url,
json: true,
body: state,
},
this.logger,
);
}
public async getTargetState(): Promise<any> {
const url = this.getUrlForAction('getTargetState');
return DeviceAPI.promisifiedRequest(
request.get,
{
url,
json: true,
},
this.logger,
).then(body => {
return body.state;
});
}
public async getDeviceInformation(): Promise<DeviceInfo> {
const url = this.getUrlForAction('getDeviceInformation');
return DeviceAPI.promisifiedRequest(
request.get,
{
url,
json: true,
},
this.logger,
).then(body => {
return body.info;
});
}
public async ping(): Promise<void> {
const url = this.getUrlForAction('ping');
return DeviceAPI.promisifiedRequest(
request.get,
{
url,
},
this.logger,
);
}
public getVersion(): Promise<string> {
const url = this.getUrlForAction('version');
return DeviceAPI.promisifiedRequest(request.get, {
url,
json: true,
}).then(body => {
if (body.status !== 'success') {
throw new ApiErrors.DeviceAPIError(
'Non-successful response from supervisor version endpoint',
);
}
return body.version;
});
}
public getLogStream(): Bluebird<Stream.Readable> {
const url = this.getUrlForAction('logs');
// Don't use the promisified version here as we want to stream the output
return new Bluebird((resolve, reject) => {
const req = request.get(url);
req.on('error', reject).on('response', res => {
if (res.statusCode !== 200) {
reject(
new ApiErrors.DeviceAPIError(
'Non-200 response from log streaming endpoint',
),
);
}
resolve(res);
});
});
}
private getUrlForAction(action: keyof typeof deviceEndpoints): string {
return `${this.deviceAddress}${deviceEndpoints[action]}`;
}
// A helper method for promisifying general (non-streaming) requests. Streaming
// requests should use a seperate setup
private static async promisifiedRequest<T>(
requestMethod: (
opts: T,
cb: (err?: any, res?: any, body?: any) => void,
) => void,
opts: T,
logger?: Logger,
): Promise<any> {
const Bluebird = await import('bluebird');
const _ = await import('lodash');
type ObjectWithUrl = { url?: string };
if (logger != null) {
let url: string | null = null;
if (_.isObject(opts) && (opts as ObjectWithUrl).url != null) {
// the `as string` shouldn't be necessary, but the type system
// is getting a little confused
url = (opts as ObjectWithUrl).url as string;
} else if (_.isString(opts)) {
url = opts;
}
if (url != null) {
logger.logDebug(`Sending request to ${url}`);
}
}
return Bluebird.fromCallback(
cb => {
return requestMethod(opts, cb);
},
{ multiArgs: true },
).then(([response, body]) => {
switch (response.statusCode) {
case 200:
return body;
case 400:
throw new ApiErrors.BadRequestDeviceAPIError(body.message);
case 503:
throw new ApiErrors.ServiceUnavailableAPIError(body.message);
default:
throw new ApiErrors.DeviceAPIError(body.message);
}
});
}
}
export default DeviceAPI;

306
lib/utils/device/deploy.ts Normal file

@ -0,0 +1,306 @@
import * as Bluebird from 'bluebird';
import * as Docker from 'dockerode';
import * as _ from 'lodash';
import { Composition } from 'resin-compose-parse';
import { BuildTask, LocalImage } from 'resin-multibuild';
import * as semver from 'resin-semver';
import { Readable } from 'stream';
import Logger = require('../logger');
import { displayBuildLog } from './logs';
import { DeviceInfo } from './api';
import * as LocalPushErrors from './errors';
// Define the logger here so the debug output
// can be used everywhere
const logger = new Logger();
export interface DeviceDeployOptions {
source: string;
deviceHost: string;
devicePort?: number;
}
async function checkSource(source: string): Promise<boolean> {
const { fs } = await import('mz');
return (await fs.exists(source)) && (await fs.stat(source)).isDirectory();
}
export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
const { loadProject, tarDirectory } = await import('../compose');
const { exitWithExpectedError } = await import('../patterns');
const { DeviceAPI } = await import('./api');
const { displayDeviceLogs } = await import('./logs');
if (!(await checkSource(opts.source))) {
exitWithExpectedError(`Could not access source directory: ${opts.source}`);
}
const api = new DeviceAPI(logger, opts.deviceHost);
// First check that we can access the device with a ping
try {
await api.ping();
} catch (e) {
exitWithExpectedError(
`Could not communicate with local mode device at address ${
opts.deviceHost
}`,
);
}
const versionError = new Error(
'The supervisor version on this remote device does not support multicontainer local mode. ' +
'Please update your device to resinOS v2.20.0 or greater from the dashboard.',
);
try {
const version = await api.getVersion();
if (!semver.satisfies(version, '>=7.21.4')) {
exitWithExpectedError(versionError);
}
} catch {
exitWithExpectedError(versionError);
}
logger.logInfo(`Starting build on device ${opts.deviceHost}`);
const project = await loadProject(logger, opts.source, 'local');
// Attempt to attach to the device's docker daemon
const docker = connectToDocker(
opts.deviceHost,
opts.devicePort != null ? opts.devicePort : 2375,
);
const tarStream = await tarDirectory(opts.source);
// Try to detect the device information
const deviceInfo = await api.getDeviceInformation();
await performBuilds(
project.composition,
tarStream,
docker,
deviceInfo,
logger,
);
logger.logDebug('Setting device state...');
// Now set the target state on the device
const currentTargetState = await api.getTargetState();
const targetState = generateTargetState(
currentTargetState,
project.composition,
);
logger.logDebug(`Sending target state: ${JSON.stringify(targetState)}`);
await api.setTargetState(targetState);
// Print an empty newline to seperate the build output
// from the device output
console.log();
logger.logInfo('Streaming device logs...');
// Now all we need to do is stream back the logs
const logStream = await api.getLogStream();
await displayDeviceLogs(logStream, logger);
}
function connectToDocker(host: string, port: number): Docker {
return new Docker({
host,
port,
Promise: Bluebird as any,
});
}
export async function performBuilds(
composition: Composition,
tarStream: Readable,
docker: Docker,
deviceInfo: DeviceInfo,
logger: Logger,
): Promise<void> {
const multibuild = await import('resin-multibuild');
const buildTasks = await multibuild.splitBuildStream(composition, tarStream);
logger.logDebug('Found build tasks:');
_.each(buildTasks, task => {
let infoStr: string;
if (task.external) {
infoStr = `image pull [${task.imageName}]`;
} else {
infoStr = `build [${task.context}]`;
}
logger.logDebug(` ${task.serviceName}: ${infoStr}`);
});
logger.logDebug(
`Resolving services with [${deviceInfo.deviceType}|${deviceInfo.arch}]`,
);
await multibuild.performResolution(
buildTasks,
deviceInfo.arch,
deviceInfo.deviceType,
);
logger.logDebug('Found project types:');
_.each(buildTasks, task => {
if (!task.external) {
logger.logDebug(` ${task.serviceName}: ${task.projectType}`);
} else {
logger.logDebug(` ${task.serviceName}: External image`);
}
});
logger.logDebug('Probing remote daemon for cache images');
await assignDockerBuildOpts(docker, buildTasks);
logger.logDebug('Starting builds...');
await assignOutputHandlers(buildTasks, logger);
const localImages = await multibuild.performBuilds(buildTasks, docker);
// Check for failures
await inspectBuildResults(localImages);
// Now tag any external images with the correct name that they should be,
// as this won't be done by resin-multibuild
await Bluebird.map(localImages, async localImage => {
if (localImage.external) {
// We can be sure that localImage.name is set here, because of the failure code above
const image = docker.getImage(localImage.name!);
await image.tag({
repo: generateImageName(localImage.serviceName),
force: true,
});
await image.remove({ force: true });
}
});
}
function assignOutputHandlers(buildTasks: BuildTask[], logger: Logger) {
_.each(buildTasks, task => {
if (task.external) {
task.progressHook = progressObj => {
displayBuildLog(
{ serviceName: task.serviceName, message: progressObj.progress },
logger,
);
};
} else {
task.streamHook = stream => {
stream.on('data', (buf: Buffer) => {
const str = buf.toString().trimRight();
if (str !== '') {
displayBuildLog(
{ serviceName: task.serviceName, message: str },
logger,
);
}
});
};
}
});
}
async function getDeviceDockerImages(docker: Docker): Promise<string[]> {
const images = await docker.listImages();
return _.map(images, 'Id');
}
// Mutates buildTasks
async function assignDockerBuildOpts(
docker: Docker,
buildTasks: BuildTask[],
): Promise<void> {
// Get all of the images on the remote docker daemon, so
// that we can use all of them for cache
const images = await getDeviceDockerImages(docker);
logger.logDebug(`Using ${images.length} on-device images for cache...`);
_.each(buildTasks, (task: BuildTask) => {
task.dockerOpts = {
cachefrom: images,
labels: {
'io.resin.local.image': '1',
'io.resin.local.service': task.serviceName,
},
t: generateImageName(task.serviceName),
};
});
}
function generateImageName(serviceName: string): string {
return `local_image_${serviceName}:latest`;
}
function generateTargetState(
currentTargetState: any,
composition: Composition,
): any {
const services: { [serviceId: string]: any } = {};
let idx = 1;
_.each(composition.services, (opts, name) => {
// Get rid of any build specific stuff
opts = _.cloneDeep(opts);
delete opts.build;
delete opts.image;
const defaults = {
environment: {},
labels: {},
};
services[idx] = _.merge(defaults, opts, {
imageId: idx,
serviceName: name,
serviceId: idx,
image: generateImageName(name),
running: true,
});
idx += 1;
});
const targetState = _.cloneDeep(currentTargetState);
delete targetState.local.apps;
targetState.local.apps = {
1: {
name: 'localapp',
commit: 'localcommit',
releaseId: '1',
services,
volumes: composition.volumes || {},
networks: composition.networks || {},
},
};
return targetState;
}
async function inspectBuildResults(images: LocalImage[]): Promise<void> {
const { exitWithExpectedError } = await import('../patterns');
const failures: LocalPushErrors.BuildFailure[] = [];
_.each(images, image => {
if (!image.successful) {
failures.push({
error: image.error!,
serviceName: image.serviceName,
});
}
});
if (failures.length > 0) {
exitWithExpectedError(new LocalPushErrors.BuildError(failures));
}
}

30
lib/utils/device/errors.ts Normal file

@ -0,0 +1,30 @@
import * as _ from 'lodash';
import { TypedError } from 'typed-error';
export interface BuildFailure {
error: Error;
serviceName: string;
}
export class BuildError extends TypedError {
private failures: BuildFailure[];
public constructor(failures: BuildFailure[]) {
super('Build error');
this.failures = failures;
}
public toString(): string {
let str = 'Some services failed to build:\n';
_.each(this.failures, failure => {
str += `\t${failure.serviceName}: ${failure.error.message}\n`;
});
return str;
}
}
export class DeviceAPIError extends TypedError {}
export class BadRequestDeviceAPIError extends DeviceAPIError {}
export class ServiceUnavailableAPIError extends DeviceAPIError {}

83
lib/utils/device/logs.ts Normal file

@ -0,0 +1,83 @@
import * as Bluebird from 'bluebird';
import chalk from 'chalk';
import ColorHash = require('color-hash');
import * as _ from 'lodash';
import { Readable } from 'stream';
import Logger = require('../logger');
interface Log {
message: string;
timestamp?: number;
serviceName?: string;
// There's also a serviceId and imageId, but they're
// meaningless in local mode
}
interface BuildLog {
serviceName: string;
message: string;
}
/**
* Display logs from a device logging stream. This function will return
* when the log stream ends.
*
* @param logs A stream which produces newline seperated log objects
*/
export function displayDeviceLogs(
logs: Readable,
logger: Logger,
): Bluebird<void> {
return new Bluebird((resolve, reject) => {
logs.on('data', log => {
displayLogLine(log, logger);
});
logs.on('error', reject);
logs.on('end', resolve);
});
}
export function displayBuildLog(log: BuildLog, logger: Logger): void {
const toPrint = `${getServiceColourFn(log.serviceName)(
`[${log.serviceName}]`,
)} ${log.message}`;
logger.logBuild(toPrint);
}
// mutates serviceColours
function displayLogLine(log: string | Buffer, logger: Logger): void {
try {
const obj: Log = JSON.parse(log.toString());
let toPrint: string;
if (obj.timestamp != null) {
toPrint = `[${new Date(obj.timestamp).toLocaleString()}]`;
} else {
toPrint = `[${new Date().toLocaleString()}]`;
}
if (obj.serviceName != null) {
const colourFn = getServiceColourFn(obj.serviceName);
toPrint += ` ${colourFn(`[${obj.serviceName}]`)}`;
}
toPrint += ` ${obj.message}`;
logger.logLogs(toPrint);
} catch (e) {
logger.logDebug(`Dropping device log due to failed parsing: ${e}`);
}
}
const getServiceColourFn = _.memoize(_getServiceColourFn);
const colorHash = new ColorHash();
function _getServiceColourFn(serviceName: string): (msg: string) => string {
const [r, g, b] = colorHash.rgb(serviceName);
return chalk.rgb(r, g, b);
}

149
lib/utils/ignore.ts Normal file

@ -0,0 +1,149 @@
import * as _ from 'lodash';
import { fs } from 'mz';
import * as path from 'path';
import dockerIgnore = require('@zeit/dockerignore');
import ignore from 'ignore';
export enum IgnoreFileType {
DockerIgnore,
GitIgnore,
}
interface IgnoreEntry {
pattern: string;
// The relative file path from the base path of the build context
filePath: string;
}
export class FileIgnorer {
private dockerIgnoreEntries: IgnoreEntry[];
private gitIgnoreEntries: IgnoreEntry[];
private static ignoreFiles: Array<{
pattern: string;
type: IgnoreFileType;
allowSubdirs: boolean;
}> = [
{
pattern: '.gitignore',
type: IgnoreFileType.GitIgnore,
allowSubdirs: true,
},
{
pattern: '.dockerignore',
type: IgnoreFileType.DockerIgnore,
allowSubdirs: false,
},
];
public constructor(public basePath: string) {
this.dockerIgnoreEntries = [];
this.gitIgnoreEntries = [];
}
/**
* @param {string} relativePath
* The relative pathname from the build context, for example a root level .gitignore should be
* ./.gitignore
* @returns IgnoreFileType
* The type of ignore file, or null
*/
public getIgnoreFileType(relativePath: string): IgnoreFileType | null {
for (const { pattern, type, allowSubdirs } of FileIgnorer.ignoreFiles) {
if (
path.basename(relativePath) === pattern &&
(allowSubdirs || path.dirname(relativePath) === '.')
) {
return type;
}
}
return null;
}
/**
* @param {string} fullPath
* The full path on disk of the ignore file
* @param {IgnoreFileType} type
* @returns Promise
*/
public async addIgnoreFile(
fullPath: string,
type: IgnoreFileType,
): Promise<void> {
const contents = await fs.readFile(fullPath, 'utf8');
contents.split('\n').forEach(line => {
// ignore empty lines and comments
if (/\s*#/.test(line) || _.isEmpty(line)) {
return;
}
this.addEntry(line, fullPath, type);
});
return;
}
// Pass this function as a predicate to a filter function, and it will filter
// any ignored files
public filter = (filename: string): boolean => {
const dockerIgnoreHandle = dockerIgnore();
const gitIgnoreHandle = ignore();
interface IgnoreHandle {
add: (pattern: string) => void;
ignores: (file: string) => boolean;
}
const ignoreTypes: Array<{
handle: IgnoreHandle;
entries: IgnoreEntry[];
}> = [
{ handle: dockerIgnoreHandle, entries: this.dockerIgnoreEntries },
{ handle: gitIgnoreHandle, entries: this.gitIgnoreEntries },
];
const relFile = path.relative(this.basePath, filename);
_.each(ignoreTypes, ({ handle, entries }) => {
_.each(entries, ({ pattern, filePath }) => {
if (FileIgnorer.contains(path.posix.dirname(filePath), filename)) {
handle.add(pattern);
}
});
});
return !_.some(ignoreTypes, ({ handle }) => handle.ignores(relFile));
};
private addEntry(
pattern: string,
filePath: string,
type: IgnoreFileType,
): void {
const entry: IgnoreEntry = { pattern, filePath };
switch (type) {
case IgnoreFileType.DockerIgnore:
this.dockerIgnoreEntries.push(entry);
break;
case IgnoreFileType.GitIgnore:
this.gitIgnoreEntries.push(entry);
break;
}
}
/**
* Given two paths, check whether the first contains the second
* @param path1 The potentially containing path
* @param path2 The potentially contained path
* @return A boolean indicating whether `path1` contains `path2`
*/
private static contains(path1: string, path2: string): boolean {
// First normalise the input, to remove any path weirdness
path1 = path.posix.normalize(path1);
path2 = path.posix.normalize(path2);
// Now test if the start of the relative path contains ../ ,
// which would tell us that path1 is not part of path2
return !/^\.\.\//.test(path.posix.relative(path1, path2));
}
}

15
lib/utils/logger.ts

@ -1,6 +1,6 @@
import { EOL as eol } from 'os';
import _ = require('lodash');
import chalk from 'chalk';
import _ = require('lodash');
import { EOL as eol } from 'os';
import { StreamLogger } from 'resin-stream-logger';
class Logger {
@ -11,6 +11,7 @@ class Logger {
success: NodeJS.ReadWriteStream;
warn: NodeJS.ReadWriteStream;
error: NodeJS.ReadWriteStream;
logs: NodeJS.ReadWriteStream;
};
public formatMessage: (name: string, message: string) => string;
@ -23,6 +24,7 @@ class Logger {
logger.addPrefix('success', chalk.green('[Success]'));
logger.addPrefix('warn', chalk.yellow('[Warn]'));
logger.addPrefix('error', chalk.red('[Error]'));
logger.addPrefix('logs', chalk.green('[Logs]'));
this.streams = {
build: logger.createLogStream('build'),
@ -31,6 +33,7 @@ class Logger {
success: logger.createLogStream('success'),
warn: logger.createLogStream('warn'),
error: logger.createLogStream('error'),
logs: logger.createLogStream('logs'),
};
_.forEach(this.streams, function(stream, key) {
@ -61,6 +64,14 @@ class Logger {
logError(msg: string) {
return this.streams.error.write(msg + eol);
}
logBuild(msg: string) {
return this.streams.build.write(msg + eol);
}
logLogs(msg: string) {
return this.streams.logs.write(msg + eol);
}
}
export = Logger;

4
lib/utils/patterns.ts

@ -229,7 +229,9 @@ export function inferOrSelectDevice(preferredUuid: string) {
throw new Error("You don't have any devices online");
}
const defaultUuid = _.map(onlineDevices, 'uuid').includes(preferredUuid)
const defaultUuid = _(onlineDevices)
.map('uuid')
.includes(preferredUuid)
? preferredUuid
: onlineDevices[0].uuid;

29
lib/utils/remote-build.ts

@ -18,6 +18,7 @@ import * as JSONStream from 'JSONStream';
import * as request from 'request';
import { ResinSDK } from 'resin-sdk';
import * as Stream from 'stream';
import { TypedError } from 'typed-error';
import { tarDirectory } from './compose';
@ -43,17 +44,25 @@ export interface RemoteBuild {
// For internal use
releaseId?: number;
hadError?: boolean;
}
interface BuilderMessage {
message: string;
type?: string;
replace?: boolean;
isError?: boolean;
// These will be set when the type === 'metadata'
resource?: string;
value?: string;
}
export class RemoteBuildFailedError extends TypedError {
public constructor(message = 'Remote build failed') {
super(message);
}
}
async function getBuilderEndpoint(
baseUrl: string,
owner: string,
@ -105,7 +114,11 @@ export async function startRemoteBuild(build: RemoteBuild): Promise<void> {
stream.on('data', getBuilderMessageHandler(build));
stream.on('end', resolve);
stream.on('error', reject);
}).return();
}).then(() => {
if (build.hadError) {
throw new RemoteBuildFailedError();
}
});
}
async function handleBuilderMetadata(obj: BuilderMessage, build: RemoteBuild) {
@ -178,6 +191,9 @@ function getBuilderMessageHandler(
process.stdout.write(`\r${message}\n`);
}
}
if (obj.isError) {
build.hadError = true;
}
};
}
@ -197,6 +213,7 @@ async function cancelBuildIfNecessary(build: RemoteBuild): Promise<void> {
async function getRequestStream(build: RemoteBuild): Promise<Stream.Duplex> {
const path = await import('path');
const visuals = await import('resin-cli-visuals');
const zlib = await import('zlib');
const tarSpinner = new visuals.Spinner('Packaging the project source...');
tarSpinner.start();
@ -217,6 +234,14 @@ async function getRequestStream(build: RemoteBuild): Promise<Stream.Duplex> {
auth: {
bearer: build.auth,
},
headers: {
'Content-Encoding': 'gzip',
},
body: tarStream.pipe(
zlib.createGzip({
level: 6,
}),
),
});
const uploadSpinner = new visuals.Spinner(
@ -224,8 +249,6 @@ async function getRequestStream(build: RemoteBuild): Promise<Stream.Duplex> {
);
uploadSpinner.start();
tarStream.pipe(post);
const parseStream = post.pipe(JSONStream.parse('*'));
parseStream.on('data', () => uploadSpinner.stop());
return parseStream as Stream.Duplex;

19
package.json

@ -40,9 +40,10 @@
"release": "npm run build && ts-node --type-check -P automation automation/deploy-bin.ts",
"pretest": "npm run build",
"test": "gulp test",
"test:fast": "npm run build:fast && gulp test",
"ci": "npm run test && catch-uncommitted",
"watch": "gulp watch",
"prettify": "prettier --write \"{lib,tests,automation,typings}/**/*.ts\"",
"prettify": "prettier --write \"{lib,tests,automation,typings}/**/*.ts\" --config ./node_modules/resin-lint/config/.prettierrc",
"lint": "resin-lint lib/ tests/ && resin-lint --typescript automation/ lib/ typings/ tests/",
"prepublish": "require-npm4-to-publish",
"prepublishOnly": "npm run build"
@ -79,10 +80,10 @@
"gulp-shell": "^0.5.2",
"mochainon": "^2.0.0",
"pkg": "^4.3.0-beta.1",
"prettier": "1.13.5",
"prettier": "^1.14.2",
"publish-release": "^1.3.3",
"require-npm4-to-publish": "^1.0.0",
"resin-lint": "^1.5.0",
"resin-lint": "^2.0.0",
"rewire": "^3.0.2",
"ts-node": "^4.0.1",
"typescript": "2.8.1"
@ -91,6 +92,7 @@
"@resin.io/valid-email": "^0.1.0",
"@types/stream-to-promise": "2.2.0",
"@types/through2": "^2.0.33",
"@zeit/dockerignore": "0.0.1",
"JSONStream": "^1.0.3",
"ansi-escapes": "^2.0.0",
"any-promise": "^1.3.0",
@ -102,6 +104,7 @@
"chalk": "^2.3.0",
"cli-truncate": "^1.1.0",
"coffeescript": "^1.12.6",
"color-hash": "^1.0.3",
"columnify": "^1.5.2",
"common-tags": "^1.7.2",
"denymount": "^2.2.0",
@ -118,11 +121,13 @@
"global-tunnel-ng": "^2.1.1",
"hasbin": "^1.2.3",
"humanize": "0.0.9",
"ignore": "^5.0.2",
"inquirer": "^3.1.1",
"is-root": "^1.0.0",
"js-yaml": "^3.10.0",
"klaw": "^3.0.0",
"lodash": "^4.17.4",
"minimatch": "^3.0.4",
"mixpanel": "^0.4.0",
"mkdirp": "^0.5.1",
"moment": "^2.20.1",
@ -135,18 +140,18 @@
"raven": "^2.5.0",
"reconfix": "^0.1.0",
"request": "^2.81.0",
"resin-bundle-resolve": "^0.5.3",
"resin-bundle-resolve": "^0.6.0",
"resin-cli-form": "^2.0.0",
"resin-cli-visuals": "^1.4.0",
"resin-compose-parse": "^1.10.2",
"resin-compose-parse": "^2.0.0",
"resin-config-json": "^1.0.0",
"resin-device-config": "^4.0.0",
"resin-device-init": "^4.0.0",
"resin-doodles": "0.0.1",
"resin-image-fs": "^5.0.2",
"resin-image-manager": "^5.0.0",
"resin-multibuild": "^0.5.1",
"resin-preload": "^6.3.0",
"resin-multibuild": "^0.9.0",
"resin-preload": "^7.0.0",
"resin-release": "^1.2.0",
"resin-sdk": "10.0.0-beta2",
"resin-sdk-preconfigured": "^6.9.0",

85
tests/utils/ignore.spec.coffee Normal file

@ -0,0 +1,85 @@
require 'mocha'
chai = require 'chai'
_ = require 'lodash'
path = require('path')
expect = chai.expect
{ FileIgnorer, IgnoreFileType } = require '../../build/utils/ignore'
describe 'File ignorer', ->
it 'should detect ignore files', ->
f = new FileIgnorer('.' + path.sep)
expect(f.getIgnoreFileType('.gitignore')).to.equal(IgnoreFileType.GitIgnore)
expect(f.getIgnoreFileType('.dockerignore')).to.equal(IgnoreFileType.DockerIgnore)
expect(f.getIgnoreFileType('./.gitignore')).to.equal(IgnoreFileType.GitIgnore)
expect(f.getIgnoreFileType('./.dockerignore')).to.equal(IgnoreFileType.DockerIgnore)
# gitignore files can appear in subdirectories, but dockerignore files cannot
expect(f.getIgnoreFileType('./subdir/.gitignore')).to.equal(IgnoreFileType.GitIgnore)
expect(f.getIgnoreFileType('./subdir/.dockerignore')).to.equal(null)
expect(f.getIgnoreFileType('./subdir/subdir2/.gitignore')).to.equal(IgnoreFileType.GitIgnore)
expect(f.getIgnoreFileType('file')).to.equal(null)
expect(f.getIgnoreFileType('./file')).to.equal(null)
it 'should filter files from the root directory', ->
ignore = new FileIgnorer('.' + path.sep)
ignore.gitIgnoreEntries = [
{ pattern: '*.ignore', filePath: '.gitignore' }
]
ignore.dockerIgnoreEntries = [
{ pattern: '*.ignore2', filePath: '.dockerignore' }
]
files = [
'a'
'a/b'
'a/b/c'
'file.ignore'
'file2.ignore'
'file.ignore2'
'file2.ignore'
]
expect(_.filter(files, ignore.filter.bind(ignore))).to.deep.equal([
'a'
'a/b'
'a/b/c'
])
it 'should filter files from subdirectories', ->
ignore = new FileIgnorer('.' + path.sep)
ignore.gitIgnoreEntries = [
{ pattern: '*.ignore', filePath: 'lib/.gitignore' }
]
files = [
'test.ignore'
'root.ignore'
'lib/normal-file'
'lib/should.ignore'
'lib/thistoo.ignore'
]
expect(_.filter(files, ignore.filter.bind(ignore))).to.deep.equal([
'test.ignore'
'root.ignore'
'lib/normal-file'
])
ignore.gitIgnoreEntries = [
{ pattern: '*.ignore', filePath: './lib/.gitignore' }
]
files = [
'test.ignore'
'root.ignore'
'lib/normal-file'
'lib/should.ignore'
'lib/thistoo.ignore'
]
expect(_.filter(files, ignore.filter.bind(ignore))).to.deep.equal([
'test.ignore'
'root.ignore'
'lib/normal-file'
])

3
tsconfig.json

@ -1,7 +1,7 @@
{
"compilerOptions": {
"module": "commonjs",
"target": "es5",
"target": "es6",
"outDir": "build",
"strict": true,
"strictPropertyInitialization": false,
@ -10,6 +10,7 @@
"preserveConstEnums": true,
"removeComments": true,
"sourceMap": true,
"skipLibCheck": true,
"lib": [
// es5 defaults:
"dom",

12
typings/color-hash.d.ts vendored Normal file

@ -0,0 +1,12 @@
declare module 'color-hash' {
interface Hasher {
hex(text: string): string;
}
class ColorHash {
hex(text: string): string;
rgb(text: string): [number, number, number];
}
export = ColorHash;
}

13
typings/dockerfile-template.d.ts vendored Normal file

@ -0,0 +1,13 @@
declare module 'dockerfile-template' {
/**
* Variables which define what will be replaced, and what they will be replaced with.
*/
export interface TemplateVariables {
[key: string]: string;
}
export function process(
content: string,
variables: TemplateVariables,
): string;
}