ExternalVendorCode/balena-cli
1
0
Fork 0
mirror of https://github.com/balena-io/balena-cli.git synced 2025-02-20 17:33:18 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1125 from balena-io/1010-retry-image-push-unknown-blob

Browse Source 
Handle 'unknown blob' errors and retry image pushing
This commit is contained in:
Paulo Castro 2019-03-05 15:17:55 +00:00 committed by GitHub
commit 709af3e92b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 174 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/actions/push.ts
View File

@ -194,11 +194,11 @@ export const push: CommandDefinition<
switch (buildTarget) {
case BuildTarget.Cloud:
const app = appOrDevice;
Bluebird.join(
await Bluebird.join(
sdk.auth.getToken(),
sdk.settings.get('balenaUrl'),
getAppOwner(sdk, app),
(token, baseUrl, owner) => {
async (token, baseUrl, owner) => {
const opts = {
emulated: options.emulated,
nocache: options.nocache,
@ -214,14 +214,14 @@ export const push: CommandDefinition<
opts,
};
return remote.startRemoteBuild(args);
return await remote.startRemoteBuild(args);
},
).nodeify(done);
break;
case BuildTarget.Device:
const device = appOrDevice;
// TODO: Support passing a different port
Bluebird.resolve(
await Bluebird.resolve(
deviceDeploy.deployToDevice({
source,
deviceHost: device,

9
lib/utils/compose.coffee
View File

@ -409,6 +409,7 @@ authorizePush = (sdk, logger, tokenAuthEndpoint, registry, images, previousRepos
pushAndUpdateServiceImages = (docker, token, images, afterEach) ->
chalk = require('chalk')
{ DockerProgress } = require('docker-progress')
{ retry } = require('./helpers')
tty = require('./tty')(process.stdout)
opts = { authconfig: registrytoken: token }
@ -421,7 +422,13 @@ pushAndUpdateServiceImages = (docker, token, images, afterEach) ->
Promise.map images, ({ serviceImage, localImage, props, logs }, index) ->
Promise.join(
localImage.inspect().get('Size')
progress.push(localImage.name, reporters[index], opts).finally(renderer.end)
retry(
-> progress.push(localImage.name, reporters[index], opts)
3 # `times` - retry 3 times
localImage.name # `label` included in retry log messages
2000 # `delayMs` - wait 2 seconds before the 1st retry
1.4 # `backoffScaler` - wait multiplier for each retry
).finally(renderer.end)
(size, digest) ->
serviceImage.image_size = size
serviceImage.content_hash = digest

12
lib/utils/compose_ts.ts
View File

@ -35,7 +35,8 @@ export interface RegistrySecrets {
export async function parseRegistrySecrets(
secretsFilename: string,
): Promise<RegistrySecrets> {
const { fs } = require('mz');
const { fs } = await import('mz');
const { exitWithExpectedError } = await import('../utils/patterns');
try {
let isYaml = false;
if (/.+\.ya?ml$/i.test(secretsFilename)) {
@ -50,10 +51,11 @@ export async function parseRegistrySecrets(
MultiBuild.addCanonicalDockerHubEntry(registrySecrets);
return registrySecrets;
} catch (error) {
error.message =
`Error validating registry secrets file "${secretsFilename}":\n` +
error.message;
throw error;
return exitWithExpectedError(
`Error validating registry secrets file "${secretsFilename}":\n${
error.message
}`,
);
}
}

39
lib/utils/helpers.ts
View File

@ -1,5 +1,5 @@
/*
Copyright 2016-2017 Balena
Copyright 2016-2019 Balena
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -187,3 +187,40 @@ export function getSubShellCommand(command: string) {
};
}
}
/**
* Call `func`, and if func() throws an error or returns a promise that
* eventually rejects, retry it `times` many times, each time printing a
* log message including the given `label` and the error that led to
* retrying. Wait delayMs before the first retry, multiplying the wait
* by backoffScaler for each further attempt.
* @param func: The function to call and, if needed, retry calling
* @param times: How many times to retry calling func()
* @param label: Label to include in the retry log message
* @param delayMs: How long to wait before the first retry
* @param backoffScaler: Multiplier to previous wait time
* @param count: Used "internally" for the recursive calls
*/
export function retry<T>(
func: () => T,
times: number,
label: string,
delayMs = 1000,
backoffScaler = 2,
count = 0,
): Bluebird<T> {
let promise = Bluebird.try(func);
if (count < times) {
promise = promise.catch((err: Error) => {
const delay = backoffScaler ** count * delayMs;
console.log(
`Retrying "${label}" after ${(delay / 1000).toFixed(2)}s (${count +
1} of ${times}) due to: ${err}`,
);
return Bluebird.delay(delay).then(() =>
retry(func, times, label, delayMs, backoffScaler, count + 1),
);
});
}
return promise;
}

188
lib/utils/remote-build.ts
View File

@ -13,15 +13,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import * as Bluebird from 'bluebird';
import * as JSONStream from 'JSONStream';
import * as readline from 'readline';
import * as request from 'request';
import { BalenaSDK } from 'balena-sdk';
import * as Stream from 'stream';
import { BalenaSDK } from 'balena-sdk';
import { Pack } from 'tar-stream';
import { RegistrySecrets } from 'resin-multibuild';
import { TypedError } from 'typed-error';
import { RegistrySecrets } from 'resin-multibuild';
import { exitWithExpectedError } from '../utils/patterns';
import { tarDirectory } from './compose';
const DEBUG_MODE = !!process.env.DEBUG;
@ -85,18 +87,16 @@ async function getBuilderEndpoint(
}
export async function startRemoteBuild(build: RemoteBuild): Promise<void> {
const Bluebird = await import('bluebird');
const stream = await getRequestStream(build);
const stream = await getRemoteBuildStream(build);
// Special windows handling (win64 also reports win32)
if (process.platform === 'win32') {
const readline = (await import('readline')).createInterface({
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
});
readline.on('SIGINT', () => process.emit('SIGINT'));
rl.on('SIGINT', () => process.emit('SIGINT'));
}
return new Bluebird((resolve, reject) => {
@ -126,13 +126,11 @@ export async function startRemoteBuild(build: RemoteBuild): Promise<void> {
});
}
async function handleBuilderMetadata(obj: BuilderMessage, build: RemoteBuild) {
const { stripIndent } = await import('common-tags');
function handleBuilderMetadata(obj: BuilderMessage, build: RemoteBuild) {
const { stripIndent } = require('common-tags');
switch (obj.resource) {
case 'cursor':
const readline = await import('readline');
if (obj.value == null) {
return;
}
@ -177,8 +175,8 @@ async function handleBuilderMetadata(obj: BuilderMessage, build: RemoteBuild) {
function getBuilderMessageHandler(
build: RemoteBuild,
): (obj: BuilderMessage) => Promise<void> {
return async (obj: BuilderMessage) => {
): (obj: BuilderMessage) => void {
return (obj: BuilderMessage) => {
if (DEBUG_MODE) {
console.log(`[debug] handling message: ${JSON.stringify(obj)}`);
}
@ -186,7 +184,6 @@ function getBuilderMessageHandler(
return handleBuilderMetadata(obj, build);
}
if (obj.message) {
const readline = await import('readline');
readline.clearLine(process.stdout, 0);
const message = obj.message.replace(TRIM_REGEX, '');
@ -216,72 +213,119 @@ async function cancelBuildIfNecessary(build: RemoteBuild): Promise<void> {
}
/**
* Return a callback function that takes a tar-stream Pack object as argument
* and uses it to add the '.balena/registry-secrets.json' metadata file that
* contains usernames and passwords to private docker registries. The builder
* will remove the file from the tar stream and use the secrets to pull base
* images from users' private registries.
* @param registrySecrets JS object containing registry usernames and passwords
* @returns A callback function, or undefined if registrySecrets is empty
* Call tarDirectory() with a suitable callback to insert registry secrets in
* the tar stream, and return the stream.
*/
function getTarStreamCallbackForRegistrySecrets(
registrySecrets: RegistrySecrets,
): ((pack: Pack) => void) | undefined {
if (Object.keys(registrySecrets).length > 0) {
return (pack: Pack) => {
pack.entry(
{ name: '.balena/registry-secrets.json' },
JSON.stringify(registrySecrets),
);
};
async function getTarStream(build: RemoteBuild): Promise<Stream.Readable> {
const path = await import('path');
const visuals = await import('resin-cli-visuals');
const tarSpinner = new visuals.Spinner('Packaging the project source...');
const preFinalizeCallback = (pack: Pack) => {
pack.entry(
{ name: '.balena/registry-secrets.json' },
JSON.stringify(build.opts.registrySecrets),
);
};
try {
tarSpinner.start();
return await tarDirectory(
path.resolve(build.source),
Object.keys(build.opts.registrySecrets).length > 0
? preFinalizeCallback
: undefined,
);
} finally {
tarSpinner.stop();
}
}
async function getRequestStream(build: RemoteBuild): Promise<Stream.Duplex> {
const path = await import('path');
const visuals = await import('resin-cli-visuals');
const zlib = await import('zlib');
const tarSpinner = new visuals.Spinner('Packaging the project source...');
tarSpinner.start();
// Tar the directory so that we can send it to the builder
const tarStream = await tarDirectory(
path.resolve(build.source),
getTarStreamCallbackForRegistrySecrets(build.opts.registrySecrets),
);
tarSpinner.stop();
const url = await getBuilderEndpoint(
build.baseUrl,
build.owner,
build.app,
build.opts,
);
/**
* Initiate a POST HTTP request to the remote builder and add some event
* listeners.
*
* ¡! Note: this function must be synchronous because of a bug in the `request`
* library that requires the following two steps to take place in the same
* iteration of Node's event loop: (1) adding a listener for the 'response'
* event and (2) calling request.pipe():
* https://github.com/request/request/issues/887
*/
function createRemoteBuildRequest(
build: RemoteBuild,
tarStream: Stream.Readable,
builderUrl: string,
onError: (error: Error) => void,
): request.Request {
const zlib = require('zlib');
if (DEBUG_MODE) {
console.log(`[debug] Connecting to builder at ${url}`);
console.log(`[debug] Connecting to builder at ${builderUrl}`);
}
const post = request.post({
url,
auth: {
bearer: build.auth,
},
headers: {
'Content-Encoding': 'gzip',
},
body: tarStream.pipe(
zlib.createGzip({
level: 6,
}),
),
});
return request
.post({
url: builderUrl,
auth: { bearer: build.auth },
headers: { 'Content-Encoding': 'gzip' },
body: tarStream.pipe(zlib.createGzip({ level: 6 })),
})
.on('error', onError)
.once('response', (response: request.RequestResponse) => {
if (response.statusCode >= 100 && response.statusCode < 400) {
if (DEBUG_MODE) {
console.log(
`[debug] received HTTP ${response.statusCode} ${
response.statusMessage
}`,
);
}
} else {
let msgArr = [
'Remote builder responded with HTTP error:',
`${response.statusCode} ${response.statusMessage}`,
];
if (response.body) {
msgArr.push(response.body);
}
onError(new Error(msgArr.join('\n')));
}
});
}
async function getRemoteBuildStream(
build: RemoteBuild,
): Promise<NodeJS.ReadWriteStream> {
const tarStream = await getTarStream(build);
const visuals = await import('resin-cli-visuals');
const uploadSpinner = new visuals.Spinner(
'Uploading source package to balena cloud',
);
uploadSpinner.start();
const exitOnError = (error: Error): never => {
uploadSpinner.stop();
return exitWithExpectedError(error);
};
const parseStream = post.pipe(JSONStream.parse('*'));
parseStream.on('data', () => uploadSpinner.stop());
return parseStream as Stream.Duplex;
try {
uploadSpinner.start();
const builderUrl = await getBuilderEndpoint(
build.baseUrl,
build.owner,
build.app,
build.opts,
);
const buildRequest = createRemoteBuildRequest(
build,
tarStream,
builderUrl,
exitOnError,
);
return buildRequest.pipe(
JSONStream.parse('*')
.once('close', () => uploadSpinner.stop())
.once('data', () => uploadSpinner.stop())
.once('end', () => uploadSpinner.stop())
.once('error', () => uploadSpinner.stop())
.once('finish', () => uploadSpinner.stop()),
);
} catch (error) {
return exitOnError(error);
}
}

2
package.json
View File

@ -123,7 +123,7 @@
"columnify": "^1.5.2",
"common-tags": "^1.7.2",
"denymount": "^2.2.0",
"docker-progress": "^3.0.1",
"docker-progress": "^3.0.4",
"docker-qemu-transpose": "^0.5.3",
"docker-toolbelt": "^3.3.5",
"dockerode": "^2.5.5",