Retry image push a few times (balena deploy, 'unknown blob')

Change-type: patch
Signed-off-by: Paulo Castro <paulo@balena.io>

lib/utils/compose.coffee

@@ -409,6 +409,7 @@ authorizePush = (sdk, logger, tokenAuthEndpoint, registry, images, previousRepos
 pushAndUpdateServiceImages = (docker, token, images, afterEach) ->
 	chalk = require('chalk')
 	{ DockerProgress } = require('docker-progress')
+	{ retry } = require('./helpers')
 	tty = require('./tty')(process.stdout)
 
 	opts = { authconfig: registrytoken: token }
@@ -421,7 +422,13 @@ pushAndUpdateServiceImages = (docker, token, images, afterEach) ->
 		Promise.map images, ({ serviceImage, localImage, props, logs }, index) ->
 			Promise.join(
 				localImage.inspect().get('Size')
-				progress.push(localImage.name, reporters[index], opts).finally(renderer.end)
+				retry(
+					-> progress.push(localImage.name, reporters[index], opts)
+					3  # `times` - retry 3 times
+					localImage.name  # `label` included in retry log messages
+					2000  # `delayMs` - wait 2 seconds before the 1st retry
+					1.4  # `backoffScaler` - wait multiplier for each retry
+				).finally(renderer.end)
 				(size, digest) ->
 					serviceImage.image_size = size
 					serviceImage.content_hash = digest

lib/utils/compose_ts.ts

@@ -35,7 +35,8 @@ export interface RegistrySecrets {
 export async function parseRegistrySecrets(
 	secretsFilename: string,
 ): Promise<RegistrySecrets> {
-	const { fs } = require('mz');
+	const { fs } = await import('mz');
+	const { exitWithExpectedError } = await import('../utils/patterns');
 	try {
 		let isYaml = false;
 		if (/.+\.ya?ml$/i.test(secretsFilename)) {
@@ -50,10 +51,11 @@ export async function parseRegistrySecrets(
 		MultiBuild.addCanonicalDockerHubEntry(registrySecrets);
 		return registrySecrets;
 	} catch (error) {
-		error.message =
-			`Error validating registry secrets file "${secretsFilename}":\n` +
-			error.message;
-		throw error;
+		return exitWithExpectedError(
+			`Error validating registry secrets file "${secretsFilename}":\n${
+				error.message
+			}`,
+		);
 	}
 }
 

lib/utils/helpers.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2016-2017 Balena
+Copyright 2016-2019 Balena
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -187,3 +187,40 @@ export function getSubShellCommand(command: string) {
 		};
 	}
 }
+
+/**
+ * Call `func`, and if func() throws an error or returns a promise that
+ * eventually rejects, retry it `times` many times, each time printing a
+ * log message including the given `label` and the error that led to
+ * retrying. Wait delayMs before the first retry, multiplying the wait
+ * by backoffScaler for each further attempt.
+ * @param func: The function to call and, if needed, retry calling
+ * @param times: How many times to retry calling func()
+ * @param label: Label to include in the retry log message
+ * @param delayMs: How long to wait before the first retry
+ * @param backoffScaler: Multiplier to previous wait time
+ * @param count: Used "internally" for the recursive calls
+ */
+export function retry<T>(
+	func: () => T,
+	times: number,
+	label: string,
+	delayMs = 1000,
+	backoffScaler = 2,
+	count = 0,
+): Bluebird<T> {
+	let promise = Bluebird.try(func);
+	if (count < times) {
+		promise = promise.catch((err: Error) => {
+			const delay = backoffScaler ** count * delayMs;
+			console.log(
+				`Retrying "${label}" after ${(delay / 1000).toFixed(2)}s (${count +
+					1} of ${times}) due to: ${err}`,
+			);
+			return Bluebird.delay(delay).then(() =>
+				retry(func, times, label, delayMs, backoffScaler, count + 1),
+			);
+		});
+	}
+	return promise;
+}

package.json

@@ -123,7 +123,7 @@
     "columnify": "^1.5.2",
     "common-tags": "^1.7.2",
     "denymount": "^2.2.0",
-    "docker-progress": "^3.0.1",
+    "docker-progress": "^3.0.4",
     "docker-qemu-transpose": "^0.5.3",
     "docker-toolbelt": "^3.3.5",
     "dockerode": "^2.5.5",