Merge pull request #1888 from balena-os/lift-report-throttle

Move report throttle out of reporting logic

package-lock.json

@@ -373,7 +373,6 @@
       "version": "1.8.3",
       "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
       "integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==",
-      "dev": true,
       "requires": {
         "type-detect": "4.0.8"
       }
@@ -382,16 +381,14 @@
       "version": "7.0.5",
       "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-7.0.5.tgz",
       "integrity": "sha512-fUt6b15bjV/VW93UP5opNXJxdwZSbK1EdiwnhN7XrQrcpaOhMJpZ/CjwFpM3THpxwA+YviBUJKSuEqKlCK5alw==",
-      "dev": true,
       "requires": {
         "@sinonjs/commons": "^1.7.0"
       }
     },
     "@sinonjs/samsam": {
-      "version": "5.3.1",
+      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.3.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-6.1.1.tgz",
-      "integrity": "sha512-1Hc0b1TtyfBu8ixF/tpfSHTVWKwCBLY4QJbkgnE7HcwyvT2xArDxb4K7dMgqRm3szI+LJbzmW/s4xxEhv6hwDg==",
+      "integrity": "sha512-cZ7rKJTLiE7u7Wi/v9Hc2fs3Ucc3jrWeMgPHbbTCeVAB2S0wOBbYlkJVeNSL04i7fdhT8wIbDq1zhC/PXTD2SA==",
-      "dev": true,
       "requires": {
         "@sinonjs/commons": "^1.6.0",
         "lodash.get": "^4.4.2",
@@ -401,8 +398,7 @@
     "@sinonjs/text-encoding": {
       "version": "0.7.1",
       "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz",
-      "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
+      "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ=="
-      "dev": true
     },
     "@szmarczak/http-timer": {
       "version": "1.1.2",
@@ -777,12 +773,12 @@
       "dev": true
     },
     "@types/sinon": {
-      "version": "10.0.0",
+      "version": "10.0.11",
-      "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.11.tgz",
-      "integrity": "sha512-jDZ55oCKxqlDmoTBBbBBEx+N8ZraUVhggMZ9T5t+6/Dh8/4NiOjSUfpLrPiEwxQDlAe3wpAkoXhWvE6LibtsMQ==",
+      "integrity": "sha512-dmZsHlBsKUtBpHriNjlK0ndlvEh8dcb9uV9Afsbt89QIyydpC7NcR+nWlAhASfy3GHnxTl4FX/aKE7XZUt/B4g==",
       "dev": true,
       "requires": {
-        "@sinonjs/fake-timers": "^7.0.4"
+        "@types/sinonjs__fake-timers": "*"
       }
     },
     "@types/sinon-chai": {
@@ -795,6 +791,12 @@
         "@types/sinon": "*"
       }
     },
+    "@types/sinonjs__fake-timers": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz",
+      "integrity": "sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==",
+      "dev": true
+    },
     "@types/source-list-map": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz",
@@ -3313,8 +3315,7 @@
     "diff": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz",
-      "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==",
+      "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w=="
-      "dev": true
     },
     "diffie-hellman": {
       "version": "5.0.3",
@@ -6230,8 +6231,7 @@
     "just-extend": {
       "version": "4.2.1",
       "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
-      "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
+      "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg=="
-      "dev": true
     },
     "keyv": {
       "version": "3.1.0",
@@ -6634,8 +6634,7 @@
     "lodash.get": {
       "version": "4.4.2",
       "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
-      "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=",
+      "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
-      "dev": true
     },
     "log-symbols": {
       "version": "4.0.0",
@@ -7620,38 +7619,26 @@
       "dev": true
     },
     "nise": {
-      "version": "4.1.0",
+      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/nise/-/nise-4.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.1.tgz",
-      "integrity": "sha512-eQMEmGN/8arp0xsvGoQ+B1qvSkR73B1nWSCh7nOt5neMCtwcQVYQGdzQMhcNscktTsWB54xnlSQFzOAPJD8nXA==",
+      "integrity": "sha512-yr5kW2THW1AkxVmCnKEh4nbYkJdB3I7LUkiUgOvEkOp414mc2UMaHMA7pjq1nYowhdoJZGwEKGaQVbxfpWj10A==",
-      "dev": true,
       "requires": {
-        "@sinonjs/commons": "^1.7.0",
+        "@sinonjs/commons": "^1.8.3",
-        "@sinonjs/fake-timers": "^6.0.0",
+        "@sinonjs/fake-timers": ">=5",
         "@sinonjs/text-encoding": "^0.7.1",
         "just-extend": "^4.0.2",
         "path-to-regexp": "^1.7.0"
       },
       "dependencies": {
-        "@sinonjs/fake-timers": {
-          "version": "6.0.1",
-          "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz",
-          "integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==",
-          "dev": true,
-          "requires": {
-            "@sinonjs/commons": "^1.7.0"
-          }
-        },
         "isarray": {
           "version": "0.0.1",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
+          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
-          "dev": true
         },
         "path-to-regexp": {
           "version": "1.8.0",
           "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
           "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
-          "dev": true,
           "requires": {
             "isarray": "0.0.1"
           }
@@ -9280,45 +9267,35 @@
       }
     },
     "sinon": {
-      "version": "10.0.0",
+      "version": "11.1.2",
-      "resolved": "https://registry.npmjs.org/sinon/-/sinon-10.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/sinon/-/sinon-11.1.2.tgz",
-      "integrity": "sha512-XAn5DxtGVJBlBWYrcYKEhWCz7FLwZGdyvANRyK06419hyEpdT0dMc5A8Vcxg5SCGHc40CsqoKsc1bt1CbJPfNw==",
+      "integrity": "sha512-59237HChms4kg7/sXhiRcUzdSkKuydDeTiamT/jesUVHshBgL8XAmhgFo0GfK6RruMDM/iRSij1EybmMog9cJw==",
-      "dev": true,
       "requires": {
-        "@sinonjs/commons": "^1.8.1",
+        "@sinonjs/commons": "^1.8.3",
-        "@sinonjs/fake-timers": "^6.0.1",
+        "@sinonjs/fake-timers": "^7.1.2",
-        "@sinonjs/samsam": "^5.3.1",
+        "@sinonjs/samsam": "^6.0.2",
-        "diff": "^4.0.2",
+        "diff": "^5.0.0",
-        "nise": "^4.1.0",
+        "nise": "^5.1.0",
-        "supports-color": "^7.1.0"
+        "supports-color": "^7.2.0"
       },
       "dependencies": {
         "@sinonjs/fake-timers": {
-          "version": "6.0.1",
+          "version": "7.1.2",
-          "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz",
+          "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-7.1.2.tgz",
-          "integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==",
+          "integrity": "sha512-iQADsW4LBMISqZ6Ci1dupJL9pprqwcVFTcOsEmQOEhW+KLCVn/Y4Jrvg2k19fIHCp+iFprriYPTdRcQR8NbUPg==",
-          "dev": true,
           "requires": {
             "@sinonjs/commons": "^1.7.0"
           }
         },
-        "diff": {
-          "version": "4.0.2",
-          "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
-          "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
-          "dev": true
-        },
         "has-flag": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
-          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
-          "dev": true
         },
         "supports-color": {
           "version": "7.2.0",
           "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
           "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-          "dev": true,
           "requires": {
             "has-flag": "^4.0.0"
           }
@@ -10579,8 +10556,7 @@
     "type-detect": {
       "version": "4.0.8",
       "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
-      "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
+      "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g=="
-      "dev": true
     },
     "type-fest": {
       "version": "0.11.0",

package.json

@@ -32,6 +32,7 @@
     "dbus": "^1.0.7",
     "mdns-resolver": "^1.0.0",
     "semver": "^7.3.2",
+    "sinon": "^11.1.2",
     "sqlite3": "^4.1.1",
     "systeminformation": "^5.6.10"
   },
@@ -66,7 +67,7 @@
     "@types/rwlock": "^5.0.2",
     "@types/semver": "^7.3.3",
     "@types/shell-quote": "^1.7.0",
-    "@types/sinon": "^10.0.0",
+    "@types/sinon": "^10.0.11",
     "@types/sinon-chai": "^3.2.5",
     "@types/supertest": "^2.0.11",
     "@types/terser-webpack-plugin": "^3.0.0",
@@ -117,7 +118,6 @@
     "rimraf": "^2.7.1",
     "rwlock": "^5.0.0",
     "shell-quote": "^1.7.2",
-    "sinon": "^10.0.0",
     "sinon-chai": "^3.6.0",
     "strict-event-emitter-types": "^2.0.0",
     "supertest": "^6.1.3",

src/device-state/current-state.ts

@@ -3,6 +3,7 @@ import * as url from 'url';
 import { CoreOptions } from 'request';
 
 import * as constants from '../lib/constants';
+import { withBackoff, OnFailureInfo } from '../lib/backoff';
 import { log } from '../lib/supervisor-console';
 import { InternalInconsistencyError, StatusError } from '../lib/errors';
 import { getRequestInstance } from '../lib/request';
@@ -14,9 +15,6 @@ import { SchemaTypeKey, SchemaReturn } from '../config/schema-type';
 import * as eventTracker from '../event-tracker';
 import * as deviceState from '../device-state';
 
-// The exponential backoff starts at 15s
-const MINIMUM_BACKOFF_DELAY = 15000;
-
 const INTERNAL_STATE_KEYS = [
 	'update_pending',
 	'update_downloaded',
@@ -28,10 +26,6 @@ const lastReportedState: DeviceStatus = {
 	local: {},
 	dependent: {},
 };
-const stateForReport: DeviceStatus = {
-	local: {},
-	dependent: {},
-};
 let reportPending = false;
 
 type CurrentStateReportConf = {
@@ -43,6 +37,7 @@ type CurrentStateReportConf = {
 		| 'deviceApiKey'
 		| 'deviceId'
 		| 'localMode'
+		| 'appUpdatePollInterval'
 		| 'hardwareMetrics'
 	>]: SchemaReturn<key>;
 };
@@ -64,7 +59,7 @@ const stripDeviceStateInLocalMode = (state: DeviceStatus): DeviceStatus => {
 	};
 };
 
-const report = async ({ stateDiff, conf }: StateReport) => {
+async function report({ stateDiff, conf }: StateReport): Promise<boolean> {
 	let body = stateDiff;
 	const { apiEndpoint, apiTimeout, deviceApiKey, localMode, uuid } = conf;
 	if (localMode) {
@@ -73,7 +68,7 @@ const report = async ({ stateDiff, conf }: StateReport) => {
 
 	if (_.isEmpty(body.local)) {
 		// Nothing to send.
-		return;
+		return false;
 	}
 
 	if (conf.uuid == null || conf.apiEndpoint == null) {
@@ -104,9 +99,11 @@ const report = async ({ stateDiff, conf }: StateReport) => {
 			headers['retry-after'] ? parseInt(headers['retry-after'], 10) : undefined,
 		);
 	}
-};
+	// State was reported
+	return true;
+}
 
-const getStateDiff = (): DeviceStatus => {
+function newStateDiff(stateForReport: DeviceStatus): DeviceStatus {
 	const lastReportedLocal = lastReportedState.local;
 	const lastReportedDependent = lastReportedState.dependent;
 	if (lastReportedLocal == null || lastReportedDependent == null) {
@@ -138,51 +135,64 @@ const getStateDiff = (): DeviceStatus => {
 	};
 
 	return _.omitBy(diff, _.isEmpty);
-};
+}
-
-const throttledReport = _.throttle(
-	// We define the throttled function this way to avoid UncaughtPromise exceptions
-	// for exceptions thrown from the report function
-	(opts: StateReport, resolve: () => void, reject: (e: Error) => void) =>
-		report(opts).then(resolve).catch(reject),
-	constants.maxReportFrequency,
-);
-
-/**
- * Perform exponential backoff on the function, increasing the attempts
- * counter on each call
- *
- * If attempts is 0 then, it will delay for min(minDelay, maxDelay)
- */
-const backoff = (
-	retry: (attempts: number) => void,
-	attempts = 0,
-	maxDelay: number,
-	minDelay = MINIMUM_BACKOFF_DELAY,
-) => {
-	const delay = Math.min(2 ** attempts * minDelay, maxDelay);
-	log.info(`Retrying request in ${delay / 1000} seconds`);
-	setTimeout(() => retry(attempts + 1), delay);
-};
-
-function reportCurrentState(attempts = 0) {
-	(async () => {
-		const {
-			hardwareMetrics,
-			appUpdatePollInterval: maxDelay,
-			...conf
-		} = await config.getMany([
-			'deviceApiKey',
-			'apiTimeout',
-			'apiEndpoint',
-			'uuid',
-			'localMode',
-			'appUpdatePollInterval',
-			'hardwareMetrics',
-		]);
 
+async function reportCurrentState(conf: CurrentStateReportConf) {
+	// Ensure no other report starts
 	reportPending = true;
+	// Wrap the report with fetching of state so report always has the latest state diff
+	const getStateAndReport = async () => {
+		// Get state to report
+		const stateToReport = await generateStateForReport();
+		// Get diff from last reported state
+		const stateDiff = newStateDiff(stateToReport);
+		// Report diff
+		if (await report({ stateDiff, conf })) {
+			// Update lastReportedState
+			_.assign(lastReportedState.local, stateDiff.local);
+			_.assign(lastReportedState.dependent, stateDiff.dependent);
+			// Log that we successfully reported the current state
+			log.info('Reported current state to the cloud');
+		}
+	};
+	// Create a report that will backoff on errors
+	const reportWithBackoff = withBackoff(getStateAndReport, {
+		maxDelay: conf.appUpdatePollInterval,
+		minDelay: 15000,
+		onFailure: handleRetry,
+	});
+	// Run in try block to avoid throwing any exceptions
+	try {
+		await reportWithBackoff();
+		stateReportErrors = 0;
+	} catch (e) {
+		log.error(e);
+	}
+	reportPending = false;
+}
+
+function handleRetry(retryInfo: OnFailureInfo) {
+	if (retryInfo.error instanceof StatusError) {
+		log.error(
+			`Device state report failure! Status code: ${retryInfo.error.statusCode} - message:`,
+			retryInfo.error?.message ?? retryInfo.error,
+		);
+	} else {
+		eventTracker.track('Device state report failure', {
+			error: retryInfo.error?.message ?? retryInfo.error,
+		});
+	}
+	stateReportErrors++;
+	log.info(
+		`Retrying current state report in ${retryInfo.delay / 1000} seconds`,
+	);
+}
+
+async function generateStateForReport() {
+	const { hardwareMetrics } = await config.getMany(['hardwareMetrics']);
+
 	const currentDeviceState = await deviceState.getStatus();
+
 	// If hardwareMetrics is false, send null patch for system metrics to cloud API
 	const info = {
 		...(hardwareMetrics
@@ -200,66 +210,35 @@ function reportCurrentState(attempts = 0) {
 		...(await sysInfo.getSystemChecks()),
 	};
 
-		stateForReport.local = {
+	return {
-			...stateForReport.local,
+		local: {
 			...currentDeviceState.local,
 			...info,
-		};
-		stateForReport.dependent = {
-			...stateForReport.dependent,
-			...currentDeviceState.dependent,
-		};
-
-		const stateDiff = getStateDiff();
-
-		// report state diff
-		throttledReport(
-			{ stateDiff, conf },
-			() => {
-				// If the patch succeeds update lastReport and reset the error counter
-				_.assign(lastReportedState.local, stateDiff.local);
-				_.assign(lastReportedState.dependent, stateDiff.dependent);
-				stateReportErrors = 0;
-
-				reportPending = false;
 		},
-			(e) => {
+		dependent: currentDeviceState.dependent,
-				if (e instanceof StatusError) {
+	};
-					// We don't want these errors to be classed as a report error, as this will cause
-					// the watchdog to kill the supervisor - and killing the supervisor will
-					// not help in this situation
-					log.error(
-						`Device state report failure! Status code: ${e.statusCode} - message:`,
-						e.message,
-					);
-
-					// We want to backoff on all errors, but without having the healthchecks
-					// get triggered.
-					// This will use retryAfter as maxDelay if the header is present in the
-					// response by the API
-					backoff(
-						reportCurrentState,
-						// Do not do exponential backoff if the API reported a retryAfter
-						e.retryAfter ? 0 : attempts,
-						maxDelay,
-						// Start the polling at the value given by the API if any
-						e.retryAfter ?? MINIMUM_BACKOFF_DELAY,
-					);
-				} else {
-					eventTracker.track('Device state report failure', {
-						error: e.message,
-					});
-					backoff(reportCurrentState, stateReportErrors++, maxDelay);
-				}
-			},
-		);
-	})();
 }
 
-export const startReporting = () => {
+export async function startReporting() {
-	const doReport = () => {
+	// Get configs needed to make a report
+	const reportConfigs = (await config.getMany([
+		'uuid',
+		'apiEndpoint',
+		'apiTimeout',
+		'deviceApiKey',
+		'deviceId',
+		'localMode',
+		'appUpdatePollInterval',
+		'hardwareMetrics',
+	])) as CurrentStateReportConf;
+	// Throttle reportCurrentState so we don't query device or hit API excessively
+	const throttledReport = _.throttle(
+		reportCurrentState,
+		constants.maxReportFrequency,
+	);
+	const doReport = async () => {
 		if (!reportPending) {
-			reportCurrentState();
+			throttledReport(reportConfigs);
 		}
 	};
 
@@ -270,4 +249,4 @@ export const startReporting = () => {
 	setInterval(doReport, constants.maxReportFrequency);
 	// Try to perform a report right away
 	return doReport();
-};
+}

src/lib/backoff.ts

@@ -0,0 +1,105 @@
+import { strict as assert } from 'assert';
+import { promisify } from 'util';
+
+export type OnFailureInfo = {
+	failures: number;
+	delay: number;
+	error: any;
+};
+
+export type Options = {
+	retryCount: number;
+	maxDelay: number;
+	minDelay: number;
+	maxRetries: number;
+	onFailure?: (info: OnFailureInfo) => void;
+};
+
+const DEFAULT_OPTIONS: Partial<Options> = {
+	retryCount: 0,
+	maxDelay: 15000,
+	minDelay: 15000,
+	maxRetries: Infinity,
+};
+
+/**
+ * Retries a function with exponential delay between calls.
+ * If attempts is 0 then, it will delay for min(minDelay, maxDelay)
+ *
+ * Supports exceptions which return `retryAfter` value to specify the backoff duration.
+ * Will call `retryCallback` if passed in options as a way for callers to react to retries
+ */
+export function withBackoff<T extends (...args: any[]) => any>(
+	func: T,
+	options: Partial<Options> = {},
+) {
+	assert(typeof func === 'function', 'expected a function as parameter');
+	// If the function returns a promise, unwrap the promise return type
+	// otherwise use the actual return
+	type TReturn = ReturnType<T> extends Promise<infer R> ? R : ReturnType<T>;
+
+	// TODO use standard lib async setTimout (requires node 16)
+	const sleep = promisify(setTimeout);
+
+	const normalizedOptions: Options = {
+		...DEFAULT_OPTIONS,
+		...options,
+	} as Options;
+
+	return async function wrapped(...args: Parameters<T>): Promise<TReturn> {
+		try {
+			return await func(...args);
+		} catch (error) {
+			if (normalizedOptions.retryCount >= normalizedOptions.maxRetries) {
+				throw new Error(
+					`Reached max number of retries: ${normalizedOptions.maxRetries}`,
+				);
+			}
+
+			const wait = isRetryError(error)
+				? error.retryAfter
+				: exponentialRange(
+						normalizedOptions.retryCount,
+						normalizedOptions.minDelay,
+						normalizedOptions.maxDelay,
+				  );
+
+			normalizedOptions.retryCount++;
+
+			if (normalizedOptions.onFailure) {
+				normalizedOptions.onFailure({
+					failures: normalizedOptions.retryCount,
+					delay: wait,
+					error,
+				});
+			}
+
+			await sleep(wait);
+
+			return wrapped(...args);
+		}
+	};
+}
+
+// This is the expected interface for the error type
+export interface RetryError extends Error {
+	retryAfter: number;
+}
+
+// Type guard only checks that retryAfter is present
+const isRetryError = (x: unknown): x is RetryError =>
+	x != null &&
+	x instanceof Error &&
+	Number.isInteger((x as RetryError).retryAfter);
+
+/**
+ * Calculates a number with exponential growth given {retryCount} within
+ * a range of {minDelay} and {maxDelay} using {minDelay} as base.
+ */
+function exponentialRange(
+	retryCount: number,
+	minDelay: number,
+	maxDelay: number,
+): number {
+	return Math.min(2 ** retryCount * minDelay, maxDelay);
+}

test/src/lib/backoff.spec.ts

@@ -0,0 +1,216 @@
+import { assert, expect } from 'chai';
+import * as sinon from 'sinon';
+
+import { StatusError } from '../../../src/lib/errors';
+import { withBackoff, OnFailureInfo } from '../../../src/lib/backoff';
+
+const DEFAULT_OPTIONS = {
+	maxRetries: 5,
+	maxDelay: 900000, // 15 minutes
+	minDelay: 10000, // 10 seconds
+};
+
+describe('lib/backoff', async () => {
+	let clock: sinon.SinonFakeTimers;
+
+	beforeEach(() => {
+		clock = sinon.useFakeTimers();
+	});
+
+	afterEach(() => {
+		clock.restore();
+	});
+
+	it('resolves after 3 retries', async () => {
+		// Create a function that will fail 3 times so it succeeds on the 4th
+		const failer = new Failer(3);
+		// Wrap function withBackoff
+		const fnWithBackoff = withBackoff(async () => {
+			return failer.willResolve('fails 3 times then resolves on 4th');
+		}, DEFAULT_OPTIONS);
+		// Call function and allow clock to trigger all events
+		clock.runAllAsync();
+		await expect(fnWithBackoff()).to.eventually.equal(
+			'fails 3 times then resolves on 4th',
+		);
+		// Check that function was called 4 times (failed 3 times, succeeds on 4th)
+		expect(failer.callCount).to.equal(4);
+	});
+
+	it('should not call the function before minDelay', async () => {
+		const failer = new Failer(3);
+		const minDelay = Math.floor(Math.random() * 1000);
+		const myBackoffFunc = withBackoff(
+			async () => {
+				return failer.willResolve('ok');
+			},
+			{
+				minDelay,
+			},
+		);
+		// Function should have been called 0 times to start
+		expect(failer.callCount).to.equal(0);
+		// Call function
+		myBackoffFunc();
+		// Check that function was run at least once
+		expect(failer.callCount).to.equal(1);
+		// Elapse some time but not enough to be minDelay
+		await clock.tickAsync(minDelay - 1);
+		// Check that the function still has only been called 1 time
+		expect(failer.callCount).to.equal(1);
+		// Elapse exactly minDelay so function is called once more
+		await clock.tickAsync(minDelay + 1);
+		// Check that function was called twice
+		expect(failer.callCount).to.equal(2);
+	});
+
+	it('backs off with exponential delay', async () => {
+		const failer = new Failer(3);
+		const minDelay = Math.floor(Math.random() * 1000);
+		const myBackoffFunc = withBackoff(
+			async () => {
+				return failer.willResolve('ok');
+			},
+			{
+				minDelay,
+				maxDelay: 5000000,
+			},
+		);
+		expect(failer.callCount).to.equal(0);
+		// Call function
+		const p = myBackoffFunc();
+		// Function is called immediately
+		expect(failer.callCount).to.equal(1);
+		// First delay is equal to minDelay
+		await clock.tickAsync(minDelay);
+		// Should have been called again
+		expect(failer.callCount).to.equal(2);
+		// Tick exponential time
+		await clock.tickAsync(minDelay * 2);
+		// Should have been called again
+		expect(failer.callCount).to.equal(3);
+		// Tick exponential time
+		await clock.tickAsync(minDelay * 2 * 2);
+		// Function should be fulfilled by now
+		await expect(p).to.be.fulfilled;
+		expect(failer.callCount).to.equal(4);
+	});
+
+	it('never exceeds maxRetries', async () => {
+		// Make the failer fail 100 times (more then maxRetries)
+		const failer = new Failer(100);
+		const fnWithBackoff = withBackoff(async () => {
+			return failer.willResolve('ok');
+		}, DEFAULT_OPTIONS);
+		clock.runAllAsync();
+		// Call the function
+		await expect(fnWithBackoff()).to.eventually.be.rejectedWith(
+			`Reached max number of retries: ${DEFAULT_OPTIONS.maxRetries}`,
+		);
+	});
+
+	it('provides correct info within onFailure callback', async () => {
+		// Create a function that will fail 3 times so it succeeds on the 4th
+		const failer = new Failer(3);
+		const minDelay = Math.floor(Math.random() * 1000);
+		let counter = 0;
+		// Wrap function withBackoff
+		const fnWithBackoff = withBackoff(
+			async () => {
+				return failer.willResolve('ok');
+			},
+			{
+				minDelay,
+				onFailure: (data: OnFailureInfo) => {
+					counter++;
+					expect(data).to.deep.equal({
+						failures: counter,
+						delay:
+							counter === 1 ? minDelay : exponentialize(minDelay, counter - 1),
+						error: 'Not ready!',
+					});
+				},
+			},
+		);
+		// Call function and allow clock to trigger all events
+		clock.runAllAsync();
+		await fnWithBackoff();
+	});
+
+	it('uses RetryAfter from exception thrown', async () => {
+		const retryAfter = 50000;
+		const minDelay = 1000;
+		const failer = new Failer(
+			1,
+			new StatusError(503, 'Service Unavailable', retryAfter),
+		);
+		const fnWithBackoff = withBackoff(
+			async () => {
+				return failer.willResolve('ok');
+			},
+			{
+				minDelay,
+			},
+		);
+		assert(retryAfter > minDelay, 'retryAfter must be greater than minDelay');
+		expect(failer.callCount).to.equal(0);
+		// Start calling function that fails with retryAfter
+		const p = fnWithBackoff();
+		// Check that function was only called once (it runs right away)
+		expect(failer.callCount).to.equal(1);
+		// Tick clock by minDelay
+		// This will not be enough to call the function because retryAfter was in the
+		// exception thrown and it is greater then minDelay
+		await clock.tickAsync(minDelay);
+		// Check that function wasn't called yet
+		expect(failer.callCount).to.equal(1);
+		// Tick clock again just before retryAfter
+		await clock.tickAsync(retryAfter - minDelay - 1);
+		// Check that call count is still only 1 since retryAfter time has not elapsed
+		expect(failer.callCount).to.equal(1);
+		// Elapse enough time to trigger function call
+		await clock.tickAsync(1);
+		// Check that function has now been called once more
+		expect(failer.callCount).to.equal(2);
+		// Failure was set to only fail once so function should be fulfilled after 2 executions
+		await expect(p).to.be.fulfilled;
+	});
+});
+
+function exponentialize(n: number, timesToExpo: number): number {
+	let product = 0;
+	let exponentialized = 0;
+	function expo(): number {
+		if (exponentialized >= timesToExpo) {
+			return product;
+		}
+		exponentialized++;
+		product = product + n * 2;
+		return expo();
+	}
+	return expo();
+}
+
+class Failer {
+	public maxFails: number;
+	public callCount: number;
+	public customError?: Error;
+
+	public constructor(maxFails: number, error?: Error) {
+		this.maxFails = maxFails;
+		this.callCount = 0;
+		this.customError = error;
+	}
+
+	public async willResolve(resolvesWith: string): Promise<string> {
+		return new Promise((resolve, reject) => {
+			if (++this.callCount <= this.maxFails) {
+				if (this.customError) {
+					return reject(this.customError);
+				}
+				return reject('Not ready!');
+			}
+			return resolve(resolvesWith);
+		});
+	}
+}