Merge pull request #1845 from balena-os/logs-backoff

Add exponential backoff on errors for logs reporting

src/logging/balena-backend.ts

@@ -10,7 +10,8 @@ import { LogBackend, LogMessage } from './log-backend';
 import log from '../lib/supervisor-console';
 
 const ZLIB_TIMEOUT = 100;
-const COOLDOWN_PERIOD = 5 * 1000;
+const MIN_COOLDOWN_PERIOD = 5 * 1000; // 5 seconds
+const MAX_COOLDOWN_PERIOD = 300 * 1000; // 5 minutes
 const KEEPALIVE_TIMEOUT = 60 * 1000;
 const RESPONSE_GRACE_PERIOD = 5 * 1000;
 
@@ -118,7 +119,35 @@ export class BalenaLogBackend extends LogBackend {
 		this.initialised = true;
 	}
 
-	private setup = _.throttle(() => {
+	private lastSetupAttempt = 0;
+	private setupFailures = 0;
+	private setupPending = false;
+	private setup() {
+		if (this.setupPending || this.req != null) {
+			// If we already have a setup pending, or we are already setup, then do nothing
+			return;
+		}
+		this.setupPending = true;
+
+		// Work out the total delay we need
+		const totalDelay = Math.min(
+			2 ** this.setupFailures * MIN_COOLDOWN_PERIOD,
+			MAX_COOLDOWN_PERIOD,
+		);
+		// Work out how much of a delay has already occurred since the last attempt
+		const alreadyDelayedBy = this.lastSetupAttempt - Date.now();
+		// The difference between the two is the actual delay we want
+		const delay = Math.max(totalDelay - alreadyDelayedBy, 0);
+
+		setTimeout(() => {
+			this.setupPending = false;
+			this.lastSetupAttempt = Date.now();
+
+			const setupFailed = () => {
+				this.setupFailures++;
+				this.teardown();
+			};
+
 			this.req = https.request(this.opts);
 
 			// Since we haven't sent the request body yet, and never will,the
@@ -129,14 +158,14 @@ export class BalenaLogBackend extends LogBackend {
 					'LogBackend: server responded with status code:',
 					res.statusCode,
 				);
-			this.teardown();
+				setupFailed();
 			});
 
-		this.req.on('timeout', () => this.teardown());
+			this.req.on('timeout', setupFailed);
-		this.req.on('close', () => this.teardown());
+			this.req.on('close', setupFailed);
 			this.req.on('error', (err) => {
 				log.error('LogBackend: unexpected error:', err);
-			this.teardown();
+				setupFailed();
 			});
 
 			// Immediately flush the headers. This gives a chance to the server to
@@ -151,20 +180,22 @@ export class BalenaLogBackend extends LogBackend {
 			// to write all of them with Z_NO_FLUSH and only afterwards do a
 			// Z_SYNC_FLUSH to maximize compression
 			this.gzip = zlib.createGzip({ writableHighWaterMark: 1024 });
-		this.gzip.on('error', () => this.teardown());
+			this.gzip.on('error', setupFailed);
 			this.gzip.pipe(this.req);
 
 			// Only start piping if there has been no error after the header flush.
-		// Doing it immediately would potentialy lose logs if it turned out that
+			// Doing it immediately would potentially lose logs if it turned out that
 			// the server is unavailalbe because @_req stream would consume our
 			// passthrough buffer
 			this.timeout = setTimeout(() => {
 				if (this.gzip != null) {
+					this.setupFailures = 0;
 					this.stream.pipe(this.gzip);
 					setImmediate(this.flush);
 				}
 			}, RESPONSE_GRACE_PERIOD);
-	}, COOLDOWN_PERIOD);
+		}, delay);
+	}
 
 	private snooze = _.debounce(this.teardown, KEEPALIVE_TIMEOUT);
 
@@ -196,9 +227,7 @@ export class BalenaLogBackend extends LogBackend {
 
 	private write(message: LogMessage) {
 		this.snooze();
-		if (this.req == null) {
 		this.setup();
-		}
 
 		if (this.writable) {
 			try {