mirror of
https://github.com/balena-os/balena-supervisor.git
synced 2024-12-19 21:57:54 +00:00
Merge pull request #1888 from balena-os/lift-report-throttle
Move report throttle out of reporting logic
This commit is contained in:
bulldozer-balena[bot]
GitHub
commit
a5d13902ec
98
package-lock.json
generated
98
package-lock.json
generated
@ -373,7 +373,6 @@
|
||||
"version": "1.8.3",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
|
||||
"integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"type-detect": "4.0.8"
|
||||
}
|
||||
@ -382,16 +381,14 @@
|
||||
"version": "7.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-7.0.5.tgz",
|
||||
"integrity": "sha512-fUt6b15bjV/VW93UP5opNXJxdwZSbK1EdiwnhN7XrQrcpaOhMJpZ/CjwFpM3THpxwA+YviBUJKSuEqKlCK5alw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"@sinonjs/samsam": {
|
||||
"version": "5.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.3.1.tgz",
|
||||
"integrity": "sha512-1Hc0b1TtyfBu8ixF/tpfSHTVWKwCBLY4QJbkgnE7HcwyvT2xArDxb4K7dMgqRm3szI+LJbzmW/s4xxEhv6hwDg==",
|
||||
"dev": true,
|
||||
"version": "6.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-6.1.1.tgz",
|
||||
"integrity": "sha512-cZ7rKJTLiE7u7Wi/v9Hc2fs3Ucc3jrWeMgPHbbTCeVAB2S0wOBbYlkJVeNSL04i7fdhT8wIbDq1zhC/PXTD2SA==",
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^1.6.0",
|
||||
"lodash.get": "^4.4.2",
|
||||
@ -401,8 +398,7 @@
|
||||
"@sinonjs/text-encoding": {
|
||||
"version": "0.7.1",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz",
|
||||
"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
|
||||
"dev": true
|
||||
"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ=="
|
||||
},
|
||||
"@szmarczak/http-timer": {
|
||||
"version": "1.1.2",
|
||||
@ -777,12 +773,12 @@
|
||||
"dev": true
|
||||
},
|
||||
"@types/sinon": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.0.tgz",
|
||||
"integrity": "sha512-jDZ55oCKxqlDmoTBBbBBEx+N8ZraUVhggMZ9T5t+6/Dh8/4NiOjSUfpLrPiEwxQDlAe3wpAkoXhWvE6LibtsMQ==",
|
||||
"version": "10.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.11.tgz",
|
||||
"integrity": "sha512-dmZsHlBsKUtBpHriNjlK0ndlvEh8dcb9uV9Afsbt89QIyydpC7NcR+nWlAhASfy3GHnxTl4FX/aKE7XZUt/B4g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/fake-timers": "^7.0.4"
|
||||
"@types/sinonjs__fake-timers": "*"
|
||||
}
|
||||
},
|
||||
"@types/sinon-chai": {
|
||||
@ -795,6 +791,12 @@
|
||||
"@types/sinon": "*"
|
||||
}
|
||||
},
|
||||
"@types/sinonjs__fake-timers": {
|
||||
"version": "8.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.1.tgz",
|
||||
"integrity": "sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/source-list-map": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz",
|
||||
@ -3313,8 +3315,7 @@
|
||||
"diff": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz",
|
||||
"integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==",
|
||||
"dev": true
|
||||
"integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w=="
|
||||
},
|
||||
"diffie-hellman": {
|
||||
"version": "5.0.3",
|
||||
@ -6230,8 +6231,7 @@
|
||||
"just-extend": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
|
||||
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
|
||||
"dev": true
|
||||
"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg=="
|
||||
},
|
||||
"keyv": {
|
||||
"version": "3.1.0",
|
||||
@ -6634,8 +6634,7 @@
|
||||
"lodash.get": {
|
||||
"version": "4.4.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
|
||||
"integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=",
|
||||
"dev": true
|
||||
"integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk="
|
||||
},
|
||||
"log-symbols": {
|
||||
"version": "4.0.0",
|
||||
@ -7620,38 +7619,26 @@
|
||||
"dev": true
|
||||
},
|
||||
"nise": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/nise/-/nise-4.1.0.tgz",
|
||||
"integrity": "sha512-eQMEmGN/8arp0xsvGoQ+B1qvSkR73B1nWSCh7nOt5neMCtwcQVYQGdzQMhcNscktTsWB54xnlSQFzOAPJD8nXA==",
|
||||
"dev": true,
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nise/-/nise-5.1.1.tgz",
|
||||
"integrity": "sha512-yr5kW2THW1AkxVmCnKEh4nbYkJdB3I7LUkiUgOvEkOp414mc2UMaHMA7pjq1nYowhdoJZGwEKGaQVbxfpWj10A==",
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^1.7.0",
|
||||
"@sinonjs/fake-timers": "^6.0.0",
|
||||
"@sinonjs/commons": "^1.8.3",
|
||||
"@sinonjs/fake-timers": ">=5",
|
||||
"@sinonjs/text-encoding": "^0.7.1",
|
||||
"just-extend": "^4.0.2",
|
||||
"path-to-regexp": "^1.7.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sinonjs/fake-timers": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz",
|
||||
"integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"isarray": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
|
||||
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
|
||||
"dev": true
|
||||
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
|
||||
},
|
||||
"path-to-regexp": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
|
||||
"integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"isarray": "0.0.1"
|
||||
}
|
||||
@ -9280,45 +9267,35 @@
|
||||
}
|
||||
},
|
||||
"sinon": {
|
||||
"version": "10.0.0",
|
||||
"resolved": "https://registry.npmjs.org/sinon/-/sinon-10.0.0.tgz",
|
||||
"integrity": "sha512-XAn5DxtGVJBlBWYrcYKEhWCz7FLwZGdyvANRyK06419hyEpdT0dMc5A8Vcxg5SCGHc40CsqoKsc1bt1CbJPfNw==",
|
||||
"dev": true,
|
||||
"version": "11.1.2",
|
||||
"resolved": "https://registry.npmjs.org/sinon/-/sinon-11.1.2.tgz",
|
||||
"integrity": "sha512-59237HChms4kg7/sXhiRcUzdSkKuydDeTiamT/jesUVHshBgL8XAmhgFo0GfK6RruMDM/iRSij1EybmMog9cJw==",
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^1.8.1",
|
||||
"@sinonjs/fake-timers": "^6.0.1",
|
||||
"@sinonjs/samsam": "^5.3.1",
|
||||
"diff": "^4.0.2",
|
||||
"nise": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
"@sinonjs/commons": "^1.8.3",
|
||||
"@sinonjs/fake-timers": "^7.1.2",
|
||||
"@sinonjs/samsam": "^6.0.2",
|
||||
"diff": "^5.0.0",
|
||||
"nise": "^5.1.0",
|
||||
"supports-color": "^7.2.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@sinonjs/fake-timers": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz",
|
||||
"integrity": "sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA==",
|
||||
"dev": true,
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-7.1.2.tgz",
|
||||
"integrity": "sha512-iQADsW4LBMISqZ6Ci1dupJL9pprqwcVFTcOsEmQOEhW+KLCVn/Y4Jrvg2k19fIHCp+iFprriYPTdRcQR8NbUPg==",
|
||||
"requires": {
|
||||
"@sinonjs/commons": "^1.7.0"
|
||||
}
|
||||
},
|
||||
"diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
|
||||
"dev": true
|
||||
},
|
||||
"has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"dev": true
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"has-flag": "^4.0.0"
|
||||
}
|
||||
@ -10579,8 +10556,7 @@
|
||||
"type-detect": {
|
||||
"version": "4.0.8",
|
||||
"resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
|
||||
"integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
|
||||
"dev": true
|
||||
"integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g=="
|
||||
},
|
||||
"type-fest": {
|
||||
"version": "0.11.0",
|
||||
|
||||
@ -32,6 +32,7 @@
|
||||
"dbus": "^1.0.7",
|
||||
"mdns-resolver": "^1.0.0",
|
||||
"semver": "^7.3.2",
|
||||
"sinon": "^11.1.2",
|
||||
"sqlite3": "^4.1.1",
|
||||
"systeminformation": "^5.6.10"
|
||||
},
|
||||
@ -66,7 +67,7 @@
|
||||
"@types/rwlock": "^5.0.2",
|
||||
"@types/semver": "^7.3.3",
|
||||
"@types/shell-quote": "^1.7.0",
|
||||
"@types/sinon": "^10.0.0",
|
||||
"@types/sinon": "^10.0.11",
|
||||
"@types/sinon-chai": "^3.2.5",
|
||||
"@types/supertest": "^2.0.11",
|
||||
"@types/terser-webpack-plugin": "^3.0.0",
|
||||
@ -117,7 +118,6 @@
|
||||
"rimraf": "^2.7.1",
|
||||
"rwlock": "^5.0.0",
|
||||
"shell-quote": "^1.7.2",
|
||||
"sinon": "^10.0.0",
|
||||
"sinon-chai": "^3.6.0",
|
||||
"strict-event-emitter-types": "^2.0.0",
|
||||
"supertest": "^6.1.3",
|
||||
|
||||
@ -3,6 +3,7 @@ import * as url from 'url';
|
||||
import { CoreOptions } from 'request';
|
||||
|
||||
import * as constants from '../lib/constants';
|
||||
import { withBackoff, OnFailureInfo } from '../lib/backoff';
|
||||
import { log } from '../lib/supervisor-console';
|
||||
import { InternalInconsistencyError, StatusError } from '../lib/errors';
|
||||
import { getRequestInstance } from '../lib/request';
|
||||
@ -14,9 +15,6 @@ import { SchemaTypeKey, SchemaReturn } from '../config/schema-type';
|
||||
import * as eventTracker from '../event-tracker';
|
||||
import * as deviceState from '../device-state';
|
||||
|
||||
// The exponential backoff starts at 15s
|
||||
const MINIMUM_BACKOFF_DELAY = 15000;
|
||||
|
||||
const INTERNAL_STATE_KEYS = [
|
||||
'update_pending',
|
||||
'update_downloaded',
|
||||
@ -28,10 +26,6 @@ const lastReportedState: DeviceStatus = {
|
||||
local: {},
|
||||
dependent: {},
|
||||
};
|
||||
const stateForReport: DeviceStatus = {
|
||||
local: {},
|
||||
dependent: {},
|
||||
};
|
||||
let reportPending = false;
|
||||
|
||||
type CurrentStateReportConf = {
|
||||
@ -43,6 +37,7 @@ type CurrentStateReportConf = {
|
||||
| 'deviceApiKey'
|
||||
| 'deviceId'
|
||||
| 'localMode'
|
||||
| 'appUpdatePollInterval'
|
||||
| 'hardwareMetrics'
|
||||
>]: SchemaReturn<key>;
|
||||
};
|
||||
@ -64,7 +59,7 @@ const stripDeviceStateInLocalMode = (state: DeviceStatus): DeviceStatus => {
|
||||
};
|
||||
};
|
||||
|
||||
const report = async ({ stateDiff, conf }: StateReport) => {
|
||||
async function report({ stateDiff, conf }: StateReport): Promise<boolean> {
|
||||
let body = stateDiff;
|
||||
const { apiEndpoint, apiTimeout, deviceApiKey, localMode, uuid } = conf;
|
||||
if (localMode) {
|
||||
@ -73,7 +68,7 @@ const report = async ({ stateDiff, conf }: StateReport) => {
|
||||
|
||||
if (_.isEmpty(body.local)) {
|
||||
// Nothing to send.
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (conf.uuid == null || conf.apiEndpoint == null) {
|
||||
@ -104,9 +99,11 @@ const report = async ({ stateDiff, conf }: StateReport) => {
|
||||
headers['retry-after'] ? parseInt(headers['retry-after'], 10) : undefined,
|
||||
);
|
||||
}
|
||||
};
|
||||
// State was reported
|
||||
return true;
|
||||
}
|
||||
|
||||
const getStateDiff = (): DeviceStatus => {
|
||||
function newStateDiff(stateForReport: DeviceStatus): DeviceStatus {
|
||||
const lastReportedLocal = lastReportedState.local;
|
||||
const lastReportedDependent = lastReportedState.dependent;
|
||||
if (lastReportedLocal == null || lastReportedDependent == null) {
|
||||
@ -138,128 +135,110 @@ const getStateDiff = (): DeviceStatus => {
|
||||
};
|
||||
|
||||
return _.omitBy(diff, _.isEmpty);
|
||||
};
|
||||
|
||||
const throttledReport = _.throttle(
|
||||
// We define the throttled function this way to avoid UncaughtPromise exceptions
|
||||
// for exceptions thrown from the report function
|
||||
(opts: StateReport, resolve: () => void, reject: (e: Error) => void) =>
|
||||
report(opts).then(resolve).catch(reject),
|
||||
constants.maxReportFrequency,
|
||||
);
|
||||
|
||||
/**
|
||||
* Perform exponential backoff on the function, increasing the attempts
|
||||
* counter on each call
|
||||
*
|
||||
* If attempts is 0 then, it will delay for min(minDelay, maxDelay)
|
||||
*/
|
||||
const backoff = (
|
||||
retry: (attempts: number) => void,
|
||||
attempts = 0,
|
||||
maxDelay: number,
|
||||
minDelay = MINIMUM_BACKOFF_DELAY,
|
||||
) => {
|
||||
const delay = Math.min(2 ** attempts * minDelay, maxDelay);
|
||||
log.info(`Retrying request in ${delay / 1000} seconds`);
|
||||
setTimeout(() => retry(attempts + 1), delay);
|
||||
};
|
||||
|
||||
function reportCurrentState(attempts = 0) {
|
||||
(async () => {
|
||||
const {
|
||||
hardwareMetrics,
|
||||
appUpdatePollInterval: maxDelay,
|
||||
...conf
|
||||
} = await config.getMany([
|
||||
'deviceApiKey',
|
||||
'apiTimeout',
|
||||
'apiEndpoint',
|
||||
'uuid',
|
||||
'localMode',
|
||||
'appUpdatePollInterval',
|
||||
'hardwareMetrics',
|
||||
]);
|
||||
|
||||
reportPending = true;
|
||||
const currentDeviceState = await deviceState.getStatus();
|
||||
// If hardwareMetrics is false, send null patch for system metrics to cloud API
|
||||
const info = {
|
||||
...(hardwareMetrics
|
||||
? await sysInfo.getSystemMetrics()
|
||||
: {
|
||||
cpu_usage: null,
|
||||
memory_usage: null,
|
||||
memory_total: null,
|
||||
storage_usage: null,
|
||||
storage_total: null,
|
||||
storage_block_device: null,
|
||||
cpu_temp: null,
|
||||
cpu_id: null,
|
||||
}),
|
||||
...(await sysInfo.getSystemChecks()),
|
||||
};
|
||||
|
||||
stateForReport.local = {
|
||||
...stateForReport.local,
|
||||
...currentDeviceState.local,
|
||||
...info,
|
||||
};
|
||||
stateForReport.dependent = {
|
||||
...stateForReport.dependent,
|
||||
...currentDeviceState.dependent,
|
||||
};
|
||||
|
||||
const stateDiff = getStateDiff();
|
||||
|
||||
// report state diff
|
||||
throttledReport(
|
||||
{ stateDiff, conf },
|
||||
() => {
|
||||
// If the patch succeeds update lastReport and reset the error counter
|
||||
_.assign(lastReportedState.local, stateDiff.local);
|
||||
_.assign(lastReportedState.dependent, stateDiff.dependent);
|
||||
stateReportErrors = 0;
|
||||
|
||||
reportPending = false;
|
||||
},
|
||||
(e) => {
|
||||
if (e instanceof StatusError) {
|
||||
// We don't want these errors to be classed as a report error, as this will cause
|
||||
// the watchdog to kill the supervisor - and killing the supervisor will
|
||||
// not help in this situation
|
||||
log.error(
|
||||
`Device state report failure! Status code: ${e.statusCode} - message:`,
|
||||
e.message,
|
||||
);
|
||||
|
||||
// We want to backoff on all errors, but without having the healthchecks
|
||||
// get triggered.
|
||||
// This will use retryAfter as maxDelay if the header is present in the
|
||||
// response by the API
|
||||
backoff(
|
||||
reportCurrentState,
|
||||
// Do not do exponential backoff if the API reported a retryAfter
|
||||
e.retryAfter ? 0 : attempts,
|
||||
maxDelay,
|
||||
// Start the polling at the value given by the API if any
|
||||
e.retryAfter ?? MINIMUM_BACKOFF_DELAY,
|
||||
);
|
||||
} else {
|
||||
eventTracker.track('Device state report failure', {
|
||||
error: e.message,
|
||||
});
|
||||
backoff(reportCurrentState, stateReportErrors++, maxDelay);
|
||||
}
|
||||
},
|
||||
);
|
||||
})();
|
||||
}
|
||||
|
||||
export const startReporting = () => {
|
||||
const doReport = () => {
|
||||
async function reportCurrentState(conf: CurrentStateReportConf) {
|
||||
// Ensure no other report starts
|
||||
reportPending = true;
|
||||
// Wrap the report with fetching of state so report always has the latest state diff
|
||||
const getStateAndReport = async () => {
|
||||
// Get state to report
|
||||
const stateToReport = await generateStateForReport();
|
||||
// Get diff from last reported state
|
||||
const stateDiff = newStateDiff(stateToReport);
|
||||
// Report diff
|
||||
if (await report({ stateDiff, conf })) {
|
||||
// Update lastReportedState
|
||||
_.assign(lastReportedState.local, stateDiff.local);
|
||||
_.assign(lastReportedState.dependent, stateDiff.dependent);
|
||||
// Log that we successfully reported the current state
|
||||
log.info('Reported current state to the cloud');
|
||||
}
|
||||
};
|
||||
// Create a report that will backoff on errors
|
||||
const reportWithBackoff = withBackoff(getStateAndReport, {
|
||||
maxDelay: conf.appUpdatePollInterval,
|
||||
minDelay: 15000,
|
||||
onFailure: handleRetry,
|
||||
});
|
||||
// Run in try block to avoid throwing any exceptions
|
||||
try {
|
||||
await reportWithBackoff();
|
||||
stateReportErrors = 0;
|
||||
} catch (e) {
|
||||
log.error(e);
|
||||
}
|
||||
reportPending = false;
|
||||
}
|
||||
|
||||
function handleRetry(retryInfo: OnFailureInfo) {
|
||||
if (retryInfo.error instanceof StatusError) {
|
||||
log.error(
|
||||
`Device state report failure! Status code: ${retryInfo.error.statusCode} - message:`,
|
||||
retryInfo.error?.message ?? retryInfo.error,
|
||||
);
|
||||
} else {
|
||||
eventTracker.track('Device state report failure', {
|
||||
error: retryInfo.error?.message ?? retryInfo.error,
|
||||
});
|
||||
}
|
||||
stateReportErrors++;
|
||||
log.info(
|
||||
`Retrying current state report in ${retryInfo.delay / 1000} seconds`,
|
||||
);
|
||||
}
|
||||
|
||||
async function generateStateForReport() {
|
||||
const { hardwareMetrics } = await config.getMany(['hardwareMetrics']);
|
||||
|
||||
const currentDeviceState = await deviceState.getStatus();
|
||||
|
||||
// If hardwareMetrics is false, send null patch for system metrics to cloud API
|
||||
const info = {
|
||||
...(hardwareMetrics
|
||||
? await sysInfo.getSystemMetrics()
|
||||
: {
|
||||
cpu_usage: null,
|
||||
memory_usage: null,
|
||||
memory_total: null,
|
||||
storage_usage: null,
|
||||
storage_total: null,
|
||||
storage_block_device: null,
|
||||
cpu_temp: null,
|
||||
cpu_id: null,
|
||||
}),
|
||||
...(await sysInfo.getSystemChecks()),
|
||||
};
|
||||
|
||||
return {
|
||||
local: {
|
||||
...currentDeviceState.local,
|
||||
...info,
|
||||
},
|
||||
dependent: currentDeviceState.dependent,
|
||||
};
|
||||
}
|
||||
|
||||
export async function startReporting() {
|
||||
// Get configs needed to make a report
|
||||
const reportConfigs = (await config.getMany([
|
||||
'uuid',
|
||||
'apiEndpoint',
|
||||
'apiTimeout',
|
||||
'deviceApiKey',
|
||||
'deviceId',
|
||||
'localMode',
|
||||
'appUpdatePollInterval',
|
||||
'hardwareMetrics',
|
||||
])) as CurrentStateReportConf;
|
||||
// Throttle reportCurrentState so we don't query device or hit API excessively
|
||||
const throttledReport = _.throttle(
|
||||
reportCurrentState,
|
||||
constants.maxReportFrequency,
|
||||
);
|
||||
const doReport = async () => {
|
||||
if (!reportPending) {
|
||||
reportCurrentState();
|
||||
throttledReport(reportConfigs);
|
||||
}
|
||||
};
|
||||
|
||||
@ -270,4 +249,4 @@ export const startReporting = () => {
|
||||
setInterval(doReport, constants.maxReportFrequency);
|
||||
// Try to perform a report right away
|
||||
return doReport();
|
||||
};
|
||||
}
|
||||
|
||||
105
src/lib/backoff.ts
Normal file
105
src/lib/backoff.ts
Normal file
@ -0,0 +1,105 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { promisify } from 'util';
|
||||
|
||||
export type OnFailureInfo = {
|
||||
failures: number;
|
||||
delay: number;
|
||||
error: any;
|
||||
};
|
||||
|
||||
export type Options = {
|
||||
retryCount: number;
|
||||
maxDelay: number;
|
||||
minDelay: number;
|
||||
maxRetries: number;
|
||||
onFailure?: (info: OnFailureInfo) => void;
|
||||
};
|
||||
|
||||
const DEFAULT_OPTIONS: Partial<Options> = {
|
||||
retryCount: 0,
|
||||
maxDelay: 15000,
|
||||
minDelay: 15000,
|
||||
maxRetries: Infinity,
|
||||
};
|
||||
|
||||
/**
|
||||
* Retries a function with exponential delay between calls.
|
||||
* If attempts is 0 then, it will delay for min(minDelay, maxDelay)
|
||||
*
|
||||
* Supports exceptions which return `retryAfter` value to specify the backoff duration.
|
||||
* Will call `retryCallback` if passed in options as a way for callers to react to retries
|
||||
*/
|
||||
export function withBackoff<T extends (...args: any[]) => any>(
|
||||
func: T,
|
||||
options: Partial<Options> = {},
|
||||
) {
|
||||
assert(typeof func === 'function', 'expected a function as parameter');
|
||||
// If the function returns a promise, unwrap the promise return type
|
||||
// otherwise use the actual return
|
||||
type TReturn = ReturnType<T> extends Promise<infer R> ? R : ReturnType<T>;
|
||||
|
||||
// TODO use standard lib async setTimout (requires node 16)
|
||||
const sleep = promisify(setTimeout);
|
||||
|
||||
const normalizedOptions: Options = {
|
||||
...DEFAULT_OPTIONS,
|
||||
...options,
|
||||
} as Options;
|
||||
|
||||
return async function wrapped(...args: Parameters<T>): Promise<TReturn> {
|
||||
try {
|
||||
return await func(...args);
|
||||
} catch (error) {
|
||||
if (normalizedOptions.retryCount >= normalizedOptions.maxRetries) {
|
||||
throw new Error(
|
||||
`Reached max number of retries: ${normalizedOptions.maxRetries}`,
|
||||
);
|
||||
}
|
||||
|
||||
const wait = isRetryError(error)
|
||||
? error.retryAfter
|
||||
: exponentialRange(
|
||||
normalizedOptions.retryCount,
|
||||
normalizedOptions.minDelay,
|
||||
normalizedOptions.maxDelay,
|
||||
);
|
||||
|
||||
normalizedOptions.retryCount++;
|
||||
|
||||
if (normalizedOptions.onFailure) {
|
||||
normalizedOptions.onFailure({
|
||||
failures: normalizedOptions.retryCount,
|
||||
delay: wait,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
await sleep(wait);
|
||||
|
||||
return wrapped(...args);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// This is the expected interface for the error type
|
||||
export interface RetryError extends Error {
|
||||
retryAfter: number;
|
||||
}
|
||||
|
||||
// Type guard only checks that retryAfter is present
|
||||
const isRetryError = (x: unknown): x is RetryError =>
|
||||
x != null &&
|
||||
x instanceof Error &&
|
||||
Number.isInteger((x as RetryError).retryAfter);
|
||||
|
||||
/**
|
||||
* Calculates a number with exponential growth given {retryCount} within
|
||||
* a range of {minDelay} and {maxDelay} using {minDelay} as base.
|
||||
*/
|
||||
function exponentialRange(
|
||||
retryCount: number,
|
||||
minDelay: number,
|
||||
maxDelay: number,
|
||||
): number {
|
||||
return Math.min(2 ** retryCount * minDelay, maxDelay);
|
||||
}
|
||||
216
test/src/lib/backoff.spec.ts
Normal file
216
test/src/lib/backoff.spec.ts
Normal file
@ -0,0 +1,216 @@
|
||||
import { assert, expect } from 'chai';
|
||||
import * as sinon from 'sinon';
|
||||
|
||||
import { StatusError } from '../../../src/lib/errors';
|
||||
import { withBackoff, OnFailureInfo } from '../../../src/lib/backoff';
|
||||
|
||||
const DEFAULT_OPTIONS = {
|
||||
maxRetries: 5,
|
||||
maxDelay: 900000, // 15 minutes
|
||||
minDelay: 10000, // 10 seconds
|
||||
};
|
||||
|
||||
describe('lib/backoff', async () => {
|
||||
let clock: sinon.SinonFakeTimers;
|
||||
|
||||
beforeEach(() => {
|
||||
clock = sinon.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clock.restore();
|
||||
});
|
||||
|
||||
it('resolves after 3 retries', async () => {
|
||||
// Create a function that will fail 3 times so it succeeds on the 4th
|
||||
const failer = new Failer(3);
|
||||
// Wrap function withBackoff
|
||||
const fnWithBackoff = withBackoff(async () => {
|
||||
return failer.willResolve('fails 3 times then resolves on 4th');
|
||||
}, DEFAULT_OPTIONS);
|
||||
// Call function and allow clock to trigger all events
|
||||
clock.runAllAsync();
|
||||
await expect(fnWithBackoff()).to.eventually.equal(
|
||||
'fails 3 times then resolves on 4th',
|
||||
);
|
||||
// Check that function was called 4 times (failed 3 times, succeeds on 4th)
|
||||
expect(failer.callCount).to.equal(4);
|
||||
});
|
||||
|
||||
it('should not call the function before minDelay', async () => {
|
||||
const failer = new Failer(3);
|
||||
const minDelay = Math.floor(Math.random() * 1000);
|
||||
const myBackoffFunc = withBackoff(
|
||||
async () => {
|
||||
return failer.willResolve('ok');
|
||||
},
|
||||
{
|
||||
minDelay,
|
||||
},
|
||||
);
|
||||
// Function should have been called 0 times to start
|
||||
expect(failer.callCount).to.equal(0);
|
||||
// Call function
|
||||
myBackoffFunc();
|
||||
// Check that function was run at least once
|
||||
expect(failer.callCount).to.equal(1);
|
||||
// Elapse some time but not enough to be minDelay
|
||||
await clock.tickAsync(minDelay - 1);
|
||||
// Check that the function still has only been called 1 time
|
||||
expect(failer.callCount).to.equal(1);
|
||||
// Elapse exactly minDelay so function is called once more
|
||||
await clock.tickAsync(minDelay + 1);
|
||||
// Check that function was called twice
|
||||
expect(failer.callCount).to.equal(2);
|
||||
});
|
||||
|
||||
it('backs off with exponential delay', async () => {
|
||||
const failer = new Failer(3);
|
||||
const minDelay = Math.floor(Math.random() * 1000);
|
||||
const myBackoffFunc = withBackoff(
|
||||
async () => {
|
||||
return failer.willResolve('ok');
|
||||
},
|
||||
{
|
||||
minDelay,
|
||||
maxDelay: 5000000,
|
||||
},
|
||||
);
|
||||
expect(failer.callCount).to.equal(0);
|
||||
// Call function
|
||||
const p = myBackoffFunc();
|
||||
// Function is called immediately
|
||||
expect(failer.callCount).to.equal(1);
|
||||
// First delay is equal to minDelay
|
||||
await clock.tickAsync(minDelay);
|
||||
// Should have been called again
|
||||
expect(failer.callCount).to.equal(2);
|
||||
// Tick exponential time
|
||||
await clock.tickAsync(minDelay * 2);
|
||||
// Should have been called again
|
||||
expect(failer.callCount).to.equal(3);
|
||||
// Tick exponential time
|
||||
await clock.tickAsync(minDelay * 2 * 2);
|
||||
// Function should be fulfilled by now
|
||||
await expect(p).to.be.fulfilled;
|
||||
expect(failer.callCount).to.equal(4);
|
||||
});
|
||||
|
||||
it('never exceeds maxRetries', async () => {
|
||||
// Make the failer fail 100 times (more then maxRetries)
|
||||
const failer = new Failer(100);
|
||||
const fnWithBackoff = withBackoff(async () => {
|
||||
return failer.willResolve('ok');
|
||||
}, DEFAULT_OPTIONS);
|
||||
clock.runAllAsync();
|
||||
// Call the function
|
||||
await expect(fnWithBackoff()).to.eventually.be.rejectedWith(
|
||||
`Reached max number of retries: ${DEFAULT_OPTIONS.maxRetries}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('provides correct info within onFailure callback', async () => {
|
||||
// Create a function that will fail 3 times so it succeeds on the 4th
|
||||
const failer = new Failer(3);
|
||||
const minDelay = Math.floor(Math.random() * 1000);
|
||||
let counter = 0;
|
||||
// Wrap function withBackoff
|
||||
const fnWithBackoff = withBackoff(
|
||||
async () => {
|
||||
return failer.willResolve('ok');
|
||||
},
|
||||
{
|
||||
minDelay,
|
||||
onFailure: (data: OnFailureInfo) => {
|
||||
counter++;
|
||||
expect(data).to.deep.equal({
|
||||
failures: counter,
|
||||
delay:
|
||||
counter === 1 ? minDelay : exponentialize(minDelay, counter - 1),
|
||||
error: 'Not ready!',
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
// Call function and allow clock to trigger all events
|
||||
clock.runAllAsync();
|
||||
await fnWithBackoff();
|
||||
});
|
||||
|
||||
it('uses RetryAfter from exception thrown', async () => {
|
||||
const retryAfter = 50000;
|
||||
const minDelay = 1000;
|
||||
const failer = new Failer(
|
||||
1,
|
||||
new StatusError(503, 'Service Unavailable', retryAfter),
|
||||
);
|
||||
const fnWithBackoff = withBackoff(
|
||||
async () => {
|
||||
return failer.willResolve('ok');
|
||||
},
|
||||
{
|
||||
minDelay,
|
||||
},
|
||||
);
|
||||
assert(retryAfter > minDelay, 'retryAfter must be greater than minDelay');
|
||||
expect(failer.callCount).to.equal(0);
|
||||
// Start calling function that fails with retryAfter
|
||||
const p = fnWithBackoff();
|
||||
// Check that function was only called once (it runs right away)
|
||||
expect(failer.callCount).to.equal(1);
|
||||
// Tick clock by minDelay
|
||||
// This will not be enough to call the function because retryAfter was in the
|
||||
// exception thrown and it is greater then minDelay
|
||||
await clock.tickAsync(minDelay);
|
||||
// Check that function wasn't called yet
|
||||
expect(failer.callCount).to.equal(1);
|
||||
// Tick clock again just before retryAfter
|
||||
await clock.tickAsync(retryAfter - minDelay - 1);
|
||||
// Check that call count is still only 1 since retryAfter time has not elapsed
|
||||
expect(failer.callCount).to.equal(1);
|
||||
// Elapse enough time to trigger function call
|
||||
await clock.tickAsync(1);
|
||||
// Check that function has now been called once more
|
||||
expect(failer.callCount).to.equal(2);
|
||||
// Failure was set to only fail once so function should be fulfilled after 2 executions
|
||||
await expect(p).to.be.fulfilled;
|
||||
});
|
||||
});
|
||||
|
||||
function exponentialize(n: number, timesToExpo: number): number {
|
||||
let product = 0;
|
||||
let exponentialized = 0;
|
||||
function expo(): number {
|
||||
if (exponentialized >= timesToExpo) {
|
||||
return product;
|
||||
}
|
||||
exponentialized++;
|
||||
product = product + n * 2;
|
||||
return expo();
|
||||
}
|
||||
return expo();
|
||||
}
|
||||
|
||||
class Failer {
|
||||
public maxFails: number;
|
||||
public callCount: number;
|
||||
public customError?: Error;
|
||||
|
||||
public constructor(maxFails: number, error?: Error) {
|
||||
this.maxFails = maxFails;
|
||||
this.callCount = 0;
|
||||
this.customError = error;
|
||||
}
|
||||
|
||||
public async willResolve(resolvesWith: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (++this.callCount <= this.maxFails) {
|
||||
if (this.customError) {
|
||||
return reject(this.customError);
|
||||
}
|
||||
return reject('Not ready!');
|
||||
}
|
||||
return resolve(resolvesWith);
|
||||
});
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user