From 8f07bf62de4a509ac74380af8be42b2acf3c1b76 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Thu, 14 Mar 2019 15:01:51 -0700 Subject: [PATCH] Add a random jitter to target state polls, and a config var to ignore update notifications and not poll immediately after startup This commit does two related things: * We make the poll interval a random time between 0.5 and 1.5 times the configured interval. * We introduce the BALENA_SUPERVISOR_INSTANT_UPDATE_TRIGGER configuration variable, that defaults to true. If this variable is set to false, then calls to /v1/update are ignored, and on startup the supervisor waits for a poll interval before getting the target state. This will help especially on cases where there's a large number of devices on a single network. By disabling instant updates and setting a large poll interval, we can now achieve a sitation where not all devices apply an update at the same time, which can help avoid overwhelming the network. Change-type: minor Signed-off-by: Pablo Carranza Velez --- src/api-binder.ts | 42 ++++++++++++++++++++++---------- src/config/schema-type.ts | 4 +++ src/config/schema.ts | 5 ++++ src/device-config.ts | 5 ++++ test/05-device-state.spec.coffee | 3 +++ 5 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/api-binder.ts b/src/api-binder.ts index 957720e5..9451c642 100644 --- a/src/api-binder.ts +++ b/src/api-binder.ts @@ -363,7 +363,7 @@ export class APIBinder { 'Trying to start poll without initializing API client', ); } - this.pollTargetState(); + this.pollTargetState(true); return null; }); } @@ -568,19 +568,27 @@ export class APIBinder { }); } - private async pollTargetState(): Promise { + private async pollTargetState(isInitialCall: boolean = false): Promise { // TODO: Remove the checkInt here with the config changes - let pollInterval = await this.config.get('appUpdatePollInterval'); + const { appUpdatePollInterval, instantUpdates } = await this.config.getMany( + ['appUpdatePollInterval', 'instantUpdates'], + ); - try { - await this.getAndSetTargetState(false); - this.targetStateFetchErrors = 0; - } catch (e) { - pollInterval = Math.min( - pollInterval, - 15000 * 2 ** this.targetStateFetchErrors, - ); - ++this.targetStateFetchErrors; + // We add jitter to the poll interval so that it's between 0.5 and 1.5 times + // the configured interval + let pollInterval = (0.5 + Math.random()) * appUpdatePollInterval; + + if (instantUpdates || !isInitialCall) { + try { + await this.getAndSetTargetState(false); + this.targetStateFetchErrors = 0; + } catch (e) { + pollInterval = Math.min( + appUpdatePollInterval, + 15000 * 2 ** this.targetStateFetchErrors, + ); + ++this.targetStateFetchErrors; + } } await Bluebird.delay(pollInterval); @@ -912,7 +920,15 @@ export class APIBinder { router.post('/v1/update', (req, res) => { apiBinder.eventTracker.track('Update notification'); if (apiBinder.readyForUpdates) { - apiBinder.getAndSetTargetState(req.body.force, true).catch(_.noop); + this.config.get('instantUpdates').then(instantUpdates => { + if (instantUpdates) { + apiBinder.getAndSetTargetState(req.body.force, true).catch(_.noop); + } else { + console.log( + 'Ignoring update notification because instant updates are disabled', + ); + } + }); } res.sendStatus(204); }); diff --git a/src/config/schema-type.ts b/src/config/schema-type.ts index 6b1223f9..02468f15 100644 --- a/src/config/schema-type.ts +++ b/src/config/schema-type.ts @@ -66,6 +66,10 @@ export const schemaTypes = { type: PermissiveNumber, default: 60000, }, + instantUpdates: { + type: PermissiveBoolean, + default: true, + }, mixpanelToken: { type: t.string, default: constants.defaultMixpanelToken, diff --git a/src/config/schema.ts b/src/config/schema.ts index f77276a1..bf0ec2be 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -185,6 +185,11 @@ export const schema = { mutable: true, removeIfNull: false, }, + instantUpdates: { + source: 'db', + mutable: true, + removeIfNull: false, + }, }; export type Schema = typeof schema; diff --git a/src/device-config.ts b/src/device-config.ts index 5f51d691..5cf2a7f7 100644 --- a/src/device-config.ts +++ b/src/device-config.ts @@ -68,6 +68,11 @@ export class DeviceConfig { varType: 'int', defaultValue: '60000', }, + instantUpdates: { + envVarName: 'SUPERVISOR_INSTANT_UPDATE_TRIGGER', + varType: 'bool', + defaultValue: 'true', + }, localMode: { envVarName: 'SUPERVISOR_LOCAL_MODE', varType: 'bool', diff --git a/test/05-device-state.spec.coffee b/test/05-device-state.spec.coffee index 7ce4d413..a87436a8 100644 --- a/test/05-device-state.spec.coffee +++ b/test/05-device-state.spec.coffee @@ -21,6 +21,7 @@ mockedInitialConfig = { 'RESIN_SUPERVISOR_DELTA_RETRY_COUNT': '30' 'RESIN_SUPERVISOR_DELTA_RETRY_INTERVAL': '10000' 'RESIN_SUPERVISOR_DELTA_VERSION': '2' + 'RESIN_SUPERVISOR_INSTANT_UPDATE_TRIGGER': 'true' 'RESIN_SUPERVISOR_LOCAL_MODE': 'false' 'RESIN_SUPERVISOR_LOG_CONTROL': 'true' 'RESIN_SUPERVISOR_OVERRIDE_LOCK': 'false' @@ -40,6 +41,7 @@ testTarget1 = { 'SUPERVISOR_DELTA_RETRY_COUNT': '30' 'SUPERVISOR_DELTA_RETRY_INTERVAL': '10000' 'SUPERVISOR_DELTA_VERSION': '2' + 'SUPERVISOR_INSTANT_UPDATE_TRIGGER': 'true' 'SUPERVISOR_LOCAL_MODE': 'false' 'SUPERVISOR_LOG_CONTROL': 'true' 'SUPERVISOR_OVERRIDE_LOCK': 'false' @@ -122,6 +124,7 @@ testTargetWithDefaults2 = { 'SUPERVISOR_DELTA_RETRY_COUNT': '30' 'SUPERVISOR_DELTA_RETRY_INTERVAL': '10000' 'SUPERVISOR_DELTA_VERSION': '2' + 'SUPERVISOR_INSTANT_UPDATE_TRIGGER': 'true' 'SUPERVISOR_LOCAL_MODE': 'false' 'SUPERVISOR_LOG_CONTROL': 'true' 'SUPERVISOR_OVERRIDE_LOCK': 'false'