state-engine: Add an exponential backoff for device-config noops

To avoid unnecesarilly using resources, we add an exponential backoff
when the noops explicitly come from the device-config module.

Change-type: patch
Signed-off-by: Cameron Diver <cameron@balena.io>
This commit is contained in:
Cameron Diver 2019-03-07 18:20:52 +00:00
parent ea1b247d3f
commit 1aa58fd7b9
No known key found for this signature in database
GPG Key ID: 49690ED87032539F

View File

@ -578,8 +578,7 @@ module.exports = class DeviceState extends EventEmitter
console.log('Scheduling another update attempt due to failure: ', delay, err) console.log('Scheduling another update attempt due to failure: ', delay, err)
@triggerApplyTarget({ force, delay, initial }) @triggerApplyTarget({ force, delay, initial })
applyTarget: ({ force = false, initial = false, intermediate = false, skipLock = false } = {}) => applyTarget: ({ force = false, initial = false, intermediate = false, skipLock = false, nextDelay = 200, retryCount = 0 } = {}) =>
nextDelay = 200
Promise.try => Promise.try =>
if !intermediate if !intermediate
@applyBlocker @applyBlocker
@ -596,15 +595,21 @@ module.exports = class DeviceState extends EventEmitter
.then (deviceConfigSteps) => .then (deviceConfigSteps) =>
noConfigSteps = _.every(deviceConfigSteps, ({ action }) -> action is 'noop') noConfigSteps = _.every(deviceConfigSteps, ({ action }) -> action is 'noop')
if not noConfigSteps if not noConfigSteps
return deviceConfigSteps return [false, deviceConfigSteps]
else else
@applications.getRequiredSteps(currentState, targetState, extraState, intermediate) @applications.getRequiredSteps(currentState, targetState, extraState, intermediate)
.then (appSteps) -> .then (appSteps) ->
# We need to forward to no-ops to the next step if the application state is done # We need to forward to no-ops to the next step if the application state is done
# The true and false values below represent whether we should add an exponential
# backoff if the steps are all no-ops. the reason that this has to be different
# is that a noop step from the application manager means that we should keep running
# in a tight loop, waiting for an image to download (for example). The device-config
# steps generally have a much higher time to wait before the no-ops will stop, so we
# should try to reduce the effect that this will have
if _.isEmpty(appSteps) and noConfigSteps if _.isEmpty(appSteps) and noConfigSteps
return deviceConfigSteps return [true, deviceConfigSteps]
return appSteps return [false, appSteps]
.then (steps) => .then ([backoff, steps]) =>
if _.isEmpty(steps) if _.isEmpty(steps)
@emitAsync('apply-target-state-end', null) @emitAsync('apply-target-state-end', null)
if !intermediate if !intermediate
@ -617,12 +622,17 @@ module.exports = class DeviceState extends EventEmitter
if !intermediate if !intermediate
@reportCurrentState(update_pending: true) @reportCurrentState(update_pending: true)
if _.every(steps, (step) -> step.action == 'noop') if _.every(steps, (step) -> step.action == 'noop')
nextDelay = 1000 if backoff
retryCount += 1
# Backoff to a maximum of 10 minutes
nextDelay = Math.min(2 ** retryCount * 1000, 60 * 10 * 1000)
else
nextDelay = 1000
Promise.map steps, (step) => Promise.map steps, (step) =>
@applyStep(step, { force, initial, intermediate, skipLock }) @applyStep(step, { force, initial, intermediate, skipLock })
.delay(nextDelay) .delay(nextDelay)
.then => .then =>
@applyTarget({ force, initial, intermediate, skipLock }) @applyTarget({ force, initial, intermediate, skipLock, nextDelay, retryCount })
.catch (err) => .catch (err) =>
@applyError(err, { force, initial, intermediate }) @applyError(err, { force, initial, intermediate })