Avoid starting services that exit repeatedly

Signed-off-by: Pablo Carranza Velez <pablo@resin.io>
This commit is contained in:
Pablo Carranza Velez 2018-02-06 17:40:59 -08:00
parent ba829412e1
commit 38c3a8bdf3
3 changed files with 27 additions and 5 deletions

View File

@ -284,15 +284,19 @@ module.exports = class ApplicationManager extends EventEmitter
@timeSpentFetching = 0 @timeSpentFetching = 0
@fetchesInProgress = 0 @fetchesInProgress = 0
@_targetVolatilePerImageId = {} @_targetVolatilePerImageId = {}
@_containerStarted = {}
@actionExecutors = { @actionExecutors = {
stop: (step, { force = false, skipLock = false } = {}) => stop: (step, { force = false, skipLock = false } = {}) =>
@_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, => @_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, =>
wait = step.options?.wait ? false wait = step.options?.wait ? false
@services.kill(step.current, { removeContainer: false, wait }) @services.kill(step.current, { removeContainer: false, wait })
.then =>
delete @_containerStarted[step.current.containerId]
kill: (step, { force = false, skipLock = false } = {}) => kill: (step, { force = false, skipLock = false } = {}) =>
@_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, => @_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, =>
@services.kill(step.current) @services.kill(step.current)
.then => .then =>
delete @_containerStarted[step.current.containerId]
if step.options?.removeImage if step.options?.removeImage
@images.removeByDockerId(step.current.image) @images.removeByDockerId(step.current.image)
updateMetadata: (step, { force = false, skipLock = false } = {}) => updateMetadata: (step, { force = false, skipLock = false } = {}) =>
@ -302,12 +306,18 @@ module.exports = class ApplicationManager extends EventEmitter
@_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, => @_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, =>
Promise.try => Promise.try =>
@services.kill(step.current, { wait: true }) @services.kill(step.current, { wait: true })
.then =>
delete @_containerStarted[step.current.containerId]
.then => .then =>
@services.start(step.target) @services.start(step.target)
.then (container) =>
@_containerStarted[container.id] = true
stopAll: (step, { force = false, skipLock = false } = {}) => stopAll: (step, { force = false, skipLock = false } = {}) =>
@stopAll({ force, skipLock }) @stopAll({ force, skipLock })
start: (step) => start: (step) =>
@services.start(step.target) @services.start(step.target)
.then (container) =>
@_containerStarted[container.id] = true
handover: (step, { force = false, skipLock = false } = {}) => handover: (step, { force = false, skipLock = false } = {}) =>
@_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, => @_lockingIfNecessary step.current.appId, { force, skipLock: skipLock or step.options?.skipLock }, =>
@services.handover(step.current, step.target) @services.handover(step.current, step.target)
@ -467,7 +477,7 @@ module.exports = class ApplicationManager extends EventEmitter
# Compares current and target services and returns a list of service pairs to be updated/removed/installed. # Compares current and target services and returns a list of service pairs to be updated/removed/installed.
# The returned list is an array of objects where the "current" and "target" properties define the update pair, and either can be null # The returned list is an array of objects where the "current" and "target" properties define the update pair, and either can be null
# (in the case of an install or removal). # (in the case of an install or removal).
compareServicesForUpdate: (currentServices, targetServices) -> compareServicesForUpdate: (currentServices, targetServices) =>
removePairs = [] removePairs = []
installPairs = [] installPairs = []
updatePairs = [] updatePairs = []
@ -512,8 +522,15 @@ module.exports = class ApplicationManager extends EventEmitter
else else
currentServicesPerId[serviceId] = currentServiceContainers[0] currentServicesPerId[serviceId] = currentServiceContainers[0]
# Returns true if a service matches its target except it should be running and it is not, but we've
# already started it before. In this case it means it just exited so we don't want to start it again.
alreadyStarted = (serviceId) =>
currentServicesPerId[serviceId].isEqualExceptForRunningState(targetServicesPerId[serviceId]) and
targetServicesPerId[serviceId].running and
@_containerStarted[currentServicesPerId[serviceId].containerId]
needUpdate = _.filter toBeMaybeUpdated, (serviceId) -> needUpdate = _.filter toBeMaybeUpdated, (serviceId) ->
return !currentServicesPerId[serviceId].isEqual(targetServicesPerId[serviceId]) return !currentServicesPerId[serviceId].isEqual(targetServicesPerId[serviceId]) and !alreadyStarted(serviceId)
for serviceId in needUpdate for serviceId in needUpdate
updatePairs.push({ updatePairs.push({
current: currentServicesPerId[serviceId] current: currentServicesPerId[serviceId]
@ -980,6 +997,8 @@ module.exports = class ApplicationManager extends EventEmitter
.map (service) => .map (service) =>
@_lockingIfNecessary service.appId, { force, skipLock }, => @_lockingIfNecessary service.appId, { force, skipLock }, =>
@services.kill(service, { removeContainer: false, wait: true }) @services.kill(service, { removeContainer: false, wait: true })
.then =>
delete @_containerStarted[service.containerId]
_lockingIfNecessary: (appId, { force = false, skipLock = false } = {}, fn) => _lockingIfNecessary: (appId, { force = false, skipLock = false } = {}, fn) =>
if skipLock if skipLock

View File

@ -148,7 +148,7 @@ module.exports = class ServiceManager extends EventEmitter
@logger.logSystemEvent(logTypes.startServiceNoop, { service }) @logger.logSystemEvent(logTypes.startServiceNoop, { service })
else else
@logger.logSystemEvent(logTypes.startServiceSuccess, { service }) @logger.logSystemEvent(logTypes.startServiceSuccess, { service })
.then (container) -> .tap (container) ->
service.running = true service.running = true
.finally => .finally =>
@reportChange(containerId) @reportChange(containerId)

View File

@ -635,8 +635,11 @@ module.exports = class Service
return isEq return isEq
isEqual: (otherService) => isEqualExceptForRunningState: (otherService) =>
return @isSameContainer(otherService) and return @isSameContainer(otherService) and
@running == otherService.running and
@releaseId == otherService.releaseId and @releaseId == otherService.releaseId and
@imageId == otherService.imageId @imageId == otherService.imageId
isEqual: (otherService) =>
return @isEqualExceptForRunningState(otherService) and
@running == otherService.running