mirror of
https://github.com/balena-os/balena-supervisor.git
synced 2025-02-21 18:06:47 +00:00
Use env var for force, and don't touch lockfile on force
Also: * Force lock when purging * Use updateStatus object to keep update state
This commit is contained in:
parent
c7c4aed746
commit
3da961d4fd
@ -67,20 +67,20 @@ module.exports = (secret) ->
|
|||||||
utils.mixpanelTrack('Purge /data', appId)
|
utils.mixpanelTrack('Purge /data', appId)
|
||||||
if !appId?
|
if !appId?
|
||||||
return res.status(400).send('Missing app id')
|
return res.status(400).send('Missing app id')
|
||||||
Promise.using application.lockUpdates(app), ->
|
Promise.using application.lockUpdates(appId, true), ->
|
||||||
knex('app').select().where({ appId })
|
knex('app').select().where({ appId })
|
||||||
.then ([ app ]) ->
|
.then ([ app ]) ->
|
||||||
if !app?
|
if !app?
|
||||||
throw new Error('App not found')
|
throw new Error('App not found')
|
||||||
application.kill(app)
|
application.kill(app)
|
||||||
.then ->
|
.then ->
|
||||||
new Promise (resolve, reject) ->
|
new Promise (resolve, reject) ->
|
||||||
request.post(config.gosuperAddress + '/v1/purge', { json: true, body: applicationId: appId.toString() })
|
request.post(config.gosuperAddress + '/v1/purge', { json: true, body: applicationId: appId.toString() })
|
||||||
.on 'error', reject
|
.on 'error', reject
|
||||||
.on 'response', -> resolve()
|
.on 'response', -> resolve()
|
||||||
.pipe(res)
|
.pipe(res)
|
||||||
.finally ->
|
.finally ->
|
||||||
application.start(app)
|
application.start(app)
|
||||||
.catch (err) ->
|
.catch (err) ->
|
||||||
res.status(503).send(err?.message or err or 'Unknown error')
|
res.status(503).send(err?.message or err or 'Unknown error')
|
||||||
|
|
||||||
|
@ -66,6 +66,9 @@ logTypes =
|
|||||||
updateApp:
|
updateApp:
|
||||||
eventName: 'Application update'
|
eventName: 'Application update'
|
||||||
humanName: 'Updating application'
|
humanName: 'Updating application'
|
||||||
|
updateAppError:
|
||||||
|
eventName: 'Application update error'
|
||||||
|
humanName: 'Failed to update application'
|
||||||
|
|
||||||
logSystemEvent = (logType, app, error) ->
|
logSystemEvent = (logType, app, error) ->
|
||||||
message = "#{logType.humanName} '#{app.imageId}'"
|
message = "#{logType.humanName} '#{app.imageId}'"
|
||||||
@ -109,6 +112,8 @@ exports.kill = kill = (app) ->
|
|||||||
throw err
|
throw err
|
||||||
.tap ->
|
.tap ->
|
||||||
logSystemEvent(logTypes.stopAppSuccess, app)
|
logSystemEvent(logTypes.stopAppSuccess, app)
|
||||||
|
app.containerId = null
|
||||||
|
knex('app').update(app).where(appId: app.appId)
|
||||||
.catch (err) ->
|
.catch (err) ->
|
||||||
logSystemEvent(logTypes.stopAppError, app, err)
|
logSystemEvent(logTypes.stopAppError, app, err)
|
||||||
throw err
|
throw err
|
||||||
@ -242,8 +247,8 @@ getEnvironment = do ->
|
|||||||
throw err
|
throw err
|
||||||
|
|
||||||
lockPath = (app) ->
|
lockPath = (app) ->
|
||||||
appId = app.appId or app
|
appId = app.appId ? app
|
||||||
return "/mnt/root/resin-data/#{appId}/resin_updates.lock"
|
return "/mnt/root/resin-data/#{appId}/resin-updates.lock"
|
||||||
|
|
||||||
# At boot, all apps should be unlocked *before* start to prevent a deadlock
|
# At boot, all apps should be unlocked *before* start to prevent a deadlock
|
||||||
exports.unlockAndStart = unlockAndStart = (app) ->
|
exports.unlockAndStart = unlockAndStart = (app) ->
|
||||||
@ -255,14 +260,18 @@ exports.lockUpdates = lockUpdates = do ->
|
|||||||
_lock = new Lock()
|
_lock = new Lock()
|
||||||
_writeLock = Promise.promisify(_lock.async.writeLock)
|
_writeLock = Promise.promisify(_lock.async.writeLock)
|
||||||
return (app, force) ->
|
return (app, force) ->
|
||||||
_writeLock(lockPath(app))
|
lockName = lockPath(app)
|
||||||
.tap ->
|
_writeLock(lockName)
|
||||||
lockFile.unlockAsync(lockPath(app)) if force == true
|
.tap (release) ->
|
||||||
.tap ->
|
if force != true
|
||||||
lockFile.lockAsync(lockPath(app))
|
lockFile.lockAsync(lockName)
|
||||||
|
.catch (err) ->
|
||||||
|
release()
|
||||||
|
throw new Error("Updates are locked: #{err.message}")
|
||||||
.disposer (release) ->
|
.disposer (release) ->
|
||||||
lockFile.unlockAsync(lockPath(app))
|
Promise.try ->
|
||||||
.then ->
|
lockFile.unlockAsync(lockName) if force != true
|
||||||
|
.finally ->
|
||||||
release()
|
release()
|
||||||
|
|
||||||
joinErrorMessages = (failures) ->
|
joinErrorMessages = (failures) ->
|
||||||
@ -271,19 +280,21 @@ joinErrorMessages = (failures) ->
|
|||||||
err.message or err
|
err.message or err
|
||||||
"#{failures.length} error#{s}: #{messages.join(' - ')}"
|
"#{failures.length} error#{s}: #{messages.join(' - ')}"
|
||||||
|
|
||||||
# 0 - Idle
|
UPDATE_IDLE = 0
|
||||||
# 1 - Updating
|
UPDATE_UPDATING = 1
|
||||||
# 2 - Update required
|
UPDATE_REQUIRED = 2
|
||||||
currentlyUpdating = 0
|
|
||||||
failedUpdates = 0
|
updateStatus =
|
||||||
forceNextUpdate = false
|
currentState: UPDATE_IDLE
|
||||||
|
failedUpdates: 0
|
||||||
|
forceNextUpdate: false
|
||||||
exports.update = update = (force) ->
|
exports.update = update = (force) ->
|
||||||
if currentlyUpdating isnt 0
|
if updateStatus.currentState isnt UPDATE_IDLE
|
||||||
# Mark an update required after the current.
|
# Mark an update required after the current.
|
||||||
forceNextUpdate = force
|
updateStatus.forceNextUpdate = force
|
||||||
currentlyUpdating = 2
|
updateStatus.currentState = UPDATE_REQUIRED
|
||||||
return
|
return
|
||||||
currentlyUpdating = 1
|
updateStatus.currentState = UPDATE_UPDATING
|
||||||
Promise.all([
|
Promise.all([
|
||||||
knex('config').select('value').where(key: 'apiKey')
|
knex('config').select('value').where(key: 'apiKey')
|
||||||
knex('config').select('value').where(key: 'uuid')
|
knex('config').select('value').where(key: 'uuid')
|
||||||
@ -317,6 +328,7 @@ exports.update = update = (force) ->
|
|||||||
remoteApp.environment_variable = environment
|
remoteApp.environment_variable = environment
|
||||||
return remoteApp
|
return remoteApp
|
||||||
.then (remoteApps) ->
|
.then (remoteApps) ->
|
||||||
|
remoteAppEnvs = {}
|
||||||
remoteApps = _.map remoteApps, (app) ->
|
remoteApps = _.map remoteApps, (app) ->
|
||||||
env =
|
env =
|
||||||
RESIN_DEVICE_UUID: uuid
|
RESIN_DEVICE_UUID: uuid
|
||||||
@ -325,6 +337,7 @@ exports.update = update = (force) ->
|
|||||||
|
|
||||||
if app.environment_variable?
|
if app.environment_variable?
|
||||||
_.extend(env, app.environment_variable)
|
_.extend(env, app.environment_variable)
|
||||||
|
remoteAppEnvs[app.id] = env
|
||||||
return {
|
return {
|
||||||
appId: '' + app.id
|
appId: '' + app.id
|
||||||
commit: app.commit
|
commit: app.commit
|
||||||
@ -370,11 +383,12 @@ exports.update = update = (force) ->
|
|||||||
.then ->
|
.then ->
|
||||||
knex('app').where('appId', appId).delete()
|
knex('app').where('appId', appId).delete()
|
||||||
.catch (err) ->
|
.catch (err) ->
|
||||||
|
logSystemEvent(logTypes.stopAppError, apps[appId], err)
|
||||||
failures.push(err)
|
failures.push(err)
|
||||||
.then ->
|
.then ->
|
||||||
# Then install the apps and add each to the db as they succeed
|
# Then install the apps and add each to the db as they succeed
|
||||||
installingPromises = toBeInstalled.map (imageId) ->
|
installingPromises = toBeInstalled.map (appId) ->
|
||||||
app = remoteApps[imageId]
|
app = remoteApps[appId]
|
||||||
start(app)
|
start(app)
|
||||||
.catch (err) ->
|
.catch (err) ->
|
||||||
failures.push(err)
|
failures.push(err)
|
||||||
@ -383,7 +397,8 @@ exports.update = update = (force) ->
|
|||||||
localApp = apps[appId]
|
localApp = apps[appId]
|
||||||
app = remoteApps[appId]
|
app = remoteApps[appId]
|
||||||
logSystemEvent(logTypes.updateApp, app) if localApp.imageId == app.imageId
|
logSystemEvent(logTypes.updateApp, app) if localApp.imageId == app.imageId
|
||||||
Promise.using lockUpdates(localApp, force), ->
|
forceThisApp = remoteAppEnvs[appId]['RESIN_OVERRIDE_LOCK'] == '1'
|
||||||
|
Promise.using lockUpdates(localApp, force || forceThisApp), ->
|
||||||
knex('app').select().where({ appId })
|
knex('app').select().where({ appId })
|
||||||
.then ([ localApp ]) ->
|
.then ([ localApp ]) ->
|
||||||
if !localApp?
|
if !localApp?
|
||||||
@ -392,29 +407,30 @@ exports.update = update = (force) ->
|
|||||||
.then ->
|
.then ->
|
||||||
start(app)
|
start(app)
|
||||||
.catch (err) ->
|
.catch (err) ->
|
||||||
|
logSystemEvent(logTypes.updateAppError, apps[appId], err)
|
||||||
failures.push(err)
|
failures.push(err)
|
||||||
Promise.all(installingPromises.concat(updatingPromises))
|
Promise.all(installingPromises.concat(updatingPromises))
|
||||||
.then ->
|
.then ->
|
||||||
throw new Error(joinErrorMessages(failures)) if failures.length > 0
|
throw new Error(joinErrorMessages(failures)) if failures.length > 0
|
||||||
.then ->
|
.then ->
|
||||||
failedUpdates = 0
|
updateStatus.failedUpdates = 0
|
||||||
# We cleanup here as we want a point when we have a consistent apps/images state, rather than potentially at a
|
# We cleanup here as we want a point when we have a consistent apps/images state, rather than potentially at a
|
||||||
# point where we might clean up an image we still want.
|
# point where we might clean up an image we still want.
|
||||||
dockerUtils.cleanupContainersAndImages()
|
dockerUtils.cleanupContainersAndImages()
|
||||||
.catch (err) ->
|
.catch (err) ->
|
||||||
failedUpdates++
|
updateStatus.failedUpdates++
|
||||||
if currentlyUpdating is 2
|
if updateStatus.currentState is UPDATE_REQUIRED
|
||||||
console.log('Updating failed, but there is already another update scheduled immediately: ', err)
|
console.log('Updating failed, but there is already another update scheduled immediately: ', err)
|
||||||
return
|
return
|
||||||
delayTime = Math.min(failedUpdates * 500, 30000)
|
delayTime = Math.min(updateStatus.failedUpdates * 500, 30000)
|
||||||
# If there was an error then schedule another attempt briefly in the future.
|
# If there was an error then schedule another attempt briefly in the future.
|
||||||
console.log('Scheduling another update attempt due to failure: ', delayTime, err)
|
console.log('Scheduling another update attempt due to failure: ', delayTime, err)
|
||||||
setTimeout(update, delayTime, force)
|
setTimeout(update, delayTime, force)
|
||||||
.finally ->
|
.finally ->
|
||||||
device.updateState(status: 'Idle')
|
device.updateState(status: 'Idle')
|
||||||
if currentlyUpdating is 2
|
if updateStatus.currentState is UPDATE_REQUIRED
|
||||||
# If an update is required then schedule it
|
# If an update is required then schedule it
|
||||||
setTimeout(update, 1, forceNextUpdate)
|
setTimeout(update, 1, updateStatus.forceNextUpdate)
|
||||||
.finally ->
|
.finally ->
|
||||||
# Set the updating as finished in its own block, so it never has to worry about other code stopping this.
|
# Set the updating as finished in its own block, so it never has to worry about other code stopping this.
|
||||||
currentlyUpdating = 0
|
updateStatus.currentState = UPDATE_IDLE
|
||||||
|
Loading…
x
Reference in New Issue
Block a user