Use env var for force, and don't touch lockfile on force

Also:

* Force lock when purging

* Use updateStatus object to keep update state
This commit is contained in:
Pablo Carranza Vélez 2015-08-24 18:42:21 +00:00
parent c7c4aed746
commit 3da961d4fd
2 changed files with 54 additions and 38 deletions

View File

@ -67,20 +67,20 @@ module.exports = (secret) ->
utils.mixpanelTrack('Purge /data', appId) utils.mixpanelTrack('Purge /data', appId)
if !appId? if !appId?
return res.status(400).send('Missing app id') return res.status(400).send('Missing app id')
Promise.using application.lockUpdates(app), -> Promise.using application.lockUpdates(appId, true), ->
knex('app').select().where({ appId }) knex('app').select().where({ appId })
.then ([ app ]) -> .then ([ app ]) ->
if !app? if !app?
throw new Error('App not found') throw new Error('App not found')
application.kill(app) application.kill(app)
.then -> .then ->
new Promise (resolve, reject) -> new Promise (resolve, reject) ->
request.post(config.gosuperAddress + '/v1/purge', { json: true, body: applicationId: appId.toString() }) request.post(config.gosuperAddress + '/v1/purge', { json: true, body: applicationId: appId.toString() })
.on 'error', reject .on 'error', reject
.on 'response', -> resolve() .on 'response', -> resolve()
.pipe(res) .pipe(res)
.finally -> .finally ->
application.start(app) application.start(app)
.catch (err) -> .catch (err) ->
res.status(503).send(err?.message or err or 'Unknown error') res.status(503).send(err?.message or err or 'Unknown error')

View File

@ -66,6 +66,9 @@ logTypes =
updateApp: updateApp:
eventName: 'Application update' eventName: 'Application update'
humanName: 'Updating application' humanName: 'Updating application'
updateAppError:
eventName: 'Application update error'
humanName: 'Failed to update application'
logSystemEvent = (logType, app, error) -> logSystemEvent = (logType, app, error) ->
message = "#{logType.humanName} '#{app.imageId}'" message = "#{logType.humanName} '#{app.imageId}'"
@ -109,6 +112,8 @@ exports.kill = kill = (app) ->
throw err throw err
.tap -> .tap ->
logSystemEvent(logTypes.stopAppSuccess, app) logSystemEvent(logTypes.stopAppSuccess, app)
app.containerId = null
knex('app').update(app).where(appId: app.appId)
.catch (err) -> .catch (err) ->
logSystemEvent(logTypes.stopAppError, app, err) logSystemEvent(logTypes.stopAppError, app, err)
throw err throw err
@ -242,8 +247,8 @@ getEnvironment = do ->
throw err throw err
lockPath = (app) -> lockPath = (app) ->
appId = app.appId or app appId = app.appId ? app
return "/mnt/root/resin-data/#{appId}/resin_updates.lock" return "/mnt/root/resin-data/#{appId}/resin-updates.lock"
# At boot, all apps should be unlocked *before* start to prevent a deadlock # At boot, all apps should be unlocked *before* start to prevent a deadlock
exports.unlockAndStart = unlockAndStart = (app) -> exports.unlockAndStart = unlockAndStart = (app) ->
@ -255,14 +260,18 @@ exports.lockUpdates = lockUpdates = do ->
_lock = new Lock() _lock = new Lock()
_writeLock = Promise.promisify(_lock.async.writeLock) _writeLock = Promise.promisify(_lock.async.writeLock)
return (app, force) -> return (app, force) ->
_writeLock(lockPath(app)) lockName = lockPath(app)
.tap -> _writeLock(lockName)
lockFile.unlockAsync(lockPath(app)) if force == true .tap (release) ->
.tap -> if force != true
lockFile.lockAsync(lockPath(app)) lockFile.lockAsync(lockName)
.catch (err) ->
release()
throw new Error("Updates are locked: #{err.message}")
.disposer (release) -> .disposer (release) ->
lockFile.unlockAsync(lockPath(app)) Promise.try ->
.then -> lockFile.unlockAsync(lockName) if force != true
.finally ->
release() release()
joinErrorMessages = (failures) -> joinErrorMessages = (failures) ->
@ -271,19 +280,21 @@ joinErrorMessages = (failures) ->
err.message or err err.message or err
"#{failures.length} error#{s}: #{messages.join(' - ')}" "#{failures.length} error#{s}: #{messages.join(' - ')}"
# 0 - Idle UPDATE_IDLE = 0
# 1 - Updating UPDATE_UPDATING = 1
# 2 - Update required UPDATE_REQUIRED = 2
currentlyUpdating = 0
failedUpdates = 0 updateStatus =
forceNextUpdate = false currentState: UPDATE_IDLE
failedUpdates: 0
forceNextUpdate: false
exports.update = update = (force) -> exports.update = update = (force) ->
if currentlyUpdating isnt 0 if updateStatus.currentState isnt UPDATE_IDLE
# Mark an update required after the current. # Mark an update required after the current.
forceNextUpdate = force updateStatus.forceNextUpdate = force
currentlyUpdating = 2 updateStatus.currentState = UPDATE_REQUIRED
return return
currentlyUpdating = 1 updateStatus.currentState = UPDATE_UPDATING
Promise.all([ Promise.all([
knex('config').select('value').where(key: 'apiKey') knex('config').select('value').where(key: 'apiKey')
knex('config').select('value').where(key: 'uuid') knex('config').select('value').where(key: 'uuid')
@ -317,6 +328,7 @@ exports.update = update = (force) ->
remoteApp.environment_variable = environment remoteApp.environment_variable = environment
return remoteApp return remoteApp
.then (remoteApps) -> .then (remoteApps) ->
remoteAppEnvs = {}
remoteApps = _.map remoteApps, (app) -> remoteApps = _.map remoteApps, (app) ->
env = env =
RESIN_DEVICE_UUID: uuid RESIN_DEVICE_UUID: uuid
@ -325,6 +337,7 @@ exports.update = update = (force) ->
if app.environment_variable? if app.environment_variable?
_.extend(env, app.environment_variable) _.extend(env, app.environment_variable)
remoteAppEnvs[app.id] = env
return { return {
appId: '' + app.id appId: '' + app.id
commit: app.commit commit: app.commit
@ -370,11 +383,12 @@ exports.update = update = (force) ->
.then -> .then ->
knex('app').where('appId', appId).delete() knex('app').where('appId', appId).delete()
.catch (err) -> .catch (err) ->
logSystemEvent(logTypes.stopAppError, apps[appId], err)
failures.push(err) failures.push(err)
.then -> .then ->
# Then install the apps and add each to the db as they succeed # Then install the apps and add each to the db as they succeed
installingPromises = toBeInstalled.map (imageId) -> installingPromises = toBeInstalled.map (appId) ->
app = remoteApps[imageId] app = remoteApps[appId]
start(app) start(app)
.catch (err) -> .catch (err) ->
failures.push(err) failures.push(err)
@ -383,7 +397,8 @@ exports.update = update = (force) ->
localApp = apps[appId] localApp = apps[appId]
app = remoteApps[appId] app = remoteApps[appId]
logSystemEvent(logTypes.updateApp, app) if localApp.imageId == app.imageId logSystemEvent(logTypes.updateApp, app) if localApp.imageId == app.imageId
Promise.using lockUpdates(localApp, force), -> forceThisApp = remoteAppEnvs[appId]['RESIN_OVERRIDE_LOCK'] == '1'
Promise.using lockUpdates(localApp, force || forceThisApp), ->
knex('app').select().where({ appId }) knex('app').select().where({ appId })
.then ([ localApp ]) -> .then ([ localApp ]) ->
if !localApp? if !localApp?
@ -392,29 +407,30 @@ exports.update = update = (force) ->
.then -> .then ->
start(app) start(app)
.catch (err) -> .catch (err) ->
logSystemEvent(logTypes.updateAppError, apps[appId], err)
failures.push(err) failures.push(err)
Promise.all(installingPromises.concat(updatingPromises)) Promise.all(installingPromises.concat(updatingPromises))
.then -> .then ->
throw new Error(joinErrorMessages(failures)) if failures.length > 0 throw new Error(joinErrorMessages(failures)) if failures.length > 0
.then -> .then ->
failedUpdates = 0 updateStatus.failedUpdates = 0
# We cleanup here as we want a point when we have a consistent apps/images state, rather than potentially at a # We cleanup here as we want a point when we have a consistent apps/images state, rather than potentially at a
# point where we might clean up an image we still want. # point where we might clean up an image we still want.
dockerUtils.cleanupContainersAndImages() dockerUtils.cleanupContainersAndImages()
.catch (err) -> .catch (err) ->
failedUpdates++ updateStatus.failedUpdates++
if currentlyUpdating is 2 if updateStatus.currentState is UPDATE_REQUIRED
console.log('Updating failed, but there is already another update scheduled immediately: ', err) console.log('Updating failed, but there is already another update scheduled immediately: ', err)
return return
delayTime = Math.min(failedUpdates * 500, 30000) delayTime = Math.min(updateStatus.failedUpdates * 500, 30000)
# If there was an error then schedule another attempt briefly in the future. # If there was an error then schedule another attempt briefly in the future.
console.log('Scheduling another update attempt due to failure: ', delayTime, err) console.log('Scheduling another update attempt due to failure: ', delayTime, err)
setTimeout(update, delayTime, force) setTimeout(update, delayTime, force)
.finally -> .finally ->
device.updateState(status: 'Idle') device.updateState(status: 'Idle')
if currentlyUpdating is 2 if updateStatus.currentState is UPDATE_REQUIRED
# If an update is required then schedule it # If an update is required then schedule it
setTimeout(update, 1, forceNextUpdate) setTimeout(update, 1, updateStatus.forceNextUpdate)
.finally -> .finally ->
# Set the updating as finished in its own block, so it never has to worry about other code stopping this. # Set the updating as finished in its own block, so it never has to worry about other code stopping this.
currentlyUpdating = 0 updateStatus.currentState = UPDATE_IDLE