From c52c2c0bd9a835a38752ae1f85d333d4cadb1d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Carranza=20V=C3=A9lez?= Date: Wed, 12 Aug 2015 20:24:18 +0000 Subject: [PATCH] First attempt at locking updates with files --- package.json | 1 + src/api.coffee | 6 +++--- src/app.coffee | 2 +- src/application.coffee | 40 ++++++++++++++++++++++++++++++++++------ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 58b285ee..217434c2 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "event-stream": "^3.0.20", "express": "^4.0.0", "knex": "~0.8.3", + "lockfile": "^1.0.1", "lodash": "^3.0.0", "mixpanel": "0.0.20", "network-checker": "~0.0.5", diff --git a/src/api.coffee b/src/api.coffee index 4e1e114b..c4233753 100644 --- a/src/api.coffee +++ b/src/api.coffee @@ -29,7 +29,7 @@ module.exports = (secret) -> api.post '/v1/update', (req, res) -> utils.mixpanelTrack('Update notification') - application.update() + application.update(req.body.force) res.sendStatus(204) api.post '/v1/spawn-tty', (req, res) -> @@ -72,7 +72,7 @@ module.exports = (secret) -> if !app? throw new Error('App not found') Promise.using application.lockUpdates(), -> - application.kill(app) + application.lockAndKill(app) .then -> new Promise (resolve, reject) -> request.post(config.gosuperAddress + '/v1/purge', { json: true, body: applicationId: appId }) @@ -80,7 +80,7 @@ module.exports = (secret) -> .on 'response', -> resolve() .pipe(res) .finally -> - application.start(app) + application.startAndUnlock(app) .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') diff --git a/src/app.coffee b/src/app.coffee index 7558631c..f7c6d0f0 100644 --- a/src/app.coffee +++ b/src/app.coffee @@ -53,7 +53,7 @@ knex.init.then -> console.log('Starting Apps..') knex('app').select() .then (apps) -> - Promise.all(apps.map(application.start)) + Promise.all(apps.map(application.startAndUnlock)) .catch (error) -> console.error('Error starting apps:', error) .then -> diff --git a/src/application.coffee b/src/application.coffee index 42229f33..a1c82473 100644 --- a/src/application.coffee +++ b/src/application.coffee @@ -11,6 +11,8 @@ tty = require './lib/tty' logger = require './lib/logger' { cachedResinApi } = require './request' device = require './device' +lockFile = Promise.promisifyAll(require('lockfile')) +fs = Promise.promisifyAll(require('fs')) { docker } = dockerUtils @@ -30,7 +32,7 @@ logTypes = humanName: 'Killed application' stopAppError: eventName: 'Application stop error' - humanName: 'Killed application' + humanName: 'Failed to kill application' downloadApp: eventName: 'Application download' @@ -244,6 +246,32 @@ getEnvironment = do -> console.error("Failed to get environment for device #{deviceId}, app #{appId}. #{err}") throw err +lockPath = (app) -> "/mnt/root/resin-data/#{app.appId}/resin_updates.lock" +locked = {} +exports.lockAndKill = lockAndKill = (app, force) -> + Promise.try -> + fs.unlinkAsync(lockPath(app)) if force == true + .then -> + locked[app.appId] = true + lockFile.lockAsync(lockPath(app)) + .catch (err) -> + if err.code != 'ENOENT' + locked[app.appId] = false + message = 'Updates are locked by application' + logSystemEvent(logTypes.stopAppError, app, { message }) + throw message + .then -> + kill(app) + +exports.startAndUnlock = startAndUnlock = (app) -> + Promise.try -> + throw "Cannot start app because we couldn't acquire lock" if locked[app.appId] == false + .then -> + locked[app.appId] = null + start(app) + .then -> + lockFile.unlockAsync(lockPath(app)) + exports.lockUpdates = lockUpdates = do -> _lock = new Lock() _lockUpdates = Promise.promisify(_lock.async.writeLock) @@ -254,7 +282,7 @@ exports.lockUpdates = lockUpdates = do -> # 2 - Update required currentlyUpdating = 0 failedUpdates = 0 -exports.update = update = -> +exports.update = update = (force) -> if currentlyUpdating isnt 0 # Mark an update required after the current. currentlyUpdating = 2 @@ -331,20 +359,20 @@ exports.update = update = -> Promise.using lockUpdates(), -> # Then delete all the ones to remove in one go Promise.map toBeRemoved, (imageId) -> - kill(apps[imageId]) + lockAndKill(apps[imageId], force) .then -> # Then install the apps and add each to the db as they succeed installingPromises = toBeInstalled.map (imageId) -> app = remoteApps[imageId] - start(app) + startAndUnlock(app) # And remove/recreate updated apps and update db as they succeed updatingPromises = toBeUpdated.map (imageId) -> localApp = apps[imageId] app = remoteApps[imageId] logSystemEvent(logTypes.updateApp, app) - kill(localApp) + lockAndKill(localApp, force) .then -> - start(app) + startAndUnlock(app) Promise.all(installingPromises.concat(updatingPromises)) .then -> failedUpdates = 0