From 42ac7487e7a9229f367514fb7e8a82be7d9b5f51 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez <pablo@resin.io> Date: Mon, 10 Jul 2017 06:31:50 -0700 Subject: [PATCH] When the device is about to reboot or shutdown, close the API server and avoid applying updates We mark when the device is rebooting and avoid some steps in the update cycle that change the device state, similarly to when the device is in local mode, to avoid problems with non-atomic operations. This doesn't solve *all* the potential scenarios of a reboot happening in the middle of an update, but at least should prevent the case where we start an app container and reboot the device before saving the containerId, potentially causing a duplicated container issue. We also correct the API docs to reflect the 202 response when reboot or shutdown are successful. Change-Type: patch Signed-off-by: Pablo Carranza Velez <pablo@resin.io> --- docs/API.md | 6 +++--- src/api.coffee | 24 ++++++++++-------------- src/app.coffee | 3 ++- src/application.coffee | 8 ++++---- src/device.coffee | 21 +++++++++++++++++++-- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/docs/API.md b/docs/API.md index 22e01448..22138682 100644 --- a/docs/API.md +++ b/docs/API.md @@ -77,7 +77,7 @@ $ curl -X POST --header "Content-Type:application/json" \ Triggers an update check on the supervisor. Optionally, forces an update when updates are locked. -Responds with an empty 204 (Accepted) response. +Responds with an empty 204 (No Content) response. #### Request body Can be a JSON object with a `force` property. If this property is true, the update lock will be overridden. @@ -110,7 +110,7 @@ $ curl -X POST --header "Content-Type:application/json" \ Reboots the device -When successful, responds with 204 accepted and a JSON object: +When successful, responds with 202 accepted and a JSON object: ```json { "Data": "OK", @@ -144,7 +144,7 @@ $ curl -X POST --header "Content-Type:application/json" \ **Dangerous**. Shuts down the device. -When successful, responds with 204 accepted and a JSON object: +When successful, responds with 202 accepted and a JSON object: ```json { "Data": "OK", diff --git a/src/api.coffee b/src/api.coffee index 37df8364..c0d0058e 100644 --- a/src/api.coffee +++ b/src/api.coffee @@ -57,18 +57,16 @@ module.exports = (application) -> .then (app) -> application.kill(app, removeContainer: false) if app? .then -> - new Promise (resolve, reject) -> - application.logSystemMessage('Rebooting', {}, 'Reboot') - utils.gosuper.post('/v1/reboot') - .on('error', reject) - .on('response', -> resolve()) - .pipe(res) + application.logSystemMessage('Rebooting', {}, 'Reboot') + device.reboot() + .then (response) -> + res.status(202).json(response) .catch (err) -> if err instanceof application.UpdatesLockedError status = 423 else status = 500 - res.status(status).send(err?.message or err or 'Unknown error') + res.status(status).json({ Data: '', Error: err?.message or err or 'Unknown error' }) parsedRouter.post '/v1/shutdown', (req, res) -> force = req.body.force @@ -80,18 +78,16 @@ module.exports = (application) -> .then (app) -> application.kill(app, removeContainer: false) if app? .then -> - new Promise (resolve, reject) -> - application.logSystemMessage('Shutting down', {}, 'Shutdown') - utils.gosuper.post('/v1/shutdown') - .on('error', reject) - .on('response', -> resolve()) - .pipe(res) + application.logSystemMessage('Shutting down', {}, 'Shutdown') + device.shutdown() + .then (response) -> + res.status(202).json(response) .catch (err) -> if err instanceof application.UpdatesLockedError status = 423 else status = 500 - res.status(status).send(err?.message or err or 'Unknown error') + res.status(status).json({ Data: '', Error: err?.message or err or 'Unknown error' }) parsedRouter.post '/v1/purge', (req, res) -> appId = req.body.appId diff --git a/src/app.coffee b/src/app.coffee index 847d813d..ce00369d 100644 --- a/src/app.coffee +++ b/src/app.coffee @@ -29,7 +29,8 @@ knex.init.then -> .then -> apiServer = api(application).listen(config.listenPort) apiServer.timeout = config.apiTimeout - + device.events.on 'shutdown', -> + apiServer.close() bootstrap.done .then -> Promise.join( diff --git a/src/application.coffee b/src/application.coffee index 6d77344e..8ab06021 100644 --- a/src/application.coffee +++ b/src/application.coffee @@ -709,7 +709,7 @@ application.update = update = (force, scheduled = false) -> .tap (remoteApps) -> # Before running the updates, try to clean up any images that aren't in use # and will not be used in the target state - return if application.localMode + return if application.localMode or device.shuttingDown dockerUtils.cleanupContainersAndImages(_.map(remoteApps, 'imageId')) .catch (err) -> console.log('Cleanup failed: ', err, err.stack) @@ -737,7 +737,7 @@ application.update = update = (force, scheduled = false) -> logSystemMessage("Error fetching/applying device configuration: #{err}", { error: err }, 'Set device configuration error') .return(allAppIds) .map (appId) -> - return if application.localMode + return if application.localMode or device.shuttingDown Promise.try -> needsDownload = _.includes(toBeDownloaded, appId) if _.includes(toBeRemoved, appId) @@ -792,9 +792,9 @@ application.update = update = (force, scheduled = false) -> _.each(failures, (err) -> console.error('Error:', err, err.stack)) throw new Error(joinErrorMessages(failures)) if failures.length > 0 .then -> - proxyvisor.sendUpdates() + proxyvisor.sendUpdates() if !device.shuttingDown .then -> - return if application.localMode + return if application.localMode or device.shuttingDown updateStatus.failed = 0 device.setUpdateState(update_pending: false, update_downloaded: false, update_failed: false) # We cleanup here as we want a point when we have a consistent apps/images state, rather than potentially at a diff --git a/src/device.coffee b/src/device.coffee index 3abf868c..3e75039e 100644 --- a/src/device.coffee +++ b/src/device.coffee @@ -1,7 +1,6 @@ _ = require 'lodash' Promise = require 'bluebird' memoizee = require 'memoizee' -knex = require './db' utils = require './utils' { resinApi } = require './request' device = exports @@ -12,6 +11,7 @@ fs = Promise.promisifyAll(require('fs')) bootstrap = require './bootstrap' { checkTruthy } = require './lib/validation' osRelease = require './lib/os-release' +EventEmitter = require 'events' # If we don't use promise: 'then', exceptions will crash the program memoizePromise = _.partial(memoizee, _, promise: 'then') @@ -40,8 +40,25 @@ exports.getID = memoizePromise -> throw new Error('Could not find this device?!') return devices[0].id +exports.shuttingDown = false +exports.events = new EventEmitter() exports.reboot = -> - utils.gosuper.postAsync('/v1/reboot') + utils.gosuper.postAsync('/v1/reboot', { json: true }) + .spread (res, body) -> + if res.statusCode != 202 + throw new Error(body.Error) + exports.shuttingDown = true + exports.events.emit('shutdown') + return body + +exports.shutdown = -> + utils.gosuper.postAsync('/v1/shutdown', { json: true }) + .spread (res, body) -> + if res.statusCode != 202 + throw new Error(body.Error) + exports.shuttingDown = true + exports.events.emit('shutdown') + return body exports.hostConfigConfigVarPrefix = 'RESIN_HOST_' bootConfigEnvVarPrefix = 'RESIN_HOST_CONFIG_'