diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b2a5cca..34f8527b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +* Add endpoints to manage images and containers locally [Pablo] +* Only use bodyParser for endpoints that need it [Pablo] * Add RESIN_APP_ID variable [Pablo] * Increase delta request timeout to 15 minutes [Pablo] diff --git a/docs/API.md b/docs/API.md index 52cd5ec6..35c00cb7 100644 --- a/docs/API.md +++ b/docs/API.md @@ -484,3 +484,128 @@ $ curl -X POST --header "Content-Type:application/json" \ --data '{"deviceId": , "appId": , "method": "GET"}' \ "https://api.resin.io/supervisor/v1/apps/" ``` + +
+ +## API endpoints for image and container management + +Supervisor 1.9 introduces these endpoints to allow users to create images and containers from the app. These endpoints are designed to be +as close as possible to the corresponding endpoints from the Docker Remote API. We chose this approach instead of just exposing the docker socket +because we need the supervisor to track the containers and images to avoid cleaning them up unwantedly. + +All examples are shown as posted from the app on the device, as this is the intended use, but the API proxy works like with the other endpoints. + +Refer to the [Docker API docs](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22) for detailed descriptions of request parameters and response formats. + +### POST /v1/images/create + +Works like [/images/create from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#create-an-image). +Allows the creation of images by pulling them from a registry or from a tar archive. + +#### Example: + +Creating an image from a tarball: +```bash +$ curl -X POST --data-binary @hello-master.tar \ + "$RESIN_SUPERVISOR_ADDRESS/v1/images/create?fromSrc=-&repo=hello&tag=master&apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +Pulling from DockerHub: +```bash +$ curl -X POST \ + "$RESIN_SUPERVISOR_ADDRESS/v1/images/create?fromImage=alpine&tag=latest&apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +
+ +### DELETE /v1/images/:name + +Deletes image with name `name`. Works like [DELETE /images/(name) from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#remove-an-image). +Will only work if `name` is sent as the `repo:tag` used for creating the image. + +#### Example: + +```bash +$ curl -X DELETE \ + "$RESIN_SUPERVISOR_ADDRESS/v1/images/hello:master?apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +
+ +### GET /v1/images + +Works like [GET /images/json from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#list-images). + +#### Example: + +```bash +$ curl -X GET \ + "$RESIN_SUPERVISOR_ADDRESS/v1/images?apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +
+ +### POST /v1/containers/create + +Works like [/containers/create from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#create-a-container). +Can only be used with `Image` specifying a `repo:tag` created with /v1/images/create. + +#### Example: + +``` +$ curl -X POST --data '{"Image":"alpine:latest","Cmd":["sh","-c", "while true; do echo hi; sleep 5; done"]}' -H "Content-Type: application/json" \ + "$RESIN_SUPERVISOR_ADDRESS/v1/containers/create?apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +
+ +### POST /v1/containers/:id/start + +Starts a container. Works like [/containers/(id)/start from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#start-a-container). +The id can be extracted from the response of /v1/containers/create. + +#### Example: + +``` +$ curl -X POST \ + "$RESIN_SUPERVISOR_ADDRESS/v1/containers/ac072860f31a9df31dea72f8418d193e6af257b4fed4008f86097200fba45966/start?apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +
+ +### POST /v1/containers/:id/stop + +Stops a container. Works like [/containers/(id)/stop from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#stop-a-container). +As with start, the id can be extracted from the response of /v1/containers/create. + +#### Example: + +``` +$ curl -X POST \ + "$RESIN_SUPERVISOR_ADDRESS/v1/containers/ac072860f31a9df31dea72f8418d193e6af257b4fed4008f86097200fba45966/stop?apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +
+ +### DELETE /v1/containers/:id + +Deletes a container. Works like [DELETE /containers/(id) from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#remove-a-container). +The id can be extracted from the response of /v1/containers/create. + +#### Example: + +``` +$ curl -X DELETE \ + "$RESIN_SUPERVISOR_ADDRESS/v1/containers/ac072860f31a9df31dea72f8418d193e6af257b4fed4008f86097200fba45966?apikey=$RESIN_SUPERVISOR_API_KEY" +``` + +### GET /v1/containers + +Lists containers. Works like [GET /containers/json from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#list-containers). + +#### Example: + +``` +$ curl -X GET \ + "$RESIN_SUPERVISOR_ADDRESS/v1/containers?apikey=$RESIN_SUPERVISOR_API_KEY" +``` diff --git a/src/api.coffee b/src/api.coffee index c5d9c867..28e46a68 100644 --- a/src/api.coffee +++ b/src/api.coffee @@ -7,6 +7,7 @@ bodyParser = require 'body-parser' request = require 'request' config = require './config' device = require './device' +dockerUtils = require './docker-utils' _ = require 'lodash' privateAppEnvVars = [ @@ -16,7 +17,10 @@ privateAppEnvVars = [ module.exports = (application) -> api = express() - api.use(bodyParser()) + unparsedRouter = express.Router() + parsedRouter = express.Router() + parsedRouter.use(bodyParser()) + api.use (req, res, next) -> utils.getOrGenerateSecret('api') .then (secret) -> @@ -28,31 +32,31 @@ module.exports = (application) -> # This should never happen... res.status(503).send('Invalid API key in supervisor') - api.get '/ping', (req, res) -> + unparsedRouter.get '/ping', (req, res) -> res.send('OK') - api.post '/v1/blink', (req, res) -> + unparsedRouter.post '/v1/blink', (req, res) -> utils.mixpanelTrack('Device blink') utils.blink.pattern.start() setTimeout(utils.blink.pattern.stop, 15000) res.sendStatus(200) - api.post '/v1/update', (req, res) -> + parsedRouter.post '/v1/update', (req, res) -> utils.mixpanelTrack('Update notification') application.update(req.body.force) res.sendStatus(204) - api.post '/v1/reboot', (req, res) -> + unparsedRouter.post '/v1/reboot', (req, res) -> utils.mixpanelTrack('Reboot') request.post(config.gosuperAddress + '/v1/reboot') .pipe(res) - api.post '/v1/shutdown', (req, res) -> + unparsedRouter.post '/v1/shutdown', (req, res) -> utils.mixpanelTrack('Shutdown') request.post(config.gosuperAddress + '/v1/shutdown') .pipe(res) - api.post '/v1/purge', (req, res) -> + parsedRouter.post '/v1/purge',(req, res) -> appId = req.body.appId utils.mixpanelTrack('Purge /data', appId) if !appId? @@ -74,15 +78,15 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') - api.post '/v1/tcp-ping', (req, res) -> + unparsedRouter.post '/v1/tcp-ping', (req, res) -> utils.disableCheck(false) res.sendStatus(204) - api.delete '/v1/tcp-ping', (req, res) -> + unparsedRouter.delete '/v1/tcp-ping', (req, res) -> utils.disableCheck(true) res.sendStatus(204) - api.post '/v1/restart', (req, res) -> + parsedRouter.post '/v1/restart', (req, res) -> appId = req.body.appId force = req.body.force utils.mixpanelTrack('Restart container', appId) @@ -101,7 +105,7 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') - api.post '/v1/apps/:appId/stop', (req, res) -> + parsedRouter.post '/v1/apps/:appId/stop', (req, res) -> { appId } = req.params { force } = req.body utils.mixpanelTrack('Stop container', appId) @@ -118,7 +122,7 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') - api.post '/v1/apps/:appId/start', (req, res) -> + unparsedRouter.post '/v1/apps/:appId/start', (req, res) -> { appId } = req.params utils.mixpanelTrack('Start container', appId) if !appId? @@ -134,7 +138,7 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') - api.get '/v1/apps/:appId', (req, res) -> + unparsedRouter.get '/v1/apps/:appId', (req, res) -> { appId } = req.params utils.mixpanelTrack('GET app', appId) if !appId? @@ -155,7 +159,7 @@ module.exports = (application) -> # Expires the supervisor's API key and generates a new one. # It also communicates the new key to the Resin API. - api.post '/v1/regenerate-api-key', (req, res) -> + unparsedRouter.post '/v1/regenerate-api-key', (req, res) -> utils.newSecret('api') .then (secret) -> device.updateState(api_secret: secret) @@ -163,7 +167,19 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') - api.get '/v1/device', (req, res) -> + unparsedRouter.get '/v1/device', (req, res) -> res.json(device.getState()) + unparsedRouter.post '/v1/images/create', dockerUtils.createImage + unparsedRouter.delete '/v1/images/*', dockerUtils.deleteImage + unparsedRouter.get '/v1/images', dockerUtils.listImages + parsedRouter.post '/v1/containers/create', dockerUtils.createContainer + parsedRouter.post '/v1/containers/:id/start', dockerUtils.startContainer + unparsedRouter.post '/v1/containers/:id/stop', dockerUtils.stopContainer + unparsedRouter.delete '/v1/containers/:id', dockerUtils.deleteContainer + unparsedRouter.get '/v1/containers', dockerUtils.listContainers + + api.use(unparsedRouter) + api.use(parsedRouter) + return api diff --git a/src/db.coffee b/src/db.coffee index b8eb308e..333bd95a 100644 --- a/src/db.coffee +++ b/src/db.coffee @@ -40,6 +40,13 @@ knex.init = Promise.all([ addColumn('app', 'appId', 'string') ] + knex.schema.hasTable('image') + .then (exists) -> + if not exists + knex.schema.createTable 'image', (t) -> + t.increments('id').primary() + t.string('repoTag') + ]) module.exports = knex diff --git a/src/docker-utils.coffee b/src/docker-utils.coffee index 45866196..1ab4bc1f 100644 --- a/src/docker-utils.coffee +++ b/src/docker-utils.coffee @@ -9,7 +9,7 @@ knex = require './db' TypedError = require 'typed-error' { request } = require './request' fs = Promise.promisifyAll require 'fs' - +Lock = require 'rwlock' class OutOfSyncError extends TypedError docker = Promise.promisifyAll(new Docker(socketPath: config.dockerSocket)) @@ -149,6 +149,13 @@ dockerSync = (imgSrc, imgDest, rsyncDiff, conf) -> docker.getImage(destId).tagAsync({ repo, tag, force: true }) do -> + _lock = new Lock() + _writeLock = Promise.promisify(_lock.async.writeLock) + lockImages = -> + _writeLock('images') + .disposer (release) -> + release() + # Keep track of the images being fetched, so we don't clean them up whilst fetching. imagesBeingFetched = 0 exports.fetchImageWithProgress = (image, onProgress) -> @@ -162,51 +169,179 @@ do -> # If there is no tag then mark it as latest supervisorTag += ':latest' exports.cleanupContainersAndImages = -> - Promise.join( - knex('app').select() - .map (app) -> - app.imageId + ':latest' - docker.listImagesAsync() - (apps, images) -> - imageTags = _.map(images, 'RepoTags') - supervisorTags = _.filter imageTags, (tags) -> - _.contains(tags, supervisorTag) - appTags = _.filter imageTags, (tags) -> - _.any tags, (tag) -> - _.contains(apps, tag) - supervisorTags = _.flatten(supervisorTags) - appTags = _.flatten(appTags) - return { images, supervisorTags, appTags } - ) - .then ({ images, supervisorTags, appTags }) -> - # Cleanup containers first, so that they don't block image removal. - docker.listContainersAsync(all: true) - .filter (containerInfo) -> - # Do not remove user apps. - if _.contains(appTags, containerInfo.Image) - return false - if !_.contains(supervisorTags, containerInfo.Image) - return true - return containerHasExited(containerInfo.Id) - .map (containerInfo) -> - docker.getContainer(containerInfo.Id).removeAsync() + Promise.using lockImages(), -> + Promise.join( + knex('image').select('repoTag') + .map (image) -> + image.repoTag + knex('app').select() + .map (app) -> + app.imageId + ':latest' + docker.listImagesAsync() + (localTags, apps, images) -> + imageTags = _.map(images, 'RepoTags') + supervisorTags = _.filter imageTags, (tags) -> + _.contains(tags, supervisorTag) + appTags = _.filter imageTags, (tags) -> + _.any tags, (tag) -> + _.contains(apps, tag) + locallyCreatedTags = _.filter imageTags, (tags) -> + _.any tags, (tag) -> + _.contains(localTags, tag) + supervisorTags = _.flatten(supervisorTags) + appTags = _.flatten(appTags) + locallyCreatedTags = _.flatten(locallyCreatedTags) + return { images, supervisorTags, appTags, locallyCreatedTags } + ) + .then ({ images, supervisorTags, appTags, locallyCreatedTags }) -> + # Cleanup containers first, so that they don't block image removal. + docker.listContainersAsync(all: true) + .filter (containerInfo) -> + # Do not remove user apps. + if _.contains(appTags, containerInfo.Image) + return false + if _.contains(locallyCreatedTags, containerInfo.Image) + return false + if !_.contains(supervisorTags, containerInfo.Image) + return true + return containerHasExited(containerInfo.Id) + .map (containerInfo) -> + docker.getContainer(containerInfo.Id).removeAsync() + .then -> + console.log('Deleted container:', containerInfo.Id, containerInfo.Image) + .catch(_.noop) .then -> - console.log('Deleted container:', containerInfo.Id, containerInfo.Image) - .catch(_.noop) - .then -> - # And then clean up the images, as long as we aren't currently trying to fetch any. - return if imagesBeingFetched > 0 - imagesToClean = _.reject images, (image) -> - _.any image.RepoTags, (tag) -> - return _.contains(appTags, tag) or _.contains(supervisorTags, tag) - Promise.map imagesToClean, (image) -> - Promise.map image.RepoTags.concat(image.Id), (tag) -> - docker.getImage(tag).removeAsync() - .then -> - console.log('Deleted image:', tag, image.Id, image.RepoTags) - .catch(_.noop) + # And then clean up the images, as long as we aren't currently trying to fetch any. + return if imagesBeingFetched > 0 + imagesToClean = _.reject images, (image) -> + _.any image.RepoTags, (tag) -> + return _.contains(appTags, tag) or _.contains(supervisorTags, tag) or _.contains(locallyCreatedTags, tag) + Promise.map imagesToClean, (image) -> + Promise.map image.RepoTags.concat(image.Id), (tag) -> + docker.getImage(tag).removeAsync() + .then -> + console.log('Deleted image:', tag, image.Id, image.RepoTags) + .catch(_.noop) containerHasExited = (id) -> docker.getContainer(id).inspectAsync() .then (data) -> return not data.State.Running + + buildRepoTag = (repo, tag, registry) -> + repoTag = '' + if registry? + repoTag += registry + '/' + repoTag += repo + if tag? + repoTag += ':' + tag + else + repoTag += ':latest' + return repoTag + + sanitizeQuery = (query) -> + _.omit(query, 'apikey') + + exports.createImage = (req, res) -> + { registry, repo, tag, fromImage, fromSrc } = req.query + if fromImage? + repoTag = fromImage + repoTag += ':' + tag if tag? + else + repoTag = buildRepoTag(repo, tag, registry) + Promise.using lockImages(), -> + knex('image').insert({ repoTag }) + .then -> + if fromImage? + docker.createImageAsync({ fromImage, tag }) + else + docker.importImageAsync(req, { repo, tag, registry }) + .then (stream) -> + stream.pipe(res) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + exports.deleteImage = (req, res) -> + imageName = req.params[0] + Promise.using lockImages(), -> + knex('image').select().where('repoTag', imageName) + .then (images) -> + throw new Error('Only images created via the Supervisor can be deleted.') if images.length == 0 + knex('image').where('repoTag', imageName).delete() + .then -> + docker.getImage(imageName).removeAsync(sanitizeQuery(req.query)) + .then (data) -> + res.json(data) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + exports.listImages = (req, res) -> + docker.listImagesAsync(sanitizeQuery(req.query)) + .then (images) -> + res.json(images) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + docker.modem.dialAsync = Promise.promisify(docker.modem.dial) + exports.createContainer = (req, res) -> + Promise.using lockImages(), -> + knex('image').select().where('repoTag', req.body.Image) + .then (images) -> + throw new Error('Only images created via the Supervisor can be used for creating containers.') if images.length == 0 + optsf = + path: '/containers/create?' + method: 'POST' + options: req.body + statusCodes: + 200: true + 201: true + 404: 'no such container' + 406: 'impossible to attach' + 500: 'server error' + docker.modem.dialAsync(optsf) + .then (data) -> + res.json(data) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + exports.startContainer = (req, res) -> + docker.getContainer(req.params.id).startAsync(req.body) + .then (data) -> + res.json(data) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + exports.stopContainer = (req, res) -> + container = docker.getContainer(req.params.id) + knex('app').select() + .then (apps) -> + throw new Error('Cannot stop an app container') if _.any(apps, containerId: req.params.id) + container.inspectAsync() + .then (cont) -> + throw new Error('Cannot stop supervisor container') if cont.Name == '/resin_supervisor' or _.any(cont.Names, (n) -> n == '/resin_supervisor') + container.stopAsync(sanitizeQuery(req.query)) + .then (data) -> + res.json(data) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + exports.deleteContainer = (req, res) -> + container = docker.getContainer(req.params.id) + knex('app').select() + .then (apps) -> + throw new Error('Cannot remove an app container') if _.any(apps, containerId: req.params.id) + container.inspectAsync() + .then (cont) -> + throw new Error('Cannot remove supervisor container') if cont.Name == '/resin_supervisor' or _.any(cont.Names, (n) -> n == '/resin_supervisor') + container.removeAsync(sanitizeQuery(req.query)) + .then (data) -> + res.json(data) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error') + + exports.listContainers = (req, res) -> + docker.listContainersAsync(sanitizeQuery(req.query)) + .then (containers) -> + res.json(containers) + .catch (err) -> + res.status(500).send(err?.message or err or 'Unknown error')