Merge pull request #143 from resin-io/142-local-management-api

Add endpoints to manage images and containers locally
This commit is contained in:
Pablo Carranza Vélez 2016-05-17 16:35:52 -03:00
commit 6748eb3c41
5 changed files with 343 additions and 58 deletions

View File

@ -1,3 +1,5 @@
* Add endpoints to manage images and containers locally [Pablo]
* Only use bodyParser for endpoints that need it [Pablo]
* Add RESIN_APP_ID variable [Pablo]
* Increase delta request timeout to 15 minutes [Pablo]

View File

@ -484,3 +484,128 @@ $ curl -X POST --header "Content-Type:application/json" \
--data '{"deviceId": <deviceId>, "appId": <appId>, "method": "GET"}' \
"https://api.resin.io/supervisor/v1/apps/<appId>"
```
<hr>
## API endpoints for image and container management
Supervisor 1.9 introduces these endpoints to allow users to create images and containers from the app. These endpoints are designed to be
as close as possible to the corresponding endpoints from the Docker Remote API. We chose this approach instead of just exposing the docker socket
because we need the supervisor to track the containers and images to avoid cleaning them up unwantedly.
All examples are shown as posted from the app on the device, as this is the intended use, but the API proxy works like with the other endpoints.
Refer to the [Docker API docs](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22) for detailed descriptions of request parameters and response formats.
### POST /v1/images/create
Works like [/images/create from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#create-an-image).
Allows the creation of images by pulling them from a registry or from a tar archive.
#### Example:
Creating an image from a tarball:
```bash
$ curl -X POST --data-binary @hello-master.tar \
"$RESIN_SUPERVISOR_ADDRESS/v1/images/create?fromSrc=-&repo=hello&tag=master&apikey=$RESIN_SUPERVISOR_API_KEY"
```
Pulling from DockerHub:
```bash
$ curl -X POST \
"$RESIN_SUPERVISOR_ADDRESS/v1/images/create?fromImage=alpine&tag=latest&apikey=$RESIN_SUPERVISOR_API_KEY"
```
<hr>
### DELETE /v1/images/:name
Deletes image with name `name`. Works like [DELETE /images/(name) from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#remove-an-image).
Will only work if `name` is sent as the `repo:tag` used for creating the image.
#### Example:
```bash
$ curl -X DELETE \
"$RESIN_SUPERVISOR_ADDRESS/v1/images/hello:master?apikey=$RESIN_SUPERVISOR_API_KEY"
```
<hr>
### GET /v1/images
Works like [GET /images/json from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#list-images).
#### Example:
```bash
$ curl -X GET \
"$RESIN_SUPERVISOR_ADDRESS/v1/images?apikey=$RESIN_SUPERVISOR_API_KEY"
```
<hr>
### POST /v1/containers/create
Works like [/containers/create from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#create-a-container).
Can only be used with `Image` specifying a `repo:tag` created with /v1/images/create.
#### Example:
```
$ curl -X POST --data '{"Image":"alpine:latest","Cmd":["sh","-c", "while true; do echo hi; sleep 5; done"]}' -H "Content-Type: application/json" \
"$RESIN_SUPERVISOR_ADDRESS/v1/containers/create?apikey=$RESIN_SUPERVISOR_API_KEY"
```
<hr>
### POST /v1/containers/:id/start
Starts a container. Works like [/containers/(id)/start from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#start-a-container).
The id can be extracted from the response of /v1/containers/create.
#### Example:
```
$ curl -X POST \
"$RESIN_SUPERVISOR_ADDRESS/v1/containers/ac072860f31a9df31dea72f8418d193e6af257b4fed4008f86097200fba45966/start?apikey=$RESIN_SUPERVISOR_API_KEY"
```
<hr>
### POST /v1/containers/:id/stop
Stops a container. Works like [/containers/(id)/stop from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#stop-a-container).
As with start, the id can be extracted from the response of /v1/containers/create.
#### Example:
```
$ curl -X POST \
"$RESIN_SUPERVISOR_ADDRESS/v1/containers/ac072860f31a9df31dea72f8418d193e6af257b4fed4008f86097200fba45966/stop?apikey=$RESIN_SUPERVISOR_API_KEY"
```
<hr>
### DELETE /v1/containers/:id
Deletes a container. Works like [DELETE /containers/(id) from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#remove-a-container).
The id can be extracted from the response of /v1/containers/create.
#### Example:
```
$ curl -X DELETE \
"$RESIN_SUPERVISOR_ADDRESS/v1/containers/ac072860f31a9df31dea72f8418d193e6af257b4fed4008f86097200fba45966?apikey=$RESIN_SUPERVISOR_API_KEY"
```
### GET /v1/containers
Lists containers. Works like [GET /containers/json from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#list-containers).
#### Example:
```
$ curl -X GET \
"$RESIN_SUPERVISOR_ADDRESS/v1/containers?apikey=$RESIN_SUPERVISOR_API_KEY"
```

View File

@ -7,6 +7,7 @@ bodyParser = require 'body-parser'
request = require 'request'
config = require './config'
device = require './device'
dockerUtils = require './docker-utils'
_ = require 'lodash'
privateAppEnvVars = [
@ -16,7 +17,10 @@ privateAppEnvVars = [
module.exports = (application) ->
api = express()
api.use(bodyParser())
unparsedRouter = express.Router()
parsedRouter = express.Router()
parsedRouter.use(bodyParser())
api.use (req, res, next) ->
utils.getOrGenerateSecret('api')
.then (secret) ->
@ -28,31 +32,31 @@ module.exports = (application) ->
# This should never happen...
res.status(503).send('Invalid API key in supervisor')
api.get '/ping', (req, res) ->
unparsedRouter.get '/ping', (req, res) ->
res.send('OK')
api.post '/v1/blink', (req, res) ->
unparsedRouter.post '/v1/blink', (req, res) ->
utils.mixpanelTrack('Device blink')
utils.blink.pattern.start()
setTimeout(utils.blink.pattern.stop, 15000)
res.sendStatus(200)
api.post '/v1/update', (req, res) ->
parsedRouter.post '/v1/update', (req, res) ->
utils.mixpanelTrack('Update notification')
application.update(req.body.force)
res.sendStatus(204)
api.post '/v1/reboot', (req, res) ->
unparsedRouter.post '/v1/reboot', (req, res) ->
utils.mixpanelTrack('Reboot')
request.post(config.gosuperAddress + '/v1/reboot')
.pipe(res)
api.post '/v1/shutdown', (req, res) ->
unparsedRouter.post '/v1/shutdown', (req, res) ->
utils.mixpanelTrack('Shutdown')
request.post(config.gosuperAddress + '/v1/shutdown')
.pipe(res)
api.post '/v1/purge', (req, res) ->
parsedRouter.post '/v1/purge',(req, res) ->
appId = req.body.appId
utils.mixpanelTrack('Purge /data', appId)
if !appId?
@ -74,15 +78,15 @@ module.exports = (application) ->
.catch (err) ->
res.status(503).send(err?.message or err or 'Unknown error')
api.post '/v1/tcp-ping', (req, res) ->
unparsedRouter.post '/v1/tcp-ping', (req, res) ->
utils.disableCheck(false)
res.sendStatus(204)
api.delete '/v1/tcp-ping', (req, res) ->
unparsedRouter.delete '/v1/tcp-ping', (req, res) ->
utils.disableCheck(true)
res.sendStatus(204)
api.post '/v1/restart', (req, res) ->
parsedRouter.post '/v1/restart', (req, res) ->
appId = req.body.appId
force = req.body.force
utils.mixpanelTrack('Restart container', appId)
@ -101,7 +105,7 @@ module.exports = (application) ->
.catch (err) ->
res.status(503).send(err?.message or err or 'Unknown error')
api.post '/v1/apps/:appId/stop', (req, res) ->
parsedRouter.post '/v1/apps/:appId/stop', (req, res) ->
{ appId } = req.params
{ force } = req.body
utils.mixpanelTrack('Stop container', appId)
@ -118,7 +122,7 @@ module.exports = (application) ->
.catch (err) ->
res.status(503).send(err?.message or err or 'Unknown error')
api.post '/v1/apps/:appId/start', (req, res) ->
unparsedRouter.post '/v1/apps/:appId/start', (req, res) ->
{ appId } = req.params
utils.mixpanelTrack('Start container', appId)
if !appId?
@ -134,7 +138,7 @@ module.exports = (application) ->
.catch (err) ->
res.status(503).send(err?.message or err or 'Unknown error')
api.get '/v1/apps/:appId', (req, res) ->
unparsedRouter.get '/v1/apps/:appId', (req, res) ->
{ appId } = req.params
utils.mixpanelTrack('GET app', appId)
if !appId?
@ -155,7 +159,7 @@ module.exports = (application) ->
# Expires the supervisor's API key and generates a new one.
# It also communicates the new key to the Resin API.
api.post '/v1/regenerate-api-key', (req, res) ->
unparsedRouter.post '/v1/regenerate-api-key', (req, res) ->
utils.newSecret('api')
.then (secret) ->
device.updateState(api_secret: secret)
@ -163,7 +167,19 @@ module.exports = (application) ->
.catch (err) ->
res.status(503).send(err?.message or err or 'Unknown error')
api.get '/v1/device', (req, res) ->
unparsedRouter.get '/v1/device', (req, res) ->
res.json(device.getState())
unparsedRouter.post '/v1/images/create', dockerUtils.createImage
unparsedRouter.delete '/v1/images/*', dockerUtils.deleteImage
unparsedRouter.get '/v1/images', dockerUtils.listImages
parsedRouter.post '/v1/containers/create', dockerUtils.createContainer
parsedRouter.post '/v1/containers/:id/start', dockerUtils.startContainer
unparsedRouter.post '/v1/containers/:id/stop', dockerUtils.stopContainer
unparsedRouter.delete '/v1/containers/:id', dockerUtils.deleteContainer
unparsedRouter.get '/v1/containers', dockerUtils.listContainers
api.use(unparsedRouter)
api.use(parsedRouter)
return api

View File

@ -40,6 +40,13 @@ knex.init = Promise.all([
addColumn('app', 'appId', 'string')
]
knex.schema.hasTable('image')
.then (exists) ->
if not exists
knex.schema.createTable 'image', (t) ->
t.increments('id').primary()
t.string('repoTag')
])
module.exports = knex

View File

@ -9,7 +9,7 @@ knex = require './db'
TypedError = require 'typed-error'
{ request } = require './request'
fs = Promise.promisifyAll require 'fs'
Lock = require 'rwlock'
class OutOfSyncError extends TypedError
docker = Promise.promisifyAll(new Docker(socketPath: config.dockerSocket))
@ -149,6 +149,13 @@ dockerSync = (imgSrc, imgDest, rsyncDiff, conf) ->
docker.getImage(destId).tagAsync({ repo, tag, force: true })
do ->
_lock = new Lock()
_writeLock = Promise.promisify(_lock.async.writeLock)
lockImages = ->
_writeLock('images')
.disposer (release) ->
release()
# Keep track of the images being fetched, so we don't clean them up whilst fetching.
imagesBeingFetched = 0
exports.fetchImageWithProgress = (image, onProgress) ->
@ -162,51 +169,179 @@ do ->
# If there is no tag then mark it as latest
supervisorTag += ':latest'
exports.cleanupContainersAndImages = ->
Promise.join(
knex('app').select()
.map (app) ->
app.imageId + ':latest'
docker.listImagesAsync()
(apps, images) ->
imageTags = _.map(images, 'RepoTags')
supervisorTags = _.filter imageTags, (tags) ->
_.contains(tags, supervisorTag)
appTags = _.filter imageTags, (tags) ->
_.any tags, (tag) ->
_.contains(apps, tag)
supervisorTags = _.flatten(supervisorTags)
appTags = _.flatten(appTags)
return { images, supervisorTags, appTags }
)
.then ({ images, supervisorTags, appTags }) ->
# Cleanup containers first, so that they don't block image removal.
docker.listContainersAsync(all: true)
.filter (containerInfo) ->
# Do not remove user apps.
if _.contains(appTags, containerInfo.Image)
return false
if !_.contains(supervisorTags, containerInfo.Image)
return true
return containerHasExited(containerInfo.Id)
.map (containerInfo) ->
docker.getContainer(containerInfo.Id).removeAsync()
Promise.using lockImages(), ->
Promise.join(
knex('image').select('repoTag')
.map (image) ->
image.repoTag
knex('app').select()
.map (app) ->
app.imageId + ':latest'
docker.listImagesAsync()
(localTags, apps, images) ->
imageTags = _.map(images, 'RepoTags')
supervisorTags = _.filter imageTags, (tags) ->
_.contains(tags, supervisorTag)
appTags = _.filter imageTags, (tags) ->
_.any tags, (tag) ->
_.contains(apps, tag)
locallyCreatedTags = _.filter imageTags, (tags) ->
_.any tags, (tag) ->
_.contains(localTags, tag)
supervisorTags = _.flatten(supervisorTags)
appTags = _.flatten(appTags)
locallyCreatedTags = _.flatten(locallyCreatedTags)
return { images, supervisorTags, appTags, locallyCreatedTags }
)
.then ({ images, supervisorTags, appTags, locallyCreatedTags }) ->
# Cleanup containers first, so that they don't block image removal.
docker.listContainersAsync(all: true)
.filter (containerInfo) ->
# Do not remove user apps.
if _.contains(appTags, containerInfo.Image)
return false
if _.contains(locallyCreatedTags, containerInfo.Image)
return false
if !_.contains(supervisorTags, containerInfo.Image)
return true
return containerHasExited(containerInfo.Id)
.map (containerInfo) ->
docker.getContainer(containerInfo.Id).removeAsync()
.then ->
console.log('Deleted container:', containerInfo.Id, containerInfo.Image)
.catch(_.noop)
.then ->
console.log('Deleted container:', containerInfo.Id, containerInfo.Image)
.catch(_.noop)
.then ->
# And then clean up the images, as long as we aren't currently trying to fetch any.
return if imagesBeingFetched > 0
imagesToClean = _.reject images, (image) ->
_.any image.RepoTags, (tag) ->
return _.contains(appTags, tag) or _.contains(supervisorTags, tag)
Promise.map imagesToClean, (image) ->
Promise.map image.RepoTags.concat(image.Id), (tag) ->
docker.getImage(tag).removeAsync()
.then ->
console.log('Deleted image:', tag, image.Id, image.RepoTags)
.catch(_.noop)
# And then clean up the images, as long as we aren't currently trying to fetch any.
return if imagesBeingFetched > 0
imagesToClean = _.reject images, (image) ->
_.any image.RepoTags, (tag) ->
return _.contains(appTags, tag) or _.contains(supervisorTags, tag) or _.contains(locallyCreatedTags, tag)
Promise.map imagesToClean, (image) ->
Promise.map image.RepoTags.concat(image.Id), (tag) ->
docker.getImage(tag).removeAsync()
.then ->
console.log('Deleted image:', tag, image.Id, image.RepoTags)
.catch(_.noop)
containerHasExited = (id) ->
docker.getContainer(id).inspectAsync()
.then (data) ->
return not data.State.Running
buildRepoTag = (repo, tag, registry) ->
repoTag = ''
if registry?
repoTag += registry + '/'
repoTag += repo
if tag?
repoTag += ':' + tag
else
repoTag += ':latest'
return repoTag
sanitizeQuery = (query) ->
_.omit(query, 'apikey')
exports.createImage = (req, res) ->
{ registry, repo, tag, fromImage, fromSrc } = req.query
if fromImage?
repoTag = fromImage
repoTag += ':' + tag if tag?
else
repoTag = buildRepoTag(repo, tag, registry)
Promise.using lockImages(), ->
knex('image').insert({ repoTag })
.then ->
if fromImage?
docker.createImageAsync({ fromImage, tag })
else
docker.importImageAsync(req, { repo, tag, registry })
.then (stream) ->
stream.pipe(res)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
exports.deleteImage = (req, res) ->
imageName = req.params[0]
Promise.using lockImages(), ->
knex('image').select().where('repoTag', imageName)
.then (images) ->
throw new Error('Only images created via the Supervisor can be deleted.') if images.length == 0
knex('image').where('repoTag', imageName).delete()
.then ->
docker.getImage(imageName).removeAsync(sanitizeQuery(req.query))
.then (data) ->
res.json(data)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
exports.listImages = (req, res) ->
docker.listImagesAsync(sanitizeQuery(req.query))
.then (images) ->
res.json(images)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
docker.modem.dialAsync = Promise.promisify(docker.modem.dial)
exports.createContainer = (req, res) ->
Promise.using lockImages(), ->
knex('image').select().where('repoTag', req.body.Image)
.then (images) ->
throw new Error('Only images created via the Supervisor can be used for creating containers.') if images.length == 0
optsf =
path: '/containers/create?'
method: 'POST'
options: req.body
statusCodes:
200: true
201: true
404: 'no such container'
406: 'impossible to attach'
500: 'server error'
docker.modem.dialAsync(optsf)
.then (data) ->
res.json(data)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
exports.startContainer = (req, res) ->
docker.getContainer(req.params.id).startAsync(req.body)
.then (data) ->
res.json(data)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
exports.stopContainer = (req, res) ->
container = docker.getContainer(req.params.id)
knex('app').select()
.then (apps) ->
throw new Error('Cannot stop an app container') if _.any(apps, containerId: req.params.id)
container.inspectAsync()
.then (cont) ->
throw new Error('Cannot stop supervisor container') if cont.Name == '/resin_supervisor' or _.any(cont.Names, (n) -> n == '/resin_supervisor')
container.stopAsync(sanitizeQuery(req.query))
.then (data) ->
res.json(data)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
exports.deleteContainer = (req, res) ->
container = docker.getContainer(req.params.id)
knex('app').select()
.then (apps) ->
throw new Error('Cannot remove an app container') if _.any(apps, containerId: req.params.id)
container.inspectAsync()
.then (cont) ->
throw new Error('Cannot remove supervisor container') if cont.Name == '/resin_supervisor' or _.any(cont.Names, (n) -> n == '/resin_supervisor')
container.removeAsync(sanitizeQuery(req.query))
.then (data) ->
res.json(data)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')
exports.listContainers = (req, res) ->
docker.listContainersAsync(sanitizeQuery(req.query))
.then (containers) ->
res.json(containers)
.catch (err) ->
res.status(500).send(err?.message or err or 'Unknown error')