Merge pull request #146 from resin-io/145-load-image-endpoint

Add /v1/images/load endpoint

CHANGELOG.md

@@ -1,3 +1,6 @@
+* Use rwlock to block when pulling images [Pablo]
+* Increase API timeout to 15 minutes, and make it configurable [Pablo]
+* Add endpoint to load images from a docker save tar [Pablo]
 * Add endpoints to manage images and containers locally [Pablo]
 * Only use bodyParser for endpoints that need it [Pablo]
 * Add RESIN_APP_ID variable [Pablo]

docs/API.md

@@ -518,6 +518,20 @@ $ curl -X POST \
 
 <hr>
 
+### POST /v1/images/load
+
+Works like [/images/load from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#load-a-tarball-with-a-set-of-images-and-tags-into-docker).
+Allows the creation of images from a tar archive produced by `docker save`.
+
+#### Example:
+
+```bash
+$ curl -X POST --data-binary @images.tar \
+	"$RESIN_SUPERVISOR_ADDRESS/v1/images/load?apikey=$RESIN_SUPERVISOR_API_KEY"
+```
+
+<hr>
+
 ### DELETE /v1/images/:name
 
 Deletes image with name `name`. Works like [DELETE /images/(name) from the Docker API](https://docs.docker.com/engine/reference/api/docker_remote_api_v1.22/#remove-an-image).

src/api.coffee

@@ -171,6 +171,7 @@ module.exports = (application) ->
 		res.json(device.getState())
 
 	unparsedRouter.post '/v1/images/create', dockerUtils.createImage
+	unparsedRouter.post '/v1/images/load', dockerUtils.loadImage
 	unparsedRouter.delete '/v1/images/*', dockerUtils.deleteImage
 	unparsedRouter.get '/v1/images', dockerUtils.listImages
 	parsedRouter.post '/v1/containers/create', dockerUtils.createContainer

src/app.coffee

@@ -27,7 +27,8 @@ knex.init.then ->
 			device.getOSVersion()
 		.then (osVersion) ->
 			console.log('Starting API server..')
-			api(application).listen(config.listenPort)
+			apiServer = api(application).listen(config.listenPort)
+			apiServer.timeout = config.apiTimeout
 			# Let API know what version we are, and our api connection info.
 			console.log('Updating supervisor version and api info')
 			device.updateState(

src/config.coffee

@@ -19,6 +19,7 @@ dockerRoot = checkString(process.env.DOCKER_ROOT) ? '/mnt/root/var/lib/rce'
 # Defaults needed for both gosuper and node supervisor are declared in entry.sh
 module.exports = config =
 	apiEndpoint: checkString(process.env.API_ENDPOINT) ? 'https://api.resin.io'
+	apiTimeout: checkInt(process.env.API_TIMEOUT) ? 15 * 60 * 1000
 	listenPort: checkInt(process.env.LISTEN_PORT) ? 80
 	gosuperAddress: "http://unix:#{process.env.GOSUPER_SOCKET}:"
 	deltaHost: checkString(process.env.DELTA_ENDPOINT) ? 'https://delta.resin.io'

src/docker-utils.coffee

@@ -61,39 +61,6 @@ findSimilarImage = (repoTag) ->
 DELTA_OUT_OF_SYNC_CODES = [23, 24]
 DELTA_REQUEST_TIMEOUT = 15 * 60 * 1000
 
-exports.rsyncImageWithProgress = (imgDest, onProgress, startFromEmpty = false) ->
-	Promise.try ->
-		if startFromEmpty
-			return 'resin/scratch'
-		findSimilarImage(imgDest)
-	.then (imgSrc) ->
-		rsyncDiff = new Promise (resolve, reject) ->
-			progress request.get("#{config.deltaHost}/api/v1/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: DELTA_REQUEST_TIMEOUT)
-			.on 'progress', (progress) ->
-				onProgress(percentage: progress.percent)
-			.on 'end', ->
-				onProgress(percentage: 100)
-			.on 'response', (res) ->
-				if res.statusCode isnt 200
-					reject(new Error("Got #{res.statusCode} when requesting image from delta server."))
-				else
-					resolve(res)
-			.on 'error', reject
-			.pause()
-
-		imageConfig = request.getAsync("#{config.deltaHost}/api/v1/config?image=#{imgDest}", {json: true, timeout: 0})
-		.spread ({statusCode}, imageConfig) ->
-			if statusCode isnt 200
-				throw new Error("Invalid configuration: #{imageConfig}")
-			return imageConfig
-
-		return [ rsyncDiff, imageConfig, imgSrc ]
-	.spread (rsyncDiff, imageConfig, imgSrc) ->
-		dockerSync(imgSrc, imgDest, rsyncDiff, imageConfig)
-	.catch OutOfSyncError, (err) ->
-		console.log('Falling back to delta-from-empty')
-		exports.rsyncImageWithProgress(imgDest, onProgress, true)
-
 getRepoAndTag = (image) ->
 	getRegistryAndName(image)
 	.then ({ registry, imageName, tagName }) ->
@@ -151,25 +118,60 @@ dockerSync = (imgSrc, imgDest, rsyncDiff, conf) ->
 do ->
 	_lock = new Lock()
 	_writeLock = Promise.promisify(_lock.async.writeLock)
-	lockImages = ->
+	_readLock = Promise.promisify(_lock.async.readLock)
+	writeLockImages = ->
 		_writeLock('images')
 		.disposer (release) ->
 			release()
+	readLockImages = ->
+		_readLock('images')
+		.disposer (release) ->
+			release()
+
+	exports.rsyncImageWithProgress = (imgDest, onProgress, startFromEmpty = false) ->
+		Promise.using readLockImages(), ->
+			Promise.try ->
+				if startFromEmpty
+					return 'resin/scratch'
+				findSimilarImage(imgDest)
+			.then (imgSrc) ->
+				rsyncDiff = new Promise (resolve, reject) ->
+					progress request.get("#{config.deltaHost}/api/v1/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: DELTA_REQUEST_TIMEOUT)
+					.on 'progress', (progress) ->
+						onProgress(percentage: progress.percent)
+					.on 'end', ->
+						onProgress(percentage: 100)
+					.on 'response', (res) ->
+						if res.statusCode isnt 200
+							reject(new Error("Got #{res.statusCode} when requesting image from delta server."))
+						else
+							resolve(res)
+					.on 'error', reject
+					.pause()
+
+				imageConfig = request.getAsync("#{config.deltaHost}/api/v1/config?image=#{imgDest}", {json: true, timeout: 0})
+				.spread ({statusCode}, imageConfig) ->
+					if statusCode isnt 200
+						throw new Error("Invalid configuration: #{imageConfig}")
+					return imageConfig
+
+				return [ rsyncDiff, imageConfig, imgSrc ]
+			.spread (rsyncDiff, imageConfig, imgSrc) ->
+				dockerSync(imgSrc, imgDest, rsyncDiff, imageConfig)
+			.catch OutOfSyncError, (err) ->
+				console.log('Falling back to delta-from-empty')
+				exports.rsyncImageWithProgress(imgDest, onProgress, true)
 
-	# Keep track of the images being fetched, so we don't clean them up whilst fetching.
-	imagesBeingFetched = 0
 	exports.fetchImageWithProgress = (image, onProgress) ->
-		imagesBeingFetched++
+		Promise.using readLockImages(), ->
-		dockerProgress.pull(image, onProgress)
+			dockerProgress.pull(image, onProgress)
-		.finally ->
-			imagesBeingFetched--
 
 	supervisorTag = config.supervisorImage
 	if !/:/g.test(supervisorTag)
 		# If there is no tag then mark it as latest
 		supervisorTag += ':latest'
 	exports.cleanupContainersAndImages = ->
-		Promise.using lockImages(), ->
+		Promise.using writeLockImages(), ->
 			Promise.join(
 				knex('image').select('repoTag')
 				.map (image) ->
@@ -211,8 +213,6 @@ do ->
 						console.log('Deleted container:', containerInfo.Id, containerInfo.Image)
 					.catch(_.noop)
 				.then ->
-					# And then clean up the images, as long as we aren't currently trying to fetch any.
-					return if imagesBeingFetched > 0
 					imagesToClean = _.reject images, (image) ->
 						_.any image.RepoTags, (tag) ->
 							return _.contains(appTags, tag) or _.contains(supervisorTags, tag) or _.contains(locallyCreatedTags, tag)
@@ -249,7 +249,7 @@ do ->
 			repoTag += ':' + tag if tag?
 		else
 			repoTag = buildRepoTag(repo, tag, registry)
-		Promise.using lockImages(), ->
+		Promise.using writeLockImages(), ->
 			knex('image').insert({ repoTag })
 			.then ->
 				if fromImage?
@@ -261,9 +261,27 @@ do ->
 		.catch (err) ->
 			res.status(500).send(err?.message or err or 'Unknown error')
 
+	exports.loadImage = (req, res) ->
+		Promise.using writeLockImages(), ->
+			docker.listImagesAsync()
+			.then (oldImages) ->
+				docker.loadImageAsync(req)
+				.then ->
+					docker.listImagesAsync()
+				.then (newImages) ->
+					oldTags = _.flatten(_.map(oldImages, 'RepoTags'))
+					newTags = _.flatten(_.map(newImages, 'RepoTags'))
+					createdTags = _.difference(newTags, oldTags)
+					Promise.map createdTags, (repoTag) ->
+						knex('image').insert({ repoTag })
+			.then ->
+				res.sendStatus(200)
+		.catch (err) ->
+			res.status(500).send(err?.message or err or 'Unknown error')
+
 	exports.deleteImage = (req, res) ->
 		imageName = req.params[0]
-		Promise.using lockImages(), ->
+		Promise.using writeLockImages(), ->
 			knex('image').select().where('repoTag', imageName)
 			.then (images) ->
 				throw new Error('Only images created via the Supervisor can be deleted.') if images.length == 0
@@ -284,7 +302,7 @@ do ->
 
 	docker.modem.dialAsync = Promise.promisify(docker.modem.dial)
 	exports.createContainer = (req, res) ->
-		Promise.using lockImages(), ->
+		Promise.using writeLockImages(), ->
 			knex('image').select().where('repoTag', req.body.Image)
 			.then (images) ->
 				throw new Error('Only images created via the Supervisor can be used for creating containers.') if images.length == 0