download progress bars

src/application.coffee

@@ -99,7 +99,15 @@ exports.start = start = (app) ->
 				utils.mixpanelTrack('Application install', app)
 				logSystemEvent('Installing application ' + app.imageId)
 				updateDeviceInfo(status: 'Downloading')
-				dockerUtils.fetchImage(app.imageId)
+				dockerUtils.fetchImageWithProgress(app.imageId)
+			.then (stream) ->
+				return new Promise (resolve, reject) ->
+					stream.on 'data', (d) ->
+						data = JSON.parse(d)
+						if data.progress?
+							updateDeviceInfo(download_progress: data.progress.percentage)
+					stream.on('error', reject)
+					stream.on('end', resolve)
 			.then ->
 				console.log('Creating container:', app.imageId)
 				updateDeviceInfo(status: 'Starting')
@@ -155,7 +163,7 @@ exports.start = start = (app) ->
 		utils.mixpanelTrack('Application start', app.imageId)
 		logSystemEvent('Starting application ' + app.imageId)
 	.finally ->
-		updateDeviceInfo(status: 'Idle')
+		updateDeviceInfo(status: 'Idle', download_progress: null, provisioning_progress: null)
 
 # 0 - Idle
 # 1 - Updating

src/docker-utils.coffee

@@ -6,6 +6,8 @@ es = require 'event-stream'
 _ = require 'lodash'
 knex = require './db'
 
+request = Promise.promisify(require 'request')
+
 docker = Promise.promisifyAll(new Docker(socketPath: config.dockerSocket))
 # Hack dockerode to promisify internal classes' prototypes
 Promise.promisifyAll(docker.getImage().constructor.prototype)
@@ -33,6 +35,15 @@ do ->
 		.finally ->
 			imagesBeingFetched--
 
+	# Pull docker image returning a stream that reports progress
+	# This is less safe than fetchImage and shouldnt be used for supervisor updates
+	exports.fetchImageWithProgress = (image) ->
+		imagesBeingFetched++
+		pullImage(image)
+		.tap (stream) ->
+			stream.once 'end', ->
+				imagesBeingFetched--
+
 	exports.cleanupContainersAndImages = ->
 		knex('app').select()
 		.map (app) ->
@@ -65,3 +76,100 @@ do ->
 						console.log('Deleted image:', image.Id, image.RepoTags)
 					.catch (err) ->
 						console.log('Error deleting image:', image.Id, image.RepoTags, err)
+
+	exports.imageExists = imageExists = (imageId) ->
+		image = docker.getImage(imageId)
+		image.inspectAsync().then (data) ->
+			return true
+		.catch (e) ->
+			return false
+
+	exports.getImageId = getImageId = (registry, imageName, tag='latest') ->
+		request("http://#{registry}/v1/repositories/#{imageName}/tags")
+		.spread (res, data) ->
+			if res.statusCode == 404
+				throw new Error("No such image #{imageName} on registry #{registry}")
+			if res.statusCode >= 400
+				throw new Error("Failed to get image tags of #{imageName} from #{registry}. Status code: #{res.statusCode}")
+			tags = JSON.parse(data)
+			return tags[tag]
+
+	exports.getImageHistory = getImageHistory = (registry, imageId) ->
+		request("http://#{registry}/v1/images/#{imageId}/ancestry")
+		.spread (res, data) ->
+			if res.statusCode >= 400
+				throw new Error("Failed to get image ancestry of #{imageId} from #{registry}. Status code: #{res.statusCode}")
+			history = JSON.parse(data)
+			return history
+
+	exports.getImageDownloadSize = getImageDownloadSize = (registry, imageId) ->
+		imageExists(imageId)
+		.then (exists) ->
+			if exists
+				return 0
+			else
+				request("http://#{registry}/v1/images/#{imageId}/json")
+				.spread (res, data) ->
+					if res.statusCode >= 400
+						throw new Error("Failed to get image download size of #{imageId} from #{registry}. Status code: #{res.statusCode}")
+					return parseInt(res.headers['x-docker-size'])
+
+	exports.getLayerDownloadSizes = getLayerDownloadSizes = (image, tag='latest') ->
+		[ registry, imageName ] = getRegistryAndName(image)
+		imageSizes = {}
+		getImageId(registry, imageName, tag)
+		.then (imageId) ->
+			getImageHistory(registry, imageId)
+		.map (layerId) ->
+			getImageDownloadSize(registry, layerId).then (size) ->
+				imageSizes[layerId] = size
+		.return(imageSizes)
+
+	calculatePercentage = (completed, total) ->
+		if not total?
+			percentage = 0 # report 0% if unknown total size
+		else if total is 0
+			percentage = 100 # report 100% if 0 total size
+		else
+			percentage = (100 * completed) // total
+		return percentage
+
+	getRegistryAndName = (image) ->
+		slashPos = image.indexOf('/')
+		if slashPos < 0
+			throw new Error("Expected image name including registry domain name")
+		registry = image.substr(0,slashPos)
+		imageName = image.substr(slashPos+1)
+		return [registry,imageName]
+
+	# a wrapper for dockerode pull/createImage 
+	# streaming progress of total download size
+	# instead of per layer progress output of `docker pull`
+	#
+	# image has to have the form registry.domain/repository/image
+	# (it has to include the registry you are pulling from)
+	exports.pullImage = pullImage = (image) ->
+		getLayerDownloadSizes(image).then (layerSizes) ->
+			totalSize = _.reduce(layerSizes, ((t,x) -> t+x), 0)
+			completedSize = 0
+			currentSize = 0
+			docker.pullAsync(image)
+			.then (inStream) ->
+				inStream.pipe es.through (data) ->
+					pull = JSON.parse(data)
+					if pull.status == 'Downloading'
+						currentSize = pull.progressDetail.current
+					else if pull.status == 'Download complete'
+						shortId = pull.id
+						longId = _.find(_.keys(layerSizes), (id) -> id.indexOf(shortId) is 0)
+						if longId?
+							completedSize += layerSizes[longId]
+							currentSize = 0
+							layerSizes[longId] = 0 # make sure we don't count this layer again
+						else
+							console.log 'Progress error: Unknown layer ' + id + ' downloaded by docker. Progress not correct.'
+							totalSize = null
+					downloadedSize = completedSize + currentSize
+					percentage = calculatePercentage downloadedSize, totalSize
+					progress = { downloadedSize, totalSize, percentage }
+					this.emit 'data', JSON.stringify({ progress, data })