calculate progress in a transform stream and expose a callback for progress events

src/application.coffee

@@ -99,15 +99,8 @@ exports.start = start = (app) ->
 				utils.mixpanelTrack('Application install', app)
 				logSystemEvent('Installing application ' + app.imageId)
 				updateDeviceInfo(status: 'Downloading')
-				dockerUtils.fetchImageWithProgress(app.imageId)
+				dockerUtils.fetchImageWithProgress app.imageId, (progress) ->
-			.then (stream) ->
+					updateDeviceInfo(download_progress: progress.percentage)
-				return new Promise (resolve, reject) ->
-					stream.on 'data', (d) ->
-						data = JSON.parse(d)
-						if data.progress?
-							updateDeviceInfo(download_progress: data.progress.percentage)
-					stream.on('error', reject)
-					stream.on('end', resolve)
 			.then ->
 				console.log('Creating container:', app.imageId)
 				updateDeviceInfo(status: 'Starting')

src/docker-utils.coffee

@@ -37,11 +37,22 @@ do ->
 
 	# Pull docker image returning a stream that reports progress
 	# This is less safe than fetchImage and shouldnt be used for supervisor updates
-	exports.fetchImageWithProgress = (image) ->
+	exports.fetchImageWithProgress = (image, progressUpdates) ->
 		imagesBeingFetched++
-		pullImage(image)
+		docker.createImageAsync(fromImage: image)
-		.tap (stream) ->
+		.then (stream) ->
-			stream.once 'end', ->
+			return new Promise (resolve, reject) ->
+				stream.on('error', reject)
+				.pipe(JSONStream.parse())
+				.on('error', reject)
+				.pipe(pullProgressStream(image))
+				.pipe es.mapSync (data) ->
+					if data.error?
+						reject(data.error)
+					else if data.totalProgress?
+						progressUpdates?(data.totalProgress)
+				.on('end', resolve)
+		.finally ->
 			imagesBeingFetched--
 
 	exports.cleanupContainersAndImages = ->
@@ -79,7 +90,7 @@ do ->
 
 	exports.imageExists = imageExists = (imageId) ->
 		image = docker.getImage(imageId)
-		image.inspectAsync().then (data) ->
+		image.inspectAsync().then ->
 			return true
 		.catch (e) ->
 			return false
@@ -107,7 +118,6 @@ do ->
 		.then (exists) ->
 			if exists
 				return 0
-			else
 			request("http://#{registry}/v1/images/#{imageId}/json")
 			.spread (res, data) ->
 				if res.statusCode >= 400
@@ -115,7 +125,7 @@ do ->
 				return parseInt(res.headers['x-docker-size'])
 
 	exports.getLayerDownloadSizes = getLayerDownloadSizes = (image, tag='latest') ->
-		[ registry, imageName ] = getRegistryAndName(image)
+		{registry, imageName} = getRegistryAndName(image)
 		imageSizes = {}
 		getImageId(registry, imageName, tag)
 		.then (imageId) ->
@@ -140,23 +150,20 @@ do ->
 			throw new Error("Expected image name including registry domain name")
 		registry = image.substr(0,slashPos)
 		imageName = image.substr(slashPos+1)
-		return [registry,imageName]
+		return {registry, imageName}
 	
-	# a wrapper for dockerode pull/createImage 
+	# Create a stream that transforms docker pull output 
-	# streaming progress of total download size
+	# to include total progress metrics.
-	# instead of per layer progress output of `docker pull`
 	# 
-	# image has to have the form registry.domain/repository/image
+	# The docker pull output should be piped to this stream as separate javascript objects.
-	# (it has to include the registry you are pulling from)
+	pullProgressStream = (image) ->
-	exports.pullImage = pullImage = (image) ->
+		totalSize = 0
-		getLayerDownloadSizes(image).then (layerSizes) ->
-			totalSize = _.reduce(layerSizes, ((t,x) -> t+x), 0)
 		completedSize = 0
 		currentSize = 0
-			docker.pullAsync(image)
+		sizePromise = getLayerDownloadSizes(image).tap (layerSizes) ->
-			.then (inStream) ->
+			totalSize = _.reduce(layerSizes, ((t,x) -> t+x), 0)
-				inStream.pipe es.through (data) ->
+		es.map (pull, callback) ->
-					pull = JSON.parse(data)
+			sizePromise.then (layerSizes) ->
 				if pull.status == 'Downloading'
 					currentSize = pull.progressDetail.current
 				else if pull.status == 'Download complete'
@@ -170,6 +177,8 @@ do ->
 						console.log 'Progress error: Unknown layer ' + id + ' downloaded by docker. Progress not correct.'
 						totalSize = null
 				downloadedSize = completedSize + currentSize
-					percentage = calculatePercentage downloadedSize, totalSize
+				percentage = calculatePercentage(downloadedSize, totalSize)
-					progress = { downloadedSize, totalSize, percentage }
+				pull.totalProgress = { downloadedSize, totalSize, percentage }
-					this.emit 'data', JSON.stringify({ progress, data })
+				callback(null, pull)
+			.catch (error) =>
+				callback(null, { error })