Implement rsync diff fetching method

Dockerfile.amd64

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

Dockerfile.armv7hf

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

Dockerfile.i386

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

Dockerfile.rpi

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

package.json

@@ -24,6 +24,7 @@
     "pubnub": "^3.7.13",
     "request": "^2.51.0",
     "resin-register-device": "^2.0.0",
+    "request-progress": "^0.3.1",
     "rwlock": "^5.0.0",
     "server-destroy": "^1.0.0",
     "sqlite3": "~3.0.4",

src/application.coffee

@@ -126,12 +126,16 @@ isValidPort = (port) ->
 	return parseFloat(port) is maybePort and maybePort > 0 and maybePort < 65535
 
 fetch = (app) ->
+	onProgress = (progress) ->
+		device.updateState(download_progress: progress.percentage)
+
 	docker.getImage(app.imageId).inspectAsync()
 	.catch (error) ->
 		logSystemEvent(logTypes.downloadApp, app)
 		device.updateState(status: 'Downloading', download_progress: 0)
-		dockerUtils.fetchImageWithProgress app.imageId, (progress) ->
-			device.updateState(download_progress: progress.percentage)
+		dockerUtils.rsyncImageWithProgress(app.imageId, onProgress)
+		.catch ->
+			dockerUtils.fetchImageWithProgress(app.imageId, onProgress)
 		.then ->
 			logSystemEvent(logTypes.downloadAppSuccess, app)
 			device.updateState(status: 'Idle', download_progress: null)

src/config.coffee

@@ -18,6 +18,7 @@ module.exports = config =
 	apiEndpoint: process.env.API_ENDPOINT ? 'https://api.resin.io'
 	listenPort: process.env.LISTEN_PORT ? 80
 	gosuperAddress: "http://unix:#{process.env.GOSUPER_SOCKET}:"
+	deltaEndpoint: process.env.DELTA_ENDPOINT ? 'https://delta.staging.resin.io'
 	registryEndpoint: process.env.REGISTRY_ENDPOINT ? 'registry.resin.io'
 	pubnub:
 		subscribe_key: checkValidKey(process.env.PUBNUB_SUBSCRIBE_KEY) ? process.env.DEFAULT_PUBNUB_SUBSCRIBE_KEY

src/docker-utils.coffee

@@ -1,6 +1,9 @@
 Docker = require 'dockerode'
 DockerProgress = require 'docker-progress'
 Promise = require 'bluebird'
+request = Promise.promisifyAll require('request')
+{ spawn } = require 'child_process'
+progress = require 'request-progress'
 config = require './config'
 _ = require 'lodash'
 knex = require './db'
@@ -13,6 +16,92 @@ Promise.promisifyAll(docker.getContainer().constructor.prototype)
 exports.docker = docker
 dockerProgress = new DockerProgress(socketPath: config.dockerSocket)
 
+createContainerDisposed = (config) ->
+	docker.createContainerAsync(config)
+	.tap (container) ->
+		container.startAsync()
+	.disposer (container) ->
+		container.removeAsync(force: true)
+
+# Trailing slashes are important to rsync
+rootDir = ({Driver, Id}) ->
+	switch Driver
+		when 'aufs'
+			"/var/lib/docker/#{Driver}/mnt/#{Id}/"
+		when 'btrfs'
+			"/var/lib/docker/#{Driver}/subvolumes/#{Id}/"
+		when 'devicemapper'
+			"/var/lib/docker/#{Driver}/mnt/#{Id}/rootfs/"
+		when 'overlay'
+			"/var/lib/docker/#{Driver}/#{Id}/merged/"
+		when 'vfs'
+			"/var/lib/docker/#{Driver}/dir/#{Id}/"
+		else
+			throw new Error("Unsupported driver: #{Driver}")
+
+# Create an array of (repoTag, image_id, created) tuples like the output of `docker images`
+listRepoTagsAsync = ->
+	docker.listImagesAsync()
+	.then (images) ->
+		images = _.sortByOrder(images, 'Created', [ false ])
+		ret = []
+		for image in images
+			for repoTag in image.RepoTags
+				ret.push [ repoTag, image.Id, image.Created ]
+		return ret
+
+# Find either the most recent image of the same app or the image of the supervisor
+findSimilarImage = (repoTag) ->
+	application = repoTag.split('/')[1]
+
+	listRepoTagsAsync()
+	.then (repoTags) ->
+		# Find the most recent image of the same application
+		for repoTag in repoTags
+			otherApplication = repoTag[0].split('/')[1]
+			if otherApplication is application
+				return repoTag
+
+		return config.supervisorImage
+
+exports.rsyncImageWithProgress = (image, onProgress) ->
+	findSimilarImage(image)
+	.spread (repoTag, id) ->
+		config =
+			Image: id
+			Cmd: [ '/bin/sh', '-c', 'sleep 1000000' ]
+			NetworkDisabled: true
+
+		Promise.using createContainerDisposed(config), (container) ->
+			container.inspectAsync()
+			.then(rootDir)
+			.then (dest) ->
+				delta = new Promise (resolve, reject) ->
+					rsync = spawn('rsync', [ '--archive', '--read-batch=-', dest ])
+					.on 'error', reject
+					.on 'exit', (code, signal) ->
+						if code isnt 0
+							reject(new Error("rsync exited. code: #{code} signal: #{signal}"))
+						else
+							resolve()
+
+					progress request.get("#{config.deltaEndpoint}/api/v1/delta?src=#{repoTag}&dest=#{image}")
+					.on 'progress', onProgress
+					.on 'response', ({statusCode}) -> reject() if statusCode isnt 200
+					.on 'error', reject
+					.pipe rsync.stdin
+
+				config = request.getAsync("#{config.deltaEndpoint}/api/v1/config?image=#{image}", json: true)
+
+				Promise.all [ config, delta ]
+			.get(0)
+			.spread ({statusCode}, config) ->
+				if statusCode isnt 200
+					throw new Error("Invalid configuration: #{config}")
+
+				config.repo = image
+				container.commitAsync(config)
+
 do ->
 	# Keep track of the images being fetched, so we don't clean them up whilst fetching.
 	imagesBeingFetched = 0
@@ -77,4 +166,3 @@ do ->
 		docker.getContainer(id).inspectAsync()
 		.then (data) ->
 			return not data.State.Running
-