Speed up rsync diff application

Dockerfile.amd64

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

Dockerfile.armv7hf

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

Dockerfile.i386

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

Dockerfile.rpi

@@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
 
 # Supervisor apt dependencies
 RUN apt-get -q update \
-	&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
+	&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
 	&& apt-get clean \
 	&& rm -rf /var/lib/apt/lists/
 

src/docker-utils.coffee

@@ -63,20 +63,31 @@ findSimilarImage = (repoTag) ->
 
 		return config.supervisorImage
 
-exports.rsyncImageWithProgress = (image, onProgress) ->
+exports.rsyncImageWithProgress = (imgDest, onProgress) ->
-	findSimilarImage(image)
+	findSimilarImage(imgDest)
-	.spread (repoTag, id) ->
+	.spread (imgSrc) ->
-		containerConfig =
+		rsyncDiff = new Promise (resolve, reject) ->
-			Image: id
+			console.log("#{config.deltaHost}/api/v1/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: 0)
-			Cmd: [ '/bin/sh', '-c', 'sleep 1000000' ]
+			progress request.get("#{config.deltaHost}/api/v1/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: 0)
-			NetworkDisabled: true
+			.on 'progress', onProgress
+			.on 'response', (res) ->
+				if res.statusCode isnt 200
+					reject()
+				else
+					resolve(res)
+			.on 'error', reject
+			.pause()
 
-		Promise.using createContainerDisposed(containerConfig), (container) ->
+		imageConfig = request.getAsync("#{config.deltaHost}/api/v1/config?image=#{imgDest}", {json: true, timeout: 0})
-			container.inspectAsync()
+		.spread ({statusCode}, imageConfig) ->
-			.then(rootDir)
+			if statusCode isnt 200
-			.then (dest) ->
+				throw new Error("Invalid configuration: #{imageConfig}")
-				delta = new Promise (resolve, reject) ->
+			return imageConfig
-					rsync = spawn('rsync', [ '--archive', '--read-batch=-', dest ])
+
+		return [ rsyncDiff, imageConfig, imgSrc ]
+	.spread (rsyncDiff, imageConfig, imgSrc) ->
+		new Promise (resolve, reject) ->
+			dockersync = spawn('/app/src/dockersync.sh', [ imgSrc, imgDest, JSON.stringify(imageConfig) ])
 			.on 'error', reject
 			.on 'exit', (code, signal) ->
 				if code isnt 0
@@ -84,22 +95,7 @@ exports.rsyncImageWithProgress = (image, onProgress) ->
 				else
 					resolve()
 
-					progress request.get("#{config.deltaEndpoint}/api/v1/delta?src=#{repoTag}&dest=#{image}", timeout: 0)
+			rsyncDiff.pipe(dockersync.stdin).resume()
-					.on 'progress', onProgress
-					.on 'response', ({statusCode}) -> reject() if statusCode isnt 200
-					.on 'error', reject
-					.pipe rsync.stdin
-
-				imageConfig = request.getAsync("#{config.deltaEndpoint}/api/v1/config?image=#{image}", {json: true, timeout: 0})
-
-				Promise.all [ imageConfig, delta ]
-			.get(0)
-			.spread ({statusCode}, imageConfig) ->
-				if statusCode isnt 200
-					throw new Error("Invalid configuration: #{imageConfig}")
-
-				imageConfig.repo = image
-				container.commitAsync(imageConfig)
 
 do ->
 	# Keep track of the images being fetched, so we don't clean them up whilst fetching.

src/dockersync.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+DOCKER_ROOT=${DOCKER_ROOT:-/var/lib/docker}
+BTRFS_ROOT=${BTRFS_ROOT:-$DOCKER_ROOT/btrfs/subvolumes}
+
+src=$1
+dest=$2
+config=$3
+
+src_id=$(docker inspect -f '{{ .Id }}' "$src")
+dest_id=$(cat /app/empty.tar | docker import - "$dest")
+
+jq ".config=$config" "$DOCKER_ROOT/graph/$dest_id/json" > "$DOCKER_ROOT/graph/$dest_id/json.tmp"
+mv "$DOCKER_ROOT/graph/$dest_id/json.tmp" "$DOCKER_ROOT/graph/$dest_id/json"
+
+btrfs subvolume delete "$BTRFS_ROOT/$dest_id"
+btrfs subvolume snapshot "$BTRFS_ROOT/$src_id" "$BTRFS_ROOT/$dest_id"
+
+rsync --archive --read-batch=- "$BTRFS_ROOT/$dest_id"