ExternalVendorCode/balena-supervisor
1
0
Fork 0
mirror of https://github.com/balena-os/balena-supervisor.git synced 2024-12-19 05:37:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Speed up rsync diff application

Browse Source
This commit is contained in:
Petros Angelatos 2015-05-02 01:07:23 +00:00 committed by Pablo Carranza Velez
parent c13e16c96a
commit 10e166020e
7 changed files with 56 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile.amd64
View File

@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
# Supervisor apt dependencies
RUN apt-get -q update \
&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/

2
Dockerfile.armv7hf
View File

@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
# Supervisor apt dependencies
RUN apt-get -q update \
&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/

2
Dockerfile.i386
View File

@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
# Supervisor apt dependencies
RUN apt-get -q update \
&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/

2
Dockerfile.rpi
View File

@ -4,7 +4,7 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/
# Supervisor apt dependencies
RUN apt-get -q update \
&& apt-get install -qqy socat supervisor rsync --no-install-recommends \
&& apt-get install -qqy socat supervisor rsync btrfs-tools jq --no-install-recommends \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/

BIN
empty.tar Normal file
View File

Binary file not shown.

64
src/docker-utils.coffee
View File

@ -63,43 +63,39 @@ findSimilarImage = (repoTag) ->
return config.supervisorImage
exports.rsyncImageWithProgress = (image, onProgress) ->
findSimilarImage(image)
.spread (repoTag, id) ->
containerConfig =
Image: id
Cmd: [ '/bin/sh', '-c', 'sleep 1000000' ]
NetworkDisabled: true
exports.rsyncImageWithProgress = (imgDest, onProgress) ->
findSimilarImage(imgDest)
.spread (imgSrc) ->
rsyncDiff = new Promise (resolve, reject) ->
console.log("#{config.deltaHost}/api/v1/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: 0)
progress request.get("#{config.deltaHost}/api/v1/delta?src=#{imgSrc}&dest=#{imgDest}", timeout: 0)
.on 'progress', onProgress
.on 'response', (res) ->
if res.statusCode isnt 200
reject()
else
resolve(res)
.on 'error', reject
.pause()
Promise.using createContainerDisposed(containerConfig), (container) ->
container.inspectAsync()
.then(rootDir)
.then (dest) ->
delta = new Promise (resolve, reject) ->
rsync = spawn('rsync', [ '--archive', '--read-batch=-', dest ])
.on 'error', reject
.on 'exit', (code, signal) ->
if code isnt 0
reject(new Error("rsync exited. code: #{code} signal: #{signal}"))
else
resolve()
imageConfig = request.getAsync("#{config.deltaHost}/api/v1/config?image=#{imgDest}", {json: true, timeout: 0})
.spread ({statusCode}, imageConfig) ->
if statusCode isnt 200
throw new Error("Invalid configuration: #{imageConfig}")
return imageConfig
progress request.get("#{config.deltaEndpoint}/api/v1/delta?src=#{repoTag}&dest=#{image}", timeout: 0)
.on 'progress', onProgress
.on 'response', ({statusCode}) -> reject() if statusCode isnt 200
.on 'error', reject
.pipe rsync.stdin
return [ rsyncDiff, imageConfig, imgSrc ]
.spread (rsyncDiff, imageConfig, imgSrc) ->
new Promise (resolve, reject) ->
dockersync = spawn('/app/src/dockersync.sh', [ imgSrc, imgDest, JSON.stringify(imageConfig) ])
.on 'error', reject
.on 'exit', (code, signal) ->
if code isnt 0
reject(new Error("rsync exited. code: #{code} signal: #{signal}"))
else
resolve()
imageConfig = request.getAsync("#{config.deltaEndpoint}/api/v1/config?image=#{image}", {json: true, timeout: 0})
Promise.all [ imageConfig, delta ]
.get(0)
.spread ({statusCode}, imageConfig) ->
if statusCode isnt 200
throw new Error("Invalid configuration: #{imageConfig}")
imageConfig.repo = image
container.commitAsync(imageConfig)
rsyncDiff.pipe(dockersync.stdin).resume()
do ->
# Keep track of the images being fetched, so we don't clean them up whilst fetching.

22
src/dockersync.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -o errexit
set -o pipefail
DOCKER_ROOT=${DOCKER_ROOT:-/var/lib/docker}
BTRFS_ROOT=${BTRFS_ROOT:-$DOCKER_ROOT/btrfs/subvolumes}
src=$1
dest=$2
config=$3
src_id=$(docker inspect -f '{{ .Id }}' "$src")
dest_id=$(cat /app/empty.tar | docker import - "$dest")
jq ".config=$config" "$DOCKER_ROOT/graph/$dest_id/json" > "$DOCKER_ROOT/graph/$dest_id/json.tmp"
mv "$DOCKER_ROOT/graph/$dest_id/json.tmp" "$DOCKER_ROOT/graph/$dest_id/json"
btrfs subvolume delete "$BTRFS_ROOT/$dest_id"
btrfs subvolume snapshot "$BTRFS_ROOT/$src_id" "$BTRFS_ROOT/$dest_id"
rsync --archive --read-batch=- "$BTRFS_ROOT/$dest_id"