From 78f0df281b012b3821fe1391941f287060b409cf Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Wed, 20 Jan 2016 14:56:09 +0000 Subject: [PATCH] Replace dockersync.sh with node and dockerode stuff. Don't install docker. Add Dockerfile changes for armel. --- Dockerfile.amd64 | 6 ---- Dockerfile.armel | 8 ++++- Dockerfile.armv7hf | 7 ---- Dockerfile.i386 | 7 ---- Dockerfile.rpi | 4 --- package.json | 4 +-- src/config.coffee | 4 +++ src/docker-utils.coffee | 74 +++++++++++++++++++++++++++++++---------- src/dockersync.sh | 26 --------------- 9 files changed, 70 insertions(+), 70 deletions(-) delete mode 100755 src/dockersync.sh diff --git a/Dockerfile.amd64 b/Dockerfile.amd64 index e1fb0991..9c249b38 100644 --- a/Dockerfile.amd64 +++ b/Dockerfile.amd64 @@ -7,8 +7,6 @@ RUN apt-get -q update \ && apt-get install -qqy \ btrfs-tools \ ca-certificates \ - curl \ - jq \ rsync \ socat \ supervisor \ @@ -16,9 +14,6 @@ RUN apt-get -q update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/ -ENV DOCKER_VERSION 1.6.2 -RUN curl -sL https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz | tar xzv --strip-components=3 -C /usr/bin - # Copy supervisord configuration files COPY config/supervisor/ /etc/supervisor/ @@ -41,7 +36,6 @@ COPY . /app/ RUN chmod +x /app/src/enterContainer.sh \ && /app/node_modules/.bin/coffee -c /app/src \ - && ln -sf /mnt/root/var/lib/rce /var/lib/docker \ && ln -sf /app/entry.sh /start # Needed for legacy RUN chmod +x /app/bin/gosuper diff --git a/Dockerfile.armel b/Dockerfile.armel index e68e1190..eeef7035 100644 --- a/Dockerfile.armel +++ b/Dockerfile.armel @@ -4,7 +4,13 @@ COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/ # Supervisor apt dependencies RUN apt-get -q update \ - && apt-get install -qqy socat supervisor --no-install-recommends \ + && apt-get install -qqy \ + btrfs-tools \ + ca-certificates \ + rsync \ + socat \ + supervisor \ + --no-install-recommends \ && apt-get clean \ && rm -rf /var/lib/apt/lists/ diff --git a/Dockerfile.armv7hf b/Dockerfile.armv7hf index 56a86ec6..972b2dbf 100644 --- a/Dockerfile.armv7hf +++ b/Dockerfile.armv7hf @@ -7,19 +7,13 @@ RUN apt-get -q update \ && apt-get install -qqy \ btrfs-tools \ ca-certificates \ - curl \ - jq \ rsync \ socat \ supervisor \ - xz-utils \ --no-install-recommends \ && apt-get clean \ && rm -rf /var/lib/apt/lists/ -ENV DOCKER_VERSION 1.6.2 -RUN curl -sL https://github.com/resin-io/docker/releases/download/${DOCKER_VERSION}/docker-arm-${DOCKER_VERSION}.tar.xz | tar xJv --strip-components=1 -C /usr/bin - # Copy supervisord configuration files COPY config/supervisor/ /etc/supervisor/ @@ -42,7 +36,6 @@ COPY . /app/ RUN chmod +x /app/src/enterContainer.sh \ && /app/node_modules/.bin/coffee -c /app/src \ - && ln -sf /mnt/root/var/lib/rce /var/lib/docker \ && ln -sf /app/entry.sh /start # Needed for legacy RUN chmod +x /app/bin/gosuper diff --git a/Dockerfile.i386 b/Dockerfile.i386 index f5ad78e8..137e487f 100644 --- a/Dockerfile.i386 +++ b/Dockerfile.i386 @@ -7,19 +7,13 @@ RUN apt-get -q update \ && apt-get install -qqy \ btrfs-tools \ ca-certificates \ - curl \ - jq \ rsync \ socat \ supervisor \ - xz-utils \ --no-install-recommends \ && apt-get clean \ && rm -rf /var/lib/apt/lists/ -ENV DOCKER_VERSION 1.6.2 -RUN curl -sL https://github.com/resin-io/docker/releases/download/${DOCKER_VERSION}/docker-386-${DOCKER_VERSION}.tar.xz | tar xJv --strip-components=1 -C /usr/bin - # Copy supervisord configuration files COPY config/supervisor/ /etc/supervisor/ @@ -42,7 +36,6 @@ COPY . /app/ RUN chmod +x /app/src/enterContainer.sh \ && /app/node_modules/.bin/coffee -c /app/src \ - && ln -sf /mnt/root/var/lib/rce /var/lib/docker \ && ln -sf /app/entry.sh /start # Needed for legacy RUN chmod +x /app/bin/gosuper diff --git a/Dockerfile.rpi b/Dockerfile.rpi index f1bf4246..4eedf1be 100644 --- a/Dockerfile.rpi +++ b/Dockerfile.rpi @@ -7,9 +7,6 @@ RUN apt-get -q update \ && apt-get install -qqy \ btrfs-tools \ ca-certificates \ - curl \ - docker.io \ - jq \ rsync \ socat \ supervisor \ @@ -39,7 +36,6 @@ COPY . /app/ RUN chmod +x /app/src/enterContainer.sh \ && /app/node_modules/.bin/coffee -c /app/src \ - && ln -sf /mnt/root/var/lib/rce /var/lib/docker \ && ln -sf /app/entry.sh /start # Needed for legacy RUN chmod +x /app/bin/gosuper diff --git a/package.json b/package.json index 10de7afe..d1e02c07 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,8 @@ "bluebird": "^2.9.24", "body-parser": "^1.12.0", "coffee-script": "~1.9.1", - "docker-progress": "^1.1.0", - "dockerode": "~2.2.1", + "docker-progress": "^2.0.0", + "dockerode": "~2.2.9", "event-stream": "^3.0.20", "express": "^4.0.0", "knex": "~0.8.3", diff --git a/src/config.coffee b/src/config.coffee index b0efeefb..857d92b1 100644 --- a/src/config.coffee +++ b/src/config.coffee @@ -13,6 +13,8 @@ checkValidKey = (s) -> return return s +dockerRoot = process.env.DOCKER_ROOT ? '/mnt/root/var/lib/rce' + # Defaults needed for both gosuper and node supervisor are declared in entry.sh module.exports = config = apiEndpoint: process.env.API_ENDPOINT ? 'https://api.resin.io' @@ -38,3 +40,5 @@ module.exports = config = vpnStatusPath: process.env.VPN_STATUS_PATH ? '/mnt/root/run/openvpn/vpn_status' checkInt: checkInt hostOsVersionPath: process.env.HOST_OS_VERSION_PATH ? '/mnt/root/etc/os-release' + dockerRoot: dockerRoot + btrfsRoot: process.env.BTRFS_ROOT ? "#{dockerRoot}/btrfs/subvolumes" diff --git a/src/docker-utils.coffee b/src/docker-utils.coffee index 787dae5b..9d260d12 100644 --- a/src/docker-utils.coffee +++ b/src/docker-utils.coffee @@ -1,13 +1,14 @@ Docker = require 'dockerode' -DockerProgress = require 'docker-progress' +{ getRegistryAndName, DockerProgress } = require 'docker-progress' Promise = require 'bluebird' -{ spawn } = require 'child_process' +{ spawn, execAsync } = Promise.promisifyAll require 'child_process' progress = require 'request-progress' config = require './config' _ = require 'lodash' knex = require './db' TypedError = require 'typed-error' { request } = require './request' +fs = Promise.promisifyAll require 'fs' class OutOfSyncError extends TypedError @@ -87,25 +88,64 @@ exports.rsyncImageWithProgress = (imgDest, onProgress, startFromEmpty = false) - return [ rsyncDiff, imageConfig, imgSrc ] .spread (rsyncDiff, imageConfig, imgSrc) -> - new Promise (resolve, reject) -> - dockersync = spawn('/app/src/dockersync.sh', [ imgSrc, imgDest, JSON.stringify(imageConfig) ], stdio: 'pipe') - .on 'error', reject - .on 'exit', (code, signal) -> - if code in DELTA_OUT_OF_SYNC_CODES - reject(new OutOfSyncError("Incompatible image")) - else if code isnt 0 - reject(new Error("rsync exited. code: #{code} signal: #{signal}")) - else - resolve() - - rsyncDiff.pipe(dockersync.stdin) - dockersync.stdout.pipe(process.stdout) - dockersync.stderr.pipe(process.stdout) - rsyncDiff.resume() + dockerSync(imgSrc, imgDest, rsyncDiff, imageConfig) .catch OutOfSyncError, (err) -> console.log('Falling back to delta-from-empty') exports.rsyncImageWithProgress(imgDest, onProgress, true) +getRepoAndTag = (image) -> + getRegistryAndName(image) + .then ({ registry, imageName, tagName }) -> + return { repo: "#{registry}/#{imageName}", tag: tagName } + +dockerSync = (imgSrc, imgDest, rsyncDiff, conf) -> + docker.importImageAsync('/app/empty.tar') + .then (stream) -> + new Promise (resolve, reject) -> + streamOutput = '' + stream.on 'data', (data) -> + streamOutput += data + stream.on 'error', reject + stream.on 'end', -> + resolve(JSON.parse(streamOutput).status) + .then (destId) -> + jsonPath = "#{config.dockerRoot}/graph/#{destId}/json" + fs.readFileAsync(jsonPath) + .then(JSON.parse) + .then (destJson) -> + destJson.config = conf + fs.writeFileAsync(jsonPath + '.tmp', JSON.stringify(destJson)) + .then -> + fs.renameAsync(jsonPath + '.tmp', jsonPath) + .then -> + if imgSrc isnt 'resin/scratch' + execAsync("btrfs subvolume delete \"#{config.btrfsRoot}/#{destId}\"") + .then -> + docker.getImage(imgSrc).inspectAsync().get('Id') + .then (srcId) -> + execAsync("btrfs subvolume snapshot \"#{config.btrfsRoot}/#{srcId}\" \"#{config.btrfsRoot}/#{destId}\"") + .then -> + new Promise (resolve, reject) -> + rsync = spawn('rsync', ['--timeout=300', '--archive', '--delete' , '--read-batch=-', "#{config.btrfsRoot}/#{destId}"], stdio: 'pipe') + .on 'error', reject + .on 'exit', (code, signal) -> + if code in DELTA_OUT_OF_SYNC_CODES + reject(new OutOfSyncError('Incompatible image')) + else if code isnt 0 + reject(new Error("rsync exited. code: #{code} signal: #{signal}")) + else + resolve() + rsyncDiff.pipe(rsync.stdin) + rsync.stdout.pipe(process.stdout) + rsync.stderr.pipe(process.stdout) + rsyncDiff.resume() + .then -> + execAsync('sync') + .then -> + getRepoAndTag(imgDest) + .then ({ repo, tag }) -> + docker.getImage(destId).tagAsync({ repo, tag, force: true }) + do -> # Keep track of the images being fetched, so we don't clean them up whilst fetching. imagesBeingFetched = 0 diff --git a/src/dockersync.sh b/src/dockersync.sh deleted file mode 100755 index 4c3adbd7..00000000 --- a/src/dockersync.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -DOCKER_ROOT=${DOCKER_ROOT:-/var/lib/docker} -BTRFS_ROOT=${BTRFS_ROOT:-$DOCKER_ROOT/btrfs/subvolumes} - -src=$1 -dest=$2 -config=$3 - -src_id=$(docker inspect -f '{{ .Id }}' "$src") -dest_id=$(cat /app/empty.tar | docker import -) - -jq ".config=$config" "$DOCKER_ROOT/graph/$dest_id/json" > "$DOCKER_ROOT/graph/$dest_id/json.tmp" -mv "$DOCKER_ROOT/graph/$dest_id/json.tmp" "$DOCKER_ROOT/graph/$dest_id/json" - -btrfs subvolume delete "$BTRFS_ROOT/$dest_id" -btrfs subvolume snapshot "$BTRFS_ROOT/$src_id" "$BTRFS_ROOT/$dest_id" - -rsync --timeout=300 --archive --delete --read-batch=- "$BTRFS_ROOT/$dest_id" - -sync - -docker tag -f "$dest_id" "$dest"