From 8d84facff562d065b7779d2b217b697ae243650b Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 7 Jul 2017 14:47:28 -0700 Subject: [PATCH 01/14] Add a single Dockerfile to build the supervisor as a multi-stage build Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .dockerignore | 7 - Dockerfile | 207 +++++++++++++++++++++++++ Dockerfile.build.template | 60 ------- Dockerfile.runtime.template | 20 --- base-image/.dockerignore | 4 - base-image/Dockerfile | 34 ---- base-image/automation/jenkins-build.sh | 63 -------- base-image/build.sh | 38 ++++- gosuper/Dockerfile | 63 -------- gosuper/build.sh | 35 +++++ 10 files changed, 277 insertions(+), 254 deletions(-) create mode 100644 Dockerfile delete mode 100644 Dockerfile.build.template delete mode 100644 Dockerfile.runtime.template delete mode 100644 base-image/.dockerignore delete mode 100644 base-image/Dockerfile delete mode 100755 base-image/automation/jenkins-build.sh delete mode 100644 gosuper/Dockerfile create mode 100644 gosuper/build.sh diff --git a/.dockerignore b/.dockerignore index f406d333..2c60d8bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,13 +1,6 @@ .dockerignore .git node_modules -Dockerfile -Dockerfile.* -Makefile .editorconfig -coffeelint.json -automation tools README.md -retry_docker_push.sh -base-image diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..67c144d6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,207 @@ +ARG ARCH=amd64 +FROM debian:jessie as base +ARG ARCH +# Install the following utilities (required by openembedded) +# http://www.openembedded.org/wiki/Getting_started#Ubuntu_.2F_Debian +RUN apt-get -qq update \ + && apt-get -qq install -y \ + build-essential \ + chrpath \ + cpio \ + curl \ + diffstat \ + file \ + gawk \ + git-core \ + libsdl1.2-dev \ + locales \ + python3 \ + texinfo \ + unzip \ + wget \ + xterm \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +ENV SOURCE_DIR /source +ENV DEST_DIR /dest +ENV SHARED_DOWNLOADS /yocto/shared-downloads +ENV SHARED_SSTATE /yocto/shared-sstate + +ARG BUILDER_UID=1000 +ARG BUILDER_GID=1000 + +COPY base-image /source +RUN cd /source && bash -ex build.sh + +############################################################################## + +# Build golang supervisor +FROM debian:jessie as gosuper +ARG ARCH + +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + curl \ + rsync \ + && rm -rf /var/lib/apt/lists/ + +ENV GOLANG_VERSION 1.8.3 +ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz +ENV GOLANG_DOWNLOAD_SHA256 1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 + +COPY gosuper/go-${GOLANG_VERSION}-patches /go-${GOLANG_VERSION}-patches + +RUN mkdir /usr/src/go \ + && cd /usr/src/go \ + && curl -L -o go.tar.gz $GOLANG_DOWNLOAD_URL \ + && echo "${GOLANG_DOWNLOAD_SHA256} go.tar.gz" | sha256sum -c - \ + && tar xzf go.tar.gz -C /usr/local \ + && cd /usr/src \ + && rm -rf go \ + && export GOROOT_BOOTSTRAP=/usr/local/go-bootstrap \ + && cp -r /usr/local/go /usr/local/go-bootstrap \ + && cd /usr/local/go/src \ + && patch -p2 -i /go-${GOLANG_VERSION}-patches/0001-dont-fail-when-no-mmx.patch \ + && patch -p2 -i /go-${GOLANG_VERSION}-patches/0002-implement-atomic-quadword-ops-with-FILD-FISTP.patch \ + && ./make.bash \ + && rm -rf /usr/local/go-bootstrap + +ENV UPX_VERSION 3.94 +# UPX doesn't provide fingerprints so I checked this one manually +ENV UPX_SHA256 e1fc0d55c88865ef758c7e4fabbc439e4b5693b9328d219e0b9b3604186abe20 + +RUN mkdir /usr/src/upx \ + && cd /usr/src/upx \ + && curl -L -o upx.tar.xz https://github.com/upx/upx/releases/download/v$UPX_VERSION/upx-$UPX_VERSION-amd64_linux.tar.xz \ + && echo "${UPX_SHA256} upx.tar.xz" | sha256sum -c - \ + && tar xf upx.tar.xz --strip-components=1 \ + && cp ./upx /usr/bin/ \ + && cd /usr/src \ + && rm -rf upx + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH + +COPY ./gosuper /go/src/resin-supervisor/gosuper + +WORKDIR /go/src/resin-supervisor/gosuper + +ENV GOOS linux +ENV GO386=387 + +RUN bash ./build.sh +RUN rsync -a --delete /go/bin/gosuper /build/ + +############################################################################## + +# The node version here should match the version of the runtime image which is +# specified in the base-image subdirectory in the project +FROM resin/rpi-node:6.5-slim as rpi-node-base +FROM resin/armv7hf-node:6.5-slim as armv7hf-node-base +FROM resin/armel-node:6.5-slim as armel-node-base +FROM resin/aarch64-node:6.5-slim as aarch64-node-base + +FROM resin/amd64-node:6.5-slim as amd64-node-base +RUN echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-start && chmod +x /usr/bin/cross-build-start \ + && echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-end && chmod +x /usr/bin/cross-build-end + +FROM resin/i386-node:6.5-slim as i386-node-base +RUN echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-start && chmod +x /usr/bin/cross-build-start \ + && echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-end && chmod +x /usr/bin/cross-build-end + +# Build nodejs dependencies +FROM $ARCH-node-base as node +ARG ARCH + +RUN [ "cross-build-start" ] + +WORKDIR /usr/src/app + +RUN apt-get update \ + && apt-get install -y \ + g++ \ + libsqlite3-dev \ + make \ + python \ + rsync \ + wget \ + && rm -rf /var/lib/apt/lists/ + +RUN mkdir -p rootfs-overlay && \ + ln -s /lib rootfs-overlay/lib64 + +COPY package.json /usr/src/app/ + +# Install only the production modules that have C extensions +RUN JOBS=MAX npm install --production --no-optional --unsafe-perm \ + && npm dedupe + +COPY webpack.config.js remove-hashbang-loader.js /usr/src/app/ +COPY src /usr/src/app/src + +# Install devDependencies, build the coffeescript and then prune the deps +RUN cp -R node_modules node_modules_prod \ + && npm install --no-optional --unsafe-perm \ + && npm run lint \ + && npm run build \ + && rm -rf node_modules \ + && mv node_modules_prod node_modules + +# Remove various uneeded filetypes in order to reduce space +RUN find . -path '*/coverage/*' -o -path '*/test/*' -o -path '*/.nyc_output/*' \ + -o -name '*.tar.*' -o -name '*.in' -o -name '*.cc' \ + -o -name '*.c' -o -name '*.coffee' -o -name '*.eslintrc' \ + -o -name '*.h' -o -name '*.html' -o -name '*.markdown' \ + -o -name '*.md' -o -name '*.patch' -o -name '*.png' \ + -o -name '*.yml' \ + -delete \ + && find . -type f -path '*/node_modules/sqlite3/deps*' -delete \ + && find . -type f -path '*/node_modules/knex/build*' -delete + +# Create /var/run/resin for the gosuper to place its socket in +RUN mkdir -p rootfs-overlay/var/run/resin + +COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/ + +COPY inittab rootfs-overlay/etc/inittab + +RUN rsync -a --delete node_modules dist rootfs-overlay /build + +RUN [ "cross-build-end" ] + +############################################################################## + +# Minimal runtime image +FROM scratch +ARG ARCH +ARG VERSION=master +ARG DEFAULT_PUBNUB_PUBLISH_KEY=pub-c-bananas +ARG DEFAULT_PUBNUB_SUBSCRIBE_KEY=sub-c-bananas +ARG DEFAULT_MIXPANEL_TOKEN=bananasbananas + +COPY --from=base /dest/ / + +WORKDIR /usr/src/app + +COPY --from=node /build/dist ./dist +COPY --from=node /build/node_modules ./node_modules +COPY --from=gosuper /build/gosuper ./gosuper +COPY --from=node /build/rootfs-overlay/ / + +VOLUME /data + +ENV CONFIG_MOUNT_POINT=/boot/config.json \ + LED_FILE=/dev/null \ + SUPERVISOR_IMAGE=resin/$ARCH-supervisor \ + VERSION=$VERSION \ + DEFAULT_PUBNUB_PUBLISH_KEY=$DEFAULT_PUBNUB_PUBLISH_KEY \ + DEFAULT_PUBNUB_SUBSCRIBE_KEY=$DEFAULT_PUBNUB_SUBSCRIBE_KEY \ + DEFAULT_MIXPANEL_TOKEN=$DEFAULT_MIXPANEL_TOKEN + +CMD [ "/sbin/init" ] diff --git a/Dockerfile.build.template b/Dockerfile.build.template deleted file mode 100644 index 6b3fb8fc..00000000 --- a/Dockerfile.build.template +++ /dev/null @@ -1,60 +0,0 @@ -# Build nodejs dependencies - -# The node version here should match the version of the runtime image which is -# specified in the base-image subdirectory in the project -FROM resin/%%ARCH%%-node:6.5-slim - -WORKDIR /usr/src/app - -RUN apt-get update \ - && apt-get install -y \ - g++ \ - libsqlite3-dev \ - make \ - python \ - rsync \ - wget \ - && rm -rf /var/lib/apt/lists/ - -RUN mkdir -p rootfs-overlay && \ - ln -s /lib rootfs-overlay/lib64 - -COPY package.json /usr/src/app/ - -# Install only the production modules that have C extensions -RUN JOBS=MAX npm install --production --no-optional --unsafe-perm \ - && npm dedupe - -COPY webpack.config.js remove-hashbang-loader.js /usr/src/app/ -COPY src /usr/src/app/src - -# Install devDependencies, build the coffeescript and then prune the deps -RUN cp -R node_modules node_modules_prod \ - && npm install --no-optional --unsafe-perm \ - && npm run lint \ - && npm run build \ - && rm -rf node_modules \ - && mv node_modules_prod node_modules - -# Remove various uneeded filetypes in order to reduce space -RUN find . -path '*/coverage/*' -o -path '*/test/*' -o -path '*/.nyc_output/*' \ - -o -name '*.tar.*' -o -name '*.in' -o -name '*.cc' \ - -o -name '*.c' -o -name '*.coffee' -o -name '*.eslintrc' \ - -o -name '*.h' -o -name '*.html' -o -name '*.markdown' \ - -o -name '*.md' -o -name '*.patch' -o -name '*.png' \ - -o -name '*.yml' \ - -delete \ - && find . -type f -path '*/node_modules/sqlite3/deps*' -delete \ - && find . -type f -path '*/node_modules/knex/build*' -delete - -# Create /var/run/resin for the gosuper to place its socket in -RUN mkdir -p rootfs-overlay/var/run/resin - -COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/ - -COPY inittab rootfs-overlay/etc/inittab - -CMD rsync -a --delete node_modules dist rootfs-overlay /build - -# -*- mode: dockerfile -*- -# vi: set ft=dockerfile : diff --git a/Dockerfile.runtime.template b/Dockerfile.runtime.template deleted file mode 100644 index 2d8b39f4..00000000 --- a/Dockerfile.runtime.template +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal runtime image -FROM %%BASE_IMAGE_TAG%% - -WORKDIR /usr/src/app - -COPY ./build/%%ARCH%%/dist ./dist -COPY ./build/%%ARCH%%/node_modules ./node_modules -COPY ./build/%%ARCH%%/gosuper ./gosuper -COPY ./build/%%ARCH%%/rootfs-overlay/ / - -VOLUME /data - -ENV CONFIG_MOUNT_POINT=/boot/config.json \ - LED_FILE=/dev/null \ - SUPERVISOR_IMAGE=resin/%%ARCH%%-supervisor - -CMD [ "/sbin/init" ] - -# -*- mode: dockerfile -*- -# vi: set ft=dockerfile : diff --git a/base-image/.dockerignore b/base-image/.dockerignore deleted file mode 100644 index 3b95ceb0..00000000 --- a/base-image/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -build -oe-core -meta-nodejs -bitbake diff --git a/base-image/Dockerfile b/base-image/Dockerfile deleted file mode 100644 index 17dc7ce4..00000000 --- a/base-image/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -FROM ubuntu:15.04 - -# Install the following utilities (required by openembedded) -# http://www.openembedded.org/wiki/Getting_started#Ubuntu_.2F_Debian -RUN apt-get update \ - && apt-get install -y \ - build-essential \ - chrpath \ - cpio \ - curl \ - diffstat \ - file \ - gawk \ - git-core \ - libsdl1.2-dev \ - python3 \ - texinfo \ - unzip \ - wget \ - xterm \ - sudo \ - && rm -rf /var/lib/apt/lists/* - -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -ENV SOURCE_DIR /source -ENV DEST_DIR /dest -ENV SHARED_DOWNLOADS /yocto/shared-downloads -ENV SHARED_SSTATE /yocto/shared-sstate - -COPY build.sh / -CMD bash -ex build.sh diff --git a/base-image/automation/jenkins-build.sh b/base-image/automation/jenkins-build.sh deleted file mode 100755 index be42f147..00000000 --- a/base-image/automation/jenkins-build.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -# ARCH, ESCAPED_BRANCH_NAME, BASE_IMAGE_REPO and BASE_IMAGE_TAG should be set before calling this script. -# This script purposefully doesn't clean up the BASE_IMAGE_TAG docker image after building it. - -JENKINS_PERSISTENT_WORKDIR=${1:-/var/lib/yocto} -DL_DIR="$JENKINS_PERSISTENT_WORKDIR/shared-downloads" -mkdir dest - -BUILDER_REPO=registry.resinstaging.io/resin/${ARCH}-supervisor-base-builder -BUILDER_IMAGE=${BUILDER_REPO}:${ESCAPED_BRANCH_NAME} - -docker pull ${BUILDER_IMAGE} || docker pull ${BUILDER_REPO}:master || true -docker build -t ${BUILDER_IMAGE} . -docker push ${BUILDER_IMAGE} || true - -case "$ARCH" in -'amd64') - machine='generic-x86-64' -;; -'i386') - machine='generic-x86' -;; -'rpi') - machine='generic-armv6' -;; -'armv7hf') - machine='generic-armv7hf' -;; -'armel') - machine='generic-armv5' -;; -'aarch64') - machine='generic-armv8' -;; -esac -SSTATE_DIR="$JENKINS_PERSISTENT_WORKDIR/$machine/sstate" -# Make sure shared directories are in place -mkdir -p $DL_DIR -mkdir -p $SSTATE_DIR - -docker run --rm \ - -e TARGET_MACHINE=$machine \ - -e BUILDER_UID=$(id -u) \ - -e BUILDER_GID=$(id -g) \ - -v `pwd`:/source \ - -v $DL_DIR:/yocto/shared-downloads \ - -v $SSTATE_DIR:/yocto/shared-sstate \ - -v `pwd`/dest:/dest \ - ${BUILDER_IMAGE} -docker rmi ${BUILDER_IMAGE} || true -docker rmi ${BUILDER_REPO}:master || true -if [ -f dest/rootfs.tar.gz ]; then - docker import dest/rootfs.tar.gz ${BASE_IMAGE_TAG} - docker tag ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME} || docker tag -f ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME} - docker push ${BASE_IMAGE_TAG} -else - echo "rootfs is missing!" - exit 1 -fi diff --git a/base-image/build.sh b/base-image/build.sh index 993dd4a1..17b87ec5 100755 --- a/base-image/build.sh +++ b/base-image/build.sh @@ -3,11 +3,43 @@ set -o errexit BUILD_DIR='/home/builder/tmp' + +case "$ARCH" in +'amd64') + export TARGET_MACHINE='generic-x86-64' +;; +'i386') + export TARGET_MACHINE='generic-x86' +;; +'rpi') + export TARGET_MACHINE='generic-armv6' +;; +'armv7hf') + export TARGET_MACHINE='generic-armv7hf' +;; +'armel') + export TARGET_MACHINE='generic-armv5' +;; +'aarch64') + export TARGET_MACHINE='generic-armv8' +;; +esac + +export SOURCE_DIR=/source +export DEST_DIR=/dest +export SHARED_DOWNLOADS=/yocto/shared-downloads +export SHARED_SSTATE=/yocto/shared-sstate +# Make sure shared directories are in place +mkdir -p $SHARED_DOWNLOADS +mkdir -p $SHARED_SSTATE +mkdir -p $DEST_DIR + groupadd -g $BUILDER_GID builder useradd -m -u $BUILDER_UID -g $BUILDER_GID builder -sudo -H -u builder /bin/bash -c "mkdir -p $BUILD_DIR \ +sudo -H -u builder \ + /bin/bash -c "mkdir -p $BUILD_DIR \ && cp -r $SOURCE_DIR/* $BUILD_DIR/ \ && cd $BUILD_DIR \ && source oe-core/oe-init-build-env build bitbake \ - && DL_DIR=$SHARED_DOWNLOADS SSTATE_DIR=$SHARED_SSTATE MACHINE=$TARGET_MACHINE $BUILD_DIR/bitbake/bin/bitbake core-image-minimal" -cp --dereference $BUILD_DIR/build/tmp-glibc/deploy/images/$TARGET_MACHINE/core-image-minimal-$TARGET_MACHINE.tar.gz $DEST_DIR/rootfs.tar.gz + && DL_DIR=$SHARED_DOWNLOADS SSTATE_DIR=$SHARED_SSTATE MACHINE=$TARGET_MACHINE $BUILD_DIR/bitbake/bin/bitbake core-image-minimal > /dev/null" +tar xzf $BUILD_DIR/build/tmp-glibc/deploy/images/$TARGET_MACHINE/core-image-minimal-$TARGET_MACHINE.tar.gz -C $DEST_DIR diff --git a/gosuper/Dockerfile b/gosuper/Dockerfile deleted file mode 100644 index 592683ff..00000000 --- a/gosuper/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -# Build golang supervisor -FROM debian:jessie - -RUN apt-get update \ - && apt-get install -y \ - build-essential \ - curl \ - rsync \ - && rm -rf /var/lib/apt/lists/ - -ENV GOLANG_VERSION 1.8.3 -ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz -ENV GOLANG_DOWNLOAD_SHA256 1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 - -COPY go-${GOLANG_VERSION}-patches /go-${GOLANG_VERSION}-patches - -RUN mkdir /usr/src/go \ - && cd /usr/src/go \ - && curl -L -o go.tar.gz $GOLANG_DOWNLOAD_URL \ - && echo "${GOLANG_DOWNLOAD_SHA256} go.tar.gz" | sha256sum -c - \ - && tar xzf go.tar.gz -C /usr/local \ - && cd /usr/src \ - && rm -rf go \ - && export GOROOT_BOOTSTRAP=/usr/local/go-bootstrap \ - && cp -r /usr/local/go /usr/local/go-bootstrap \ - && cd /usr/local/go/src \ - && patch -p2 -i /go-${GOLANG_VERSION}-patches/0001-dont-fail-when-no-mmx.patch \ - && patch -p2 -i /go-${GOLANG_VERSION}-patches/0002-implement-atomic-quadword-ops-with-FILD-FISTP.patch \ - && ./make.bash \ - && rm -rf /usr/local/go-bootstrap - -ENV UPX_VERSION 3.94 -# UPX doesn't provide fingerprints so I checked this one manually -ENV UPX_SHA256 e1fc0d55c88865ef758c7e4fabbc439e4b5693b9328d219e0b9b3604186abe20 - -RUN mkdir /usr/src/upx \ - && cd /usr/src/upx \ - && curl -L -o upx.tar.xz https://github.com/upx/upx/releases/download/v$UPX_VERSION/upx-$UPX_VERSION-amd64_linux.tar.xz \ - && echo "${UPX_SHA256} upx.tar.xz" | sha256sum -c - \ - && tar xf upx.tar.xz --strip-components=1 \ - && cp ./upx /usr/bin/ \ - && cd /usr/src \ - && rm -rf upx - -ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - -COPY . /go/src/resin-supervisor/gosuper - -WORKDIR /go/src/resin-supervisor/gosuper - -ENV GOOS linux -ENV GO386=387 - -ARG GOARCH=amd64 -ARG GOARM='' - -RUN go install -a -v ./gosuper \ - && cd /go/bin \ - && find -type f -name gosuper -exec mv {} /go/bin/gosuper \; \ - && upx --best /go/bin/gosuper - -CMD rsync -a --delete /go/bin/gosuper /build diff --git a/gosuper/build.sh b/gosuper/build.sh new file mode 100644 index 00000000..ec5ec235 --- /dev/null +++ b/gosuper/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -o errexit + +./test_formatting.sh +go test -v ./gosuper + +case "$ARCH" in +'amd64') + export GOARCH=amd64 +;; +'i386') + export GOARCH=386 +;; +'rpi') + export GOARCH=arm + export GOARM=6 +;; +'armv7hf') + export GOARCH=arm + export GOARM=7 +;; +'armel') + export GOARCH=arm + export GOARM=5 +;; +'aarch64') + export GOARCH=arm64 +;; +esac + +go install -a -v ./gosuper \ + && cd /go/bin \ + && find -type f -name gosuper -exec mv {} /go/bin/gosuper \; \ + && upx --best /go/bin/gosuper From 012b23f012a677de1bb7298fa2344f03b3974738 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 7 Jul 2017 14:48:47 -0700 Subject: [PATCH 02/14] Refactor the makefile to make it easier to use and make use of the multi-stage build Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .gitignore | 1 + Makefile | 199 ++++++++++++++++++++++++----------------------------- 2 files changed, 92 insertions(+), 108 deletions(-) diff --git a/.gitignore b/.gitignore index c71493f1..e5aa9b13 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ Dockerfile.runtime.* !Dockerfile.runtime.template /build/ /dist/ +tools/dind/config/services/docker.service.d/proxy.conf diff --git a/Makefile b/Makefile index 9e8e17a3..57d04fab 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,44 @@ +# resin-supervisor Makefile +# +# If you're looking for an easy way to develop on the supervisor, check ./tools/dev/dindctl, which provides a simplified interface +# to this makefile. +# +# Build targets (require Docker 17.05 or greater): +# * supervisor (default) - builds a resin-supervisor image +# * deploy - pushes a resin-supervisor image to the registry, retrying up to 3 times +# * base - builds the "base" component (a yocto builder with the output rootfs at /dest) +# * gosuper - builds the "gosuper" component (a golang image with the Go supervisor component at /go/bin/gosuper and /build/gosuper) +# * nodesuper - builds the node component, with the node_modules and src at /usr/src/app and /build (also includes a rootfs-overlay there) +# * supervisor-dind: build the development docker-in-docker supervisor that run-supervisor uses +# +# Variables for build targets: +# * ARCH: amd64/rpi/i386/armv7hf/armel/aarch64 architecture for which to build the supervisor - default: amd64 +# * IMAGE: image to build or deploy - default: resin/$(ARCH)-supervisor:latest +# * MIXPANEL_TOKEN, PUBNUB_SUBSCRIBE_KEY, PUBNUB_PUBLISH_KEY: (optional) default pubnub and mixpanel keys to embed in the supervisor image +# * DISABLE_CACHE: if set to true, run build with no cache - default: false +# * DOCKER_BUILD_OPTIONS: Additional options for docker build, like --cache-from parameters +# +# Test/development targets: +# * run-supervisor, stop-supervisor - build and start or stop a docker-in-docker resin-supervisor (requires aufs and ability to run privileged containers) +# * format-gosuper, test-gosuper - build a gosuper image and run formatting or unit tests +# * test-integration - run an integration test (see gosuper/supertest). Requires a docker-in-docker supervisor to be running +# +# Variables for test/dev targets: +# * IMAGE: image to build and run (either for run-supervisor or test-gosuper/integration) +# * SUPERVISOR_IMAGE: In run-supervisor, the supervisor image to run inside the docker-in-docker image +# * PRELOADED_IMAGE: If true, will preload user app image from tools/dev/apps.json and bind mount apps.json into the docker-in-docker supervisor +# * SUPERVISOR_EXTRA_MOUNTS: Additional bind mount flags for the docker-in-docker supervisor +# * PASSWORDLESS_DROPBEAR: For run-supervisor - start a passwordless ssh daemon in the docker-in-docker supervisor +# + +THIS_FILE := $(lastword $(MAKEFILE_LIST)) + +help: + @cat $(THIS_FILE) | awk '{if(/^#/)print;else exit}' | sed 's/\#//' + OS := $(shell uname) +# If we're behind a proxy, use it during build ifdef http_proxy DOCKER_HTTP_PROXY=--build-arg http_proxy=$(http_proxy) endif @@ -18,74 +57,48 @@ ifdef use_proxy_at_runtime rt_no_proxy=$(no_proxy) endif -DISABLE_CACHE = 'false' - -ARCH = rpi# rpi/amd64/i386/armv7hf/armel/aarch64 - -DEPLOY_REGISTRY = - -SUPERVISOR_VERSION = master -BASE_IMAGE_VERSION = $(shell find base-image -print0 | LC_ALL=C sort -z | tar --null -cf - --no-recursion --mtime=@0 --owner=root --group=root --numeric-owner -T - | md5sum | awk -F " " '{print $$1}') -ESCAPED_BASE_IMAGE_TAG = resin\/$(ARCH)-supervisor-base:$(BASE_IMAGE_VERSION) +DISABLE_CACHE ?= 'false' DOCKER_VERSION:=$(shell docker version --format '{{.Server.Version}}') DOCKER_MAJOR_VERSION:=$(word 1, $(subst ., ,$(DOCKER_VERSION))) DOCKER_MINOR_VERSION:=$(word 2, $(subst ., ,$(DOCKER_VERSION))) -DOCKER_GE_1_12 := $(shell [ $(DOCKER_MAJOR_VERSION) -gt 1 -o \( $(DOCKER_MAJOR_VERSION) -eq 1 -a $(DOCKER_MINOR_VERSION) -ge 12 \) ] && echo true) +DOCKER_GE_17_05 := $(shell [ $(DOCKER_MAJOR_VERSION) -gt 17 -o \( $(DOCKER_MAJOR_VERSION) -eq 17 -a $(DOCKER_MINOR_VERSION) -ge 5 \) ] && echo true) -# In docker 1.12 tag --force has been removed. -ifeq ($(DOCKER_GE_1_12),true) -DOCKER_TAG_FORCE= -else -DOCKER_TAG_FORCE=-f -endif +# Default values for Pubnub and Mixpanel keys +PUBNUB_SUBSCRIBE_KEY ?= sub-c-bananas +PUBNUB_PUBLISH_KEY ?= pub-c-bananas +MIXPANEL_TOKEN ?= bananasbananas -all: supervisor +# Default architecture and output image +ARCH ?= amd64 +IMAGE ?= resin/$(ARCH)-supervisor:master -PUBNUB_SUBSCRIBE_KEY = sub-c-bananas -PUBNUB_PUBLISH_KEY = pub-c-bananas -MIXPANEL_TOKEN = bananasbananas +# Default values for run-supervisor +SUPERVISOR_IMAGE ?= resin/$(ARCH)-supervisor:master +PASSWORDLESS_DROPBEAR ?= false -PASSWORDLESS_DROPBEAR = false - -IMAGE = "resin/$(ARCH)-supervisor:$(SUPERVISOR_VERSION)" -DOCKERFILE = $(ARCH) - -SUPERVISOR_IMAGE=$(DEPLOY_REGISTRY)$(IMAGE) - -ifeq ($(ARCH),rpi) - GOARCH = arm - GOARM = 6 -endif -ifeq ($(ARCH),armv7hf) - GOARCH = arm - GOARM = 7 -endif -ifeq ($(ARCH),armel) - GOARCH = arm - GOARM = 5 -endif -ifeq ($(ARCH),i386) - GOARCH = 386 -endif -ifeq ($(ARCH),amd64) - GOARCH = amd64 -endif -ifeq ($(ARCH),aarch64) - GOARCH = arm64 -endif +# Bind mounts and variables for the run-supervisor target SUPERVISOR_DIND_MOUNTS := -v $$(pwd)/../../:/resin-supervisor -v $$(pwd)/config.json:/mnt/conf/config.json -v $$(pwd)/config/env:/usr/src/app/config/env -v $$(pwd)/config/localenv:/usr/src/app/config/localenv ifeq ($(OS), Linux) SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /bin/kmod:/bin/kmod endif -ifdef PRELOADED_IMAGE +ifeq ($(PRELOADED_IMAGE),true) SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/apps.json:/usr/src/app/config/apps.json else PRELOADED_IMAGE= endif +SUPERVISOR_EXTRA_MOUNTS ?= -SUPERVISOR_EXTRA_MOUNTS = +ifdef TARGET_COMPONENT + DOCKER_TARGET_COMPONENT := "--target=${TARGET_COMPONENT}" +else + DOCKER_TARGET_COMPONENT := +endif +# Default target is to build the supervisor image +all: supervisor + +# Settings to make the run-supervisor target work behind a proxy DOCKERD_PROXY=tools/dind/config/services/docker.service.d/proxy.conf ${DOCKERD_PROXY}: rm -f ${DOCKERD_PROXY} @@ -113,9 +126,10 @@ supervisor-dind: ${DOCKERD_PROXY} $(DOCKER_HTTP_PROXY) \ $(DOCKER_HTTPS_PROXY) \ $(DOCKER_NO_PROXY) \ + ${DOCKER_BUILD_OPTIONS} \ --no-cache=$(DISABLE_CACHE) \ --build-arg PASSWORDLESS_DROPBEAR=$(PASSWORDLESS_DROPBEAR) \ - -t resin/resin-supervisor-dind:$(SUPERVISOR_VERSION) . + -t $(IMAGE) . run-supervisor: stop-supervisor supervisor-dind cd tools/dind \ @@ -133,7 +147,7 @@ run-supervisor: stop-supervisor supervisor-dind if [ -n "$(rt_no_proxy)" ]; then \ echo "no_proxy=$(rt_no_proxy)" >> config/localenv; \ fi \ - && docker run -d --name resin_supervisor_1 --privileged ${SUPERVISOR_DIND_MOUNTS} resin/resin-supervisor-dind:$(SUPERVISOR_VERSION) + && docker run -d --name resin_supervisor_1 --privileged ${SUPERVISOR_DIND_MOUNTS} $(IMAGE) stop-supervisor: # Stop docker and remove volumes to prevent us from running out of loopback devices, @@ -142,81 +156,53 @@ stop-supervisor: -docker stop resin_supervisor_1 > /dev/null || true -docker rm -f --volumes resin_supervisor_1 > /dev/null || true -refresh-supervisor-src: - echo " * Compiling CoffeeScript.." \ - && coffee -c ./src \ - && echo " * Restarting supervisor container.." \ - && docker exec -ti resin_supervisor_1 docker restart resin_supervisor - -supervisor: nodesuper gosuper - sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template | sed 's/%%BASE_IMAGE_TAG%%/$(ESCAPED_BASE_IMAGE_TAG)/g' > Dockerfile.runtime.$(ARCH) - echo "ENV VERSION=$(shell jq -r .version package.json) \\" >> Dockerfile.runtime.$(ARCH) - echo " DEFAULT_PUBNUB_PUBLISH_KEY=$(PUBNUB_PUBLISH_KEY) \\" >> Dockerfile.runtime.$(ARCH) - echo " DEFAULT_PUBNUB_SUBSCRIBE_KEY=$(PUBNUB_SUBSCRIBE_KEY) \\" >> Dockerfile.runtime.$(ARCH) - echo " DEFAULT_MIXPANEL_TOKEN=$(MIXPANEL_TOKEN)" >> Dockerfile.runtime.$(ARCH) -ifdef rt_https_proxy - echo "ENV HTTPS_PROXY=$(rt_https_proxy) \\" >> Dockerfile.runtime.$(ARCH) - echo " https_proxy=$(rt_https_proxy)" >> Dockerfile.runtime.$(ARCH) -endif -ifdef rt_http_proxy - echo "ENV HTTP_PROXY=$(rt_http_proxy) \\" >> Dockerfile.runtime.$(ARCH) - echo " http_proxy=$(rt_http_proxy)" >> Dockerfile.runtime.$(ARCH) -endif -ifdef rt_no_proxy - echo "ENV no_proxy=$(rt_no_proxy)" >> Dockerfile.runtime.$(ARCH) +supervisor-image: +ifneq ($(DOCKER_GE_17_05),true) + @echo "Docker >= 17.05 is needed to build the supervisor" + @exit 1 endif docker build \ $(DOCKER_HTTP_PROXY) \ $(DOCKER_HTTPS_PROXY) \ $(DOCKER_NO_PROXY) \ - -f Dockerfile.runtime.$(ARCH) \ - --pull \ + $(DOCKER_TARGET_COMPONENT) \ + $(DOCKER_BUILD_OPTIONS) \ + --no-cache=$(DISABLE_CACHE) \ + --build-arg ARCH=$(ARCH) \ + --build-arg VERSION=$(shell jq -r .version package.json) \ + --build-arg DEFAULT_PUBNUB_PUBLISH_KEY=$(PUBNUB_PUBLISH_KEY) \ + --build-arg DEFAULT_PUBNUB_SUBSCRIBE_KEY=$(PUBNUB_SUBSCRIBE_KEY) \ + --build-arg DEFAULT_MIXPANEL_TOKEN=$(MIXPANEL_TOKEN) \ -t $(IMAGE) . -lint: - docker run --rm resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) bash -c 'npm install resin-lint && npm run lint' +supervisor: + @$(MAKE) -f $(THIS_FILE) IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image -deploy: supervisor - docker tag $(DOCKER_TAG_FORCE) $(IMAGE) $(SUPERVISOR_IMAGE) - bash retry_docker_push.sh $(SUPERVISOR_IMAGE) +deploy: + @bash retry_docker_push.sh $(IMAGE) + +base: + $(MAKE) -f $(THIS_FILE) TARGET_COMPONENT=base IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image nodesuper: - sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.build.template > Dockerfile.build.$(ARCH) - docker build --pull \ - $(DOCKER_HTTP_PROXY) \ - $(DOCKER_HTTPS_PROXY) \ - $(DOCKER_NO_PROXY) \ - -f Dockerfile.build.$(ARCH) \ - -t resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) . - docker run --rm \ - -v `pwd`/build/$(ARCH):/build \ - resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) + $(MAKE) -f $(THIS_FILE) TARGET_COMPONENT=node IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image gosuper: - cd gosuper && docker build --pull \ - $(DOCKER_HTTP_PROXY) \ - $(DOCKER_HTTPS_PROXY) \ - $(DOCKER_NO_PROXY) \ - --build-arg GOARCH=$(GOARCH) \ - --build-arg GOARM=$(GOARM) \ - -t resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) . - docker run --rm \ - -v `pwd`/build/$(ARCH):/build \ - resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) + $(MAKE) -f $(THIS_FILE) TARGET_COMPONENT=gosuper IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image test-gosuper: gosuper docker run \ --rm \ -v /var/run/dbus:/mnt/root/run/dbus \ -e DBUS_SYSTEM_BUS_ADDRESS="/mnt/root/run/dbus/system_bus_socket" \ - resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) bash -c \ + $(IMAGE) bash -c \ './test_formatting.sh && go test -v ./gosuper' format-gosuper: gosuper docker run \ --rm \ -v $(shell pwd)/gosuper:/go/src/resin-supervisor/gosuper \ - resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) \ + $(IMAGE) \ go fmt ./... test-integration: gosuper @@ -227,10 +213,7 @@ test-integration: gosuper --volumes-from resin_supervisor_1 \ -v /var/run/dbus:/mnt/root/run/dbus \ -e DBUS_SYSTEM_BUS_ADDRESS="/mnt/root/run/dbus/system_bus_socket" \ - resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) \ + $(IMAGE) \ go test -v ./supertest -base-image-version: - @echo $(BASE_IMAGE_VERSION) - -.PHONY: supervisor deploy supervisor-dind run-supervisor gosuper nodesuper +.PHONY: supervisor deploy base nodesuper gosuper supervisor-dind run-supervisor From da04b3065cd01bc580c3d3a427fb952e1083e38d Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 7 Jul 2017 14:49:41 -0700 Subject: [PATCH 03/14] Make dindctl an easier to use tool that takes options instead of using env vars Change-Type: patch Signed-off-by: Pablo Carranza Velez --- README.md | 83 ++++++++++--------- tools/dev/dindctl | 197 ++++++++++++++++++++++++++++++---------------- 2 files changed, 174 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index ab106bdd..1fe1e471 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ We are using [waffle.io](https://waffle.io) to manage our tickets / issues, so i ### Deploy your local version to a Docker registry -We'll show how to use the DockerHub registry, but any other can be specified as part of the `SUPERVISOR_IMAGE` variable. +We'll show how to use the DockerHub registry, but any other can be specified as part of the `IMAGE` variable. If you haven't done so yet, login to the registry: ```bash @@ -20,21 +20,24 @@ docker login ``` Use your username and password as required. -Then deploy to a specific repo and tag, e.g. +Then build the supervisor and deploy it to a specific repo and tag, e.g. ```bash -make ARCH=amd64 SUPERVISOR_IMAGE=username/resin-supervisor:master deploy +./tools/dev/dindctl deploy --image username/resin-supervisor:master --arch amd64 ``` -This will build the Supervisor docker image if you haven't done it yet, and upload it to the registry. -As we pointed out before, a different registry can be specified with the DEPLOY_REGISTRY env var. + +This will build the Supervisor docker image and upload it to dockerhub. You can use other registries by adding to the +image name e.g. `myregistry.com/username/resin-supervisor:master`. ### Set up config.json -Add `tools/dind/config.json` file from a staging device image. + +Add a `tools/dind/config.json` file from a staging device image. It should be configured for an x86 or amd64 device type so that you can push apps to it and they run properly on your computer. + +Note: don't use resinstaging for production devices. This is for development purposes only. A production (resin.io) config.json should work just as fine for this local supervisor, but we also don't recommend using this in production scenarios - ResinOS is way better suited for that. A config.json file can be obtained in several ways, for instance: -* Download an Intel Edison image from staging, open `config.img` with an archive tool like [peazip](http://sourceforge.net/projects/peazip/files/) -* Download a Raspberry Pi 2 image, flash it to an SD card, then mount partition 5 (resin-conf). -* Install Resin CLI with `npm install -g resin-cli`, then login with `resin login` and finally run `resin config generate --app -o config.json` (choose the default settings whenever prompted). Check [this section](https://github.com/resin-io/resin-cli#how-do-i-point-the-resin-cli-to-staging) on how to point Resin CLI to a device on staging. +* Log in to the dashboard on resinstaging (https://dashboard.resinstaging.io), create or select an application, click "Download OS" and on the Advanced section select "Download configuration only". +* Install the Resin CLI with `npm install -g resin-cli`, then login with `resin login` and finally run `resin config generate --app -o config.json` (choose the default settings whenever prompted). Check [this section](https://github.com/resin-io/resin-cli#how-do-i-point-the-resin-cli-to-staging) on how to point Resin CLI to a device on staging. The config.json file should look something like this: @@ -46,31 +49,45 @@ The config.json file should look something like this: "userId": "141", /* User ID for the user who owns the app */ "username": "gh_pcarranzav", /* User name for the user who owns the app */ "deviceType": "intel-edison", /* The device type corresponding to the test application */ - "files": { /* This field is used by the host OS on devices, so the supervisor doesn't care about it */ - "network/settings": "[global]\nOfflineMode=false\n\n[WiFi]\nEnable=true\nTethering=false\n\n[Wired]\nEnable=true\nTethering=false\n\n[Bluetooth]\nEnable=true\nTethering=false", - "network/network.config": "[service_home_ethernet]\nType = ethernet\nNameservers = 8.8.8.8,8.8.4.4" - }, "apiEndpoint": "https://api.resinstaging.io", /* Endpoint for the Resin API */ - "registryEndpoint": "registry.resinstaging.io", /* Endpoint for the Resin registry */ + "deltaEndpoint": "https://delta.resinstaging.io", /* Endpoint for the delta server to download docker binary diffs */ "vpnEndpoint": "vpn.resinstaging.io", /* Endpoint for the Resin VPN server */ "pubnubSubscribeKey": "sub-c-aaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", /* Subscribe key for Pubnub for logs */ "pubnubPublishKey": "pub-c-aaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", /* Publish key for Pubnub for logs */ "listenPort": 48484, /* Listen port for the supervisor API */ "mixpanelToken": "aaaaaaaaaaaaaaaaaaaaaaaaaa", /* Mixpanel token to report events */ + "files": { /* This field is used by the host OS on devices, so the supervisor doesn't care about it */ + "network/settings": "[global]\nOfflineMode=false\n\n[WiFi]\nEnable=true\nTethering=false\n\n[Wired]\nEnable=true\nTethering=false\n\n[Bluetooth]\nEnable=true\nTethering=false", + "network/network.config": "[service_home_ethernet]\nType = ethernet\nNameservers = 8.8.8.8,8.8.4.4" + }, + "registryEndpoint": "registry.resinstaging.io", /* Endpoint for the Resin registry, not used by the latest supervisor versions */ } ``` -Additionally, the `uuid`, `registered_at` and `deviceId` fields will be added by the supervisor upon registration with the resin API. + +Additionally, the `uuid`, `registered_at` and `deviceId` fields will be added by the supervisor upon registration with the resin API. Other fields may be present (the format has evolved over time and will likely continue to do so). ### Start the supervisor instance Ensure your kernel supports aufs (in Ubuntu, install `linux-image-extra-$(uname -r)`) and the `aufs` module is loaded (if necessary, run `sudo modprobe aufs`). ```bash -ARCH=amd64 SUPERVISOR_IMAGE=username/resin-supervisor:master ./tools/dev/dindctl run +./tools/dev/dindctl run --image username/resin-supervisor:master ``` This will setup a docker-in-docker instance with an image that runs the supervisor image. +If you want to develop and test your changes, you can run: + +```bash +./tools/dev/dindctl run --image username/resin-supervisor:master --mount-dist +``` + +This will mount the ./dist folder into the supervisor container, so that any changes you make can be added to the running supervisor with: + +```bash +./tools/dev/dindctl refresh +``` + ### Testing with preloaded apps To test preloaded apps, add a `tools/dind/apps.json` file according to the preloaded apps spec. @@ -93,8 +110,7 @@ Make sure the config.json file doesn't have uuid, registered_at or deviceId popu Then run the supervisor like this: ```bash -make ARCH=amd64 PRELOADED_IMAGE=true \ - SUPERVISOR_IMAGE=username/resin-supervisor:master run-supervisor +./tools/dev/dindctl run --image username/resin-supervisor:master --preload ``` This will make the docker-in-docker instance pull the image specified in apps.json before running the supervisor. @@ -123,34 +139,26 @@ docker exec -it resin_supervisor_1 journalctl -f This will stop the container and remove it, also removing its volumes. ## Working with the Go supervisor -The Dockerfile used to build the Go supervisor is Dockerfile.gosuper, and the code for the Go supervisor lives in the `gosuper` directory. -To build it, run: +The code for the Go supervisor lives in the `gosuper` directory. + +To build it, run (with the ARCH and IMAGE you want): ```bash -make ARCH=amd64 gosuper +make ARCH=amd64 IMAGE=username/gosuper:master gosuper ``` -This will build and run the docker image that builds the Go supervisor and outputs the executable at `gosuper/bin`. +This will build a docker image that builds the Go supervisor and has the executable at /go/bin/gosuper inside the image. ### Adding Go dependencies -This project uses [Godep](https://github.com/tools/godep) to manage its Go dependencies. In order for it to work, this repo needs to be withing the `src` directory in a valid Go workspace. This can easily be achieved by having the repo as a child of a directory named `src` and setting the `GOPATH` environment variable to such directory's parent. +This project uses [glide](https://github.com/Masterminds/glide) to manage its Go dependencies. Refer to its repository for instructions on adding packages. -If these conditions are met, a new dependency can be added with: -```bash -go get github.com/path/to/dependency -``` -Then we add the corresponding import statement in our code (e.g. main.go): -```go -import "github.com/path/to/dependency" -``` -And we save it to Godeps.json with: -```bash -cd gosuper -godep save -r ./... -``` -(The -r switch will modify the import statement to use Godep's `_workspace`) +In order for go utilities to work, this repo needs to be within the `src` directory in a valid Go workspace. This can easily be achieved by having the repo as a child of a directory named `src` and setting the `GOPATH` environment variable to such directory's parent. ## Testing + +We're working on adding more tests to this repo, but here's what you can run in the meantime: + ### Gosuper + The Go supervisor can be tested by running: ```bash make ARCH=amd64 test-gosuper @@ -164,6 +172,7 @@ Once it's running, you can run the test with: ```bash make ARCH=amd64 test-integration ``` + The tests will fail if the supervisor API is down - bear in mind that the supervisor image takes a while to start the actual supervisor program, so you might have to wait a few minutes between running the supervisor and testing it. The test expects the supervisor to be already running the application (so that the app is already on the SQLite database), so check the dashboard to see if the app has already downloaded. diff --git a/tools/dev/dindctl b/tools/dev/dindctl index 3d0cb202..e49b6ddd 100755 --- a/tools/dev/dindctl +++ b/tools/dev/dindctl @@ -1,4 +1,37 @@ #!/bin/bash +# +# This script can be used to facilitate supervisor development. Its core feature is allowing +# faster development iterations by bind-mounting the local './dist' directly into the running +# supervisor container. +# +# Setting the '--mount-nm' flag in either 'run' or 'deployrun' action will bind-mount +# './node_modules/' into the running supervisor. In this case, it's up to the developer +# to make sure that the correct dependencies are installed. +# +# Usage: dindctl action [options] +# +# Actions: +# deploy build and deploy local supervisor image - you can override registry/image name with --image. +# run [options] build dind host container, run it (with name resin_supervisor_1), then pull the configured supervisor image into the dind host and run it. +# deployrun [options] run 'deploy' and then immediately 'run' the deployed container. +# refresh recompile sources in './src' and restart supervisor container on dind host - requires --mount-dist in order to work properly. +# logs [-f] print out supervisor log files - use '-f' to follow instead, or any other arguments you'd send to journalctl. +# stop stop dind supervisor host container. +# Options: +# --arch | -a [arch] architecture of the supervisor to build (default: amd64 ) +# --image | -i [image] image name for supervisor image to build/deploy/use ( default: resin/$ARCH-supervisor:master ) +# --dind-image [image] image name for the dind host container +# --mount-dist bind-mount './dist/' (where webpack stores the built js) from local development environment into supervisor container. +# --mount-nm bind-mount './node_modules/' from local development environment into supervisor container. +# --preload | -p use tools/dev/apps.json to preload an application image into the dind host. +# --ssh enable a passwordless dropbear ssh server on the dind host +# +# See README.md for examples. +# +# The script requires make and docker. +# + +THIS_FILE=$0 set -o errexit set -o pipefail @@ -6,75 +39,100 @@ set -o pipefail DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) SUPERVISOR_BASE_DIR="${DIR}/../.." -ARCH=${ARCH:-"amd64"} -SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE:-"registry.resindev.io/resin/${ARCH}-supervisor:master"} -PASSWORDLESS_DROPBEAR=${PASSWORDLESS_DROPBEAR:-"false"} -SUPERVISOR_EXTRA_MOUNTS= +ARCH="amd64" +SUPERVISOR_IMAGE="resin/${ARCH}-supervisor:master" +PASSWORDLESS_DROPBEAR="false" +SUPERVISOR_EXTRA_MOUNTS="" +DIST_MOUNTED="false" +DIND_IMAGE="resin-supervisor-dind" +PRELOADED_IMAGE="" function showHelp { - echo - echo " This script can be used to facilitate supervisor development. Its core feature is allowing" - echo " faster development iterations by bind-mounting the local './src' directly into the running" - echo " supervisor container." - echo - echo " Setting the '--mount-nm' flag in either 'run' or 'deployrun' action will bind-mount" - echo " './node_modules/' into the running supervisor. In this case, it's up to the developer" - echo " to make sure that the correct dependencies are installed." - echo - echo " Usage: [environment] $0 action [options]" - echo - echo " Environment Variables:" - echo " ARCH [=amd64]" - echo " SUPERVISOR_IMAGE [=registry.resindev.io/resin/-supervisor:master]" - echo " PASSWORDLESS_DROPBEAR [=false]" - echo " Actions:" - echo " deploy build and deploy local supervisor image - you can override registry/image name with 'SUPERVISOR_IMAGE'" - echo " run [options] build dind host container, run it, then pull the configured 'SUPERVISOR_IMAGE' into the dind host and run it" - echo " deployrun [options] run 'deploy' and then immediately 'run' the deployed container" - echo " refresh recompile sources in './src' with 'coffee -c' and restart supervisor container on dind host" - echo " logs [-f] print out supervisor log files - use '-f' to follow instead" - echo " stop stop dind supervisor host container" - echo " Options:" - echo " --mount-src bind-mount './src/' from local development environment into supervisor container" - echo " --mount-nm bind-mount './node_modules/' from local development environment into supervisor container" - echo + cat $THIS_FILE | awk '{if(/^#/)print;else exit}' | tail -n +2 | sed 's/\#//' | sed 's|dindctl|'$THIS_FILE'|' } -function deploySupervisor { - make -C "$SUPERVISOR_BASE_DIR" \ - ARCH="$ARCH" \ - SUPERVISOR_IMAGE="$SUPERVISOR_IMAGE" \ - PASSWORDLESS_DROPBEAR="$PASSWORDLESS_DROPBEAR" \ - deploy -} - -function runDind { - for arg in "$@" +function parseOptions { + while [[ $# -gt 1 ]] do - case $arg in - --mount-src) - coffee -c "$SUPERVISOR_BASE_DIR/src" - SUPERVISOR_EXTRA_MOUNTS="$SUPERVISOR_EXTRA_MOUNTS -v /resin-supervisor/src:/usr/src/app/src" - shift + case $1 in + --mount-dist) + DIST_MOUNTED="true" + SUPERVISOR_EXTRA_MOUNTS="$SUPERVISOR_EXTRA_MOUNTS -v /resin-supervisor/dist:/usr/src/app/dist" ;; --mount-nm) SUPERVISOR_EXTRA_MOUNTS="$SUPERVISOR_EXTRA_MOUNTS -v /resin-supervisor/node_modules:/usr/src/app/node_modules" - shift + ;; + -p|--preload) + PRELOADED_IMAGE="true" + ;; + --ssh) + PASSWORDLESS_DROPBEAR="true" + ;; + -i|--image) + SUPERVISOR_IMAGE="$2" + shift || { echo "--image provided not specified" && exit 1; } + ;; + --dind-image) + DIND_IMAGE="$2" + shift || { echo "--dind-image provided not specified" && exit 1; } + ;; + -a|--arch) + ARCH="$2" + shift || { echo "--arch provided not specified" && exit 1; } ;; *) echo "Warning: unknown argument: $arg" ;; esac + shift done +} +function deploySupervisor { + echo "Building and deploying for architecture $ARCH and tagging as $IMAGE" + make -C "$SUPERVISOR_BASE_DIR" \ + ARCH="$ARCH" \ + IMAGE="$SUPERVISOR_IMAGE" \ + supervisor \ + && make -C "$SUPERVISOR_BASE_DIR" \ + IMAGE="$SUPERVISOR_IMAGE" \ + deploy +} + +function buildSupervisorSrc { + echo "Rebuilding supervisor source" + ( cd "$SUPERVISOR_BASE_DIR" && npm run build ) +} + +function refreshSupervisorSrc { + buildSupervisorSrc + echo "Restarting the supervisor container" + docker exec -ti resin_supervisor_1 docker restart resin_supervisor +} + +function runDind { + if [ "$DIST_MOUNTED" = "true" ]; then + buildSupervisorSrc + echo "Running with mounted dist folder" + fi + if [ "$PRELOADED_IMAGE" = "true" ]; then + echo "Running with preloaded apps" + fi + echo "Starting dind supervisor" make -C "$SUPERVISOR_BASE_DIR" \ ARCH="$ARCH" \ SUPERVISOR_IMAGE="$SUPERVISOR_IMAGE" \ PASSWORDLESS_DROPBEAR="$PASSWORDLESS_DROPBEAR" \ SUPERVISOR_EXTRA_MOUNTS="$SUPERVISOR_EXTRA_MOUNTS" \ + IMAGE="$DIND_IMAGE" run-supervisor } +function stopDind { + echo "Stopping dind supervisor" + make -C "$SUPERVISOR_BASE_DIR" stop-supervisor +} + function logs { docker exec -ti resin_supervisor_1 journalctl $@ } @@ -82,26 +140,27 @@ function logs { action="$1" shift || true -case $action in - deploy) - deploySupervisor - ;; - run) - runDind "$@" - ;; - deployrun) - deploySupervisor && runDind "$@" - ;; - refresh) - make -C "$SUPERVISOR_BASE_DIR" refresh-supervisor-src - ;; - logs) - logs "$@" - ;; - stop) - make -C "$SUPERVISOR_BASE_DIR" stop-supervisor - ;; - *) - showHelp -esac - +if [ "$action" = "logs" ]; then + logs "$@" +else + parseOptions "$@" + case $action in + deploy) + deploySupervisor + ;; + run) + runDind + ;; + deployrun) + deploySupervisor && runDind + ;; + refresh) + refreshSupervisorSrc + ;; + stop) + stopDind + ;; + *) + showHelp + esac +fi From e28e770fd7c8bb8197ebb417c112880e71622bc5 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 7 Jul 2017 14:54:11 -0700 Subject: [PATCH 04/14] Replace the Jenkins build with a CircleCI build that uses the new multi-stage build Change-Type: patch Signed-off-by: Pablo Carranza Velez --- automation/Dockerfile | 10 +++ automation/build.sh | 135 ++++++++++++++++++++++++++++++++++++ automation/jenkins_build.sh | 62 ----------------- circle.yml | 63 +++++++++++++++++ 4 files changed, 208 insertions(+), 62 deletions(-) create mode 100644 automation/Dockerfile create mode 100755 automation/build.sh delete mode 100755 automation/jenkins_build.sh create mode 100644 circle.yml diff --git a/automation/Dockerfile b/automation/Dockerfile new file mode 100644 index 00000000..6787118c --- /dev/null +++ b/automation/Dockerfile @@ -0,0 +1,10 @@ +# Dockerfile for a docker-in-docker supervisor builder +FROM library/docker:17.06-dind +RUN apk add --no-cache \ + make \ + jq \ + git \ + bash +WORKDIR /src + +COPY . /src diff --git a/automation/build.sh b/automation/build.sh new file mode 100755 index 00000000..0019652a --- /dev/null +++ b/automation/build.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# +# resin-supervisor automated build +# +# Required variables: +# * ARCH +# * TAG +# +# Optional variables: +# * PUSH_IMAGES +# * CLEANUP +# * ENABLE_TESTS +# * PUBNUB_SUBSCRIBE_KEY, PUBNUB_PUBLISH_KEY, MIXPANEL_TOKEN: default keys to inject in the supervisor image +# +# Builds the supervisor for the architecture defined by $ARCH. +# Will produce and push an image tagged as resin/$ARCH-supervisor:$TAG +# +# It pulls intermediate images for caching, if available: +# resin/$ARCH-supervisor-base:$TAG +# resin/$ARCH-supervisor-node:$TAG +# resin/$ARCH-supervisor-go:$TAG +# +# In all of these cases it will use "master" if $TAG is not found. +# +# If PUSH_IMAGES is "true", it will also push the supervisor and intermediate images +# to the docker registry. The supervisor image will also be pushed to registry.resinstaging.io +# so that devices with older docker versions can pull it from there (it's a v1 registry). +# If CLEANUP is "true", all images will be removed after pushing - including any relevant images +# that may have been on the host from before the build, so be careful! +# If ENABLE_TESTS is "true", tests will be run. +# +# Requires docker >= 17.05, and make. +# +# If you just cloned the repo, run this to populate the yocto dependencies before running this script: +# git submodule update --init --recursive +# git clean -fxd base-image +# git submodule foreach --recursive git clean -fxd +# + +set -e + +THIS_FILE=$0 +if [ -z "$ARCH" ] || [ -z "$TAG" ]; then + cat $THIS_FILE | awk '{if(/^#/)print;else exit}' | tail -n +2 | sed 's/\#//' + exit 1 +fi + +function tryRemove() { + docker rmi $1 || true +} + +# This is the supervisor image we will produce +TARGET_IMAGE=resin/$ARCH-supervisor:$TAG + +# Intermediate images and cache +BASE_IMAGE=resin/$ARCH-supervisor-base:$TAG +NODE_IMAGE=resin/$ARCH-supervisor-node:$TAG +GO_IMAGE=resin/$ARCH-supervisor-go:$TAG + +TARGET_CACHE=$TARGET_IMAGE +BASE_CACHE=$BASE_IMAGE +GO_CACHE=$GO_IMAGE +NODE_CACHE=$NODE_IMAGE + +CACHE_FROM="" +# Attempt to pull images for cache +# Only if the pull succeeds we add a --cache-from option +docker pull $TARGET_CACHE || { + TARGET_CACHE=resin/$ARCH-supervisor:master + docker pull $TARGET_CACHE +} && { + CACHE_FROM="$CACHE_FROM --cache-from $TARGET_CACHE" +} || true + +docker pull $BASE_CACHE || { + BASE_CACHE=resin/$ARCH-supervisor-base:master + docker pull $BASE_CACHE +} && { + CACHE_FROM="$CACHE_FROM --cache-from $BASE_CACHE" +} || true + +docker pull $NODE_CACHE || { + NODE_CACHE=resin/$ARCH-supervisor-node:master + docker pull $NODE_CACHE +} && { + CACHE_FROM="$CACHE_FROM --cache-from $NODE_CACHE" +} || true + +docker pull $GO_CACHE || { + GO_CACHE=resin/$ARCH-supervisor-go:master + docker pull $GO_CACHE +} && { + CACHE_FROM="$CACHE_FROM --cache-from $GO_CACHE" +} || true + +if [ "$ENABLE_TESTS" = "true" ]; then + make ARCH=$ARCH IMAGE=$GO_IMAGE test-gosuper +fi + +export DOCKER_BUILD_OPTIONS=${CACHE_FROM} +export ARCH +export PUBNUB_PUBLISH_KEY +export PUBNUB_SUBSCRIBE_KEY +export MIXPANEL_TOKEN + +# This is the step that actually builds the supervisor +make IMAGE=$TARGET_IMAGE supervisor + +if [ "$PUSH_IMAGES" = "true" ]; then + make IMAGE=$TARGET_IMAGE deploy + docker tag $TARGET_IMAGE registry.resinstaging.io/$TARGET_IMAGE + make IMAGE=registry.resinstaging.io/$TARGET_IMAGE deploy + + # Try to push the intermediate images to improve caching in future builds + # But we don't care much if any of this fails. + ( make IMAGE=$BASE_IMAGE base && make IMAGE=$BASE_IMAGE deploy ) || true + ( make IMAGE=$GO_IMAGE gosuper && make IMAGE=$GO_IMAGE deploy ) || true + ( make IMAGE=$NODE_IMAGE node && make IMAGE=$NODE_IMAGE deploy ) || true +fi + +if [ "$CLEANUP" = "true" ]; then + tryRemove $TARGET_IMAGE + + # Only attempt to remove intermediate imaegs if we built them + if [ "$PUSH_IMAGES" = "true" ]; then + tryRemove $BASE_IMAGE + tryRemove $GO_IMAGE + tryRemove $NODE_IMAGE + fi + + tryRemove $TARGET_CACHE + tryRemove $BASE_CACHE + tryRemove $GO_CACHE + tryRemove $NODE_CACHE +fi diff --git a/automation/jenkins_build.sh b/automation/jenkins_build.sh deleted file mode 100755 index d6ee6682..00000000 --- a/automation/jenkins_build.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -set -e - -# Jenkins build steps -export ESCAPED_BRANCH_NAME=$(echo $sourceBranch | sed 's/[^a-z0-9A-Z_.-]/-/g') -git submodule update --init --recursive -git clean -fxd base-image -git submodule foreach --recursive git clean -fxd -BASE_IMAGE_VERSION=$(make base-image-version) -export BASE_IMAGE_REPO=resin/$ARCH-supervisor-base -export BASE_IMAGE_TAG=resin/$ARCH-supervisor-base:$BASE_IMAGE_VERSION - -# Try to pull the base image according to the contents of the base-image folder, otherwise build it -docker pull $BASE_IMAGE_TAG || (cd base-image && bash -ex automation/jenkins-build.sh) - -# Try pulling the old build first for caching purposes. -docker pull resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME} || docker pull resin/${ARCH}-supervisor:master || true -# Also pull the intermediate images, if possible, to improve caching -NODE_SUPERVISOR_REPO=registry.resinstaging.io/resin/node-supervisor-${ARCH} -GO_SUPERVISOR_REPO=registry.resinstaging.io/resin/go-supervisor-${ARCH} -docker pull ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || docker pull ${NODE_SUPERVISOR_REPO}:master || true -docker pull ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || docker pull ${GO_SUPERVISOR_REPO}:master || true - -# Test the gosuper -make SUPERVISOR_VERSION=${ESCAPED_BRANCH_NAME} JOB_NAME=${JOB_NAME} test-gosuper - -MAKE_ARGS="ARCH=${ARCH} \ - ESCAPED_BASE_IMAGE_TAG=$(echo $BASE_IMAGE_TAG | sed -e 's/\//\\\//g; s/\./\\\./g') \ - PUBNUB_SUBSCRIBE_KEY=${PUBNUB_SUBSCRIBE_KEY} \ - PUBNUB_PUBLISH_KEY=${PUBNUB_PUBLISH_KEY} \ - MIXPANEL_TOKEN=${MIXPANEL_TOKEN} \ - SUPERVISOR_VERSION=${ESCAPED_BRANCH_NAME}" - -make ${MAKE_ARGS} \ - DEPLOY_REGISTRY= \ - deploy - -# Try to push the intermediate images to improve caching in future builds -docker tag resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \ - || docker tag -f resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \ - || true -docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \ - || docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \ - || true -docker push ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true -docker push ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true - -make ${MAKE_ARGS} \ - DEPLOY_REGISTRY=registry.resinstaging.io/ \ - deploy - -docker rmi registry.resinstaging.io/resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME} || true -docker rmi resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME} || true -docker rmi resin/${ARCH}-supervisor:master || true -docker rmi ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true -docker rmi ${NODE_SUPERVISOR_REPO}:master || true -docker rmi resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true -docker rmi ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true -docker rmi ${GO_SUPERVISOR_REPO}:master || true -docker rmi resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true -docker rmi ${BASE_IMAGE_TAG} || true -docker rmi ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME} || true diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000..efd416e8 --- /dev/null +++ b/circle.yml @@ -0,0 +1,63 @@ +--- +defaults: &defaults + docker: + - image: library/docker:stable + working_directory: /tmp/build + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + steps: + - setup_remote_docker + - run: + name: Check docker is running and install git + command: | + docker info + apk update && apk upgrade && apk add --nocache git + - checkout + - run: + name: Initialize the submodules (yocto layers) + command: | + git submodule update --init --recursive + git clean -fxd base-image + git submodule foreach --recursive git clean -fxd + - run: + name: Build and start the docker-in-docker builder + command: | + docker build --rm=false --tag builder -f automation/Dockerfile . + - run: + name: Start the docker-in-docker builder and build $ARCH-supervisor + no_output_timeout: 10800 + command: | + # build the dind image + docker build --rm=false --tag builder -f automation/Dockerfile . + # start the dind container + dind=$(docker run --privileged -d builder) + # confirm it's running + docker ps + # start the build for this architecture + docker exec -it -e TAG=${CIRCLE_BRANCH} -e ARCH=${ARCH} -e PUSH_IMAGES=${PUSH_IMAGES} ${dind} bash automation/build.sh + docker exec -ti ${dind} docker images + +version: 2 +jobs: + amd64: + <<: *defaults + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + ARCH: amd64 + PUSH_IMAGES: "false" + i386: + <<: *defaults + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + ARCH: i386 + PUSH_IMAGES: "false" + +workflows: + version: 2 + build_them_all: + jobs: + - amd64 + - i386 From 5bfeccccef2ff1e84567cdafa7fbfddb98681352 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Mon, 10 Jul 2017 08:53:31 -0700 Subject: [PATCH 05/14] Add builds for all architectures, and actually push the built images Signed-off-by: Pablo Carranza Velez --- circle.yml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index efd416e8..724bd749 100644 --- a/circle.yml +++ b/circle.yml @@ -46,14 +46,42 @@ jobs: DOCKER_USERNAME: travisciresin DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: amd64 - PUSH_IMAGES: "false" + PUSH_IMAGES: "true" i386: <<: *defaults environment: DOCKER_USERNAME: travisciresin DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: i386 - PUSH_IMAGES: "false" + PUSH_IMAGES: "true" + armel: + <<: *defaults + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + ARCH: armel + PUSH_IMAGES: "true" + armv7hf: + <<: *defaults + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + ARCH: armv7hf + PUSH_IMAGES: "true" + aarch64: + <<: *defaults + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + ARCH: aarch64 + PUSH_IMAGES: "true" + rpi: + <<: *defaults + environment: + DOCKER_USERNAME: travisciresin + DOCKER_EMAIL: accounts+travisci+docker@resin.io + ARCH: rpi + PUSH_IMAGES: "true" workflows: version: 2 From 392d963348f52f6fa62c78a3df352f5c18d4ad64 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Mon, 10 Jul 2017 09:37:36 -0700 Subject: [PATCH 06/14] Use a local supervisor image for the dind supervisor We change the Makefile and dinctl so that instead of having to deploy and then pull a supervisor image, we save it from the local docker images and load it inside the dind container. We also fix dind so that it uses the dind hack script from the docker repo as official dind images do (this avoids breaking the permissions of /dev/pts/ptmx in the host as in https://github.com/kubernetes/kubernetes/issues/18230). Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .gitignore | 1 + Makefile | 11 +++++--- README.md | 21 +++++---------- tools/dev/dindctl | 27 +++++++++---------- tools/dind/Dockerfile | 6 +++++ .../services/docker.service.d/aufs.conf | 2 +- .../services/resin-supervisor-dind.service | 2 +- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index e5aa9b13..3863d57a 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ Dockerfile.runtime.* /build/ /dist/ tools/dind/config/services/docker.service.d/proxy.conf +tools/dind/supervisor-image.tar diff --git a/Makefile b/Makefile index 57d04fab..738b79fe 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ # * base - builds the "base" component (a yocto builder with the output rootfs at /dest) # * gosuper - builds the "gosuper" component (a golang image with the Go supervisor component at /go/bin/gosuper and /build/gosuper) # * nodesuper - builds the node component, with the node_modules and src at /usr/src/app and /build (also includes a rootfs-overlay there) -# * supervisor-dind: build the development docker-in-docker supervisor that run-supervisor uses +# * supervisor-dind: build the development docker-in-docker supervisor that run-supervisor uses (requires a SUPERVISOR_IMAGE to be available locally) # # Variables for build targets: # * ARCH: amd64/rpi/i386/armv7hf/armel/aarch64 architecture for which to build the supervisor - default: amd64 @@ -19,13 +19,13 @@ # * DOCKER_BUILD_OPTIONS: Additional options for docker build, like --cache-from parameters # # Test/development targets: -# * run-supervisor, stop-supervisor - build and start or stop a docker-in-docker resin-supervisor (requires aufs and ability to run privileged containers) +# * run-supervisor, stop-supervisor - build and start or stop a docker-in-docker resin-supervisor (requires aufs, ability to run privileged containers, and a SUPERVISOR_IMAGE to be available locally) # * format-gosuper, test-gosuper - build a gosuper image and run formatting or unit tests # * test-integration - run an integration test (see gosuper/supertest). Requires a docker-in-docker supervisor to be running # # Variables for test/dev targets: # * IMAGE: image to build and run (either for run-supervisor or test-gosuper/integration) -# * SUPERVISOR_IMAGE: In run-supervisor, the supervisor image to run inside the docker-in-docker image +# * SUPERVISOR_IMAGE: In run-supervisor and supervisor-dind, the supervisor image to run inside the docker-in-docker image # * PRELOADED_IMAGE: If true, will preload user app image from tools/dev/apps.json and bind mount apps.json into the docker-in-docker supervisor # * SUPERVISOR_EXTRA_MOUNTS: Additional bind mount flags for the docker-in-docker supervisor # * PASSWORDLESS_DROPBEAR: For run-supervisor - start a passwordless ssh daemon in the docker-in-docker supervisor @@ -120,7 +120,10 @@ ${DOCKERD_PROXY}: touch ${DOCKERD_PROXY}; \ fi -supervisor-dind: ${DOCKERD_PROXY} +supervisor-tar: + docker save --output tools/dind/supervisor-image.tar $(SUPERVISOR_IMAGE) + +supervisor-dind: ${DOCKERD_PROXY} supervisor-tar cd tools/dind \ && docker build \ $(DOCKER_HTTP_PROXY) \ diff --git a/README.md b/README.md index 1fe1e471..6922d40f 100644 --- a/README.md +++ b/README.md @@ -10,23 +10,15 @@ We are using [waffle.io](https://waffle.io) to manage our tickets / issues, so i ## Running supervisor locally -### Deploy your local version to a Docker registry +### Build a local supervisor image -We'll show how to use the DockerHub registry, but any other can be specified as part of the `IMAGE` variable. - -If you haven't done so yet, login to the registry: +Build the supervisor with a specific repo and tag, e.g. ```bash -docker login -``` -Use your username and password as required. - -Then build the supervisor and deploy it to a specific repo and tag, e.g. -```bash -./tools/dev/dindctl deploy --image username/resin-supervisor:master --arch amd64 +./tools/dev/dindctl build --image username/resin-supervisor:master --arch amd64 ``` -This will build the Supervisor docker image and upload it to dockerhub. You can use other registries by adding to the -image name e.g. `myregistry.com/username/resin-supervisor:master`. +This will build the Supervisor docker image locally. If you then run `docker images` you should see the repo/tag you +set there. ### Set up config.json @@ -74,7 +66,8 @@ Ensure your kernel supports aufs (in Ubuntu, install `linux-image-extra-$(uname ./tools/dev/dindctl run --image username/resin-supervisor:master ``` -This will setup a docker-in-docker instance with an image that runs the supervisor image. +This will setup a docker-in-docker instance with an image that runs the supervisor image. The image has to be available +locally, either because you built it as described above, or because you pulled it before running `dindctl run`. If you want to develop and test your changes, you can run: diff --git a/tools/dev/dindctl b/tools/dev/dindctl index e49b6ddd..57a59a2b 100755 --- a/tools/dev/dindctl +++ b/tools/dev/dindctl @@ -4,22 +4,22 @@ # faster development iterations by bind-mounting the local './dist' directly into the running # supervisor container. # -# Setting the '--mount-nm' flag in either 'run' or 'deployrun' action will bind-mount +# Setting the '--mount-nm' flag in either 'run' or 'buildrun' action will bind-mount # './node_modules/' into the running supervisor. In this case, it's up to the developer # to make sure that the correct dependencies are installed. # # Usage: dindctl action [options] # # Actions: -# deploy build and deploy local supervisor image - you can override registry/image name with --image. -# run [options] build dind host container, run it (with name resin_supervisor_1), then pull the configured supervisor image into the dind host and run it. -# deployrun [options] run 'deploy' and then immediately 'run' the deployed container. +# build build local supervisor image - you can override image name with --image. +# run [options] build dind host container, run it (with name resin_supervisor_1), which will include the specified supervisor image and run it. +# buildrun [options] run 'build' and then immediately 'run' the built container. # refresh recompile sources in './src' and restart supervisor container on dind host - requires --mount-dist in order to work properly. # logs [-f] print out supervisor log files - use '-f' to follow instead, or any other arguments you'd send to journalctl. # stop stop dind supervisor host container. # Options: # --arch | -a [arch] architecture of the supervisor to build (default: amd64 ) -# --image | -i [image] image name for supervisor image to build/deploy/use ( default: resin/$ARCH-supervisor:master ) +# --image | -i [image] image name for supervisor image to build/use ( default: resin/$ARCH-supervisor:master ) # --dind-image [image] image name for the dind host container # --mount-dist bind-mount './dist/' (where webpack stores the built js) from local development environment into supervisor container. # --mount-nm bind-mount './node_modules/' from local development environment into supervisor container. @@ -88,15 +88,12 @@ function parseOptions { done } -function deploySupervisor { +function buildSupervisor { echo "Building and deploying for architecture $ARCH and tagging as $IMAGE" make -C "$SUPERVISOR_BASE_DIR" \ ARCH="$ARCH" \ IMAGE="$SUPERVISOR_IMAGE" \ - supervisor \ - && make -C "$SUPERVISOR_BASE_DIR" \ - IMAGE="$SUPERVISOR_IMAGE" \ - deploy + supervisor } function buildSupervisorSrc { @@ -124,7 +121,7 @@ function runDind { SUPERVISOR_IMAGE="$SUPERVISOR_IMAGE" \ PASSWORDLESS_DROPBEAR="$PASSWORDLESS_DROPBEAR" \ SUPERVISOR_EXTRA_MOUNTS="$SUPERVISOR_EXTRA_MOUNTS" \ - IMAGE="$DIND_IMAGE" + IMAGE="$DIND_IMAGE" \ run-supervisor } @@ -145,14 +142,14 @@ if [ "$action" = "logs" ]; then else parseOptions "$@" case $action in - deploy) - deploySupervisor + build) + buildSupervisor ;; run) runDind ;; - deployrun) - deploySupervisor && runDind + buildrun) + buildSupervisor && runDind ;; refresh) refreshSupervisorSrc diff --git a/tools/dind/Dockerfile b/tools/dind/Dockerfile index 9fe9cb26..eb99de13 100644 --- a/tools/dind/Dockerfile +++ b/tools/dind/Dockerfile @@ -26,6 +26,10 @@ RUN apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 58118E89F3A912897C0 && apt-get --force-yes install docker-engine=${DOCKER_VERSION}-0~${RELEASE_NAME} \ && rm -rf /var/lib/apt/lists/* +ENV DIND_COMMIT 3b5fac462d21ca164b3778647420016315289034 +RUN curl -sL https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/dind > /usr/bin/dind \ + && chmod +x /usr/bin/dind + RUN passwd -d root # Change os release to a resin-sync compatible one @@ -40,3 +44,5 @@ COPY resin-vars vpn-init /usr/src/app/ RUN if [ "$PASSWORDLESS_DROPBEAR" = "true" ]; then sed -i 's/\(DROPBEAR_EXTRA_ARGS=\).*/\1"-B"/' /etc/default/dropbear; fi RUN systemctl enable resin-supervisor-dind + +COPY supervisor-image.tar /usr/src/ diff --git a/tools/dind/config/services/docker.service.d/aufs.conf b/tools/dind/config/services/docker.service.d/aufs.conf index 77f9bf5b..96276c82 100644 --- a/tools/dind/config/services/docker.service.d/aufs.conf +++ b/tools/dind/config/services/docker.service.d/aufs.conf @@ -1,3 +1,3 @@ [Service] ExecStart= -ExecStart=/usr/bin/docker daemon --storage-driver aufs --host=fd:// +ExecStart=/usr/bin/dind /usr/bin/docker daemon --storage-driver aufs --host=fd:// diff --git a/tools/dind/config/services/resin-supervisor-dind.service b/tools/dind/config/services/resin-supervisor-dind.service index 9d208adc..fb492059 100644 --- a/tools/dind/config/services/resin-supervisor-dind.service +++ b/tools/dind/config/services/resin-supervisor-dind.service @@ -9,7 +9,7 @@ WorkingDirectory=/usr/src/app EnvironmentFile=/usr/src/app/config/env EnvironmentFile=/usr/src/app/config/localenv ExecStartPre=/bin/bash -c 'if [ "${PRELOADED_IMAGE}" == "true" ]; then /usr/bin/docker pull $(jq ".[0].imageId" ${APPS_PATH}); fi' -ExecStartPre=/usr/bin/docker pull ${SUPERVISOR_IMAGE} +ExecStartPre=/usr/bin/docker load --input /usr/src/supervisor-image.tar ExecStartPre=-/usr/bin/docker kill resin_supervisor ExecStartPre=-/usr/bin/docker rm resin_supervisor ExecStartPre=-/bin/touch /etc/resolv.conf From 60a8ab85f461eed596c6c92312325eae67180e23 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Wed, 12 Jul 2017 10:18:35 -0700 Subject: [PATCH 07/14] Add a deploy-to-resin.js and automatically deploy to the Resin API on master builds Also, make all of the arch builds part of the Circle workflow. Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .gitignore | 1 + automation/Dockerfile | 9 +++- automation/build.sh | 20 +++++--- automation/deploy-to-resin.js | 91 +++++++++++++++++++++++++++++++++++ automation/package.json | 16 ++++++ circle.yml | 38 ++++++++++++--- 6 files changed, 161 insertions(+), 14 deletions(-) create mode 100644 automation/deploy-to-resin.js create mode 100644 automation/package.json diff --git a/.gitignore b/.gitignore index 3863d57a..c8807c19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /node_modules/ +/automation/node_modules/ *.swp data bin/gosuper diff --git a/automation/Dockerfile b/automation/Dockerfile index 6787118c..56a4db8f 100644 --- a/automation/Dockerfile +++ b/automation/Dockerfile @@ -4,7 +4,14 @@ RUN apk add --no-cache \ make \ jq \ git \ - bash + bash \ + nodejs \ + nodejs-npm + WORKDIR /src +COPY ./automation/package.json /src/automation/package.json +RUN cd automation \ + && JOBS=max npm install \ + && npm cache clean COPY . /src diff --git a/automation/build.sh b/automation/build.sh index 0019652a..c891f1f1 100755 --- a/automation/build.sh +++ b/automation/build.sh @@ -11,6 +11,7 @@ # * CLEANUP # * ENABLE_TESTS # * PUBNUB_SUBSCRIBE_KEY, PUBNUB_PUBLISH_KEY, MIXPANEL_TOKEN: default keys to inject in the supervisor image +# * EXTRA_TAG: when PUSH_IMAGES is true, additional tag to push to the registries # # Builds the supervisor for the architecture defined by $ARCH. # Will produce and push an image tagged as resin/$ARCH-supervisor:$TAG @@ -109,13 +110,20 @@ make IMAGE=$TARGET_IMAGE supervisor if [ "$PUSH_IMAGES" = "true" ]; then make IMAGE=$TARGET_IMAGE deploy docker tag $TARGET_IMAGE registry.resinstaging.io/$TARGET_IMAGE - make IMAGE=registry.resinstaging.io/$TARGET_IMAGE deploy + make IMAGE=registry.resinstaging.io/$TARGET_IMAGE deploy - # Try to push the intermediate images to improve caching in future builds - # But we don't care much if any of this fails. - ( make IMAGE=$BASE_IMAGE base && make IMAGE=$BASE_IMAGE deploy ) || true - ( make IMAGE=$GO_IMAGE gosuper && make IMAGE=$GO_IMAGE deploy ) || true - ( make IMAGE=$NODE_IMAGE node && make IMAGE=$NODE_IMAGE deploy ) || true + if [ -n "$EXTRA_TAG" ]; then + docker tag $TARGET_IMAGE resin/$ARCH-supervisor:$EXTRA_TAG + make IMAGE=resin/$ARCH-supervisor:$EXTRA_TAG deploy + docker tag $TARGET_IMAGE registry.resinstaging.io/resin/$ARCH-supervisor:$EXTRA_TAG + make IMAGE=registry.resinstaging.io/resin/$ARCH-supervisor:$EXTRA_TAG deploy + fi + + # Try to push the intermediate images to improve caching in future builds + # But we don't care much if any of this fails. + ( make IMAGE=$BASE_IMAGE base && make IMAGE=$BASE_IMAGE deploy ) || true + ( make IMAGE=$GO_IMAGE gosuper && make IMAGE=$GO_IMAGE deploy ) || true + ( make IMAGE=$NODE_IMAGE node && make IMAGE=$NODE_IMAGE deploy ) || true fi if [ "$CLEANUP" = "true" ]; then diff --git a/automation/deploy-to-resin.js b/automation/deploy-to-resin.js new file mode 100644 index 00000000..7bb44ce6 --- /dev/null +++ b/automation/deploy-to-resin.js @@ -0,0 +1,91 @@ +// Deploy a supervisor image as a supervisor_release in the Resin API +// +// Environment variables: +// This program deploys only for device types where the architecture matches $ARCH. +// It deploys to the API specified by $API_ENDPOINT and using a provided $API_TOKEN or $API_KEY +// (if both are set, API_TOKEN is preferred). +// The tag to deploy must be passed as $TAG. +// +const PineJsClient = require('pinejs-client'); +const Promise = require('bluebird'); +const _ = require('lodash'); +const url = require('url'); + +const apiEndpoint = process.env.API_ENDPOINT; +const apikey = process.env.API_KEY; +const arch = process.env.ARCH; +const tag = process.env.TAG; +const apiToken = process.env.API_TOKEN; + +if (_.isEmpty(apikey) && _.isEmpty(apiToken)) { + console.error('Skipping deploy due to empty API_KEY and API_TOKEN'); + process.exit(0); +} + +if (_.isEmpty(apiEndpoint)) { + console.error('Please set a valid $API_ENDPOINT'); + process.exit(1); +} + +if (_.isEmpty(tag)) { + console.error('Please set a $TAG to deploy'); + process.exit(1); +} + +const supportedArchitectures = [ 'amd64', 'rpi', 'aarch64', 'armel', 'i386', 'armv7hf' ]; +if (_.isEmpty(arch) || !_.includes(supportedArchitectures, arch)) { + console.error('Invalid architecture ' + arch); + process.exit(1); +} + +const requestOpts = { + gzip: true, + timeout: 30000 +}; + +if (!_.isEmpty(apiToken)) { + requestOpts.headers = { + Authorization: 'Bearer ' + apiToken + }; +} + + +const apiEndpointWithPrefix = url.resolve(apiEndpoint, '/v2/') +const resinApi = new PineJsClient({ + apiPrefix: apiEndpointWithPrefix, + passthrough: requestOpts +}); + +resinApi._request(_.extend({ + url: apiEndpoint + '/config/device-types', + method: 'GET' +}, resinApi.passthrough)) +.then( (deviceTypes) => { + // This is a critical step so we better do it serially + return Promise.mapSeries(deviceTypes, (deviceType) => { + if (deviceType.arch === arch) { + const customOptions = {}; + if (_.isEmpty(apiToken)) { + customOptions.apikey = apikey; + } + console.log(`Deploying ${supervisor_version} for ${device_type}`); + return resinApi.post({ + resource: 'supervisor_release', + body: { + image_name: `registry.resinstaging.io/resin/${arch}-supervisor`, + supervisor_version: tag, + device_type: deviceType.slug, + is_public: true + }, + customOptions + }); + } + }); +}) +.then( () => { + process.exit(0); +}) +.catch( (err) => { + console.error(`Error when deploying the supervisor to ${apiEndpoint}`, err, err.stack); + process.exit(1); +}); diff --git a/automation/package.json b/automation/package.json new file mode 100644 index 00000000..34974e50 --- /dev/null +++ b/automation/package.json @@ -0,0 +1,16 @@ +{ + "name": "resin-supervisor-automation", + "version": "1.0.0", + "description": "Tools to build/deploy resin-supervisor", + "main": "deploy-to-resin.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Resin Inc.", + "license": "Apache-2.0", + "dependencies": { + "bluebird": "^3.5.0", + "lodash": "^4.17.4", + "pinejs-client": "^4.0.0" + } +} diff --git a/circle.yml b/circle.yml index 724bd749..7686c05a 100644 --- a/circle.yml +++ b/circle.yml @@ -3,16 +3,13 @@ defaults: &defaults docker: - image: library/docker:stable working_directory: /tmp/build - environment: - DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io steps: - setup_remote_docker - run: name: Check docker is running and install git command: | docker info - apk update && apk upgrade && apk add --nocache git + apk update && apk upgrade && apk add --nocache git jq - checkout - run: name: Initialize the submodules (yocto layers) @@ -34,9 +31,20 @@ defaults: &defaults dind=$(docker run --privileged -d builder) # confirm it's running docker ps + VERSION_TAG=v$(jq .version package.json | sed 's/"//g') + if [ "${CIRCLE_BRANCH}" = "master" ]; then + EXTRA_BUILD_ARGS="-e EXTRA_TAG=$VERSION_TAG" + else + EXTRA_BUILD_ARGS="" + fi # start the build for this architecture - docker exec -it -e TAG=${CIRCLE_BRANCH} -e ARCH=${ARCH} -e PUSH_IMAGES=${PUSH_IMAGES} ${dind} bash automation/build.sh - docker exec -ti ${dind} docker images + docker exec -it -e TAG=${CIRCLE_BRANCH} -e ARCH=${ARCH} -e PUSH_IMAGES=${PUSH_IMAGES} $EXTRA_BUILD_ARGS ${dind} bash automation/build.sh + if [ "${CIRCLE_BRANCH}" = "master" ] && [ "${DEPLOY_TO_RESIN}" = "true" ]; then + echo "Deploying to Resin API (staging)" + docker exec -it -e ARCH=${ARCH} -e TAG=$VERSION_TAG -e API_KEY=$STAGING_API_KEY -e API_ENDPOINT=$STAGING_API_ENDPOINT ${dind} node automation/deploy-to-resin.js + echo "Deploying to Resin API (production)" + docker exec -it -e ARCH=${ARCH} -e TAG=$VERSION_TAG -e API_KEY=$PRODUCTION_API_KEY -e API_ENDPOINT=$PRODUCTION_API_ENDPOINT ${dind} node automation/deploy-to-resin.js + fi version: 2 jobs: @@ -47,6 +55,8 @@ jobs: DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: amd64 PUSH_IMAGES: "true" + STAGING_API_ENDPOINT: https://api.resinstaging.io + PRODUCTION_API_ENDPOINT: https://api.resin.io i386: <<: *defaults environment: @@ -54,6 +64,8 @@ jobs: DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: i386 PUSH_IMAGES: "true" + STAGING_API_ENDPOINT: https://api.resinstaging.io + PRODUCTION_API_ENDPOINT: https://api.resin.io armel: <<: *defaults environment: @@ -61,6 +73,8 @@ jobs: DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: armel PUSH_IMAGES: "true" + STAGING_API_ENDPOINT: https://api.resinstaging.io + PRODUCTION_API_ENDPOINT: https://api.resin.io armv7hf: <<: *defaults environment: @@ -68,6 +82,8 @@ jobs: DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: armv7hf PUSH_IMAGES: "true" + STAGING_API_ENDPOINT: https://api.resinstaging.io + PRODUCTION_API_ENDPOINT: https://api.resin.io aarch64: <<: *defaults environment: @@ -75,6 +91,8 @@ jobs: DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: aarch64 PUSH_IMAGES: "true" + STAGING_API_ENDPOINT: https://api.resinstaging.io + PRODUCTION_API_ENDPOINT: https://api.resin.io rpi: <<: *defaults environment: @@ -82,10 +100,16 @@ jobs: DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: rpi PUSH_IMAGES: "true" + STAGING_API_ENDPOINT: https://api.resinstaging.io + PRODUCTION_API_ENDPOINT: https://api.resin.io workflows: version: 2 - build_them_all: + build_and_maybe_deploy: jobs: - amd64 - i386 + - rpi + - armv7hf + - aarch64 + - armel From 29de3ceeb258048c7342a91dc3dd52efe68455ef Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 14 Jul 2017 13:36:37 -0300 Subject: [PATCH 08/14] Add a script to automate the creation of meta-resin PRs Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .gitignore | 1 + automation/pr-to-meta-resin.sh | 85 ++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100755 automation/pr-to-meta-resin.sh diff --git a/.gitignore b/.gitignore index c8807c19..0b8635fb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /node_modules/ /automation/node_modules/ +/automation/meta-resin/ *.swp data bin/gosuper diff --git a/automation/pr-to-meta-resin.sh b/automation/pr-to-meta-resin.sh new file mode 100755 index 00000000..dc2b9214 --- /dev/null +++ b/automation/pr-to-meta-resin.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# PR a supervisor release to meta-resin +# +# It clones meta-resin and opens a PR changing the supervisor version to $TAG. +# If a meta-resin folder exists, it assumes it has a meta-resin git repo. +# +# If $PR_1X is "true", an additional PR for 1.X will be created. +# +# Requires $GITHUB_USERNAME and $GITHUB_PASSWORD to push and create the pull-request. +# Requires $TAG to be set to the supervisor version to use. +# + +set -e + +THIS_FILE=$0 +if [ -z "$TAG" ]; then + cat $THIS_FILE | awk '{if(/^#/)print;else exit}' | tail -n +2 | sed 's/\#//' + exit 1 +fi + +if [[ -z "$GITHUB_PASSWORD" || -z "$GITHUB_USERNAME" ]]; then + echo "GITHUB_USERNAME and GITHUB_PASSWORD are required" + exit 1 +fi + +USER=${USER:-$(whoami)} + +function prepareBranches() { + BASE=$1 + HEAD=$2 + git checkout $BASE + git reset HEAD + git checkout . + git fetch + git merge origin/${BASE} + git checkout -b ${HEAD} +} + +function setSupervisorTag() { + sed -i "s/SUPERVISOR_TAG ?= \".*\"/SUPERVISOR_TAG ?= \"${TAG}\"/" meta-resin-common/recipes-containers/docker-disk/docker-resin-supervisor-disk.bb +} + +function commitAndPR() { + BASE=$1 + HEAD=$2 + git commit -as -m " +docker-resin-supervisor-disk: Update to ${TAG} + +Changelog-Entry: Update supervisor to ${TAG} +Change-Type: patch +" < Date: Fri, 14 Jul 2017 14:19:33 -0300 Subject: [PATCH 09/14] Move dindctl to the base of the repository This makes it easier to use, as it's meant to be used constantly when developing. No need to change PATH or type `./tools/dev/dindctl`, just run `./dindctl` now. Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .dockerignore | 1 + tools/dev/dindctl => dindctl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) rename tools/dev/dindctl => dindctl (99%) diff --git a/.dockerignore b/.dockerignore index 2c60d8bf..a17c3d5a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,5 @@ node_modules .editorconfig tools +dindctl README.md diff --git a/tools/dev/dindctl b/dindctl similarity index 99% rename from tools/dev/dindctl rename to dindctl index 57a59a2b..bc3398da 100755 --- a/tools/dev/dindctl +++ b/dindctl @@ -37,7 +37,7 @@ set -o errexit set -o pipefail DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -SUPERVISOR_BASE_DIR="${DIR}/../.." +SUPERVISOR_BASE_DIR="${DIR}" ARCH="amd64" SUPERVISOR_IMAGE="resin/${ARCH}-supervisor:master" From fa8116d97ca7b593a51fc9edd92d8278c5fe84eb Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Mon, 17 Jul 2017 17:11:24 -0300 Subject: [PATCH 10/14] Disable pushing to registry.resinstaging.io, and use build environment to login to dockerhub Also, avoid building the dind builder twice, and try to get a reusable docker instance. Change-Type: patch Signed-off-by: Pablo Carranza Velez --- automation/build.sh | 11 +++++------ automation/deploy-to-resin.js | 2 +- circle.yml | 24 ++++++++++++------------ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/automation/build.sh b/automation/build.sh index c891f1f1..4bd5461f 100755 --- a/automation/build.sh +++ b/automation/build.sh @@ -12,6 +12,7 @@ # * ENABLE_TESTS # * PUBNUB_SUBSCRIBE_KEY, PUBNUB_PUBLISH_KEY, MIXPANEL_TOKEN: default keys to inject in the supervisor image # * EXTRA_TAG: when PUSH_IMAGES is true, additional tag to push to the registries +# * DOCKER_USERNAME, DOCKER_PASSWORD: if the password is set, then these will be used to login to dockerhub before pushing # # Builds the supervisor for the architecture defined by $ARCH. # Will produce and push an image tagged as resin/$ARCH-supervisor:$TAG @@ -24,8 +25,7 @@ # In all of these cases it will use "master" if $TAG is not found. # # If PUSH_IMAGES is "true", it will also push the supervisor and intermediate images -# to the docker registry. The supervisor image will also be pushed to registry.resinstaging.io -# so that devices with older docker versions can pull it from there (it's a v1 registry). +# to the docker registry. # If CLEANUP is "true", all images will be removed after pushing - including any relevant images # that may have been on the host from before the build, so be careful! # If ENABLE_TESTS is "true", tests will be run. @@ -108,15 +108,14 @@ export MIXPANEL_TOKEN make IMAGE=$TARGET_IMAGE supervisor if [ "$PUSH_IMAGES" = "true" ]; then + if [ -n "$DOCKER_PASSWORD" ]; then + docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD + fi make IMAGE=$TARGET_IMAGE deploy - docker tag $TARGET_IMAGE registry.resinstaging.io/$TARGET_IMAGE - make IMAGE=registry.resinstaging.io/$TARGET_IMAGE deploy if [ -n "$EXTRA_TAG" ]; then docker tag $TARGET_IMAGE resin/$ARCH-supervisor:$EXTRA_TAG make IMAGE=resin/$ARCH-supervisor:$EXTRA_TAG deploy - docker tag $TARGET_IMAGE registry.resinstaging.io/resin/$ARCH-supervisor:$EXTRA_TAG - make IMAGE=registry.resinstaging.io/resin/$ARCH-supervisor:$EXTRA_TAG deploy fi # Try to push the intermediate images to improve caching in future builds diff --git a/automation/deploy-to-resin.js b/automation/deploy-to-resin.js index 7bb44ce6..15f0df80 100644 --- a/automation/deploy-to-resin.js +++ b/automation/deploy-to-resin.js @@ -72,7 +72,7 @@ resinApi._request(_.extend({ return resinApi.post({ resource: 'supervisor_release', body: { - image_name: `registry.resinstaging.io/resin/${arch}-supervisor`, + image_name: `resin/${arch}-supervisor`, supervisor_version: tag, device_type: deviceType.slug, is_public: true diff --git a/circle.yml b/circle.yml index 7686c05a..2ad69cd2 100644 --- a/circle.yml +++ b/circle.yml @@ -4,7 +4,8 @@ defaults: &defaults - image: library/docker:stable working_directory: /tmp/build steps: - - setup_remote_docker + - setup_remote_docker: + reusable: true - run: name: Check docker is running and install git command: | @@ -25,20 +26,25 @@ defaults: &defaults name: Start the docker-in-docker builder and build $ARCH-supervisor no_output_timeout: 10800 command: | - # build the dind image - docker build --rm=false --tag builder -f automation/Dockerfile . # start the dind container + echo "Starting the dind builder" dind=$(docker run --privileged -d builder) - # confirm it's running - docker ps VERSION_TAG=v$(jq .version package.json | sed 's/"//g') if [ "${CIRCLE_BRANCH}" = "master" ]; then EXTRA_BUILD_ARGS="-e EXTRA_TAG=$VERSION_TAG" else EXTRA_BUILD_ARGS="" fi + echo "Starting build.sh" # start the build for this architecture - docker exec -it -e TAG=${CIRCLE_BRANCH} -e ARCH=${ARCH} -e PUSH_IMAGES=${PUSH_IMAGES} $EXTRA_BUILD_ARGS ${dind} bash automation/build.sh + docker exec -it \ + -e TAG=${CIRCLE_BRANCH} \ + -e ARCH=${ARCH} \ + -e PUSH_IMAGES=${PUSH_IMAGES} \ + -e DOCKER_USERNAME=${DOCKER_USERNAME} \ + -e DOCKER_PASSWORD=${DOCKER_PASSWORD} \ + $EXTRA_BUILD_ARGS \ + ${dind} bash automation/build.sh if [ "${CIRCLE_BRANCH}" = "master" ] && [ "${DEPLOY_TO_RESIN}" = "true" ]; then echo "Deploying to Resin API (staging)" docker exec -it -e ARCH=${ARCH} -e TAG=$VERSION_TAG -e API_KEY=$STAGING_API_KEY -e API_ENDPOINT=$STAGING_API_ENDPOINT ${dind} node automation/deploy-to-resin.js @@ -52,7 +58,6 @@ jobs: <<: *defaults environment: DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: amd64 PUSH_IMAGES: "true" STAGING_API_ENDPOINT: https://api.resinstaging.io @@ -61,7 +66,6 @@ jobs: <<: *defaults environment: DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: i386 PUSH_IMAGES: "true" STAGING_API_ENDPOINT: https://api.resinstaging.io @@ -70,7 +74,6 @@ jobs: <<: *defaults environment: DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: armel PUSH_IMAGES: "true" STAGING_API_ENDPOINT: https://api.resinstaging.io @@ -79,7 +82,6 @@ jobs: <<: *defaults environment: DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: armv7hf PUSH_IMAGES: "true" STAGING_API_ENDPOINT: https://api.resinstaging.io @@ -88,7 +90,6 @@ jobs: <<: *defaults environment: DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: aarch64 PUSH_IMAGES: "true" STAGING_API_ENDPOINT: https://api.resinstaging.io @@ -97,7 +98,6 @@ jobs: <<: *defaults environment: DOCKER_USERNAME: travisciresin - DOCKER_EMAIL: accounts+travisci+docker@resin.io ARCH: rpi PUSH_IMAGES: "true" STAGING_API_ENDPOINT: https://api.resinstaging.io From d7f7a0bf57b10d9d6698e961b21d46d26ce66488 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Tue, 18 Jul 2017 18:30:02 -0300 Subject: [PATCH 11/14] Disable armel builds, and make the automation build without using docker-in-docker Changelog-Entry: Disable armel builds Change-Type: patch Signed-off-by: Pablo Carranza Velez --- automation/Dockerfile | 17 --------------- circle.yml | 50 ++++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 41 deletions(-) delete mode 100644 automation/Dockerfile diff --git a/automation/Dockerfile b/automation/Dockerfile deleted file mode 100644 index 56a4db8f..00000000 --- a/automation/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Dockerfile for a docker-in-docker supervisor builder -FROM library/docker:17.06-dind -RUN apk add --no-cache \ - make \ - jq \ - git \ - bash \ - nodejs \ - nodejs-npm - -WORKDIR /src -COPY ./automation/package.json /src/automation/package.json -RUN cd automation \ - && JOBS=max npm install \ - && npm cache clean - -COPY . /src diff --git a/circle.yml b/circle.yml index 2ad69cd2..753dc0b0 100644 --- a/circle.yml +++ b/circle.yml @@ -1,17 +1,31 @@ --- defaults: &defaults docker: - - image: library/docker:stable + - image: library/docker:17.06.0-ce working_directory: /tmp/build steps: - setup_remote_docker: + version: 17.06.0-ce reusable: true - run: name: Check docker is running and install git command: | docker info - apk update && apk upgrade && apk add --nocache git jq + apk update && apk upgrade && apk add --nocache \ + make \ + jq \ + git \ + bash \ + nodejs \ + nodejs-npm \ + openssh-client - checkout + - run: + name: Install npm dependencies + working_directory: /tmp/build/automation + command: | + JOBS=max npm install \ + && npm cache clean - run: name: Initialize the submodules (yocto layers) command: | @@ -19,37 +33,26 @@ defaults: &defaults git clean -fxd base-image git submodule foreach --recursive git clean -fxd - run: - name: Build and start the docker-in-docker builder - command: | - docker build --rm=false --tag builder -f automation/Dockerfile . - - run: - name: Start the docker-in-docker builder and build $ARCH-supervisor + name: Build $ARCH-supervisor no_output_timeout: 10800 command: | - # start the dind container - echo "Starting the dind builder" - dind=$(docker run --privileged -d builder) VERSION_TAG=v$(jq .version package.json | sed 's/"//g') if [ "${CIRCLE_BRANCH}" = "master" ]; then - EXTRA_BUILD_ARGS="-e EXTRA_TAG=$VERSION_TAG" - else - EXTRA_BUILD_ARGS="" + export EXTRA_TAG=$VERSION_TAG fi echo "Starting build.sh" # start the build for this architecture - docker exec -it \ - -e TAG=${CIRCLE_BRANCH} \ - -e ARCH=${ARCH} \ - -e PUSH_IMAGES=${PUSH_IMAGES} \ - -e DOCKER_USERNAME=${DOCKER_USERNAME} \ - -e DOCKER_PASSWORD=${DOCKER_PASSWORD} \ - $EXTRA_BUILD_ARGS \ - ${dind} bash automation/build.sh + export TAG=${CIRCLE_BRANCH} + export ARCH=${ARCH} + export PUSH_IMAGES=${PUSH_IMAGES} + export DOCKER_USERNAME=${DOCKER_USERNAME} + export DOCKER_PASSWORD=${DOCKER_PASSWORD} + bash automation/build.sh if [ "${CIRCLE_BRANCH}" = "master" ] && [ "${DEPLOY_TO_RESIN}" = "true" ]; then echo "Deploying to Resin API (staging)" - docker exec -it -e ARCH=${ARCH} -e TAG=$VERSION_TAG -e API_KEY=$STAGING_API_KEY -e API_ENDPOINT=$STAGING_API_ENDPOINT ${dind} node automation/deploy-to-resin.js + ARCH=${ARCH} TAG=$VERSION_TAG API_KEY=$STAGING_API_KEY API_ENDPOINT=$STAGING_API_ENDPOINT node automation/deploy-to-resin.js echo "Deploying to Resin API (production)" - docker exec -it -e ARCH=${ARCH} -e TAG=$VERSION_TAG -e API_KEY=$PRODUCTION_API_KEY -e API_ENDPOINT=$PRODUCTION_API_ENDPOINT ${dind} node automation/deploy-to-resin.js + ARCH=${ARCH} TAG=$VERSION_TAG API_KEY=$PRODUCTION_API_KEY API_ENDPOINT=$PRODUCTION_API_ENDPOINT node automation/deploy-to-resin.js fi version: 2 @@ -112,4 +115,3 @@ workflows: - rpi - armv7hf - aarch64 - - armel From d9605c71dd88542cfe434a7bfda19d7d7f61a95d Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Tue, 18 Jul 2017 18:43:00 -0300 Subject: [PATCH 12/14] automation/build.sh: build base, nodesuper and gosuper before supervisor, and use them for cache Otherwise the step where we build them doesn't reuse what was built when building the supervisor, because we're explicitly setting `--cache-from` options. We also push the images as soon as they're built. Signed-off-by: Pablo Carranza Velez --- automation/build.sh | 39 ++++++++++++++++++++++----------------- circle.yml | 3 +-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/automation/build.sh b/automation/build.sh index 4bd5461f..257e19a9 100755 --- a/automation/build.sh +++ b/automation/build.sh @@ -12,7 +12,6 @@ # * ENABLE_TESTS # * PUBNUB_SUBSCRIBE_KEY, PUBNUB_PUBLISH_KEY, MIXPANEL_TOKEN: default keys to inject in the supervisor image # * EXTRA_TAG: when PUSH_IMAGES is true, additional tag to push to the registries -# * DOCKER_USERNAME, DOCKER_PASSWORD: if the password is set, then these will be used to login to dockerhub before pushing # # Builds the supervisor for the architecture defined by $ARCH. # Will produce and push an image tagged as resin/$ARCH-supervisor:$TAG @@ -24,7 +23,7 @@ # # In all of these cases it will use "master" if $TAG is not found. # -# If PUSH_IMAGES is "true", it will also push the supervisor and intermediate images +# If PUSH_IMAGES is "true", it will also push the supervisor and intermediate images (must be logged in to dockerhub) # to the docker registry. # If CLEANUP is "true", all images will be removed after pushing - including any relevant images # that may have been on the host from before the build, so be careful! @@ -104,36 +103,42 @@ export PUBNUB_PUBLISH_KEY export PUBNUB_SUBSCRIBE_KEY export MIXPANEL_TOKEN +make IMAGE=$BASE_IMAGE base +if [ "$PUSH_IMAGES" = "true" ]; then + make IMAGE=$BASE_IMAGE deploy || true +fi +export DOCKER_BUILD_OPTIONS="${DOCKER_BUILD_OPTIONS} --cache-from ${BASE_IMAGE}" + +make IMAGE=$GO_IMAGE gosuper +if [ "$PUSH_IMAGES" = "true" ]; then + make IMAGE=$GO_IMAGE deploy || true +fi +export DOCKER_BUILD_OPTIONS="${DOCKER_BUILD_OPTIONS} --cache-from ${GO_IMAGE}" + +make IMAGE=$NODE_IMAGE nodesuper +if [ "$PUSH_IMAGES" = "true" ]; then + make IMAGE=$NODE_IMAGE deploy || true +fi +export DOCKER_BUILD_OPTIONS="${DOCKER_BUILD_OPTIONS} --cache-from ${NODE_IMAGE}" + # This is the step that actually builds the supervisor make IMAGE=$TARGET_IMAGE supervisor if [ "$PUSH_IMAGES" = "true" ]; then - if [ -n "$DOCKER_PASSWORD" ]; then - docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD - fi make IMAGE=$TARGET_IMAGE deploy if [ -n "$EXTRA_TAG" ]; then docker tag $TARGET_IMAGE resin/$ARCH-supervisor:$EXTRA_TAG make IMAGE=resin/$ARCH-supervisor:$EXTRA_TAG deploy fi - - # Try to push the intermediate images to improve caching in future builds - # But we don't care much if any of this fails. - ( make IMAGE=$BASE_IMAGE base && make IMAGE=$BASE_IMAGE deploy ) || true - ( make IMAGE=$GO_IMAGE gosuper && make IMAGE=$GO_IMAGE deploy ) || true - ( make IMAGE=$NODE_IMAGE node && make IMAGE=$NODE_IMAGE deploy ) || true fi if [ "$CLEANUP" = "true" ]; then tryRemove $TARGET_IMAGE - # Only attempt to remove intermediate imaegs if we built them - if [ "$PUSH_IMAGES" = "true" ]; then - tryRemove $BASE_IMAGE - tryRemove $GO_IMAGE - tryRemove $NODE_IMAGE - fi + tryRemove $BASE_IMAGE + tryRemove $GO_IMAGE + tryRemove $NODE_IMAGE tryRemove $TARGET_CACHE tryRemove $BASE_CACHE diff --git a/circle.yml b/circle.yml index 753dc0b0..43c8917f 100644 --- a/circle.yml +++ b/circle.yml @@ -41,12 +41,11 @@ defaults: &defaults export EXTRA_TAG=$VERSION_TAG fi echo "Starting build.sh" + docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD # start the build for this architecture export TAG=${CIRCLE_BRANCH} export ARCH=${ARCH} export PUSH_IMAGES=${PUSH_IMAGES} - export DOCKER_USERNAME=${DOCKER_USERNAME} - export DOCKER_PASSWORD=${DOCKER_PASSWORD} bash automation/build.sh if [ "${CIRCLE_BRANCH}" = "master" ] && [ "${DEPLOY_TO_RESIN}" = "true" ]; then echo "Deploying to Resin API (staging)" From b328a1e916ad10e3100e0c13fa47a5862eeeb37e Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Wed, 19 Jul 2017 12:50:27 -0300 Subject: [PATCH 13/14] Make the ARCH-supervisor-base image builder smaller by not keeping the downloads, sstate and build directories We should revisit this if we want to later copy those folders into the host and use Circle's caching system with them, or if https://github.com/moby/moby/issues/32507 gets implemented in which case we can use a bind mounted folder instead. Signed-off-by: Pablo Carranza Velez --- base-image/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/base-image/build.sh b/base-image/build.sh index 17b87ec5..c44e6132 100755 --- a/base-image/build.sh +++ b/base-image/build.sh @@ -43,3 +43,9 @@ sudo -H -u builder \ && source oe-core/oe-init-build-env build bitbake \ && DL_DIR=$SHARED_DOWNLOADS SSTATE_DIR=$SHARED_SSTATE MACHINE=$TARGET_MACHINE $BUILD_DIR/bitbake/bin/bitbake core-image-minimal > /dev/null" tar xzf $BUILD_DIR/build/tmp-glibc/deploy/images/$TARGET_MACHINE/core-image-minimal-$TARGET_MACHINE.tar.gz -C $DEST_DIR +# Delete the sstate and downloads directory so that the resulting image isn't huge +# If https://github.com/moby/moby/issues/32507 gets implemented we can start using +# a bind mounted cached directory instead +rm -rf $SHARED_DOWNLOADS +rm -rf $SHARED_SSTATE +rm -rf $BUILD_DIR From 7ee2d48706398e041b0c1f085528d690f982cfb9 Mon Sep 17 00:00:00 2001 From: "resin-io-versionbot[bot]" Date: Mon, 24 Jul 2017 20:03:30 +0000 Subject: [PATCH 14/14] v6.0.2 --- CHANGELOG.md | 13 +++++++++++++ package.json | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b8e8699..32d052bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY! This project adheres to [Semantic Versioning](http://semver.org/). +## v6.0.2 - 2017-07-24 + +* Disable armel builds [Pablo Carranza Velez] +* Disable pushing to registry.resinstaging.io, and use build environment to login to dockerhub [Pablo Carranza Velez] +* Move dindctl to the base of the repository [Pablo Carranza Velez] +* Add a script to automate the creation of meta-resin PRs [Pablo Carranza Velez] +* Add a deploy-to-resin.js and automatically deploy to the Resin API on master builds [Pablo Carranza Velez] +* Use a local supervisor image for the dind supervisor [Pablo Carranza Velez] +* Replace the Jenkins build with a CircleCI build that uses the new multi-stage build [Pablo Carranza Velez] +* Make dindctl an easier to use tool that takes options instead of using env vars [Pablo Carranza Velez] +* Refactor the makefile to make it easier to use and make use of the multi-stage build [Pablo Carranza Velez] +* Add a single Dockerfile to build the supervisor as a multi-stage build [Pablo Carranza Velez] + ## v6.0.1 - 2017-07-12 * Use webpack to join all modules [Pablo Carranza Velez] diff --git a/package.json b/package.json index 3f6b06be..d4a5bd1c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "resin-supervisor", "description": "This is resin.io's Supervisor, a program that runs on IoT devices and has the task of running user Apps (which are Docker containers), and updating them as Resin's API informs it to.", - "version": "6.0.1", + "version": "6.0.2", "license": "Apache-2.0", "repository": { "type": "git",