Add a single Dockerfile to build the supervisor as a multi-stage build

Change-Type: patch
Signed-off-by: Pablo Carranza Velez <pablo@resin.io>
This commit is contained in:
Pablo Carranza Velez 2017-07-07 14:47:28 -07:00
parent d0251a5cd3
commit 8d84facff5
10 changed files with 277 additions and 254 deletions

View File

@ -1,13 +1,6 @@
.dockerignore
.git
node_modules
Dockerfile
Dockerfile.*
Makefile
.editorconfig
coffeelint.json
automation
tools
README.md
retry_docker_push.sh
base-image

207
Dockerfile Normal file
View File

@ -0,0 +1,207 @@
ARG ARCH=amd64
FROM debian:jessie as base
ARG ARCH
# Install the following utilities (required by openembedded)
# http://www.openembedded.org/wiki/Getting_started#Ubuntu_.2F_Debian
RUN apt-get -qq update \
&& apt-get -qq install -y \
build-essential \
chrpath \
cpio \
curl \
diffstat \
file \
gawk \
git-core \
libsdl1.2-dev \
locales \
python3 \
texinfo \
unzip \
wget \
xterm \
sudo \
&& rm -rf /var/lib/apt/lists/*
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV SOURCE_DIR /source
ENV DEST_DIR /dest
ENV SHARED_DOWNLOADS /yocto/shared-downloads
ENV SHARED_SSTATE /yocto/shared-sstate
ARG BUILDER_UID=1000
ARG BUILDER_GID=1000
COPY base-image /source
RUN cd /source && bash -ex build.sh
##############################################################################
# Build golang supervisor
FROM debian:jessie as gosuper
ARG ARCH
RUN apt-get update \
&& apt-get install -y \
build-essential \
curl \
rsync \
&& rm -rf /var/lib/apt/lists/
ENV GOLANG_VERSION 1.8.3
ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
ENV GOLANG_DOWNLOAD_SHA256 1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772
COPY gosuper/go-${GOLANG_VERSION}-patches /go-${GOLANG_VERSION}-patches
RUN mkdir /usr/src/go \
&& cd /usr/src/go \
&& curl -L -o go.tar.gz $GOLANG_DOWNLOAD_URL \
&& echo "${GOLANG_DOWNLOAD_SHA256} go.tar.gz" | sha256sum -c - \
&& tar xzf go.tar.gz -C /usr/local \
&& cd /usr/src \
&& rm -rf go \
&& export GOROOT_BOOTSTRAP=/usr/local/go-bootstrap \
&& cp -r /usr/local/go /usr/local/go-bootstrap \
&& cd /usr/local/go/src \
&& patch -p2 -i /go-${GOLANG_VERSION}-patches/0001-dont-fail-when-no-mmx.patch \
&& patch -p2 -i /go-${GOLANG_VERSION}-patches/0002-implement-atomic-quadword-ops-with-FILD-FISTP.patch \
&& ./make.bash \
&& rm -rf /usr/local/go-bootstrap
ENV UPX_VERSION 3.94
# UPX doesn't provide fingerprints so I checked this one manually
ENV UPX_SHA256 e1fc0d55c88865ef758c7e4fabbc439e4b5693b9328d219e0b9b3604186abe20
RUN mkdir /usr/src/upx \
&& cd /usr/src/upx \
&& curl -L -o upx.tar.xz https://github.com/upx/upx/releases/download/v$UPX_VERSION/upx-$UPX_VERSION-amd64_linux.tar.xz \
&& echo "${UPX_SHA256} upx.tar.xz" | sha256sum -c - \
&& tar xf upx.tar.xz --strip-components=1 \
&& cp ./upx /usr/bin/ \
&& cd /usr/src \
&& rm -rf upx
ENV GOPATH /go
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
COPY ./gosuper /go/src/resin-supervisor/gosuper
WORKDIR /go/src/resin-supervisor/gosuper
ENV GOOS linux
ENV GO386=387
RUN bash ./build.sh
RUN rsync -a --delete /go/bin/gosuper /build/
##############################################################################
# The node version here should match the version of the runtime image which is
# specified in the base-image subdirectory in the project
FROM resin/rpi-node:6.5-slim as rpi-node-base
FROM resin/armv7hf-node:6.5-slim as armv7hf-node-base
FROM resin/armel-node:6.5-slim as armel-node-base
FROM resin/aarch64-node:6.5-slim as aarch64-node-base
FROM resin/amd64-node:6.5-slim as amd64-node-base
RUN echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-start && chmod +x /usr/bin/cross-build-start \
&& echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-end && chmod +x /usr/bin/cross-build-end
FROM resin/i386-node:6.5-slim as i386-node-base
RUN echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-start && chmod +x /usr/bin/cross-build-start \
&& echo '#!/bin/sh\nexit 0' > /usr/bin/cross-build-end && chmod +x /usr/bin/cross-build-end
# Build nodejs dependencies
FROM $ARCH-node-base as node
ARG ARCH
RUN [ "cross-build-start" ]
WORKDIR /usr/src/app
RUN apt-get update \
&& apt-get install -y \
g++ \
libsqlite3-dev \
make \
python \
rsync \
wget \
&& rm -rf /var/lib/apt/lists/
RUN mkdir -p rootfs-overlay && \
ln -s /lib rootfs-overlay/lib64
COPY package.json /usr/src/app/
# Install only the production modules that have C extensions
RUN JOBS=MAX npm install --production --no-optional --unsafe-perm \
&& npm dedupe
COPY webpack.config.js remove-hashbang-loader.js /usr/src/app/
COPY src /usr/src/app/src
# Install devDependencies, build the coffeescript and then prune the deps
RUN cp -R node_modules node_modules_prod \
&& npm install --no-optional --unsafe-perm \
&& npm run lint \
&& npm run build \
&& rm -rf node_modules \
&& mv node_modules_prod node_modules
# Remove various uneeded filetypes in order to reduce space
RUN find . -path '*/coverage/*' -o -path '*/test/*' -o -path '*/.nyc_output/*' \
-o -name '*.tar.*' -o -name '*.in' -o -name '*.cc' \
-o -name '*.c' -o -name '*.coffee' -o -name '*.eslintrc' \
-o -name '*.h' -o -name '*.html' -o -name '*.markdown' \
-o -name '*.md' -o -name '*.patch' -o -name '*.png' \
-o -name '*.yml' \
-delete \
&& find . -type f -path '*/node_modules/sqlite3/deps*' -delete \
&& find . -type f -path '*/node_modules/knex/build*' -delete
# Create /var/run/resin for the gosuper to place its socket in
RUN mkdir -p rootfs-overlay/var/run/resin
COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/
COPY inittab rootfs-overlay/etc/inittab
RUN rsync -a --delete node_modules dist rootfs-overlay /build
RUN [ "cross-build-end" ]
##############################################################################
# Minimal runtime image
FROM scratch
ARG ARCH
ARG VERSION=master
ARG DEFAULT_PUBNUB_PUBLISH_KEY=pub-c-bananas
ARG DEFAULT_PUBNUB_SUBSCRIBE_KEY=sub-c-bananas
ARG DEFAULT_MIXPANEL_TOKEN=bananasbananas
COPY --from=base /dest/ /
WORKDIR /usr/src/app
COPY --from=node /build/dist ./dist
COPY --from=node /build/node_modules ./node_modules
COPY --from=gosuper /build/gosuper ./gosuper
COPY --from=node /build/rootfs-overlay/ /
VOLUME /data
ENV CONFIG_MOUNT_POINT=/boot/config.json \
LED_FILE=/dev/null \
SUPERVISOR_IMAGE=resin/$ARCH-supervisor \
VERSION=$VERSION \
DEFAULT_PUBNUB_PUBLISH_KEY=$DEFAULT_PUBNUB_PUBLISH_KEY \
DEFAULT_PUBNUB_SUBSCRIBE_KEY=$DEFAULT_PUBNUB_SUBSCRIBE_KEY \
DEFAULT_MIXPANEL_TOKEN=$DEFAULT_MIXPANEL_TOKEN
CMD [ "/sbin/init" ]

View File

@ -1,60 +0,0 @@
# Build nodejs dependencies
# The node version here should match the version of the runtime image which is
# specified in the base-image subdirectory in the project
FROM resin/%%ARCH%%-node:6.5-slim
WORKDIR /usr/src/app
RUN apt-get update \
&& apt-get install -y \
g++ \
libsqlite3-dev \
make \
python \
rsync \
wget \
&& rm -rf /var/lib/apt/lists/
RUN mkdir -p rootfs-overlay && \
ln -s /lib rootfs-overlay/lib64
COPY package.json /usr/src/app/
# Install only the production modules that have C extensions
RUN JOBS=MAX npm install --production --no-optional --unsafe-perm \
&& npm dedupe
COPY webpack.config.js remove-hashbang-loader.js /usr/src/app/
COPY src /usr/src/app/src
# Install devDependencies, build the coffeescript and then prune the deps
RUN cp -R node_modules node_modules_prod \
&& npm install --no-optional --unsafe-perm \
&& npm run lint \
&& npm run build \
&& rm -rf node_modules \
&& mv node_modules_prod node_modules
# Remove various uneeded filetypes in order to reduce space
RUN find . -path '*/coverage/*' -o -path '*/test/*' -o -path '*/.nyc_output/*' \
-o -name '*.tar.*' -o -name '*.in' -o -name '*.cc' \
-o -name '*.c' -o -name '*.coffee' -o -name '*.eslintrc' \
-o -name '*.h' -o -name '*.html' -o -name '*.markdown' \
-o -name '*.md' -o -name '*.patch' -o -name '*.png' \
-o -name '*.yml' \
-delete \
&& find . -type f -path '*/node_modules/sqlite3/deps*' -delete \
&& find . -type f -path '*/node_modules/knex/build*' -delete
# Create /var/run/resin for the gosuper to place its socket in
RUN mkdir -p rootfs-overlay/var/run/resin
COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/
COPY inittab rootfs-overlay/etc/inittab
CMD rsync -a --delete node_modules dist rootfs-overlay /build
# -*- mode: dockerfile -*-
# vi: set ft=dockerfile :

View File

@ -1,20 +0,0 @@
# Minimal runtime image
FROM %%BASE_IMAGE_TAG%%
WORKDIR /usr/src/app
COPY ./build/%%ARCH%%/dist ./dist
COPY ./build/%%ARCH%%/node_modules ./node_modules
COPY ./build/%%ARCH%%/gosuper ./gosuper
COPY ./build/%%ARCH%%/rootfs-overlay/ /
VOLUME /data
ENV CONFIG_MOUNT_POINT=/boot/config.json \
LED_FILE=/dev/null \
SUPERVISOR_IMAGE=resin/%%ARCH%%-supervisor
CMD [ "/sbin/init" ]
# -*- mode: dockerfile -*-
# vi: set ft=dockerfile :

View File

@ -1,4 +0,0 @@
build
oe-core
meta-nodejs
bitbake

View File

@ -1,34 +0,0 @@
FROM ubuntu:15.04
# Install the following utilities (required by openembedded)
# http://www.openembedded.org/wiki/Getting_started#Ubuntu_.2F_Debian
RUN apt-get update \
&& apt-get install -y \
build-essential \
chrpath \
cpio \
curl \
diffstat \
file \
gawk \
git-core \
libsdl1.2-dev \
python3 \
texinfo \
unzip \
wget \
xterm \
sudo \
&& rm -rf /var/lib/apt/lists/*
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LC_ALL en_US.UTF-8
ENV SOURCE_DIR /source
ENV DEST_DIR /dest
ENV SHARED_DOWNLOADS /yocto/shared-downloads
ENV SHARED_SSTATE /yocto/shared-sstate
COPY build.sh /
CMD bash -ex build.sh

View File

@ -1,63 +0,0 @@
#!/bin/bash
set -o errexit
set -o pipefail
# ARCH, ESCAPED_BRANCH_NAME, BASE_IMAGE_REPO and BASE_IMAGE_TAG should be set before calling this script.
# This script purposefully doesn't clean up the BASE_IMAGE_TAG docker image after building it.
JENKINS_PERSISTENT_WORKDIR=${1:-/var/lib/yocto}
DL_DIR="$JENKINS_PERSISTENT_WORKDIR/shared-downloads"
mkdir dest
BUILDER_REPO=registry.resinstaging.io/resin/${ARCH}-supervisor-base-builder
BUILDER_IMAGE=${BUILDER_REPO}:${ESCAPED_BRANCH_NAME}
docker pull ${BUILDER_IMAGE} || docker pull ${BUILDER_REPO}:master || true
docker build -t ${BUILDER_IMAGE} .
docker push ${BUILDER_IMAGE} || true
case "$ARCH" in
'amd64')
machine='generic-x86-64'
;;
'i386')
machine='generic-x86'
;;
'rpi')
machine='generic-armv6'
;;
'armv7hf')
machine='generic-armv7hf'
;;
'armel')
machine='generic-armv5'
;;
'aarch64')
machine='generic-armv8'
;;
esac
SSTATE_DIR="$JENKINS_PERSISTENT_WORKDIR/$machine/sstate"
# Make sure shared directories are in place
mkdir -p $DL_DIR
mkdir -p $SSTATE_DIR
docker run --rm \
-e TARGET_MACHINE=$machine \
-e BUILDER_UID=$(id -u) \
-e BUILDER_GID=$(id -g) \
-v `pwd`:/source \
-v $DL_DIR:/yocto/shared-downloads \
-v $SSTATE_DIR:/yocto/shared-sstate \
-v `pwd`/dest:/dest \
${BUILDER_IMAGE}
docker rmi ${BUILDER_IMAGE} || true
docker rmi ${BUILDER_REPO}:master || true
if [ -f dest/rootfs.tar.gz ]; then
docker import dest/rootfs.tar.gz ${BASE_IMAGE_TAG}
docker tag ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME} || docker tag -f ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME}
docker push ${BASE_IMAGE_TAG}
else
echo "rootfs is missing!"
exit 1
fi

View File

@ -3,11 +3,43 @@
set -o errexit
BUILD_DIR='/home/builder/tmp'
case "$ARCH" in
'amd64')
export TARGET_MACHINE='generic-x86-64'
;;
'i386')
export TARGET_MACHINE='generic-x86'
;;
'rpi')
export TARGET_MACHINE='generic-armv6'
;;
'armv7hf')
export TARGET_MACHINE='generic-armv7hf'
;;
'armel')
export TARGET_MACHINE='generic-armv5'
;;
'aarch64')
export TARGET_MACHINE='generic-armv8'
;;
esac
export SOURCE_DIR=/source
export DEST_DIR=/dest
export SHARED_DOWNLOADS=/yocto/shared-downloads
export SHARED_SSTATE=/yocto/shared-sstate
# Make sure shared directories are in place
mkdir -p $SHARED_DOWNLOADS
mkdir -p $SHARED_SSTATE
mkdir -p $DEST_DIR
groupadd -g $BUILDER_GID builder
useradd -m -u $BUILDER_UID -g $BUILDER_GID builder
sudo -H -u builder /bin/bash -c "mkdir -p $BUILD_DIR \
sudo -H -u builder \
/bin/bash -c "mkdir -p $BUILD_DIR \
&& cp -r $SOURCE_DIR/* $BUILD_DIR/ \
&& cd $BUILD_DIR \
&& source oe-core/oe-init-build-env build bitbake \
&& DL_DIR=$SHARED_DOWNLOADS SSTATE_DIR=$SHARED_SSTATE MACHINE=$TARGET_MACHINE $BUILD_DIR/bitbake/bin/bitbake core-image-minimal"
cp --dereference $BUILD_DIR/build/tmp-glibc/deploy/images/$TARGET_MACHINE/core-image-minimal-$TARGET_MACHINE.tar.gz $DEST_DIR/rootfs.tar.gz
&& DL_DIR=$SHARED_DOWNLOADS SSTATE_DIR=$SHARED_SSTATE MACHINE=$TARGET_MACHINE $BUILD_DIR/bitbake/bin/bitbake core-image-minimal > /dev/null"
tar xzf $BUILD_DIR/build/tmp-glibc/deploy/images/$TARGET_MACHINE/core-image-minimal-$TARGET_MACHINE.tar.gz -C $DEST_DIR

View File

@ -1,63 +0,0 @@
# Build golang supervisor
FROM debian:jessie
RUN apt-get update \
&& apt-get install -y \
build-essential \
curl \
rsync \
&& rm -rf /var/lib/apt/lists/
ENV GOLANG_VERSION 1.8.3
ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz
ENV GOLANG_DOWNLOAD_SHA256 1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772
COPY go-${GOLANG_VERSION}-patches /go-${GOLANG_VERSION}-patches
RUN mkdir /usr/src/go \
&& cd /usr/src/go \
&& curl -L -o go.tar.gz $GOLANG_DOWNLOAD_URL \
&& echo "${GOLANG_DOWNLOAD_SHA256} go.tar.gz" | sha256sum -c - \
&& tar xzf go.tar.gz -C /usr/local \
&& cd /usr/src \
&& rm -rf go \
&& export GOROOT_BOOTSTRAP=/usr/local/go-bootstrap \
&& cp -r /usr/local/go /usr/local/go-bootstrap \
&& cd /usr/local/go/src \
&& patch -p2 -i /go-${GOLANG_VERSION}-patches/0001-dont-fail-when-no-mmx.patch \
&& patch -p2 -i /go-${GOLANG_VERSION}-patches/0002-implement-atomic-quadword-ops-with-FILD-FISTP.patch \
&& ./make.bash \
&& rm -rf /usr/local/go-bootstrap
ENV UPX_VERSION 3.94
# UPX doesn't provide fingerprints so I checked this one manually
ENV UPX_SHA256 e1fc0d55c88865ef758c7e4fabbc439e4b5693b9328d219e0b9b3604186abe20
RUN mkdir /usr/src/upx \
&& cd /usr/src/upx \
&& curl -L -o upx.tar.xz https://github.com/upx/upx/releases/download/v$UPX_VERSION/upx-$UPX_VERSION-amd64_linux.tar.xz \
&& echo "${UPX_SHA256} upx.tar.xz" | sha256sum -c - \
&& tar xf upx.tar.xz --strip-components=1 \
&& cp ./upx /usr/bin/ \
&& cd /usr/src \
&& rm -rf upx
ENV GOPATH /go
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
COPY . /go/src/resin-supervisor/gosuper
WORKDIR /go/src/resin-supervisor/gosuper
ENV GOOS linux
ENV GO386=387
ARG GOARCH=amd64
ARG GOARM=''
RUN go install -a -v ./gosuper \
&& cd /go/bin \
&& find -type f -name gosuper -exec mv {} /go/bin/gosuper \; \
&& upx --best /go/bin/gosuper
CMD rsync -a --delete /go/bin/gosuper /build

35
gosuper/build.sh Normal file
View File

@ -0,0 +1,35 @@
#!/bin/bash
set -o errexit
./test_formatting.sh
go test -v ./gosuper
case "$ARCH" in
'amd64')
export GOARCH=amd64
;;
'i386')
export GOARCH=386
;;
'rpi')
export GOARCH=arm
export GOARM=6
;;
'armv7hf')
export GOARCH=arm
export GOARM=7
;;
'armel')
export GOARCH=arm
export GOARM=5
;;
'aarch64')
export GOARCH=arm64
;;
esac
go install -a -v ./gosuper \
&& cd /go/bin \
&& find -type f -name gosuper -exec mv {} /go/bin/gosuper \; \
&& upx --best /go/bin/gosuper