Build the base image together with the supervisor

This commit changes the automation scripts so that the base image is built together with the rest of the supervisor. We use a hash of the contents of the base-image folder as tag for the base image so that unnecessary rebuilds are avoided. The build scripts are also modified to always clean up the resulting base image and the image used to build it. We use docker pull and push to enable caching. Change-Type: patch Signed-off-by: Pablo Carranza Velez <pablo@resin.io>
2025-02-20 17:52:51 +00:00 · 2016-12-15 15:14:53 -03:00 · 2016-12-15 15:14:53 -03:00 · 9e327bf7a6
commit 9e327bf7a6
parent b22470a4c8
4 changed files with 75 additions and 55 deletions
--- a/Dockerfile.runtime.template
+++ b/Dockerfile.runtime.template
@ -1,5 +1,5 @@
 # Minimal runtime image
-FROM resin/%%ARCH%%-supervisor-base:20161103
+FROM %%BASE_IMAGE_TAG%%

 WORKDIR /usr/src/app

--- a/3
+++ b/3
@ -25,6 +25,7 @@ ARCH = rpi# rpi/amd64/i386/armv7hf/armel
 DEPLOY_REGISTRY =

 SUPERVISOR_VERSION = master
+ESCAPED_BASE_IMAGE_TAG = resin\/$(ARCH)-supervisor-base:$(SUPERVISOR_VERSION)

 DOCKER_VERSION:=$(shell docker version --format '{{.Server.Version}}')
 DOCKER_MAJOR_VERSION:=$(word 1, $(subst ., ,$(DOCKER_VERSION)))
@ -144,7 +145,7 @@ refresh-supervisor-src:
 	&& docker exec -ti resin_supervisor_1 docker restart resin_supervisor

 supervisor: nodesuper gosuper
-	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template > Dockerfile.runtime.$(ARCH)
+	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template | sed 's/%%BASE_IMAGE_TAG%%/$(ESCAPED_BASE_IMAGE_TAG)/g' > Dockerfile.runtime.$(ARCH)
 	echo "ENV VERSION=$(shell jq -r .version package.json) \\" >> Dockerfile.runtime.$(ARCH)
 	echo "    DEFAULT_PUBNUB_PUBLISH_KEY=$(PUBNUB_PUBLISH_KEY) \\" >> Dockerfile.runtime.$(ARCH)
 	echo "    DEFAULT_PUBNUB_SUBSCRIBE_KEY=$(PUBNUB_SUBSCRIBE_KEY) \\" >> Dockerfile.runtime.$(ARCH)
--- a/automation/jenkins_build.sh
+++ b/automation/jenkins_build.sh
@ -2,18 +2,27 @@
 set -e

 # Jenkins build steps
-ESCAPED_BRANCH_NAME=$(echo $sourceBranch | sed 's/[^a-z0-9A-Z_.-]/-/g')
+export ESCAPED_BRANCH_NAME=$(echo $sourceBranch | sed 's/[^a-z0-9A-Z_.-]/-/g')
+BASE_IMAGE_VERSION=$(tar -c --mtime='1970-01-01' --owner=0 --group=0 -f - base-image | md5sum | awk -F " " '{print $1}')
+export BASE_IMAGE_REPO=resin/$ARCH-supervisor-base
+export BASE_IMAGE_TAG=resin/$ARCH-supervisor-base:$BASE_IMAGE_VERSION
+
+# Try to pull the base image according to the contents of the base-image folder, otherwise build it
+docker pull $BASE_IMAGE_TAG || (cd base-image && bash -ex automation/jenkins-build.sh)

 # Try pulling the old build first for caching purposes.
 docker pull resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME} || docker pull resin/${ARCH}-supervisor:master || true
 # Also pull the intermediate images, if possible, to improve caching
-docker pull registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || docker pull registry.resinstaging.io/resin/node-supervisor-${ARCH}:master || true
-docker pull registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || docker pull registry.resinstaging.io/resin/go-supervisor-${ARCH}:master || true
+NODE_SUPERVISOR_REPO=registry.resinstaging.io/resin/node-supervisor-${ARCH}
+GO_SUPERVISOR_REPO=registry.resinstaging.io/resin/go-supervisor-${ARCH}
+docker pull ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || docker pull ${NODE_SUPERVISOR_REPO}:master || true
+docker pull ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || docker pull ${GO_SUPERVISOR_REPO}:master || true

 # Test the gosuper
 make SUPERVISOR_VERSION=${ESCAPED_BRANCH_NAME} JOB_NAME=${JOB_NAME} test-gosuper

 MAKE_ARGS="ARCH=${ARCH} \
+    ESCAPED_BASE_IMAGE_TAG=$(echo $BASE_IMAGE_TAG | sed -e 's/\//\\\//g; s/\./\\\./g') \
    PUBNUB_SUBSCRIBE_KEY=${PUBNUB_SUBSCRIBE_KEY} \
    PUBNUB_PUBLISH_KEY=${PUBNUB_PUBLISH_KEY} \
    MIXPANEL_TOKEN=${MIXPANEL_TOKEN} \
@ -24,10 +33,14 @@ make ${MAKE_ARGS} \
    deploy

 # Try to push the intermediate images to improve caching in future builds
-docker tag -f resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
-docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
-docker push registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
-docker push registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
+docker tag resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| docker tag -f resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| true
+docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| true
+docker push ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true
+docker push ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true

 make ${MAKE_ARGS} \
    DEPLOY_REGISTRY=registry.resinstaging.io/ \
@ -36,5 +49,6 @@ make ${MAKE_ARGS} \
 # Cleanup removing by Id to actually remove the images rather than untagging them
 docker rmi -f $(docker inspect -f "{{.Id}}" registry.resinstaging.io/resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME}) || true
 docker rmi -f $(docker inspect -f "{{.Id}}" resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME}) || true
-docker rmi -f $(docker inspect -f "{{.Id}}" registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME}) || true
-docker rmi -f $(docker inspect -f "{{.Id}}" registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME}) || true
+docker rmi -f $(docker inspect -f "{{.Id}}" ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME}) || true
+docker rmi -f $(docker inspect -f "{{.Id}}" ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME}) || true
+docker rmi -f $(docker inspect -f "{{.Id}}" ${BASE_IMAGE_TAG}) || true
--- a/base-image/automation/jenkins-build.sh
+++ b/base-image/automation/jenkins-build.sh
@ -3,55 +3,60 @@
 set -o errexit
 set -o pipefail

-date=$(date +'%Y%m%d' -u)
+# ARCH, ESCAPED_BRANCH_NAME, BASE_IMAGE_REPO and BASE_IMAGE_TAG should be set before calling this script.
+# This script purposefully doesn't clean up the BASE_IMAGE_TAG docker image after building it.
+
 JENKINS_PERSISTENT_WORKDIR=${1:-/var/lib/yocto}
 DL_DIR="$JENKINS_PERSISTENT_WORKDIR/shared-downloads"
-# MACHINE_LIST: generic-x86-64 generic-x86 generic-armv6 generic-armv7hf generic-armv5
-# MACHINE_LIST should be set in jenkins config

 git submodule update --init --recursive
 rm -rf dest
 mkdir dest

-docker build -t supervisor-base-builder .
-for machine in $MACHINE_LIST; do
-	case "$machine" in
-	'generic-x86-64')
-		REPO='resin/amd64-supervisor-base'
-	;;
-	'generic-x86')
-		REPO='resin/i386-supervisor-base'
-	;;
-	'generic-armv6')
-		REPO='resin/rpi-supervisor-base'
-	;;
-	'generic-armv7hf')
-		REPO='resin/armv7hf-supervisor-base'
-	;;
-	'generic-armv5')
-		REPO='resin/armel-supervisor-base'
-	;;
-	esac
-	SSTATE_DIR="$JENKINS_PERSISTENT_WORKDIR/$machine/sstate"
-	# Make sure shared directories are in place
-	mkdir -p $DL_DIR
-	mkdir -p $SSTATE_DIR
+BUILDER_REPO=registry.resinstaging.io/resin/${ARCH}-supervisor-base-builder
+BUILDER_IMAGE=${BUILDER_REPO}:${ESCAPED_BRANCH_NAME}

-	docker run --rm \
-		-e TARGET_MACHINE=$machine \
-		-e BUILDER_UID=$(id -u) \
-		-e BUILDER_GID=$(id -g) \
-		-v `pwd`:/source \
-		-v $DL_DIR:/yocto/shared-downloads \
-		-v $SSTATE_DIR:/yocto/shared-sstate \
-		-v `pwd`/dest:/dest \
-		supervisor-base-builder
-	if [ -f dest/rootfs.tar.gz ]; then
-		docker import dest/rootfs.tar.gz $REPO:$date
-		docker tag -f $REPO:$date $REPO:latest
-		docker push $REPO
-	else
-		echo "rootfs is missing!"
-		exit 1
-	fi
-done
+docker pull ${BUILDER_IMAGE} || docker pull ${BUILDER_REPO}:master || true
+docker build -t ${BUILDER_IMAGE} .
+docker push ${BUILDER_IMAGE} || true
+
+case "$ARCH" in
+'amd64')
+	machine='generic-x86-64'
+;;
+'i386')
+	machine='generic-x86'
+;;
+'rpi')
+	machine='generic-armv6'
+;;
+'armv7hf')
+	machine='generic-armv7hf'
+;;
+'armel')
+	machine='generic-armv5'
+;;
+esac
+SSTATE_DIR="$JENKINS_PERSISTENT_WORKDIR/$machine/sstate"
+# Make sure shared directories are in place
+mkdir -p $DL_DIR
+mkdir -p $SSTATE_DIR
+
+docker run --rm \
+	-e TARGET_MACHINE=$machine \
+	-e BUILDER_UID=$(id -u) \
+	-e BUILDER_GID=$(id -g) \
+	-v `pwd`:/source \
+	-v $DL_DIR:/yocto/shared-downloads \
+	-v $SSTATE_DIR:/yocto/shared-sstate \
+	-v `pwd`/dest:/dest \
+	${BUILDER_IMAGE}
+docker rmi -f $(docker inspect -f "{{.Id}}" ${BUILDER_IMAGE}) || true
+if [ -f dest/rootfs.tar.gz ]; then
+	docker import dest/rootfs.tar.gz ${BASE_IMAGE_TAG}
+	docker tag ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME} || docker tag -f ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME}
+	docker push ${BASE_IMAGE_TAG}
+else
+	echo "rootfs is missing!"
+	exit 1
+fi