Merge pull request #363 from resin-io/362-base-image-build

Build the base image together with the supervisor
2025-01-31 08:25:36 +00:00 · 2016-12-16 23:58:11 +02:00 · 2016-12-16 23:58:11 +02:00 · bb1b93cdf6
commit bb1b93cdf6
parent 7602ff0bdb 9e327bf7a6
4 changed files with 75 additions and 55 deletions
--- a/Dockerfile.runtime.template
+++ b/Dockerfile.runtime.template
@ -1,5 +1,5 @@
 # Minimal runtime image
-FROM resin/%%ARCH%%-supervisor-base:20161103
+FROM %%BASE_IMAGE_TAG%%

 WORKDIR /usr/src/app

--- a/3
+++ b/3
@ -25,6 +25,7 @@ ARCH = rpi# rpi/amd64/i386/armv7hf/armel
 DEPLOY_REGISTRY =

 SUPERVISOR_VERSION = master
+ESCAPED_BASE_IMAGE_TAG = resin\/$(ARCH)-supervisor-base:$(SUPERVISOR_VERSION)

 DOCKER_VERSION:=$(shell docker version --format '{{.Server.Version}}')
 DOCKER_MAJOR_VERSION:=$(word 1, $(subst ., ,$(DOCKER_VERSION)))
@ -144,7 +145,7 @@ refresh-supervisor-src:
 	&& docker exec -ti resin_supervisor_1 docker restart resin_supervisor

 supervisor: nodesuper gosuper
-	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template > Dockerfile.runtime.$(ARCH)
+	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template | sed 's/%%BASE_IMAGE_TAG%%/$(ESCAPED_BASE_IMAGE_TAG)/g' > Dockerfile.runtime.$(ARCH)
 	echo "ENV VERSION=$(shell jq -r .version package.json) \\" >> Dockerfile.runtime.$(ARCH)
 	echo "    DEFAULT_PUBNUB_PUBLISH_KEY=$(PUBNUB_PUBLISH_KEY) \\" >> Dockerfile.runtime.$(ARCH)
 	echo "    DEFAULT_PUBNUB_SUBSCRIBE_KEY=$(PUBNUB_SUBSCRIBE_KEY) \\" >> Dockerfile.runtime.$(ARCH)
--- a/automation/jenkins_build.sh
+++ b/automation/jenkins_build.sh
@ -2,18 +2,27 @@
 set -e

 # Jenkins build steps
-ESCAPED_BRANCH_NAME=$(echo $sourceBranch | sed 's/[^a-z0-9A-Z_.-]/-/g')
+export ESCAPED_BRANCH_NAME=$(echo $sourceBranch | sed 's/[^a-z0-9A-Z_.-]/-/g')
+BASE_IMAGE_VERSION=$(tar -c --mtime='1970-01-01' --owner=0 --group=0 -f - base-image | md5sum | awk -F " " '{print $1}')
+export BASE_IMAGE_REPO=resin/$ARCH-supervisor-base
+export BASE_IMAGE_TAG=resin/$ARCH-supervisor-base:$BASE_IMAGE_VERSION
+
+# Try to pull the base image according to the contents of the base-image folder, otherwise build it
+docker pull $BASE_IMAGE_TAG || (cd base-image && bash -ex automation/jenkins-build.sh)

 # Try pulling the old build first for caching purposes.
 docker pull resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME} || docker pull resin/${ARCH}-supervisor:master || true
 # Also pull the intermediate images, if possible, to improve caching
-docker pull registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || docker pull registry.resinstaging.io/resin/node-supervisor-${ARCH}:master || true
-docker pull registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || docker pull registry.resinstaging.io/resin/go-supervisor-${ARCH}:master || true
+NODE_SUPERVISOR_REPO=registry.resinstaging.io/resin/node-supervisor-${ARCH}
+GO_SUPERVISOR_REPO=registry.resinstaging.io/resin/go-supervisor-${ARCH}
+docker pull ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || docker pull ${NODE_SUPERVISOR_REPO}:master || true
+docker pull ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || docker pull ${GO_SUPERVISOR_REPO}:master || true

 # Test the gosuper
 make SUPERVISOR_VERSION=${ESCAPED_BRANCH_NAME} JOB_NAME=${JOB_NAME} test-gosuper

 MAKE_ARGS="ARCH=${ARCH} \
+    ESCAPED_BASE_IMAGE_TAG=$(echo $BASE_IMAGE_TAG | sed -e 's/\//\\\//g; s/\./\\\./g') \
    PUBNUB_SUBSCRIBE_KEY=${PUBNUB_SUBSCRIBE_KEY} \
    PUBNUB_PUBLISH_KEY=${PUBNUB_PUBLISH_KEY} \
    MIXPANEL_TOKEN=${MIXPANEL_TOKEN} \
@ -24,10 +33,14 @@ make ${MAKE_ARGS} \
    deploy

 # Try to push the intermediate images to improve caching in future builds
-docker tag -f resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
-docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
-docker push registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
-docker push registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} || true
+docker tag resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| docker tag -f resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| true
+docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| docker tag -f resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME} ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} \
+	|| true
+docker push ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true
+docker push ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME} || true

 make ${MAKE_ARGS} \
    DEPLOY_REGISTRY=registry.resinstaging.io/ \
@ -36,5 +49,6 @@ make ${MAKE_ARGS} \
 # Cleanup removing by Id to actually remove the images rather than untagging them
 docker rmi -f $(docker inspect -f "{{.Id}}" registry.resinstaging.io/resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME}) || true
 docker rmi -f $(docker inspect -f "{{.Id}}" resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME}) || true
-docker rmi -f $(docker inspect -f "{{.Id}}" registry.resinstaging.io/resin/node-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME}) || true
-docker rmi -f $(docker inspect -f "{{.Id}}" registry.resinstaging.io/resin/go-supervisor-${ARCH}:${ESCAPED_BRANCH_NAME}) || true
+docker rmi -f $(docker inspect -f "{{.Id}}" ${NODE_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME}) || true
+docker rmi -f $(docker inspect -f "{{.Id}}" ${GO_SUPERVISOR_REPO}:${ESCAPED_BRANCH_NAME}) || true
+docker rmi -f $(docker inspect -f "{{.Id}}" ${BASE_IMAGE_TAG}) || true
--- a/base-image/automation/jenkins-build.sh
+++ b/base-image/automation/jenkins-build.sh
@ -3,55 +3,60 @@
 set -o errexit
 set -o pipefail

-date=$(date +'%Y%m%d' -u)
+# ARCH, ESCAPED_BRANCH_NAME, BASE_IMAGE_REPO and BASE_IMAGE_TAG should be set before calling this script.
+# This script purposefully doesn't clean up the BASE_IMAGE_TAG docker image after building it.
+
 JENKINS_PERSISTENT_WORKDIR=${1:-/var/lib/yocto}
 DL_DIR="$JENKINS_PERSISTENT_WORKDIR/shared-downloads"
-# MACHINE_LIST: generic-x86-64 generic-x86 generic-armv6 generic-armv7hf generic-armv5
-# MACHINE_LIST should be set in jenkins config

 git submodule update --init --recursive
 rm -rf dest
 mkdir dest

-docker build -t supervisor-base-builder .
-for machine in $MACHINE_LIST; do
-	case "$machine" in
-	'generic-x86-64')
-		REPO='resin/amd64-supervisor-base'
-	;;
-	'generic-x86')
-		REPO='resin/i386-supervisor-base'
-	;;
-	'generic-armv6')
-		REPO='resin/rpi-supervisor-base'
-	;;
-	'generic-armv7hf')
-		REPO='resin/armv7hf-supervisor-base'
-	;;
-	'generic-armv5')
-		REPO='resin/armel-supervisor-base'
-	;;
-	esac
-	SSTATE_DIR="$JENKINS_PERSISTENT_WORKDIR/$machine/sstate"
-	# Make sure shared directories are in place
-	mkdir -p $DL_DIR
-	mkdir -p $SSTATE_DIR
+BUILDER_REPO=registry.resinstaging.io/resin/${ARCH}-supervisor-base-builder
+BUILDER_IMAGE=${BUILDER_REPO}:${ESCAPED_BRANCH_NAME}

-	docker run --rm \
-		-e TARGET_MACHINE=$machine \
-		-e BUILDER_UID=$(id -u) \
-		-e BUILDER_GID=$(id -g) \
-		-v `pwd`:/source \
-		-v $DL_DIR:/yocto/shared-downloads \
-		-v $SSTATE_DIR:/yocto/shared-sstate \
-		-v `pwd`/dest:/dest \
-		supervisor-base-builder
-	if [ -f dest/rootfs.tar.gz ]; then
-		docker import dest/rootfs.tar.gz $REPO:$date
-		docker tag -f $REPO:$date $REPO:latest
-		docker push $REPO
-	else
-		echo "rootfs is missing!"
-		exit 1
-	fi
-done
+docker pull ${BUILDER_IMAGE} || docker pull ${BUILDER_REPO}:master || true
+docker build -t ${BUILDER_IMAGE} .
+docker push ${BUILDER_IMAGE} || true
+
+case "$ARCH" in
+'amd64')
+	machine='generic-x86-64'
+;;
+'i386')
+	machine='generic-x86'
+;;
+'rpi')
+	machine='generic-armv6'
+;;
+'armv7hf')
+	machine='generic-armv7hf'
+;;
+'armel')
+	machine='generic-armv5'
+;;
+esac
+SSTATE_DIR="$JENKINS_PERSISTENT_WORKDIR/$machine/sstate"
+# Make sure shared directories are in place
+mkdir -p $DL_DIR
+mkdir -p $SSTATE_DIR
+
+docker run --rm \
+	-e TARGET_MACHINE=$machine \
+	-e BUILDER_UID=$(id -u) \
+	-e BUILDER_GID=$(id -g) \
+	-v `pwd`:/source \
+	-v $DL_DIR:/yocto/shared-downloads \
+	-v $SSTATE_DIR:/yocto/shared-sstate \
+	-v `pwd`/dest:/dest \
+	${BUILDER_IMAGE}
+docker rmi -f $(docker inspect -f "{{.Id}}" ${BUILDER_IMAGE}) || true
+if [ -f dest/rootfs.tar.gz ]; then
+	docker import dest/rootfs.tar.gz ${BASE_IMAGE_TAG}
+	docker tag ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME} || docker tag -f ${BASE_IMAGE_TAG} ${BASE_IMAGE_REPO}:${ESCAPED_BRANCH_NAME}
+	docker push ${BASE_IMAGE_TAG}
+else
+	echo "rootfs is missing!"
+	exit 1
+fi