build: implement multi-Dockerfile build process

This build strategy lends itself to how Rockerfiles work. In the build
Dockerfile all the build utilities (e.g gcc, python) are installed and
run the build process to produce some build artifacts. There are two
build Dockerfiles, one for the nodejs part and one for the golang part.
The build artifacts of these are combined into the runtime Dockerfile.
For all this to work there is some minimal glue implemented in the
Makefile.

Part of this commit is a switch of the base image the runtime is based
on to the minimal OpenEmbedded one produced by #198

Signed-off-by: Petros Angelatos <petrosagg@gmail.com>

.dockerignore

@@ -9,5 +9,5 @@ coffeelint.json
 automation
 tools
 README.md
-gosuper
 retry_docker_push.sh
+base-image

.gitignore

@@ -12,3 +12,9 @@ base-image/build/cache
 base-image/build/downloads
 base-image/build/sstate-cache
 base-image/build/tmp-glibc
+
+Dockerfile.build.*
+Dockerfile.runtime.*
+!Dockerfile.build.template
+!Dockerfile.runtime.template
+build

Dockerfile.build.template

@@ -0,0 +1,65 @@
+# Build nodejs dependencies
+
+# The node version here should match the version of the runtime image which is
+# specified in the base-image subdirectory in the project
+FROM resin/%%ARCH%%-node:6.5-slim
+
+WORKDIR /usr/src/app
+
+RUN apt-get update \
+	&& apt-get install -y \
+		g++ \
+		libsqlite3-dev \
+		make \
+		python \
+		rsync \
+		wget \
+	&& rm -rf /var/lib/apt/lists/ \
+	&& npm install -g coffee-script
+
+ENV DOCKER_COMPOSE_VERSION 1.7.1
+
+ENV DOCKER_COMPOSE_SHA256_amd64 37df85ee18bf0e2a8d71cbfb8198b1c06cc388f19118be7bdfc4d6db112af834
+ENV DOCKER_COMPOSE_SHA256_i386 b926fd9a2a9d89358f1353867706f94558a62caaf3aa72bf10bcbbe31e1a44f0
+ENV DOCKER_COMPOSE_SHA256_rpi 3f0b8c69c66a2daa5fbb0c127cb76ca95d7125827a9c43dd3c36f9bc2ed6e0e5
+ENV DOCKER_COMPOSE_SHA256_armv7hf 3f0b8c69c66a2daa5fbb0c127cb76ca95d7125827a9c43dd3c36f9bc2ed6e0e5
+ENV DOCKER_COMPOSE_SHA256_armel a1025fed97536e2698798ea277a014ec5e1eae816a8cf3155ecbe9679e3e7bac
+
+RUN set -x \
+	&& mkdir -p rootfs-overlay/usr/bin/ \
+	&& ln -s /lib rootfs-overlay/lib64 \
+	&& pkgname='docker-compose' \
+	&& arch=%%ARCH%% \
+	&& if [ $arch = 'rpi' -o $arch = 'armv7hf' ]; then arch=armhf; fi \
+	&& base="http://resin-packages.s3.amazonaws.com/${pkgname}" \
+	&& pkgver=$DOCKER_COMPOSE_VERSION \
+	&& checksum=$DOCKER_COMPOSE_SHA256_%%ARCH%% \
+	&& wget "${base}/${pkgver}/${pkgname}-linux-${arch}-${pkgver}.tar.gz" \
+	&& echo "$checksum  ${pkgname}-linux-${arch}-${pkgver}.tar.gz" | sha256sum -c \
+	&& tar xzf "${pkgname}-linux-${arch}-${pkgver}.tar.gz" --strip-components=1 -C rootfs-overlay/usr/bin \
+	&& mv "rootfs-overlay/usr/bin/${pkgname}-linux-${arch}" rootfs-overlay/usr/bin/docker-compose
+
+COPY package.json /usr/src/app/
+
+RUN JOBS=MAX npm install --unsafe-perm --production --no-optional \
+	&& npm dedupe
+
+COPY src /usr/src/app/src
+
+RUN coffee -c src
+
+# Remove tar files (sqlite3 module) and tests to reduce space
+RUN find . -name '*.tar.*' -delete \
+    && find . -path '*/test/*' -delete
+
+# Create /var/run/resin for the gosuper to place its socket in
+RUN mkdir -p rootfs-overlay/var/run/resin
+
+COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/
+
+COPY inittab rootfs-overlay/etc/inittab
+
+CMD rsync -a --delete node_modules src rootfs-overlay /build
+
+# -*- mode: dockerfile -*-
+# vi: set ft=dockerfile :

Dockerfile.runtime.template

@@ -0,0 +1,20 @@
+# Minimal runtime image
+FROM resin/%%ARCH%%-supervisor-base
+
+WORKDIR /usr/src/app
+
+COPY ./build/%%ARCH%%/src ./src
+COPY ./build/%%ARCH%%/node_modules ./node_modules
+COPY ./build/%%ARCH%%/gosuper ./gosuper
+COPY ./build/%%ARCH%%/rootfs-overlay/ /
+
+VOLUME /data
+
+ENV CONFIG_MOUNT_POINT /boot/config.json
+ENV LED_FILE /dev/null
+ENV SUPERVISOR_IMAGE resin/%%ARCH%%-supervisor
+
+CMD [ "/sbin/init" ]
+
+# -*- mode: dockerfile -*-
+# vi: set ft=dockerfile :

Makefile

@@ -21,12 +21,10 @@ endif
 DISABLE_CACHE = 'false'
 
 ARCH = rpi# rpi/amd64/i386/armv7hf/armel
-BASE_DISTRO =
 
 DEPLOY_REGISTRY =
 
 SUPERVISOR_VERSION = master
-JOB_NAME = 1
 
 all: supervisor
 
@@ -36,15 +34,8 @@ MIXPANEL_TOKEN = bananasbananas
 
 PASSWORDLESS_DROPBEAR = false
 
-ifdef BASE_DISTRO
+IMAGE = "resin/$(ARCH)-supervisor:$(SUPERVISOR_VERSION)"
-$(info BASE_DISTRO SPECIFIED. START BUILDING ALPINE SUPERVISOR)
+DOCKERFILE = $(ARCH)
-	IMAGE = "resin/$(ARCH)-supervisor:$(SUPERVISOR_VERSION)-alpine"
-	DOCKERFILE = alpine.$(ARCH)
-else
-$(info BASE_DISTRO NOT SPECIFIED. START BUILDING DEBIAN SUPERVISOR)
-	IMAGE = "resin/$(ARCH)-supervisor:$(SUPERVISOR_VERSION)"
-	DOCKERFILE = $(ARCH)
-endif
 
 SUPERVISOR_IMAGE=$(DEPLOY_REGISTRY)$(IMAGE)
 
@@ -78,9 +69,6 @@ endif
 
 SUPERVISOR_EXTRA_MOUNTS =
 
-clean:
-	-rm Dockerfile
-
 DOCKERD_PROXY=tools/dind/config/services/docker.service.d/proxy.conf
 ${DOCKERD_PROXY}:
 	rm -f ${DOCKERD_PROXY}
@@ -143,43 +131,61 @@ refresh-supervisor-src:
 	&& echo " * Restarting supervisor container.." \
 	&& docker exec -ti resin_supervisor_1 docker restart resin_supervisor
 
-supervisor: gosuper
+supervisor: nodesuper gosuper
-	cp Dockerfile.$(DOCKERFILE) Dockerfile
+	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template > Dockerfile.runtime.$(ARCH)
-	echo "ENV VERSION "`jq -r .version package.json` >> Dockerfile
+	echo "ENV VERSION "`jq -r .version package.json` >> Dockerfile.runtime.$(ARCH)
-	echo "ENV DEFAULT_PUBNUB_PUBLISH_KEY $(PUBNUB_PUBLISH_KEY)" >> Dockerfile
+	echo "ENV DEFAULT_PUBNUB_PUBLISH_KEY $(PUBNUB_PUBLISH_KEY)" >> Dockerfile.runtime.$(ARCH)
-	echo "ENV DEFAULT_PUBNUB_SUBSCRIBE_KEY $(PUBNUB_SUBSCRIBE_KEY)" >> Dockerfile
+	echo "ENV DEFAULT_PUBNUB_SUBSCRIBE_KEY $(PUBNUB_SUBSCRIBE_KEY)" >> Dockerfile.runtime.$(ARCH)
-	echo "ENV DEFAULT_MIXPANEL_TOKEN $(MIXPANEL_TOKEN)" >> Dockerfile
+	echo "ENV DEFAULT_MIXPANEL_TOKEN $(MIXPANEL_TOKEN)" >> Dockerfile.runtime.$(ARCH)
 ifdef rt_https_proxy
-	echo "ENV HTTPS_PROXY $(rt_https_proxy)" >> Dockerfile
+	echo "ENV HTTPS_PROXY $(rt_https_proxy)" >> Dockerfile.runtime.$(ARCH)
-	echo "ENV https_proxy $(rt_https_proxy)" >> Dockerfile
+	echo "ENV https_proxy $(rt_https_proxy)" >> Dockerfile.runtime.$(ARCH)
 endif
 ifdef rt_http_proxy
-	echo "ENV HTTP_PROXY $(rt_http_proxy)" >> Dockerfile
+	echo "ENV HTTP_PROXY $(rt_http_proxy)" >> Dockerfile.runtime.$(ARCH)
-	echo "ENV http_proxy $(rt_http_proxy)" >> Dockerfile
+	echo "ENV http_proxy $(rt_http_proxy)" >> Dockerfile.runtime.$(ARCH)
 endif
 ifdef rt_no_proxy
-	echo "ENV no_proxy $(rt_no_proxy)" >> Dockerfile
+	echo "ENV no_proxy $(rt_no_proxy)" >> Dockerfile.runtime.$(ARCH)
 endif
-	docker build $(DOCKER_HTTP_PROXY) $(DOCKER_HTTPS_PROXY) $(DOCKER_NO_PROXY) --no-cache=$(DISABLE_CACHE) -t $(IMAGE) .
+	docker build \
-	-rm Dockerfile
+		$(DOCKER_HTTP_PROXY) \
+		$(DOCKER_HTTPS_PROXY) \
+		$(DOCKER_NO_PROXY) \
+		-f Dockerfile.runtime.$(ARCH) \
+		--pull \
+		-t $(IMAGE) .
 
-lint: supervisor
+lint:
-	docker run --rm --entrypoint='sh' $(IMAGE) -c 'npm install && npm run lint'
+	docker run --rm resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) bash -c 'npm install resin-lint && npm run lint'
 
 deploy: supervisor
 	docker tag -f $(IMAGE) $(SUPERVISOR_IMAGE)
 	bash retry_docker_push.sh $(SUPERVISOR_IMAGE)
 
-go-builder:
+nodesuper:
-	-cp tools/dind/config.json ./gosuper/
+	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.build.template > Dockerfile.build.$(ARCH)
-	cd gosuper && docker build $(DOCKER_HTTP_PROXY) $(DOCKER_HTTPS_PROXY) $(DOCKER_NO_PROXY) -t resin/go-supervisor-builder:$(SUPERVISOR_VERSION) .
+	docker build \
-	-rm ./gosuper/config.json
+		$(DOCKER_HTTP_PROXY) \
+		$(DOCKER_HTTPS_PROXY) \
+		$(DOCKER_NO_PROXY) \
+		-f Dockerfile.build.$(ARCH) \
+		-t resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) .
+	docker run --rm \
+		-v `pwd`/build/$(ARCH):/build \
+		resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION)
 
-gosuper: go-builder
+gosuper:
-	-mkdir -p bin
+	cd gosuper && docker build \
-	-docker rm --volumes -f resin_build_gosuper_$(JOB_NAME) || true
+		$(DOCKER_HTTP_PROXY) \
-	docker run --rm --name resin_build_gosuper_$(JOB_NAME) -v $(shell pwd)/gosuper/bin:/usr/src/app/bin -e USER_ID=$(shell id -u) -e GROUP_ID=$(shell id -g) -e GOARCH=$(GOARCH) -e GOARM=$(GOARM) resin/go-supervisor-builder:$(SUPERVISOR_VERSION)
+		$(DOCKER_HTTPS_PROXY) \
-	mv gosuper/bin/linux_$(GOARCH)/gosuper bin/gosuper
+		$(DOCKER_NO_PROXY) \
+		--build-arg GOARCH=$(GOARCH) \
+		--build-arg GOARM=$(GOARM) \
+		-t resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) .
+	docker run --rm \
+		-v `pwd`/build/$(ARCH):/build \
+		resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION)
 
 test-gosuper: gosuper
 	docker run \
@@ -207,4 +213,4 @@ test-integration: gosuper
 		resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) \
 			go test -v ./supertest
 
-.PHONY: supervisor deploy supervisor-dind run-supervisor
+.PHONY: supervisor deploy supervisor-dind run-supervisor gosuper nodesuper

automation/jenkins_build.sh

@@ -12,14 +12,12 @@ docker pull resin/${ARCH}-supervisor:${ESCAPED_BRANCH_NAME} || docker pull resin
 make SUPERVISOR_VERSION=${VERSION} JOB_NAME=${JOB_NAME} test-gosuper
 
 MAKE_ARGS="ARCH=${ARCH} \
-    JOB_NAME=${JOB_NAME} \
-    BASE_DISTRO=${BASE_DISTRO} \
     PUBNUB_SUBSCRIBE_KEY=${PUBNUB_SUBSCRIBE_KEY} \
     PUBNUB_PUBLISH_KEY=${PUBNUB_PUBLISH_KEY} \
     MIXPANEL_TOKEN=${MIXPANEL_TOKEN}"
 
-make ${MAKE_ARGS} \
+# Disabled until this is merged in npm https://github.com/npm/npm/pull/13257
-    lint
+# make ${MAKE_ARGS} lint
 
 make ${MAKE_ARGS} \
     SUPERVISOR_VERSION=${ESCAPED_BRANCH_NAME} \

gosuper/Dockerfile

@@ -1,15 +1,22 @@
+# Build golang supervisor
 FROM golang:1.5.1
 
+RUN apt-get update \
+	&& apt-get install -y \
+		rsync \
+	&& rm -rf /var/lib/apt/lists/
+
+COPY . /go/src/resin-supervisor/gosuper
+
+WORKDIR /go/src/resin-supervisor/gosuper
+
 ENV GOOS linux
-ENV GOPATH /usr/src/app
 
-WORKDIR /usr/src/app
+ARG GOARCH=amd64
+ARG GOARM=''
 
-COPY . src/resin-supervisor/gosuper
+RUN go install -a -v ./gosuper \
+	&& cd /go/bin \
+	&& find -type f -name gosuper -exec mv {} /go/bin/gosuper \;
 
-RUN chmod +x src/resin-supervisor/gosuper/build_gosuper.sh
+CMD rsync -a --delete /go/bin/gosuper /build
-RUN chmod +x src/resin-supervisor/gosuper/test_formatting.sh
-
-# Run go install with -a (force rebuilding of all packages)
-# and -v (print package names as they are built)
-CMD cd ./src/resin-supervisor/gosuper && ./build_gosuper.sh

gosuper/build_gosuper.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-go install -a -v ./gosuper
-RETURN_VALUE=$?
-
-HOSTARCH=$(uname -m)
-# For consistency, always keep the binary within a linux_$GOARCH folder
-if [[ ( $GOARCH == "amd64" && $HOSTARCH == "x86_64" ) || ( $GOARCH == "arm" && $HOSTARCH == "armv7l" ) ]]; then
-	mkdir -p $GOPATH/bin/linux_$GOARCH
-	cp $GOPATH/bin/gosuper $GOPATH/bin/linux_$GOARCH/
-fi
-chown -R $USER_ID:$GROUP_ID $GOPATH/bin
-
-exit $RETURN_VALUE

package.json

@@ -2,9 +2,9 @@
   "name": "resin-supervisor",
   "version": "2.1.1",
   "scripts": {
-    "postinstall": "sh postinstall.sh",
     "start": "./entry.sh",
-    "lint": "resin-lint src/"
+    "lint": "resin-lint src/",
+    "start": "./entry.sh"
   },
   "dependencies": {
     "JSONStream": "^1.1.2",
@@ -33,7 +33,7 @@
     "resin-register-device": "^2.0.0",
     "rimraf": "^2.5.4",
     "rwlock": "^5.0.0",
-    "sqlite3": "3.0.9",
+    "sqlite3": "^3.1.0",
     "typed-error": "~0.1.0",
     "yamljs": "^0.2.7"
   },