Refactor the makefile to make it easier to use and make use of the multi-stage build

Change-Type: patch
Signed-off-by: Pablo Carranza Velez <pablo@resin.io>

.gitignore

@@ -19,3 +19,4 @@ Dockerfile.runtime.*
 !Dockerfile.runtime.template
 /build/
 /dist/
+tools/dind/config/services/docker.service.d/proxy.conf

Makefile

@@ -1,5 +1,44 @@
+# resin-supervisor Makefile
+#
+# If you're looking for an easy way to develop on the supervisor, check ./tools/dev/dindctl, which provides a simplified interface
+# to this makefile.
+#
+# Build targets (require Docker 17.05 or greater):
+# * supervisor (default) - builds a resin-supervisor image
+# * deploy - pushes a resin-supervisor image to the registry, retrying up to 3 times
+# * base - builds the "base" component (a yocto builder with the output rootfs at /dest)
+# * gosuper - builds the "gosuper" component (a golang image with the Go supervisor component at /go/bin/gosuper and /build/gosuper)
+# * nodesuper - builds the node component, with the node_modules and src at /usr/src/app and /build (also includes a rootfs-overlay there)
+# * supervisor-dind: build the development docker-in-docker supervisor that run-supervisor uses
+#
+# Variables for build targets:
+# * ARCH: amd64/rpi/i386/armv7hf/armel/aarch64 architecture for which to build the supervisor - default: amd64
+# * IMAGE: image to build or deploy - default: resin/$(ARCH)-supervisor:latest
+# * MIXPANEL_TOKEN, PUBNUB_SUBSCRIBE_KEY, PUBNUB_PUBLISH_KEY: (optional) default pubnub and mixpanel keys to embed in the supervisor image
+# * DISABLE_CACHE: if set to true, run build with no cache - default: false
+# * DOCKER_BUILD_OPTIONS: Additional options for docker build, like --cache-from parameters
+#
+# Test/development targets:
+# * run-supervisor, stop-supervisor - build and start or stop a docker-in-docker resin-supervisor (requires aufs and ability to run privileged containers)
+# * format-gosuper, test-gosuper - build a gosuper image and run formatting or unit tests
+# * test-integration - run an integration test (see gosuper/supertest). Requires a docker-in-docker supervisor to be running
+#
+# Variables for test/dev targets:
+# * IMAGE: image to build and run (either for run-supervisor or test-gosuper/integration)
+# * SUPERVISOR_IMAGE: In run-supervisor, the supervisor image to run inside the docker-in-docker image
+# * PRELOADED_IMAGE: If true, will preload user app image from tools/dev/apps.json and bind mount apps.json into the docker-in-docker supervisor
+# * SUPERVISOR_EXTRA_MOUNTS: Additional bind mount flags for the docker-in-docker supervisor
+# * PASSWORDLESS_DROPBEAR: For run-supervisor - start a passwordless ssh daemon in the docker-in-docker supervisor
+#
+
+THIS_FILE := $(lastword $(MAKEFILE_LIST))
+
+help:
+	@cat $(THIS_FILE) | awk '{if(/^#/)print;else exit}' | sed 's/\#//'
+
 OS := $(shell uname)
 
+# If we're behind a proxy, use it during build
 ifdef http_proxy
 	DOCKER_HTTP_PROXY=--build-arg http_proxy=$(http_proxy)
 endif
@@ -18,74 +57,48 @@ ifdef use_proxy_at_runtime
 	rt_no_proxy=$(no_proxy)
 endif
 
-DISABLE_CACHE = 'false'
+DISABLE_CACHE ?= 'false'
-
-ARCH = rpi# rpi/amd64/i386/armv7hf/armel/aarch64
-
-DEPLOY_REGISTRY =
-
-SUPERVISOR_VERSION = master
-BASE_IMAGE_VERSION = $(shell find base-image -print0 | LC_ALL=C sort -z | tar --null -cf - --no-recursion --mtime=@0 --owner=root --group=root --numeric-owner -T - | md5sum | awk -F " " '{print $$1}')
-ESCAPED_BASE_IMAGE_TAG = resin\/$(ARCH)-supervisor-base:$(BASE_IMAGE_VERSION)
 
 DOCKER_VERSION:=$(shell docker version --format '{{.Server.Version}}')
 DOCKER_MAJOR_VERSION:=$(word 1, $(subst ., ,$(DOCKER_VERSION)))
 DOCKER_MINOR_VERSION:=$(word 2, $(subst ., ,$(DOCKER_VERSION)))
-DOCKER_GE_1_12 := $(shell [ $(DOCKER_MAJOR_VERSION) -gt 1 -o \( $(DOCKER_MAJOR_VERSION) -eq 1 -a $(DOCKER_MINOR_VERSION) -ge 12 \) ] && echo true)
+DOCKER_GE_17_05 := $(shell [ $(DOCKER_MAJOR_VERSION) -gt 17 -o \( $(DOCKER_MAJOR_VERSION) -eq 17 -a $(DOCKER_MINOR_VERSION) -ge 5 \) ] && echo true)
 
-# In docker 1.12 tag --force has been removed.
+# Default values for Pubnub and Mixpanel keys
-ifeq ($(DOCKER_GE_1_12),true)
+PUBNUB_SUBSCRIBE_KEY ?= sub-c-bananas
-DOCKER_TAG_FORCE=
+PUBNUB_PUBLISH_KEY ?= pub-c-bananas
-else
+MIXPANEL_TOKEN ?= bananasbananas
-DOCKER_TAG_FORCE=-f
-endif
 
-all: supervisor
+# Default architecture and output image
+ARCH ?= amd64
+IMAGE ?= resin/$(ARCH)-supervisor:master
 
-PUBNUB_SUBSCRIBE_KEY = sub-c-bananas
+# Default values for run-supervisor
-PUBNUB_PUBLISH_KEY = pub-c-bananas
+SUPERVISOR_IMAGE ?= resin/$(ARCH)-supervisor:master
-MIXPANEL_TOKEN = bananasbananas
+PASSWORDLESS_DROPBEAR ?= false
 
-PASSWORDLESS_DROPBEAR = false
+# Bind mounts and variables for the run-supervisor target
-
-IMAGE = "resin/$(ARCH)-supervisor:$(SUPERVISOR_VERSION)"
-DOCKERFILE = $(ARCH)
-
-SUPERVISOR_IMAGE=$(DEPLOY_REGISTRY)$(IMAGE)
-
-ifeq ($(ARCH),rpi)
-	GOARCH = arm
-	GOARM = 6
-endif
-ifeq ($(ARCH),armv7hf)
-	GOARCH = arm
-	GOARM = 7
-endif
-ifeq ($(ARCH),armel)
-	GOARCH = arm
-	GOARM = 5
-endif
-ifeq ($(ARCH),i386)
-	GOARCH = 386
-endif
-ifeq ($(ARCH),amd64)
-	GOARCH = amd64
-endif
-ifeq ($(ARCH),aarch64)
-	GOARCH = arm64
-endif
 SUPERVISOR_DIND_MOUNTS := -v $$(pwd)/../../:/resin-supervisor -v $$(pwd)/config.json:/mnt/conf/config.json -v $$(pwd)/config/env:/usr/src/app/config/env -v $$(pwd)/config/localenv:/usr/src/app/config/localenv
 ifeq ($(OS), Linux)
 	SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v /bin/kmod:/bin/kmod
 endif
-ifdef PRELOADED_IMAGE
+ifeq ($(PRELOADED_IMAGE),true)
 	SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/apps.json:/usr/src/app/config/apps.json
 else
 	PRELOADED_IMAGE=
 endif
+SUPERVISOR_EXTRA_MOUNTS ?=
 
-SUPERVISOR_EXTRA_MOUNTS =
+ifdef TARGET_COMPONENT
+	DOCKER_TARGET_COMPONENT := "--target=${TARGET_COMPONENT}"
+else
+	DOCKER_TARGET_COMPONENT :=
+endif
 
+# Default target is to build the supervisor image
+all: supervisor
+
+# Settings to make the run-supervisor target work behind a proxy
 DOCKERD_PROXY=tools/dind/config/services/docker.service.d/proxy.conf
 ${DOCKERD_PROXY}:
 	rm -f ${DOCKERD_PROXY}
@@ -113,9 +126,10 @@ supervisor-dind: ${DOCKERD_PROXY}
 		$(DOCKER_HTTP_PROXY) \
 		$(DOCKER_HTTPS_PROXY) \
 		$(DOCKER_NO_PROXY) \
+		${DOCKER_BUILD_OPTIONS} \
 		--no-cache=$(DISABLE_CACHE) \
 		--build-arg PASSWORDLESS_DROPBEAR=$(PASSWORDLESS_DROPBEAR) \
-		-t resin/resin-supervisor-dind:$(SUPERVISOR_VERSION) .
+		-t $(IMAGE) .
 
 run-supervisor: stop-supervisor supervisor-dind
 	cd tools/dind \
@@ -133,7 +147,7 @@ run-supervisor: stop-supervisor supervisor-dind
 	if [ -n "$(rt_no_proxy)" ]; then \
 		echo "no_proxy=$(rt_no_proxy)" >> config/localenv; \
 	fi \
-	&& docker run -d --name resin_supervisor_1 --privileged ${SUPERVISOR_DIND_MOUNTS} resin/resin-supervisor-dind:$(SUPERVISOR_VERSION)
+	&& docker run -d --name resin_supervisor_1 --privileged ${SUPERVISOR_DIND_MOUNTS} $(IMAGE)
 
 stop-supervisor:
 	# Stop docker and remove volumes to prevent us from running out of loopback devices,
@@ -142,81 +156,53 @@ stop-supervisor:
 	-docker stop resin_supervisor_1 > /dev/null || true
 	-docker rm -f --volumes resin_supervisor_1 > /dev/null || true
 
-refresh-supervisor-src:
+supervisor-image:
-	echo " * Compiling CoffeeScript.." \
+ifneq ($(DOCKER_GE_17_05),true)
-	&& coffee -c ./src \
+	@echo "Docker >= 17.05 is needed to build the supervisor"
-	&& echo " * Restarting supervisor container.." \
+	@exit 1
-	&& docker exec -ti resin_supervisor_1 docker restart resin_supervisor
-
-supervisor: nodesuper gosuper
-	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.runtime.template | sed 's/%%BASE_IMAGE_TAG%%/$(ESCAPED_BASE_IMAGE_TAG)/g' > Dockerfile.runtime.$(ARCH)
-	echo "ENV VERSION=$(shell jq -r .version package.json) \\" >> Dockerfile.runtime.$(ARCH)
-	echo "    DEFAULT_PUBNUB_PUBLISH_KEY=$(PUBNUB_PUBLISH_KEY) \\" >> Dockerfile.runtime.$(ARCH)
-	echo "    DEFAULT_PUBNUB_SUBSCRIBE_KEY=$(PUBNUB_SUBSCRIBE_KEY) \\" >> Dockerfile.runtime.$(ARCH)
-	echo "    DEFAULT_MIXPANEL_TOKEN=$(MIXPANEL_TOKEN)" >> Dockerfile.runtime.$(ARCH)
-ifdef rt_https_proxy
-	echo "ENV HTTPS_PROXY=$(rt_https_proxy) \\" >> Dockerfile.runtime.$(ARCH)
-	echo "    https_proxy=$(rt_https_proxy)" >> Dockerfile.runtime.$(ARCH)
-endif
-ifdef rt_http_proxy
-	echo "ENV HTTP_PROXY=$(rt_http_proxy) \\" >> Dockerfile.runtime.$(ARCH)
-	echo "    http_proxy=$(rt_http_proxy)" >> Dockerfile.runtime.$(ARCH)
-endif
-ifdef rt_no_proxy
-	echo "ENV no_proxy=$(rt_no_proxy)" >> Dockerfile.runtime.$(ARCH)
 endif
 	docker build \
 		$(DOCKER_HTTP_PROXY) \
 		$(DOCKER_HTTPS_PROXY) \
 		$(DOCKER_NO_PROXY) \
-		-f Dockerfile.runtime.$(ARCH) \
+		$(DOCKER_TARGET_COMPONENT) \
-		--pull \
+		$(DOCKER_BUILD_OPTIONS) \
+		--no-cache=$(DISABLE_CACHE) \
+		--build-arg ARCH=$(ARCH) \
+		--build-arg VERSION=$(shell jq -r .version package.json) \
+		--build-arg DEFAULT_PUBNUB_PUBLISH_KEY=$(PUBNUB_PUBLISH_KEY) \
+		--build-arg DEFAULT_PUBNUB_SUBSCRIBE_KEY=$(PUBNUB_SUBSCRIBE_KEY) \
+		--build-arg DEFAULT_MIXPANEL_TOKEN=$(MIXPANEL_TOKEN) \
 		-t $(IMAGE) .
 
-lint:
+supervisor:
-	docker run --rm resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) bash -c 'npm install resin-lint && npm run lint'
+	@$(MAKE) -f $(THIS_FILE) IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image
 
-deploy: supervisor
+deploy:
-	docker tag $(DOCKER_TAG_FORCE) $(IMAGE) $(SUPERVISOR_IMAGE)
+	@bash retry_docker_push.sh $(IMAGE)
-	bash retry_docker_push.sh $(SUPERVISOR_IMAGE)
+
+base:
+	$(MAKE) -f $(THIS_FILE) TARGET_COMPONENT=base IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image
 
 nodesuper:
-	sed 's/%%ARCH%%/$(ARCH)/g' Dockerfile.build.template > Dockerfile.build.$(ARCH)
+	$(MAKE) -f $(THIS_FILE) TARGET_COMPONENT=node IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image
-	docker build --pull \
-		$(DOCKER_HTTP_PROXY) \
-		$(DOCKER_HTTPS_PROXY) \
-		$(DOCKER_NO_PROXY) \
-		-f Dockerfile.build.$(ARCH) \
-		-t resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION) .
-	docker run --rm \
-		-v `pwd`/build/$(ARCH):/build \
-		resin/node-supervisor-$(ARCH):$(SUPERVISOR_VERSION)
 
 gosuper:
-	cd gosuper && docker build --pull \
+	$(MAKE) -f $(THIS_FILE) TARGET_COMPONENT=gosuper IMAGE=$(IMAGE) ARCH=$(ARCH) supervisor-image
-		$(DOCKER_HTTP_PROXY) \
-		$(DOCKER_HTTPS_PROXY) \
-		$(DOCKER_NO_PROXY) \
-		--build-arg GOARCH=$(GOARCH) \
-		--build-arg GOARM=$(GOARM) \
-		-t resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) .
-	docker run --rm \
-		-v `pwd`/build/$(ARCH):/build \
-		resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION)
 
 test-gosuper: gosuper
 	docker run \
 		--rm \
 		-v /var/run/dbus:/mnt/root/run/dbus \
 		-e DBUS_SYSTEM_BUS_ADDRESS="/mnt/root/run/dbus/system_bus_socket" \
-		resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) bash -c \
+		$(IMAGE) bash -c \
 			'./test_formatting.sh && go test -v ./gosuper'
 
 format-gosuper: gosuper
 	docker run \
 		--rm \
 		-v $(shell pwd)/gosuper:/go/src/resin-supervisor/gosuper \
-		resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) \
+		$(IMAGE) \
 			go fmt ./...
 
 test-integration: gosuper
@@ -227,10 +213,7 @@ test-integration: gosuper
 		--volumes-from resin_supervisor_1 \
 		-v /var/run/dbus:/mnt/root/run/dbus \
 		-e DBUS_SYSTEM_BUS_ADDRESS="/mnt/root/run/dbus/system_bus_socket" \
-		resin/go-supervisor-$(ARCH):$(SUPERVISOR_VERSION) \
+		$(IMAGE) \
 			go test -v ./supertest
 
-base-image-version:
+.PHONY: supervisor deploy base nodesuper gosuper supervisor-dind run-supervisor
-	@echo $(BASE_IMAGE_VERSION)
-
-.PHONY: supervisor deploy supervisor-dind run-supervisor gosuper nodesuper