Refactor testing and (cross)compile.

With the update to diesel 2.0.0-rc0, it is now possible to cross-compile
without emulation. Note that during each (cross)compile, we have to
install libpq-dev for the target platform, as only one version can be
installed at a time (e.g. libpq-dev:amd64 conflicts with libpq-dev:armhf).

.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.dockerignore

@@ -1,5 +1,6 @@
 .git
 .cargo
+.rust
 **/target
 **/node_modules
 Dockerfile

.github/buildkitd.toml

@@ -0,0 +1,2 @@
+[worker.oci]
+  max-parallelism = 1

.github/workflows/main.yml

@@ -14,12 +14,22 @@ jobs:
       -
         name: Checkout
         uses: actions/checkout@v2
+      -
+        name: Cargo cache
+        uses: actions/cache@v3
+        with:
+          path: .rust/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
       -
         name: Build UI
         run: make build-ui
       -
         name: Run tests
         run: make test
+      -
+        name: Make cache accessible to caching action
+        run: |
+          sudo chown $(whoami):$(whoami) -R .rust
 
   dist:
     needs: tests
@@ -30,11 +40,11 @@ jobs:
         name: Checkout
         uses: actions/checkout@v2
       -
-        name: Set up QEMU
+        name: Cargo cache
-        uses: docker/setup-qemu-action@v1
+        uses: actions/cache@v3
-      -
+        with:
-        name: Set up Docker Buildx
+          path: .rust/
-        uses: docker/setup-buildx-action@v1
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
       -
         name: Build UI
         run: make build-ui
@@ -52,6 +62,11 @@ jobs:
         name: Upload binaries to S3
         run: |
           aws s3 sync dist s3://builds.loraserver.io/chirpstack
+        if: startsWith(github.ref, 'refs/tags/v')
+      -
+        name: Make cache accessible to caching action
+        run: |
+          sudo chown $(whoami):$(whoami) -R .rust
 
   docker:
     needs: tests
@@ -78,6 +93,8 @@ jobs:
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
+        with:
+          config: .github/buildkitd.toml
       -
         name: Login to DockerHub
         uses: docker/login-action@v1 
@@ -89,7 +106,7 @@ jobs:
         id: docker_build
         uses: docker/build-push-action@v2
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
           push: ${{ startsWith(github.ref, 'refs/tags/v') }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -1,6 +1,7 @@
 # hidden files
 .*
 !/chirpstack/.rpm
+!/.cargo
 
 # Log files
 *.log

Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.15.0 AS ui-build
+# UI build stage
+FROM --platform=$BUILDPLATFORM alpine:3.15.0 AS ui-build
 
 ENV PROJECT_PATH=/chirpstack
 
@@ -9,15 +10,61 @@ COPY ./api/grpc-web $PROJECT_PATH/api/grpc-web
 COPY ./ui $PROJECT_PATH/ui
 
 RUN cd $PROJECT_PATH/ui && \
-		yarn install --network-timeout 600000 && \
+	yarn install --network-timeout 600000 && \
-		yarn build
+	yarn build
 
-FROM chirpstack/chirpstack-dev-cache:latest AS rust-build
+
+# ChirpStack build stage
+FROM --platform=$BUILDPLATFORM rust:1.62.0-buster AS rust-build
+
+ENV PROJECT_PATH=/chirpstack
+RUN mkdir -p $PROJECT_PATH
+
+RUN dpkg --add-architecture armhf
+RUN dpkg --add-architecture arm64
+
+RUN apt-get update && \
+	apt-get install -y \
+	make \
+	cmake \
+	git \
+	libpq-dev \
+	clang \
+	libclang-3.9-dev \
+	jq \
+	gcc-arm-linux-gnueabihf \
+	g++-arm-linux-gnueabihf \
+	gcc-aarch64-linux-gnu \
+	g++-aarch64-linux-gnu \
+	zlib1g-dev:armhf \
+	zlib1g-dev:arm64
+
+RUN rustup target add armv7-unknown-linux-gnueabihf
+RUN rustup target add aarch64-unknown-linux-gnu
+
+ARG TARGETPLATFORM
+RUN mkdir -p /release/$TARGETPLATFORM
 
 COPY . $PROJECT_PATH
 COPY --from=ui-build $PROJECT_PATH/ui/build $PROJECT_PATH/ui/build
-RUN cd $PROJECT_PATH/chirpstack && cargo build --release
 
+RUN case "$TARGETPLATFORM" in \
+	"linux/amd64") \
+		cd $PROJECT_PATH/chirpstack && make release-amd64; \
+		cp $PROJECT_PATH/target/release/chirpstack /release/$TARGETPLATFORM; \
+		;; \
+	"linux/arm/v7") \
+		cd $PROJECT_PATH/chirpstack && make release-armv7hf; \
+		cp $PROJECT_PATH/target/armv7-unknown-linux-gnueabihf/release/chirpstack /release/$TARGETPLATFORM; \
+		;; \
+	"linux/arm64") \
+		cd $PROJECT_PATH/chirpstack && make release-arm64; \
+		cp $PROJECT_PATH/target/aarch64-unknown-linux-gnu/release/chirpstack /release/$TARGETPLATFORM; \
+		;; \
+	esac;
+
+
+# Final stage
 FROM debian:buster-slim as production
 
 RUN apt-get update && \
@@ -26,7 +73,9 @@ RUN apt-get update && \
 		libpq5 \
 		&& rm -rf /var/lib/apt/lists/*
 
-COPY --from=rust-build /target/release/chirpstack /usr/bin/chirpstack
+ARG TARGETPLATFORM
+
+COPY --from=rust-build /release/$TARGETPLATFORM/chirpstack /usr/bin/chirpstack
 COPY --from=rust-build /chirpstack/chirpstack/configuration/* /etc/chirpstack/
 USER nobody:nogroup
 ENTRYPOINT ["/usr/bin/chirpstack"]

Dockerfile-dev-cache

@@ -1,36 +0,0 @@
-FROM rust:1.62.0-buster
-
-ENV PROJECT_PATH=/chirpstack
-ENV CARGO_TARGET_DIR=/target
-
-RUN mkdir -p $PROJECT_PATH
-RUN mkdir -p $CARGO_TARGET_DIR
-
-RUN apt-get update && \
-        apt-get install -y \
-		make \
-		cmake \
-		git \
-		bash \
-		screen \
-		postgresql-client \
-		libpq-dev \
-		mosquitto-clients \
-		redis-tools \
-		rpm \
-		clang \
-		golang-cfssl \
-		jq \
-		&& rm -rf /var/lib/apt/lists/*
-
-RUN rustup component add rustfmt clippy
-
-RUN cargo install diesel_cli --version 2.0.0-rc.0 --no-default-features --features postgres
-RUN cargo install cargo-deb
-RUN cargo install cargo-rpm
-
-COPY . $PROJECT_PATH
-
-WORKDIR $PROJECT_PATH/chirpstack
-RUN cargo build --release
-RUN rm -rf $PROJECT_PATH/*

Dockerfile-devel

@@ -0,0 +1,40 @@
+FROM rust:1.62.0-buster
+
+ENV PROJECT_PATH=/chirpstack
+RUN mkdir -p $PROJECT_PATH
+
+RUN dpkg --add-architecture armhf
+RUN dpkg --add-architecture arm64
+
+RUN apt-get update && \
+	apt-get install -y \
+	make \
+	cmake \
+	git \
+	bash \
+	screen \
+	postgresql-client \
+	libpq-dev \
+	mosquitto-clients \
+	redis-tools \
+	rpm \
+	clang \
+	libclang-3.9-dev \
+	golang-cfssl \
+	jq \
+	gcc-arm-linux-gnueabihf \
+	g++-arm-linux-gnueabihf \
+	gcc-aarch64-linux-gnu \
+	g++-aarch64-linux-gnu \
+	zlib1g-dev:armhf \
+	zlib1g-dev:arm64
+
+RUN rustup component add rustfmt clippy
+RUN rustup target add armv7-unknown-linux-gnueabihf
+RUN rustup target add aarch64-unknown-linux-gnu
+
+RUN cargo install diesel_cli --version 2.0.0-rc.0 --no-default-features --features postgres
+RUN cargo install cargo-deb
+RUN cargo install cargo-rpm
+
+WORKDIR $PROJECT_PATH/chirpstack

Makefile

@@ -1,18 +1,8 @@
 .PHONY: dist api
 
-# Builds a debug / development binary.
-build-debug:
-	docker-compose run --rm chirpstack make debug
-
-# Builds a release binary.
-build-release:
-	docker-compose run --rm chirpstack make release
-
 # Build distributable binaries.
 dist:
-	# The pull is needed as else the specified platform is not respected.
+	docker-compose run --rm --no-deps chirpstack make dist
-	docker-compose pull chirpstack-build-amd64 && docker-compose run --rm chirpstack-build-amd64 make dist
-	docker-compose pull chirpstack-build-arm64 && docker-compose run --rm chirpstack-build-arm64 make dist
 
 # Set the versions
 version:
@@ -31,7 +21,7 @@ api: version
 
 # Builds the UI.
 build-ui:
-	docker-compose run --rm chirpstack-ui make build
+	docker-compose run --rm --no-deps chirpstack-ui make build
 
 # Enters the devshell for ChirpStack development.
 devshell:
@@ -52,5 +42,5 @@ test-server: build-ui
 
 # Update the Docker development images
 update-images:
-	docker-compose pull chirpstack
+	docker-compose build chirpstack
 	docker-compose build chirpstack-ui

chirpstack/Makefile

@@ -1,29 +1,107 @@
-.PHONY: debug release release-deb test test-server dbshell dist
+.PHONY: dist
 
 TARGET_ARCH := $(shell rustc --print cfg |grep target_arch |sed 's/target_arch="\(.*\)"/\1/')
 PKG_VERSION := $(shell cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version')
 
-debug:
+debug-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	cargo build
 
-release:
+debug-armv7hf:
+	# We can not install this by default, as only one libpq-dev version can be
+	# installed at a time.
+	apt update && apt install -y libpq-dev:armhf
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo build --target armv7-unknown-linux-gnueabihf
+
+debug-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo build --target aarch64-unknown-linux-gnu
+
+release-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	cargo build --release
 
-dist: release-deb release-rpm release-targz
+release-armv7hf:
+	apt update && apt install -y libpq-dev:armhf
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo build --target armv7-unknown-linux-gnueabihf --release
 
-release-deb:
+release-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo build --target aarch64-unknown-linux-gnu --release
+
+dist: release-deb-amd64 \
+	release-deb-armv7hf \
+	release-deb-arm64 \
+	release-rpm-amd64 \
+	release-rpm-armv7hf \
+	release-rpm-arm64 \
+	release-targz-amd64 \
+	release-targz-armv7hf \
+	release-targz-arm64 \
+
+release-deb-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	mkdir -p /chirpstack/dist
 	cargo deb
-	cp /target/debian/*.deb /chirpstack/dist
+	cp ../target/debian/*.deb /chirpstack/dist
 
-release-rpm:
+release-deb-armv7hf:
+	apt update && apt install -y libpq-dev:armhf
+	mkdir -p /chirpstack/dist
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo deb --target armv7-unknown-linux-gnueabihf
+	cp ../target/armv7-unknown-linux-gnueabihf/debian/*.deb /chirpstack/dist
+
+release-deb-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	mkdir -p /chirpstack/dist
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo deb --target aarch64-unknown-linux-gnu
+	cp ../target/aarch64-unknown-linux-gnu/debian/*.deb /chirpstack/dist
+
+release-rpm-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	mkdir -p /chirpstack/dist
 	cargo rpm build
-	find /target/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
+	find ../target/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
 
-release-targz:
+release-rpm-armv7hf:
+	apt update && apt install -y libpq-dev:armhf
 	mkdir -p /chirpstack/dist
-	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_$(TARGET_ARCH).tar.gz -C /target/release chirpstack
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo rpm build --target armv7-unknown-linux-gnueabihf
+	find ../target/armv7-unknown-linux-gnueabihf/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
+
+release-rpm-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	mkdir -p /chirpstack/dist
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo rpm build --target aarch64-unknown-linux-gnu
+	find ../target/aarch64-unknown-linux-gnu/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
+
+release-targz-amd64: release-amd64
+	mkdir -p /chirpstack/dist
+	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_amd64.tar.gz -C ../target/release chirpstack
+
+release-targz-armv7hf: release-armv7hf
+	mkdir -p /chirpstack/dist
+	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_armv7hf.tar.gz -C ../target/armv7-unknown-linux-gnueabihf/release chirpstack
+
+release-targz-arm64: release-arm64
+	mkdir -p /chirpstack/dist
+	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_arm64.tar.gz -C ../target/aarch64-unknown-linux-gnu/release chirpstack
 
 test:
 	cargo fmt --check
@@ -33,8 +111,8 @@ test:
 test-lrwn:
 	cd ../lrwn && make test
 
-test-server: debug configuration/certs/ca.pem
+test-server: debug-amd64 configuration/certs/ca.pem
-	/target/debug/chirpstack -c ./configuration
+	../target/debug/chirpstack -c ./configuration
 
 dbshell:
 	psql -h postgres -U chirpstack chirpstack

docker-compose.yml

@@ -1,9 +1,14 @@
 services:
   chirpstack:
-    image: chirpstack/chirpstack-dev-cache:latest
+    build:
-    platform: linux/amd64
+      context: .
+      dockerfile: Dockerfile-devel
     volumes:
       - ./:/chirpstack
+      - ./.rust/target:/chirpstack/target
+      - ./.rust/.cargo/registry/index:/usr/local/cargo/registry/index
+      - ./.rust/.cargo/registry/cache:/usr/local/cargo/registry/cache
+      - ./.rust/.cargo/git/db:/usr/local/cargo/git/db
     depends_on:
       - postgres
       - redis