Refactor testing and (cross)compile.

With the update to diesel 2.0.0-rc0, it is now possible to cross-compile without emulation. Note that during each (cross)compile, we have to install libpq-dev for the target platform, as only one version can be installed at a time (e.g. libpq-dev:amd64 conflicts with libpq-dev:armhf).
2025-01-18 02:29:45 +00:00 · 2022-07-21 11:00:47 +01:00 · 2022-07-21 11:00:47 +01:00 · 5436582f03
commit 5436582f03
parent 9db867906d
11 changed files with 227 additions and 75 deletions
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
--- a/.dockerignore
+++ b/.dockerignore
@ -1,5 +1,6 @@
 .git
 .cargo
+.rust
 **/target
 **/node_modules
 Dockerfile
--- a/.github/buildkitd.toml
+++ b/.github/buildkitd.toml
@ -0,0 +1,2 @@
+[worker.oci]
+  max-parallelism = 1
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -14,12 +14,22 @@ jobs:
      -
        name: Checkout
        uses: actions/checkout@v2
+      -
+        name: Cargo cache
+        uses: actions/cache@v3
+        with:
+          path: .rust/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      -
        name: Build UI
        run: make build-ui
      -
        name: Run tests
        run: make test
+      -
+        name: Make cache accessible to caching action
+        run: |
+          sudo chown $(whoami):$(whoami) -R .rust

  dist:
    needs: tests
@ -30,11 +40,11 @@ jobs:
        name: Checkout
        uses: actions/checkout@v2
      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        name: Cargo cache
+        uses: actions/cache@v3
+        with:
+          path: .rust/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      -
        name: Build UI
        run: make build-ui
@ -52,6 +62,11 @@ jobs:
        name: Upload binaries to S3
        run: |
          aws s3 sync dist s3://builds.loraserver.io/chirpstack
+        if: startsWith(github.ref, 'refs/tags/v')
+      -
+        name: Make cache accessible to caching action
+        run: |
+          sudo chown $(whoami):$(whoami) -R .rust

  docker:
    needs: tests
@ -78,6 +93,8 @@ jobs:
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
+        with:
+          config: .github/buildkitd.toml
      -
        name: Login to DockerHub
        uses: docker/login-action@v1 
@ -89,7 +106,7 @@ jobs:
        id: docker_build
        uses: docker/build-push-action@v2
        with:
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64,linux/arm/v7,linux/arm64
          push: ${{ startsWith(github.ref, 'refs/tags/v') }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
--- a/.gitignore
+++ b/.gitignore
@ -1,6 +1,7 @@
 # hidden files
 .*
 !/chirpstack/.rpm
+!/.cargo

 # Log files
 *.log
--- a/61
+++ b/61
@ -1,4 +1,5 @@
-FROM alpine:3.15.0 AS ui-build
+# UI build stage
+FROM --platform=$BUILDPLATFORM alpine:3.15.0 AS ui-build

 ENV PROJECT_PATH=/chirpstack

@ -9,15 +10,61 @@ COPY ./api/grpc-web $PROJECT_PATH/api/grpc-web
 COPY ./ui $PROJECT_PATH/ui

 RUN cd $PROJECT_PATH/ui && \
-		yarn install --network-timeout 600000 && \
-		yarn build
+	yarn install --network-timeout 600000 && \
+	yarn build

-FROM chirpstack/chirpstack-dev-cache:latest AS rust-build
+
+# ChirpStack build stage
+FROM --platform=$BUILDPLATFORM rust:1.62.0-buster AS rust-build
+
+ENV PROJECT_PATH=/chirpstack
+RUN mkdir -p $PROJECT_PATH
+
+RUN dpkg --add-architecture armhf
+RUN dpkg --add-architecture arm64
+
+RUN apt-get update && \
+	apt-get install -y \
+	make \
+	cmake \
+	git \
+	libpq-dev \
+	clang \
+	libclang-3.9-dev \
+	jq \
+	gcc-arm-linux-gnueabihf \
+	g++-arm-linux-gnueabihf \
+	gcc-aarch64-linux-gnu \
+	g++-aarch64-linux-gnu \
+	zlib1g-dev:armhf \
+	zlib1g-dev:arm64
+
+RUN rustup target add armv7-unknown-linux-gnueabihf
+RUN rustup target add aarch64-unknown-linux-gnu
+
+ARG TARGETPLATFORM
+RUN mkdir -p /release/$TARGETPLATFORM

 COPY . $PROJECT_PATH
 COPY --from=ui-build $PROJECT_PATH/ui/build $PROJECT_PATH/ui/build
-RUN cd $PROJECT_PATH/chirpstack && cargo build --release

+RUN case "$TARGETPLATFORM" in \
+	"linux/amd64") \
+		cd $PROJECT_PATH/chirpstack && make release-amd64; \
+		cp $PROJECT_PATH/target/release/chirpstack /release/$TARGETPLATFORM; \
+		;; \
+	"linux/arm/v7") \
+		cd $PROJECT_PATH/chirpstack && make release-armv7hf; \
+		cp $PROJECT_PATH/target/armv7-unknown-linux-gnueabihf/release/chirpstack /release/$TARGETPLATFORM; \
+		;; \
+	"linux/arm64") \
+		cd $PROJECT_PATH/chirpstack && make release-arm64; \
+		cp $PROJECT_PATH/target/aarch64-unknown-linux-gnu/release/chirpstack /release/$TARGETPLATFORM; \
+		;; \
+	esac;
+
+
+# Final stage
 FROM debian:buster-slim as production

 RUN apt-get update && \
@ -26,7 +73,9 @@ RUN apt-get update && \
 		libpq5 \
 		&& rm -rf /var/lib/apt/lists/*

-COPY --from=rust-build /target/release/chirpstack /usr/bin/chirpstack
+ARG TARGETPLATFORM
+
+COPY --from=rust-build /release/$TARGETPLATFORM/chirpstack /usr/bin/chirpstack
 COPY --from=rust-build /chirpstack/chirpstack/configuration/* /etc/chirpstack/
 USER nobody:nogroup
 ENTRYPOINT ["/usr/bin/chirpstack"]
--- a/36
+++ b/36
@ -1,36 +0,0 @@
-FROM rust:1.62.0-buster
-
-ENV PROJECT_PATH=/chirpstack
-ENV CARGO_TARGET_DIR=/target
-
-RUN mkdir -p $PROJECT_PATH
-RUN mkdir -p $CARGO_TARGET_DIR
-
-RUN apt-get update && \
-        apt-get install -y \
-		make \
-		cmake \
-		git \
-		bash \
-		screen \
-		postgresql-client \
-		libpq-dev \
-		mosquitto-clients \
-		redis-tools \
-		rpm \
-		clang \
-		golang-cfssl \
-		jq \
-		&& rm -rf /var/lib/apt/lists/*
-
-RUN rustup component add rustfmt clippy
-
-RUN cargo install diesel_cli --version 2.0.0-rc.0 --no-default-features --features postgres
-RUN cargo install cargo-deb
-RUN cargo install cargo-rpm
-
-COPY . $PROJECT_PATH
-
-WORKDIR $PROJECT_PATH/chirpstack
-RUN cargo build --release
-RUN rm -rf $PROJECT_PATH/*
--- a/40
+++ b/40
@ -0,0 +1,40 @@
+FROM rust:1.62.0-buster
+
+ENV PROJECT_PATH=/chirpstack
+RUN mkdir -p $PROJECT_PATH
+
+RUN dpkg --add-architecture armhf
+RUN dpkg --add-architecture arm64
+
+RUN apt-get update && \
+	apt-get install -y \
+	make \
+	cmake \
+	git \
+	bash \
+	screen \
+	postgresql-client \
+	libpq-dev \
+	mosquitto-clients \
+	redis-tools \
+	rpm \
+	clang \
+	libclang-3.9-dev \
+	golang-cfssl \
+	jq \
+	gcc-arm-linux-gnueabihf \
+	g++-arm-linux-gnueabihf \
+	gcc-aarch64-linux-gnu \
+	g++-aarch64-linux-gnu \
+	zlib1g-dev:armhf \
+	zlib1g-dev:arm64
+
+RUN rustup component add rustfmt clippy
+RUN rustup target add armv7-unknown-linux-gnueabihf
+RUN rustup target add aarch64-unknown-linux-gnu
+
+RUN cargo install diesel_cli --version 2.0.0-rc.0 --no-default-features --features postgres
+RUN cargo install cargo-deb
+RUN cargo install cargo-rpm
+
+WORKDIR $PROJECT_PATH/chirpstack
--- a/16
+++ b/16
@ -1,18 +1,8 @@
 .PHONY: dist api

-# Builds a debug / development binary.
-build-debug:
-	docker-compose run --rm chirpstack make debug
-
-# Builds a release binary.
-build-release:
-	docker-compose run --rm chirpstack make release
-
 # Build distributable binaries.
 dist:
-	# The pull is needed as else the specified platform is not respected.
-	docker-compose pull chirpstack-build-amd64 && docker-compose run --rm chirpstack-build-amd64 make dist
-	docker-compose pull chirpstack-build-arm64 && docker-compose run --rm chirpstack-build-arm64 make dist
+	docker-compose run --rm --no-deps chirpstack make dist

 # Set the versions
 version:
@ -31,7 +21,7 @@ api: version

 # Builds the UI.
 build-ui:
-	docker-compose run --rm chirpstack-ui make build
+	docker-compose run --rm --no-deps chirpstack-ui make build

 # Enters the devshell for ChirpStack development.
 devshell:
@ -52,5 +42,5 @@ test-server: build-ui

 # Update the Docker development images
 update-images:
-	docker-compose pull chirpstack
+	docker-compose build chirpstack
 	docker-compose build chirpstack-ui
--- a/chirpstack/Makefile
+++ b/chirpstack/Makefile
@ -1,29 +1,107 @@
-.PHONY: debug release release-deb test test-server dbshell dist
+.PHONY: dist

 TARGET_ARCH := $(shell rustc --print cfg |grep target_arch |sed 's/target_arch="\(.*\)"/\1/')
 PKG_VERSION := $(shell cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version')

-debug:
+debug-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	cargo build

-release:
+debug-armv7hf:
+	# We can not install this by default, as only one libpq-dev version can be
+	# installed at a time.
+	apt update && apt install -y libpq-dev:armhf
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo build --target armv7-unknown-linux-gnueabihf
+
+debug-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo build --target aarch64-unknown-linux-gnu
+
+release-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	cargo build --release

-dist: release-deb release-rpm release-targz
+release-armv7hf:
+	apt update && apt install -y libpq-dev:armhf
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo build --target armv7-unknown-linux-gnueabihf --release

-release-deb:
+release-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo build --target aarch64-unknown-linux-gnu --release
+
+dist: release-deb-amd64 \
+	release-deb-armv7hf \
+	release-deb-arm64 \
+	release-rpm-amd64 \
+	release-rpm-armv7hf \
+	release-rpm-arm64 \
+	release-targz-amd64 \
+	release-targz-armv7hf \
+	release-targz-arm64 \
+
+release-deb-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	mkdir -p /chirpstack/dist
 	cargo deb
-	cp /target/debian/*.deb /chirpstack/dist
+	cp ../target/debian/*.deb /chirpstack/dist

-release-rpm:
+release-deb-armv7hf:
+	apt update && apt install -y libpq-dev:armhf
+	mkdir -p /chirpstack/dist
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo deb --target armv7-unknown-linux-gnueabihf
+	cp ../target/armv7-unknown-linux-gnueabihf/debian/*.deb /chirpstack/dist
+
+release-deb-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	mkdir -p /chirpstack/dist
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo deb --target aarch64-unknown-linux-gnu
+	cp ../target/aarch64-unknown-linux-gnu/debian/*.deb /chirpstack/dist
+
+release-rpm-amd64:
+	apt update && apt install -y libpq-dev:amd64
 	mkdir -p /chirpstack/dist
 	cargo rpm build
-	find /target/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
+	find ../target/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;

-release-targz:
+release-rpm-armv7hf:
+	apt update && apt install -y libpq-dev:armhf
 	mkdir -p /chirpstack/dist
-	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_$(TARGET_ARCH).tar.gz -C /target/release chirpstack
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/arm-linux-gnueabihf" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/arm-linux-gnueabihf" \
+		cargo rpm build --target armv7-unknown-linux-gnueabihf
+	find ../target/armv7-unknown-linux-gnueabihf/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
+
+release-rpm-arm64:
+	apt update && apt install -y libpq-dev:arm64
+	mkdir -p /chirpstack/dist
+	BINDGEN_EXTRA_CLANG_ARGS="--sysroot=/usr/aarch64-linux-gnu" \
+	PKG_CONFIG_SYSROOT_DIR="/usr/aarch64-linux-gnu" \
+		cargo rpm build --target aarch64-unknown-linux-gnu
+	find ../target/aarch64-unknown-linux-gnu/release/rpmbuild/RPMS -type f -exec cp {} /chirpstack/dist \;
+
+release-targz-amd64: release-amd64
+	mkdir -p /chirpstack/dist
+	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_amd64.tar.gz -C ../target/release chirpstack
+
+release-targz-armv7hf: release-armv7hf
+	mkdir -p /chirpstack/dist
+	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_armv7hf.tar.gz -C ../target/armv7-unknown-linux-gnueabihf/release chirpstack
+
+release-targz-arm64: release-arm64
+	mkdir -p /chirpstack/dist
+	tar -czvf /chirpstack/dist/chirpstack_$(PKG_VERSION)_arm64.tar.gz -C ../target/aarch64-unknown-linux-gnu/release chirpstack

 test:
 	cargo fmt --check
@ -33,8 +111,8 @@ test:
 test-lrwn:
 	cd ../lrwn && make test

-test-server: debug configuration/certs/ca.pem
-	/target/debug/chirpstack -c ./configuration
+test-server: debug-amd64 configuration/certs/ca.pem
+	../target/debug/chirpstack -c ./configuration

 dbshell:
 	psql -h postgres -U chirpstack chirpstack
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,9 +1,14 @@
 services:
  chirpstack:
-    image: chirpstack/chirpstack-dev-cache:latest
-    platform: linux/amd64
+    build:
+      context: .
+      dockerfile: Dockerfile-devel
    volumes:
      - ./:/chirpstack
+      - ./.rust/target:/chirpstack/target
+      - ./.rust/.cargo/registry/index:/usr/local/cargo/registry/index
+      - ./.rust/.cargo/registry/cache:/usr/local/cargo/registry/cache
+      - ./.rust/.cargo/git/db:/usr/local/cargo/git/db
    depends_on:
      - postgres
      - redis