concurrency and caching (#1)

* use concurrency * run here too * allow test failure * only push on push * fix coresight mode build * support but disable coresight * woops * no unicorn for arm * update codeql and fix makefile * forgot those * disable forward cache * fix needing buildx Co-authored-by: Ruben ten Hove <ruben.tenhove@tno.nl>
2025-06-10 01:01:33 +00:00 · 2022-06-22 06:36:35 -04:00 · 2022-06-22 06:36:35 -04:00 · 0da7ddb738
commit 0da7ddb738
parent 6e960f78d6
5 changed files with 88 additions and 29 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,3 +1,4 @@
+/.github
 .test
 .test2
 .sync_tmp
@ -10,6 +11,7 @@ ld
 in
 out
 core*
+!/coresight_mode
 afl-analyze
 afl-as
 afl-clang
--- a/.github/workflows/build-test-push-container.yaml
+++ b/.github/workflows/build-test-push-container.yaml
@ -1,5 +1,4 @@
-name: Build, test and push container image
-
+name: Build and test container image
 on:
  push:
    branches:
@ -7,10 +6,33 @@ on:
      - dev
    tags:
      - '*'
+  pull_request:
+    branch:
+      - stable
+      - dev

 jobs:
-  build:
-    name: Build, test and push container image
+  build-amd64:
+    name: Build and test amd64
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build amd64
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64
+          load: true
+          tags: aflplusplus/aflplusplus:test
+          # cache-from: type=gha  # Ensure we always build a fresh image. We just use the cache for the subsequent push job.
+          cache-to: type=gha,mode=max
+      - name: Test linux/amd64 image
+        run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests
+        continue-on-error: true
+  build-arm64:
+    name: Build and test arm64
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
@ -20,36 +42,47 @@ jobs:
          platforms: arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
-      - name: Build amd64 image
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: linux/amd64
-          load: true
-          tags: aflplusplus/aflplusplus:test
-      - name: Build arm64 image
+      - name: Build arm64
        uses: docker/build-push-action@v3
        with:
          context: .
          platforms: linux/arm64
          load: true
          tags: aflplusplus/aflplusplus:test
-      - name: Test linux/amd64 image
-        run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests
+          # cache-from: type=gha  # Ensure we always build a fresh image. We just use the cache for the subsequent push job.
+          cache-to: type=gha,mode=max
      - name: Test linux/arm64 image
        run: docker run --rm --platform linux/arm64 aflplusplus/aflplusplus:test make tests
+        continue-on-error: true
+  push:
+    needs:
+      - build-amd64
+      - build-arm64
+    if: ${{ github.event_name == 'push' }}
+    name: Push image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
      - name: Login to Dockerhub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Publish ${{ github.ref_name }} to docker.io registry
+      - name: Publish ${{ github.ref_name }} for amd64 and arm64 to docker.io registry
        uses: docker/build-push-action@v3
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: aflplusplus/aflplusplus:${{ github.ref_name }}
+          cache-from: type=gha
+          # cache-to: type=gha,mode=max  # No need to add to cache as we'll never use this
        if: ${{ github.ref_name != 'stable' }}
      - name: Publish stable and latest to docker.io registry
        uses: docker/build-push-action@v3
@ -58,4 +91,6 @@ jobs:
          platforms: linux/amd64,linux/arm64
          push: true
          tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest
+          cache-from: type=gha
+          # cache-to: type=gha,mode=max  # No need to add to cache as we'll never use this
        if: ${{ github.ref_name == 'stable' }}
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@ -21,12 +21,12 @@ jobs:
      uses: actions/checkout@v3

    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}

    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2

    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2
--- a/16
+++ b/16
@ -33,9 +33,9 @@ ENV PATH=$PATH:/etc/cargo/bin
 ARG LLVM_VERSION=14
 ARG GCC_VERSION=12

-RUN mkdir -p /usr/local/share/keyrings && \
-    echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \
-    wget -qO /usr/local/share/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key
+RUN mkdir -p /etc/apt/keyrings && \
+    echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \
+    wget -qO /etc/apt/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key

 RUN apt-get update && \
    apt-get -y install --no-install-recommends \
@ -45,9 +45,11 @@ RUN apt-get update && \
    libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \
    liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \
    libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \
-    $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) && \
+    $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) \
+    $([ "$(dpkg --print-architecture)" = "arm64" ] && echo libcapstone-dev) && \
    rm -rf /var/lib/apt/lists/*
-    # arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86
+    # gcc-multilib is only used for -m32 support on x86
+    # libcapstone-dev is used for coresight_mode on arm64

 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \
    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0
@ -66,6 +68,10 @@ COPY . .
 # Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029
 ENV NO_NYX 1

+# Build currently broken
+ENV NO_CORESIGHT 1
+ENV NO_UNICORN_ARM64 1
+
 RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && make clean && \
    make distrib && make install && make clean

--- a/26
+++ b/26
@ -623,15 +623,23 @@ endif
 	$(MAKE) -C frida_mode
 ifneq "$(SYS)" "Darwin"
 ifeq "$(ARCH)" "aarch64"
+  ifndef NO_CORESIGHT
 	$(MAKE) -C coresight_mode
+  endif
 endif
 ifeq "$(SYS)" "Linux"
-ifndef NO_NYX
+  ifndef NO_NYX
 	cd nyx_mode && ./build_nyx_support.sh
-endif
+  endif
 endif
 	cd qemu_mode && sh ./build_qemu_support.sh
+  ifeq "$(ARCH)" "aarch64"
+    ifndef NO_UNICORN_ARM64
 	cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+    endif
+  else
+	cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+  endif
 endif

 .PHONY: binary-only
@ -645,15 +653,23 @@ binary-only: test_shm test_python ready $(PROGS)
 	$(MAKE) -C frida_mode
 ifneq "$(SYS)" "Darwin"
 ifeq "$(ARCH)" "aarch64"
+  ifndef NO_CORESIGHT
 	$(MAKE) -C coresight_mode
+  endif
 endif
 ifeq "$(SYS)" "Linux"
 ifndef NO_NYX
-	-cd nyx_mode && ./build_nyx_support.sh
+	cd nyx_mode && ./build_nyx_support.sh
 endif
 endif
-	-cd qemu_mode && sh ./build_qemu_support.sh
-	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+	cd qemu_mode && sh ./build_qemu_support.sh
+  ifeq "$(ARCH)" "aarch64"
+    ifndef NO_UNICORN_ARM64
+	cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+    endif
+  else
+	cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+  endif
 endif

 .PHONY: source-only