Merge pull request #1453 from rhtenhove/dev

improve CI and container image build

.custom-format.py

@@ -19,40 +19,22 @@ import subprocess
 import sys
 import os
 import re
+import shutil
 
 # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # future use
 
 with open(".clang-format") as f:
     fmt = f.read()
 
-CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN")
+CURRENT_LLVM = os.getenv('LLVM_VERSION', 14)
-if CLANG_FORMAT_BIN is None:
+CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "")
-    o = 0
+
-    try:
+if shutil.which(CLANG_FORMAT_BIN) is None:
-        p = subprocess.Popen(["clang-format-11", "--version"], stdout=subprocess.PIPE)
+    CLANG_FORMAT_BIN = f"clang-format-{CURRENT_LLVM}"
-        o, _ = p.communicate()
+
-        o = str(o, "utf-8")
+if shutil.which(CLANG_FORMAT_BIN) is None:
-        o = re.sub(r".*ersion ", "", o)
+    print(f"[!] clang-format-{CURRENT_LLVM} is needed. Aborted.")
-        # o = o[len("clang-format version "):].strip()
+    exit(1)
-        o = o[: o.find(".")]
-        o = int(o)
-    except:
-        print("clang-format-11 is needed. Aborted.")
-        exit(1)
-    # if o < 7:
-    #    if subprocess.call(['which', 'clang-format-7'], stdout=subprocess.PIPE) == 0:
-    #        CLANG_FORMAT_BIN = 'clang-format-7'
-    #    elif subprocess.call(['which', 'clang-format-8'], stdout=subprocess.PIPE) == 0:
-    #        CLANG_FORMAT_BIN = 'clang-format-8'
-    #    elif subprocess.call(['which', 'clang-format-9'], stdout=subprocess.PIPE) == 0:
-    #        CLANG_FORMAT_BIN = 'clang-format-9'
-    #    elif subprocess.call(['which', 'clang-format-11'], stdout=subprocess.PIPE) == 0:
-    #        CLANG_FORMAT_BIN = 'clang-format-11'
-    #    else:
-    #        print ("clang-format 7 or above is needed. Aborted.")
-    #        exit(1)
-    else:
-        CLANG_FORMAT_BIN = "clang-format-11"
 
 COLUMN_LIMIT = 80
 for line in fmt.split("\n"):

.dockerignore

@@ -1,65 +1,75 @@
+!/coresight_mode
+*.dSYM
+*.o
+*.pyc
+*.so
+.sync_tmp
 .test
 .test2
-.sync_tmp
+.git
-*.o
+.dockerignore
-*.so
+.github
-*.pyc
+CITATION.cff
-*.dSYM
+CONTRIBUTING.md
-as
+Changelog.md
-ld
+Dockerfile
-in
+LICENSE
-out
+TODO.md
-core*
 afl-analyze
-afl-as
-afl-clang
-afl-clang\+\+
-afl-clang-fast
-afl-clang-fast\+\+
-afl-clang-lto
-afl-clang-lto\+\+
-afl-fuzz
-afl-g\+\+
-afl-gcc
-afl-gcc-fast
-afl-g\+\+-fast
-afl-gotcpu
-afl-ld
-afl-ld-lto
-afl-qemu-trace
-afl-showmap
-afl-tmin
 afl-analyze.8
+afl-as
 afl-as.8
-afl-clang-fast\+\+.8
+afl-clang
+afl-clang-fast
 afl-clang-fast.8
+afl-clang-fast\+\+
+afl-clang-fast\+\+.8
+afl-clang-lto
 afl-clang-lto.8
+afl-clang-lto\+\+
 afl-clang-lto\+\+.8
+afl-clang\+\+
 afl-cmin.8
 afl-cmin.bash.8
+afl-fuzz
 afl-fuzz.8
-afl-gcc.8
+afl-g\+\+
-afl-gcc-fast.8
+afl-g\+\+-fast
 afl-g\+\+-fast.8
+afl-gcc
+afl-gcc-fast
+afl-gcc-fast.8
+afl-gcc.8
+afl-gotcpu
 afl-gotcpu.8
+afl-ld
+afl-ld-lto
 afl-plot.8
+afl-qemu-trace
+afl-showmap
 afl-showmap.8
 afl-system-config.8
+afl-tmin
 afl-tmin.8
 afl-whatsup.8
+as
+core*
+examples/afl_frida/afl-frida
+examples/afl_frida/frida-gum-example.c
+examples/afl_frida/frida-gum.h
+examples/afl_frida/libtestinstr.so
+examples/afl_network_proxy/afl-network-client
+examples/afl_network_proxy/afl-network-server
+in
+ld
+out
 qemu_mode/libcompcov/compcovtest
 qemu_mode/qemu-*
+test/unittests/unit_hash
+test/unittests/unit_list
+test/unittests/unit_maybe_alloc
+test/unittests/unit_preallocable
+test/unittests/unit_rand
 unicorn_mode/samples/*/\.test-*
 unicorn_mode/samples/*/output
 unicorn_mode/unicornafl
-test/unittests/unit_maybe_alloc
-test/unittests/unit_preallocable
-test/unittests/unit_list
-test/unittests/unit_rand
-test/unittests/unit_hash
-examples/afl_network_proxy/afl-network-server
-examples/afl_network_proxy/afl-network-client
-examples/afl_frida/afl-frida
-examples/afl_frida/libtestinstr.so
-examples/afl_frida/frida-gum-example.c
-examples/afl_frida/frida-gum.h

.github/workflows/build_aflplusplus_docker.yaml

@@ -1,49 +0,0 @@
-name: Publish Docker Images
-
-on:
-  push:
-    branches:
-      - stable
-      - dev
-    tags:
-      - '*'
-
-jobs:
-  push_to_registry:
-    name: Push Docker images to Dockerhub
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-      - name: Login to Dockerhub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Publish dev as dev to docker.io registry
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: aflplusplus/aflplusplus:${{ github.ref_name }}
-        if: ${{ github.ref_name == 'dev' }}
-      - name: Publish stable as stable and latest to docker.io registry
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest
-        if: ${{ github.ref_name == 'stable' }}
-      - name: Publish tagged release to docker.io registry
-        uses: docker/build-push-action@v3
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: aflplusplus/aflplusplus:${{ github.ref_name }}
-        if: ${{ github.ref_type == 'tag' }}

.github/workflows/ci.yml

@@ -2,13 +2,16 @@ name: CI
 
 on:
   push:
-    branches: [ stable, dev ]
+    branches:
+      - stable
+      - dev
   pull_request:
-    branches: [ stable, dev ]
+    branches:
+      - dev # No need for stable-pull-request, as that equals dev-push
 
 jobs:
   linux:
-    runs-on: '${{ matrix.os }}'
+    runs-on: "${{ matrix.os }}"
     strategy:
       matrix:
         os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04]
@@ -16,7 +19,7 @@ jobs:
       AFL_SKIP_CPUFREQ: 1
       AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: debug
         run: apt-cache search plugin-dev | grep gcc-; echo; apt-cache search clang-format- | grep clang-format-
       - name: update
@@ -38,7 +41,7 @@ jobs:
       AFL_SKIP_CPUFREQ: 1
       AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: install
         run: brew install make gcc llvm
       - name: fix install

.github/workflows/code-format.yml

@@ -0,0 +1,33 @@
+name: Formatting
+
+on:
+  push:
+    branches:
+      - stable
+      - dev
+  pull_request:
+    branches:
+      - dev # No need for stable-pull-request, as that equals dev-push
+
+jobs:
+  code-format-check:
+    name: Check code format
+    if: ${{ 'false' == 'true' }} # Disable the job
+    runs-on: ubuntu-22.04
+    container: docker.io/aflplusplus/aflplusplus:dev
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Format
+        run: |
+          git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus
+          apt-get update
+          apt-get install -y clang-format-${LLVM_VERSION}
+          make code-format
+      - name: Check if code needed formatting
+        run: |
+          git --no-pager -c color.ui=always diff HEAD
+          if ! git diff HEAD --quiet; then
+            echo "[!] Please run 'make code-format' and push its changes."
+            exit 1
+          fi

.github/workflows/codeql-analysis.yml

@@ -2,31 +2,32 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ stable, dev ]
+    branches:
+      - stable
+      - dev
   pull_request:
-    branches: [ stable, dev ]
+    branches:
+      - dev # No need for stable-pull-request, as that equals dev-push
 
 jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
-
+    container: # We use a previous image as it's expected to have all the dependencies
-    strategy:
+      image: docker.io/aflplusplus/aflplusplus:dev
-      fail-fast: false
-      matrix:
-        language: [ 'cpp' ]
-
     steps:
-    - name: Checkout repository
+      - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760
-      uses: actions/checkout@v2
+        run: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus
-
+      - name: Checkout
-    - name: Initialize CodeQL
+        uses: actions/checkout@v3
-      uses: github/codeql-action/init@v1
+      - name: Initialize CodeQL
-      with:
+        uses: github/codeql-action/init@v2
-        languages: ${{ matrix.language }}
+        with:
-
+          languages: cpp, python
-    - name: Autobuild
+      - name: Build AFLplusplus # Rebuild because CodeQL needs to monitor the build process
-      uses: github/codeql-action/autobuild@v1
+        env:
-
+          CC: gcc # These are symlinked to the version used in the container build
-    - name: Perform CodeQL Analysis
+          CXX: g++
-      uses: github/codeql-action/analyze@v1
+        run: make -i all # Best effort using -i
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

.github/workflows/container.yml

@@ -0,0 +1,75 @@
+name: Container
+on:
+  push:
+    branches:
+      - stable
+      - dev
+    tags:
+      - "*"
+  pull_request:
+    branches:
+      - dev # No need for stable-pull-request, as that equals dev-push
+
+jobs:
+  build-and-test-amd64:
+    name: Test amd64 image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build amd64
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          tags: aflplusplus:test-amd64
+          load: true
+          cache-to: type=gha,mode=max
+          build-args: |
+            TEST_BUILD=1
+      - name: Test amd64
+        run: >
+          docker run --rm aflplusplus:test-amd64 bash -c "
+          apt-get update && 
+          apt-get install -y libcmocka-dev && 
+          make -i tests
+          "
+
+  push:
+    name: Push amd64 and arm64 images
+    runs-on: ubuntu-latest
+    needs:
+      - build-and-test-amd64
+    if: ${{ github.event_name == 'push' && github.repository == 'AFLplusplus/AFLplusplus' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to docker.io
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Set tags to push
+        id: push-tags
+        run: |
+          PUSH_TAGS=docker.io/aflplusplus/aflplusplus:${GITHUB_REF_NAME}
+          if [ "${GITHUB_REF_NAME}" = "stable" ]; then
+            PUSH_TAGS=${PUSH_TAGS},docker.io/aflplusplus/aflplusplus:latest
+          fi
+          export PUSH_TAGS
+          echo "::set-output name=PUSH_TAGS::${PUSH_TAGS}"
+      - name: Push to docker.io registry
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.push-tags.outputs.PUSH_TAGS }}
+          cache-from: type=gha

.github/workflows/rust_custom_mutator.yml

@@ -2,9 +2,12 @@ name: Rust Custom Mutators
 
 on:
   push:
-    branches: [ stable, dev ]
+    branches:
+      - stable
+      - dev
   pull_request:
-    branches: [ stable, dev ]
+    branches:
+      - dev # No need for stable-pull-request, as that equals dev-push
 
 jobs:
   test:
@@ -17,7 +20,7 @@ jobs:
       matrix:
         os: [ubuntu-22.04, ubuntu-20.04]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install Rust Toolchain
         uses: actions-rs/toolchain@v1
         with:

Dockerfile

@@ -7,79 +7,82 @@
 
 FROM ubuntu:22.04 AS aflplusplus
 LABEL "maintainer"="afl++ team <afl@aflplus.plus>"
-LABEL "about"="AFLplusplus docker image"
+LABEL "about"="AFLplusplus container image"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-env NO_ARCH_OPT 1
+ENV NO_ARCH_OPT=1
-
+ENV IS_DOCKER=1
-RUN apt-get update && \
-    apt-get -y install --no-install-suggests --no-install-recommends \
-    automake \
-    cmake \
-    meson \
-    ninja-build \
-    bison flex \
-    build-essential \
-    git \
-    python3 python3-dev python3-setuptools python-is-python3 \
-    libtool libtool-bin \
-    libglib2.0-dev \
-    wget vim jupp nano bash-completion less \
-    apt-utils apt-transport-https ca-certificates gnupg dialog \
-    joe vim ssh psmisc \
-    cargo libgtk-3-dev \
-    libpixman-1-dev \
-    gnuplot-nox \
-    && rm -rf /var/lib/apt/lists/*
-
-# TODO: reactivate in timely manner
-#RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" >> /etc/apt/sources.list && \
-#    wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-
-RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu jammy main" >> /etc/apt/sources.list && \
-    apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 1E9377A2BA9EF27F
 
 RUN apt-get update && apt-get full-upgrade -y && \
-    apt-get -y install --no-install-suggests --no-install-recommends \
+    apt-get install -y --no-install-recommends wget ca-certificates && \
-    gcc-11 g++-11 gcc-11-plugin-dev gdb lcov \
+    rm -rf /var/lib/apt/lists/*
-    clang-14 clang-tools-14 libc++1-14 libc++-14-dev \
-    libc++abi1-14 libc++abi-14-dev libclang1-14 libclang-14-dev \
-    libclang-common-14-dev libclang-cpp14 libclang-cpp14-dev liblld-14 \
-    liblld-14-dev liblldb-14 liblldb-14-dev libllvm14 libomp-14-dev \
-    libomp5-14 lld-14 lldb-14 llvm-14 llvm-14-dev llvm-14-runtime llvm-14-tools
 
-# arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86
+ENV LLVM_VERSION=14
-ARG TARGETPLATFORM
+ENV GCC_VERSION=11
-RUN [ "$TARGETPLATFORM" = "linux/amd64" ] && \
-    apt-get -y install --no-install-suggests --no-install-recommends \
-    gcc-10-multilib gcc-multilib || true
 
-RUN rm -rf /var/lib/apt/lists/*
+RUN echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \
+    wget -qO /etc/apt/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key
 
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 0
+RUN apt-get update && \
-RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 0
+    apt-get -y install --no-install-recommends \
+    make cmake automake meson ninja-build bison flex \
+    git xz-utils bzip2 wget jupp nano bash-completion less vim joe ssh psmisc \
+    python3 python3-dev python3-setuptools python-is-python3 \
+    libtool libtool-bin libglib2.0-dev \
+    apt-utils apt-transport-https gnupg dialog \
+    gnuplot-nox libpixman-1-dev \
+    gcc-${GCC_VERSION} g++-${GCC_VERSION} gcc-${GCC_VERSION}-plugin-dev gdb lcov \
+    clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} \
+    libc++-${LLVM_VERSION}-dev libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev \
+    libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \
+    libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} \
+    libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \
+    liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev \
+    libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev libomp5-${LLVM_VERSION} \
+    lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} \
+    llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \
+    $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) \
+    $([ "$(dpkg --print-architecture)" = "arm64" ] && echo libcapstone-dev) && \
+    rm -rf /var/lib/apt/lists/*
+    # gcc-multilib is only used for -m32 support on x86
+    # libcapstone-dev is used for coresight_mode on arm64
 
-ENV LLVM_CONFIG=llvm-config-14
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \
+    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 && \
+    update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 0 && \
+    update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 0
+
+RUN wget -qO- https://sh.rustup.rs | CARGO_HOME=/etc/cargo sh -s -- -y -q --no-modify-path
+ENV PATH=$PATH:/etc/cargo/bin
+
+ENV LLVM_CONFIG=llvm-config-${LLVM_VERSION}
 ENV AFL_SKIP_CPUFREQ=1
 ENV AFL_TRY_AFFINITY=1
 ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
 
-RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov /afl-cov
+RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov && \
-RUN cd /afl-cov && make install && cd ..
+    (cd afl-cov && make install) && rm -rf afl-cov
+
+# Build currently broken
+ENV NO_CORESIGHT=1
+ENV NO_UNICORN_ARM64=1
 
-COPY . /AFLplusplus
 WORKDIR /AFLplusplus
+COPY . .
 
-RUN export CC=gcc-11 && export CXX=g++-11 && make clean && \
+ARG CC=gcc-$GCC_VERSION
-    make distrib && make install && make clean
+ARG CXX=g++-$GCC_VERSION
 
-RUN sh -c 'echo set encoding=utf-8 > /root/.vimrc'
+# Used in CI to prevent a 'make clean' which would remove the binaries to be tested
-RUN echo '. /etc/bash_completion' >> ~/.bashrc
+ARG TEST_BUILD
-RUN echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc
-RUN echo "export PS1='"'[afl++ \h] \w$(__git_ps1) \$ '"'" >> ~/.bashrc
-ENV IS_DOCKER="1"
 
-# Disabled as there are now better alternatives
+RUN sed -i.bak 's/^	-/	/g' GNUmakefile && \
-#COPY --from=aflplusplus/afl-dyninst /usr/local/lib/libdyninstAPI_RT.so /usr/local/lib/libdyninstAPI_RT.so
+    make clean && make distrib && \
-#COPY --from=aflplusplus/afl-dyninst /afl-dyninst/libAflDyninst.so /usr/local/lib/libAflDyninst.so
+    ([ "${TEST_BUILD}" ] || (make install && make clean)) && \
+    mv GNUmakefile.bak GNUmakefile
+
+RUN echo "set encoding=utf-8" > /root/.vimrc && \
+    echo ". /etc/bash_completion" >> ~/.bashrc && \
+    echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc && \
+    echo "export PS1='"'[afl++ \h] \w$(__git_ps1) \$ '"'" >> ~/.bashrc

GNUmakefile

@@ -312,7 +312,7 @@ all:	test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu
 
 .PHONY: llvm
 llvm:
-	-$(MAKE) -j4 -f GNUmakefile.llvm
+	-$(MAKE) -j$(nproc) -f GNUmakefile.llvm
 	@test -e afl-cc || { echo "[-] Compiling afl-cc failed. You seem not to have a working compiler." ; exit 1; }
 
 .PHONY: gcc_plugin
@@ -572,7 +572,7 @@ clean:
 	-$(MAKE) -f GNUmakefile.gcc_plugin clean
 	-$(MAKE) -C utils/libdislocator clean
 	-$(MAKE) -C utils/libtokencap clean
-	$(MAKE) -C utils/aflpp_driver clean
+	-$(MAKE) -C utils/aflpp_driver clean
 	-$(MAKE) -C utils/afl_network_proxy clean
 	-$(MAKE) -C utils/socket_fuzzing clean
 	-$(MAKE) -C utils/argv_fuzzing clean
@@ -610,7 +610,7 @@ endif
 
 .PHONY: distrib
 distrib: all
-	-$(MAKE) -j4 -f GNUmakefile.llvm
+	-$(MAKE) -j$(nproc) -f GNUmakefile.llvm
 ifneq "$(SYS)" "Darwin"
 	-$(MAKE) -f GNUmakefile.gcc_plugin
 endif
@@ -623,15 +623,23 @@ endif
 	-$(MAKE) -C frida_mode
 ifneq "$(SYS)" "Darwin"
 ifeq "$(ARCH)" "aarch64"
+  ifndef NO_CORESIGHT
 	-$(MAKE) -C coresight_mode
+  endif
 endif
 ifeq "$(SYS)" "Linux"
-ifndef NO_NYX
+  ifndef NO_NYX
 	-cd nyx_mode && ./build_nyx_support.sh
-endif
+  endif
 endif
 	-cd qemu_mode && sh ./build_qemu_support.sh
+  ifeq "$(ARCH)" "aarch64"
+    ifndef NO_UNICORN_ARM64
 	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+    endif
+  else
+	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+  endif
 endif
 
 .PHONY: binary-only
@@ -645,7 +653,9 @@ binary-only: test_shm test_python ready $(PROGS)
 	-$(MAKE) -C frida_mode
 ifneq "$(SYS)" "Darwin"
 ifeq "$(ARCH)" "aarch64"
+  ifndef NO_CORESIGHT
 	-$(MAKE) -C coresight_mode
+  endif
 endif
 ifeq "$(SYS)" "Linux"
 ifndef NO_NYX
@@ -653,12 +663,18 @@ ifndef NO_NYX
 endif
 endif
 	-cd qemu_mode && sh ./build_qemu_support.sh
+  ifeq "$(ARCH)" "aarch64"
+    ifndef NO_UNICORN_ARM64
 	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+    endif
+  else
+	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+  endif
 endif
 
 .PHONY: source-only
 source-only: all
-	-$(MAKE) -j4 -f GNUmakefile.llvm
+	-$(MAKE) -j$(nproc) -f GNUmakefile.llvm
 ifneq "$(SYS)" "Darwin"
 	-$(MAKE) -f GNUmakefile.gcc_plugin
 endif

coresight_mode/GNUmakefile

@@ -54,7 +54,7 @@ $(GLIBC_LDSO): | $(GLIBC_NAME).tar.xz
 	  $(MAKE) install
 
 $(GLIBC_NAME).tar.xz:
-	wget -O $@ $(GLIBC_URL_BASE)/$@
+	wget -qO $@ $(GLIBC_URL_BASE)/$@
 
 clean:
 	$(MAKE) -C $(CS_TRACE) clean

custom_mutators/grammar_mutator/build_grammar_mutator.sh

@@ -128,7 +128,7 @@ git pull >/dev/null 2>&1
 sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null
 git checkout "$GRAMMAR_VERSION" || exit 1
 echo "[*] Downloading antlr..."
-wget -c https://www.antlr.org/download/antlr-4.8-complete.jar
+wget -q https://www.antlr.org/download/antlr-4.8-complete.jar
 cd ..
 
 echo

frida_mode/GNUmakefile

@@ -275,7 +275,7 @@ endif
 
 else
 $(GUM_DEVKIT_TARBALL): | $(FRIDA_BUILD_DIR)
-	wget -O $@ $(GUM_DEVKIT_URL) || curl -L -o $@ $(GUM_DEVKIT_URL)
+	wget -qO $@ $(GUM_DEVKIT_URL) || curl -L -o $@ $(GUM_DEVKIT_URL)
 
 $(GUM_DEVIT_LIBRARY): $(GUM_DEVKIT_TARBALL)
 	tar Jxvfm $(GUM_DEVKIT_TARBALL) -C $(FRIDA_BUILD_DIR)

nyx_mode/build_nyx_support.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+
+set -e
+
 echo "================================================="
 echo "           Nyx build script"
 echo "================================================="
@@ -6,14 +9,14 @@ echo
 
 echo "[*] Performing basic sanity checks..."
 
-if [ ! "`uname -s`" = "Linux" ]; then
+if [ ! "$(uname -s)" = "Linux" ]; then
 
   echo "[-] Error: Nyx mode is only available on Linux."
   exit 0
 
 fi
 
-if [ ! "`uname -m`" = "x86_64" ]; then
+if [ ! "$(uname -m)" = "x86_64" ]; then
 
   echo "[-] Error: Nyx mode is only available on x86_64 (yet)."
   exit 0
@@ -22,10 +25,10 @@ fi
 
 echo "[*] Making sure all Nyx is checked out"
 
-git status 1>/dev/null 2>/dev/null
-if [ $? -eq 0 ]; then
 
-  git submodule init || exit 1
+if git status 1>/dev/null 2>&1; then
+
+  git submodule init
   echo "[*] initializing QEMU-Nyx submodule"
   git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors
   echo "[*] initializing packer submodule"
@@ -47,32 +50,27 @@ test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install gi
 
 echo "[*] checking packer init.cpio.gz ..."
 if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then
-    cd packer/linux_initramfs/
+    (cd packer/linux_initramfs/ && sh pack.sh)
-    sh pack.sh || exit 1
-    cd ../../
 fi
 
 echo "[*] Checking libnyx ..."
 if [ ! -f "libnyx/libnyx/target/release/liblibnyx.a" ]; then
-    cd libnyx/libnyx
+    (cd libnyx/libnyx && cargo build --release)
-    cargo build --release || exit 1
-    cd ../../
 fi
 
 echo "[*] Checking QEMU-Nyx ..."
 if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then
-    cd QEMU-Nyx/
+    
-    ./compile_qemu_nyx.sh static || exit 1
+    if ! dpkg -s gtk3-devel > /dev/null 2>&1; then
-    cd ..
+        echo "[-] Disabling GTK because gtk3-devel is not installed."
+        sed -i 's/--enable-gtk//g' QEMU-Nyx/compile_qemu_nyx.sh
+    fi
+    (cd QEMU-Nyx && ./compile_qemu_nyx.sh static)
 fi
 
 echo "[*] Checking libnyx.so ..."
-if [ -f "libnyx/libnyx/target/release/liblibnyx.so" ]; then
+cp libnyx/libnyx/target/release/liblibnyx.so ../libnyx.so
-  cp -v libnyx/libnyx/target/release/liblibnyx.so ../libnyx.so || exit 1
+
-else
-  echo "[ ] libnyx.so not found..."
-  exit 1
-fi
 echo "[+] All done for nyx_mode, enjoy!"
 
 exit 0

qemu_mode/build_qemu_support.sh

@@ -273,7 +273,7 @@ echo "[+] Configuration complete."
 
 echo "[*] Attempting to build QEMU (fingers crossed!)..."
 
-make -j `nproc` || exit 1
+make -j$(nproc) || exit 1
 
 echo "[+] Build process successful!"