From d9c2487c3e23fd39f146baf49c89b805ea56ddb1 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 10:30:30 -0400 Subject: [PATCH 01/24] reduce layers, size and complexity --- Dockerfile | 66 ++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/Dockerfile b/Dockerfile index bdfa1c56..e845c0e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,16 +9,16 @@ LABEL "about"="AFLplusplus docker image" ARG DEBIAN_FRONTEND=noninteractive -env NO_ARCH_OPT 1 +ENV NO_ARCH_OPT 1 RUN apt-get update && \ - apt-get -y install --no-install-suggests --no-install-recommends \ + apt-get -y install --no-install-recommends \ automake \ + make \ cmake \ meson \ ninja-build \ bison flex \ - build-essential \ git \ python3 python3-dev python3-setuptools python-is-python3 \ libtool libtool-bin \ @@ -29,53 +29,51 @@ RUN apt-get update && \ gnuplot-nox \ && rm -rf /var/lib/apt/lists/* -# TODO: reactivate in timely manner -#RUN echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" >> /etc/apt/sources.list && \ -# wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - +ARG LLVM_VERSION=14 +ARG GCC_VERSION=12 -RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu jammy main" >> /etc/apt/sources.list && \ - apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 1E9377A2BA9EF27F +RUN echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ + wget -qO /usr/local/share/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key + +# RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu jammy main" >> /etc/apt/sources.list && \ +# apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 1E9377A2BA9EF27F RUN apt-get update && apt-get full-upgrade -y && \ - apt-get -y install --no-install-suggests --no-install-recommends \ - gcc-12 g++-12 gcc-12-plugin-dev gdb lcov \ - clang-14 clang-tools-14 libc++1-14 libc++-14-dev \ - libc++abi1-14 libc++abi-14-dev libclang1-14 libclang-14-dev \ - libclang-common-14-dev libclang-cpp14 libclang-cpp14-dev liblld-14 \ - liblld-14-dev liblldb-14 liblldb-14-dev libllvm14 libomp-14-dev \ - libomp5-14 lld-14 lldb-14 llvm-14 llvm-14-dev llvm-14-runtime llvm-14-tools + apt-get -y install --no-install-recommends \ + gcc-${GCC_VERSION} g++-${GCC_VERSION} gcc-${GCC_VERSION}-plugin-dev gdb lcov \ + clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} libc++-${LLVM_VERSION}-dev \ + libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \ + libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ + liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \ + libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ + && rm -rf /var/lib/apt/lists/* # arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86 ARG TARGETPLATFORM RUN [ "$TARGETPLATFORM" = "linux/amd64" ] && \ - apt-get -y install --no-install-suggests --no-install-recommends \ - gcc-10-multilib gcc-multilib || true + apt-get -y install --no-install-recommends \ + gcc-${LLVM_VERSION}-multilib gcc-multilib \ + && rm -rf /var/lib/apt/lists/* -RUN rm -rf /var/lib/apt/lists/* +# RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${LLVM_VERSION} 0 && \ +# update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${LLVM_VERSION} 0 -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 0 -RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 0 - -ENV LLVM_CONFIG=llvm-config-14 +ENV LLVM_CONFIG=llvm-config-${LLVM_VERSION} ENV AFL_SKIP_CPUFREQ=1 ENV AFL_TRY_AFFINITY=1 ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 -RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov /afl-cov -RUN cd /afl-cov && make install && cd .. +RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov && \ + (cd afl-cov && make install) && rm -rf afl-cov -COPY . /AFLplusplus WORKDIR /AFLplusplus +COPY . . -RUN export CC=gcc-12 && export CXX=g++-12 && make clean && \ +RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && make clean && \ make distrib && make install && make clean -RUN sh -c 'echo set encoding=utf-8 > /root/.vimrc' -RUN echo '. /etc/bash_completion' >> ~/.bashrc -RUN echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc -RUN echo "export PS1='"'[afl++ \h] \w$(__git_ps1) \$ '"'" >> ~/.bashrc +RUN echo "set encoding=utf-8" > /root/.vimrc && \ + echo ". /etc/bash_completion" >> ~/.bashrc && \ + echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc && \ + echo "export PS1='"'[afl++ \h] \w$(__git_ps1) \$ '"'" >> ~/.bashrc ENV IS_DOCKER="1" - -# Disabled as there are now better alternatives -#COPY --from=aflplusplus/afl-dyninst /usr/local/lib/libdyninstAPI_RT.so /usr/local/lib/libdyninstAPI_RT.so -#COPY --from=aflplusplus/afl-dyninst /afl-dyninst/libAflDyninst.so /usr/local/lib/libAflDyninst.so From 01cb84051a9d4bfd33123902577f9310bb6c251d Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 16:54:47 +0200 Subject: [PATCH 02/24] secure apt key; fix multilib --- Dockerfile | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index e845c0e7..87ef1a91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,12 +13,9 @@ ENV NO_ARCH_OPT 1 RUN apt-get update && \ apt-get -y install --no-install-recommends \ - automake \ - make \ - cmake \ - meson \ - ninja-build \ - bison flex \ + make cmake automake \ + meson ninja-build bison flex \ + xz-utils \ git \ python3 python3-dev python3-setuptools python-is-python3 \ libtool libtool-bin \ @@ -26,18 +23,16 @@ RUN apt-get update && \ wget vim jupp nano bash-completion less \ apt-utils apt-transport-https ca-certificates gnupg dialog \ libpixman-1-dev \ - gnuplot-nox \ - && rm -rf /var/lib/apt/lists/* + gnuplot-nox && \ + rm -rf /var/lib/apt/lists/* ARG LLVM_VERSION=14 ARG GCC_VERSION=12 -RUN echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ +RUN mkdir -p /usr/local/share/keyrings && \ + echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ wget -qO /usr/local/share/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key -# RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu jammy main" >> /etc/apt/sources.list && \ -# apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 1E9377A2BA9EF27F - RUN apt-get update && apt-get full-upgrade -y && \ apt-get -y install --no-install-recommends \ gcc-${GCC_VERSION} g++-${GCC_VERSION} gcc-${GCC_VERSION}-plugin-dev gdb lcov \ @@ -45,16 +40,16 @@ RUN apt-get update && apt-get full-upgrade -y && \ libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \ libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \ - libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ - && rm -rf /var/lib/apt/lists/* + libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools && \ + rm -rf /var/lib/apt/lists/* # arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86 -ARG TARGETPLATFORM -RUN [ "$TARGETPLATFORM" = "linux/amd64" ] && \ - apt-get -y install --no-install-recommends \ - gcc-${LLVM_VERSION}-multilib gcc-multilib \ - && rm -rf /var/lib/apt/lists/* - +RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ + apt-get update; \ + apt-get -y install --no-install-recommends \ + gcc-${GCC_VERSION}-multilib gcc-multilib; \ + rm -rf /var/lib/apt/lists/*; \ + fi # RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${LLVM_VERSION} 0 && \ # update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${LLVM_VERSION} 0 From a5822c7c2fc34e52defe630b418db7f856bc4f89 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 18:52:43 +0200 Subject: [PATCH 03/24] various changes --- ...er.yaml => build-push-test-container.yaml} | 35 ++++++------ .github/workflows/ci.yml | 5 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/rust_custom_mutator.yml | 4 +- Dockerfile | 14 +++-- GNUmakefile | 56 +++++++++---------- GNUmakefile.gcc_plugin | 2 +- coresight_mode/GNUmakefile | 2 +- .../grammar_mutator/build_grammar_mutator.sh | 2 +- frida_mode/GNUmakefile | 2 +- 10 files changed, 65 insertions(+), 59 deletions(-) rename .github/workflows/{build_aflplusplus_docker.yaml => build-push-test-container.yaml} (60%) diff --git a/.github/workflows/build_aflplusplus_docker.yaml b/.github/workflows/build-push-test-container.yaml similarity index 60% rename from .github/workflows/build_aflplusplus_docker.yaml rename to .github/workflows/build-push-test-container.yaml index 7245a84e..8a6d77d1 100644 --- a/.github/workflows/build_aflplusplus_docker.yaml +++ b/.github/workflows/build-push-test-container.yaml @@ -1,4 +1,4 @@ -name: Publish Docker Images +name: Build, test and push container image on: push: @@ -9,29 +9,38 @@ on: - '*' jobs: - push_to_registry: - name: Push Docker images to Dockerhub + build: + name: Build, test and push container image runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + - uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Build image + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + load: true + tags: aflplusplus/aflplusplus:test + - name: Test linux/amd64 image + run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests + - name: Test linux/arm64 image + run: docker run --rm --platform linux/arm64 aflplusplus/aflplusplus:test make tests - name: Login to Dockerhub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Publish dev as dev to docker.io registry + - name: Publish ${{ github.ref_name }} to docker.io registry uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: aflplusplus/aflplusplus:${{ github.ref_name }} - if: ${{ github.ref_name == 'dev' }} - - name: Publish stable as stable and latest to docker.io registry + if: ${{ github.ref_name != 'stable' }} + - name: Publish stable and latest to docker.io registry uses: docker/build-push-action@v3 with: context: . @@ -39,11 +48,3 @@ jobs: push: true tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest if: ${{ github.ref_name == 'stable' }} - - name: Publish tagged release to docker.io registry - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: aflplusplus/aflplusplus:${{ github.ref_name }} - if: ${{ github.ref_type == 'tag' }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 799b72e7..9f0cacca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: AFL_SKIP_CPUFREQ: 1 AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: debug run: apt-cache search plugin-dev | grep gcc-; echo; apt-cache search clang-format- | grep clang-format- - name: update @@ -38,7 +38,7 @@ jobs: AFL_SKIP_CPUFREQ: 1 AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: install run: brew install make gcc - name: fix install @@ -51,3 +51,4 @@ jobs: run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests - name: force frida test for MacOS run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr + \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index eda8dfd0..d5dce05f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Initialize CodeQL uses: github/codeql-action/init@v1 diff --git a/.github/workflows/rust_custom_mutator.yml b/.github/workflows/rust_custom_mutator.yml index c279439e..eb3e2581 100644 --- a/.github/workflows/rust_custom_mutator.yml +++ b/.github/workflows/rust_custom_mutator.yml @@ -17,7 +17,7 @@ jobs: matrix: os: [ubuntu-22.04, ubuntu-20.04] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install Rust Toolchain uses: actions-rs/toolchain@v1 with: @@ -27,4 +27,4 @@ jobs: - name: Run General Tests run: cargo test - name: Run Tests for afl_internals feature flag - run: cd custom_mutator && cargo test --features=afl_internals \ No newline at end of file + run: cd custom_mutator && cargo test --features=afl_internals diff --git a/Dockerfile b/Dockerfile index 87ef1a91..21c229be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,11 +11,11 @@ ARG DEBIAN_FRONTEND=noninteractive ENV NO_ARCH_OPT 1 -RUN apt-get update && \ +RUN apt-get update && apt-get full-upgrade -y && \ apt-get -y install --no-install-recommends \ make cmake automake \ meson ninja-build bison flex \ - xz-utils \ + xz-utils libbz2-1.0 \ git \ python3 python3-dev python3-setuptools python-is-python3 \ libtool libtool-bin \ @@ -26,6 +26,9 @@ RUN apt-get update && \ gnuplot-nox && \ rm -rf /var/lib/apt/lists/* +RUN wget -qO- https://sh.rustup.rs | CARGO_HOME=/etc/cargo sh -s -- -y -q --no-modify-path +ENV PATH=$PATH:/etc/cargo/bin + ARG LLVM_VERSION=14 ARG GCC_VERSION=12 @@ -33,7 +36,7 @@ RUN mkdir -p /usr/local/share/keyrings && \ echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ wget -qO /usr/local/share/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key -RUN apt-get update && apt-get full-upgrade -y && \ +RUN apt-get update && \ apt-get -y install --no-install-recommends \ gcc-${GCC_VERSION} g++-${GCC_VERSION} gcc-${GCC_VERSION}-plugin-dev gdb lcov \ clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} libc++-${LLVM_VERSION}-dev \ @@ -50,8 +53,9 @@ RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ gcc-${GCC_VERSION}-multilib gcc-multilib; \ rm -rf /var/lib/apt/lists/*; \ fi -# RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${LLVM_VERSION} 0 && \ -# update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${LLVM_VERSION} 0 + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 ENV LLVM_CONFIG=llvm-config-${LLVM_VERSION} ENV AFL_SKIP_CPUFREQ=1 diff --git a/GNUmakefile b/GNUmakefile index 42d48b68..fad619cf 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -610,42 +610,42 @@ endif .PHONY: distrib distrib: all - -$(MAKE) -j4 -f GNUmakefile.llvm + $(MAKE) -j4 -f GNUmakefile.llvm ifneq "$(SYS)" "Darwin" - -$(MAKE) -f GNUmakefile.gcc_plugin + $(MAKE) -f GNUmakefile.gcc_plugin endif - -$(MAKE) -C utils/libdislocator - -$(MAKE) -C utils/libtokencap - -$(MAKE) -C utils/afl_network_proxy - -$(MAKE) -C utils/socket_fuzzing - -$(MAKE) -C utils/argv_fuzzing - # -$(MAKE) -C utils/plot_ui - -$(MAKE) -C frida_mode + $(MAKE) -C utils/libdislocator + $(MAKE) -C utils/libtokencap + $(MAKE) -C utils/afl_network_proxy + $(MAKE) -C utils/socket_fuzzing + $(MAKE) -C utils/argv_fuzzing + # $(MAKE) -C utils/plot_ui + $(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" - -$(MAKE) -C coresight_mode + $(MAKE) -C coresight_mode endif ifeq "$(SYS)" "Linux" ifndef NO_NYX - -cd nyx_mode && ./build_nyx_support.sh + cd nyx_mode && ./build_nyx_support.sh endif endif - -cd qemu_mode && sh ./build_qemu_support.sh - -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + cd qemu_mode && sh ./build_qemu_support.sh + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif .PHONY: binary-only binary-only: test_shm test_python ready $(PROGS) - -$(MAKE) -C utils/libdislocator - -$(MAKE) -C utils/libtokencap - -$(MAKE) -C utils/afl_network_proxy - -$(MAKE) -C utils/socket_fuzzing - -$(MAKE) -C utils/argv_fuzzing - # -$(MAKE) -C utils/plot_ui - -$(MAKE) -C frida_mode + $(MAKE) -C utils/libdislocator + $(MAKE) -C utils/libtokencap + $(MAKE) -C utils/afl_network_proxy + $(MAKE) -C utils/socket_fuzzing + $(MAKE) -C utils/argv_fuzzing + # $(MAKE) -C utils/plot_ui + $(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" - -$(MAKE) -C coresight_mode + $(MAKE) -C coresight_mode endif ifeq "$(SYS)" "Linux" ifndef NO_NYX @@ -658,13 +658,13 @@ endif .PHONY: source-only source-only: all - -$(MAKE) -j4 -f GNUmakefile.llvm + $(MAKE) -j4 -f GNUmakefile.llvm ifneq "$(SYS)" "Darwin" - -$(MAKE) -f GNUmakefile.gcc_plugin + $(MAKE) -f GNUmakefile.gcc_plugin endif - -$(MAKE) -C utils/libdislocator - -$(MAKE) -C utils/libtokencap - # -$(MAKE) -C utils/plot_ui + $(MAKE) -C utils/libdislocator + $(MAKE) -C utils/libtokencap + # $(MAKE) -C utils/plot_ui ifeq "$(SYS)" "Linux" ifndef NO_NYX -cd nyx_mode && ./build_nyx_support.sh @@ -712,9 +712,9 @@ install: all $(MANPAGES) @if [ -f utils/afl_network_proxy/afl-network-server ]; then $(MAKE) -C utils/afl_network_proxy install; fi @if [ -f utils/aflpp_driver/libAFLDriver.a ]; then set -e; install -m 644 utils/aflpp_driver/libAFLDriver.a $${DESTDIR}$(HELPER_PATH); fi @if [ -f utils/aflpp_driver/libAFLQemuDriver.a ]; then set -e; install -m 644 utils/aflpp_driver/libAFLQemuDriver.a $${DESTDIR}$(HELPER_PATH); fi - -$(MAKE) -f GNUmakefile.llvm install + $(MAKE) -f GNUmakefile.llvm install ifneq "$(SYS)" "Darwin" - -$(MAKE) -f GNUmakefile.gcc_plugin install + $(MAKE) -f GNUmakefile.gcc_plugin install endif ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-gcc ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-g++ diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index e21203ae..5e651688 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -122,7 +122,7 @@ test_deps: # @echo "[*] Checking for gcc for plugin support..." # @$(CC) -v 2>&1 | grep -q -- --enable-plugin || ( echo "[-] Oops, this gcc has not been configured with plugin support."; exit 1 ) @echo "[*] Checking for gcc plugin development header files..." - @test -d `$(CC) -print-file-name=plugin`/include || ( echo "[-] Oops, can't find gcc header files. Be sure to install 'gcc-X-plugin-dev'."; exit 1 ) + -@test -d `$(CC) -print-file-name=plugin`/include || ( echo "[-] Oops, can't find gcc header files. Be sure to install 'gcc-X-plugin-dev'."; exit 1 ) @echo "[*] Checking for './afl-showmap'..." @test -f ./afl-showmap || ( echo "[-] Oops, can't find './afl-showmap'. Be sure to compile AFL first."; exit 1 ) @echo "[+] All set and ready to build." diff --git a/coresight_mode/GNUmakefile b/coresight_mode/GNUmakefile index 9ab30ff7..167b83fa 100644 --- a/coresight_mode/GNUmakefile +++ b/coresight_mode/GNUmakefile @@ -54,7 +54,7 @@ $(GLIBC_LDSO): | $(GLIBC_NAME).tar.xz $(MAKE) install $(GLIBC_NAME).tar.xz: - wget -O $@ $(GLIBC_URL_BASE)/$@ + wget -qO $@ $(GLIBC_URL_BASE)/$@ clean: $(MAKE) -C $(CS_TRACE) clean diff --git a/custom_mutators/grammar_mutator/build_grammar_mutator.sh b/custom_mutators/grammar_mutator/build_grammar_mutator.sh index e8594ba3..5121b07f 100755 --- a/custom_mutators/grammar_mutator/build_grammar_mutator.sh +++ b/custom_mutators/grammar_mutator/build_grammar_mutator.sh @@ -128,7 +128,7 @@ git pull >/dev/null 2>&1 sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null git checkout "$GRAMMAR_VERSION" || exit 1 echo "[*] Downloading antlr..." -wget -c https://www.antlr.org/download/antlr-4.8-complete.jar +wget -q https://www.antlr.org/download/antlr-4.8-complete.jar cd .. echo diff --git a/frida_mode/GNUmakefile b/frida_mode/GNUmakefile index 014dcca5..28e570f9 100644 --- a/frida_mode/GNUmakefile +++ b/frida_mode/GNUmakefile @@ -275,7 +275,7 @@ endif else $(GUM_DEVKIT_TARBALL): | $(FRIDA_BUILD_DIR) - wget -O $@ $(GUM_DEVKIT_URL) || curl -L -o $@ $(GUM_DEVKIT_URL) + wget -qO $@ $(GUM_DEVKIT_URL) || curl -L -o $@ $(GUM_DEVKIT_URL) $(GUM_DEVIT_LIBRARY): $(GUM_DEVKIT_TARBALL) tar Jxvfm $(GUM_DEVKIT_TARBALL) -C $(FRIDA_BUILD_DIR) From fc98d53e27861aace92b360cb51dd320d0a65867 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 18:54:54 +0200 Subject: [PATCH 04/24] separate platforms --- .github/workflows/build-push-test-container.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-push-test-container.yaml b/.github/workflows/build-push-test-container.yaml index 8a6d77d1..6f58d78c 100644 --- a/.github/workflows/build-push-test-container.yaml +++ b/.github/workflows/build-push-test-container.yaml @@ -16,11 +16,18 @@ jobs: - uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build image + - name: Build amd64 image uses: docker/build-push-action@v3 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 + load: true + tags: aflplusplus/aflplusplus:test + - name: Build arm64 image + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/arm64 load: true tags: aflplusplus/aflplusplus:test - name: Test linux/amd64 image From c7db5b67f9602cabd73e0aa1e227744e6261e000 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 21:30:43 +0200 Subject: [PATCH 05/24] just disable nyx --- Dockerfile | 19 ++++++++---------- nyx_mode/build_nyx_support.sh | 38 +++++++++++++++++------------------ 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 21c229be..e9b1bc7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,12 +10,13 @@ LABEL "about"="AFLplusplus docker image" ARG DEBIAN_FRONTEND=noninteractive ENV NO_ARCH_OPT 1 +ENV IS_DOCKER="1" RUN apt-get update && apt-get full-upgrade -y && \ apt-get -y install --no-install-recommends \ make cmake automake \ meson ninja-build bison flex \ - xz-utils libbz2-1.0 \ + xz-utils bzip2 \ git \ python3 python3-dev python3-setuptools python-is-python3 \ libtool libtool-bin \ @@ -43,16 +44,10 @@ RUN apt-get update && \ libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \ libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \ - libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools && \ + libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ + $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) && \ rm -rf /var/lib/apt/lists/* - -# arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86 -RUN if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ - apt-get update; \ - apt-get -y install --no-install-recommends \ - gcc-${GCC_VERSION}-multilib gcc-multilib; \ - rm -rf /var/lib/apt/lists/*; \ - fi + # arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 @@ -68,6 +63,9 @@ RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov && \ WORKDIR /AFLplusplus COPY . . +# Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029 +ENV NO_NYX 1 + RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && make clean && \ make distrib && make install && make clean @@ -75,4 +73,3 @@ RUN echo "set encoding=utf-8" > /root/.vimrc && \ echo ". /etc/bash_completion" >> ~/.bashrc && \ echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc && \ echo "export PS1='"'[afl++ \h] \w$(__git_ps1) \$ '"'" >> ~/.bashrc -ENV IS_DOCKER="1" diff --git a/nyx_mode/build_nyx_support.sh b/nyx_mode/build_nyx_support.sh index 83e0ae32..c4a480e6 100755 --- a/nyx_mode/build_nyx_support.sh +++ b/nyx_mode/build_nyx_support.sh @@ -1,4 +1,7 @@ #!/bin/bash + +set -e + echo "=================================================" echo " Nyx build script" echo "=================================================" @@ -6,14 +9,14 @@ echo echo "[*] Performing basic sanity checks..." -if [ ! "`uname -s`" = "Linux" ]; then +if [ ! "$(uname -s)" = "Linux" ]; then echo "[-] Error: Nyx mode is only available on Linux." exit 0 fi -if [ ! "`uname -m`" = "x86_64" ]; then +if [ ! "$(uname -m)" = "x86_64" ]; then echo "[-] Error: Nyx mode is only available on x86_64 (yet)." exit 0 @@ -22,10 +25,10 @@ fi echo "[*] Making sure all Nyx is checked out" -git status 1>/dev/null 2>/dev/null -if [ $? -eq 0 ]; then - git submodule init || exit 1 +if git status 1>/dev/null 2>&1; then + + git submodule init echo "[*] initializing QEMU-Nyx submodule" git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors echo "[*] initializing packer submodule" @@ -47,32 +50,27 @@ test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install gi echo "[*] checking packer init.cpio.gz ..." if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then - cd packer/linux_initramfs/ - sh pack.sh || exit 1 - cd ../../ + (cd packer/linux_initramfs/ && sh pack.sh) fi echo "[*] Checking libnyx ..." if [ ! -f "libnyx/libnyx/target/release/liblibnyx.a" ]; then - cd libnyx/libnyx - cargo build --release || exit 1 - cd ../../ + (cd libnyx/libnyx && cargo build --release) fi echo "[*] Checking QEMU-Nyx ..." if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then - cd QEMU-Nyx/ - ./compile_qemu_nyx.sh static || exit 1 - cd .. + + if ! [ "${IS_DOCKER}" = "" ]; then + echo "[-] Disabling GTK as we're building a container image." + sed -i 's/--enable-gtk//g' QEMU-Nyx/compile_qemu_nyx.sh + fi + (cd QEMU-Nyx && ./compile_qemu_nyx.sh static) fi echo "[*] Checking libnyx.so ..." -if [ -f "libnyx/libnyx/target/release/liblibnyx.so" ]; then - cp -v libnyx/libnyx/target/release/liblibnyx.so ../libnyx.so || exit 1 -else - echo "[ ] libnyx.so not found..." - exit 1 -fi +cp libnyx/libnyx/target/release/liblibnyx.so ../libnyx.so + echo "[+] All done for nyx_mode, enjoy!" exit 0 From 32820230471737c7fa7740df736d75d3e8282def Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 21:36:02 +0200 Subject: [PATCH 06/24] check for package instead of if in docker --- nyx_mode/build_nyx_support.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nyx_mode/build_nyx_support.sh b/nyx_mode/build_nyx_support.sh index c4a480e6..e7fca64f 100755 --- a/nyx_mode/build_nyx_support.sh +++ b/nyx_mode/build_nyx_support.sh @@ -61,8 +61,8 @@ fi echo "[*] Checking QEMU-Nyx ..." if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then - if ! [ "${IS_DOCKER}" = "" ]; then - echo "[-] Disabling GTK as we're building a container image." + if ! dpkg -s gtk3-devel > /dev/null 2>&1; then + echo "[-] Disabling GTK because gtk3-devel is not installed." sed -i 's/--enable-gtk//g' QEMU-Nyx/compile_qemu_nyx.sh fi (cd QEMU-Nyx && ./compile_qemu_nyx.sh static) From 6e960f78d69d1ac8325b93849bd2e04ae502bddb Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 21 Jun 2022 22:01:30 +0200 Subject: [PATCH 07/24] arm needs qemu indeed --- .github/workflows/build-push-test-container.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-push-test-container.yaml b/.github/workflows/build-push-test-container.yaml index 6f58d78c..e2b828ac 100644 --- a/.github/workflows/build-push-test-container.yaml +++ b/.github/workflows/build-push-test-container.yaml @@ -14,6 +14,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build amd64 image From 0da7ddb738df60b3650d36832c9ede040a7ba6b9 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Wed, 22 Jun 2022 06:36:35 -0400 Subject: [PATCH 08/24] concurrency and caching (#1) * use concurrency * run here too * allow test failure * only push on push * fix coresight mode build * support but disable coresight * woops * no unicorn for arm * update codeql and fix makefile * forgot those * disable forward cache * fix needing buildx Co-authored-by: Ruben ten Hove --- .dockerignore | 4 +- ...er.yaml => build-test-push-container.yaml} | 65 ++++++++++++++----- .github/workflows/codeql-analysis.yml | 6 +- Dockerfile | 16 +++-- GNUmakefile | 26 ++++++-- 5 files changed, 88 insertions(+), 29 deletions(-) rename .github/workflows/{build-push-test-container.yaml => build-test-push-container.yaml} (53%) diff --git a/.dockerignore b/.dockerignore index d05bf1c6..9d3d434b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +/.github .test .test2 .sync_tmp @@ -10,6 +11,7 @@ ld in out core* +!/coresight_mode afl-analyze afl-as afl-clang @@ -62,4 +64,4 @@ examples/afl_network_proxy/afl-network-client examples/afl_frida/afl-frida examples/afl_frida/libtestinstr.so examples/afl_frida/frida-gum-example.c -examples/afl_frida/frida-gum.h \ No newline at end of file +examples/afl_frida/frida-gum.h diff --git a/.github/workflows/build-push-test-container.yaml b/.github/workflows/build-test-push-container.yaml similarity index 53% rename from .github/workflows/build-push-test-container.yaml rename to .github/workflows/build-test-push-container.yaml index e2b828ac..5d9e69e4 100644 --- a/.github/workflows/build-push-test-container.yaml +++ b/.github/workflows/build-test-push-container.yaml @@ -1,5 +1,4 @@ -name: Build, test and push container image - +name: Build and test container image on: push: branches: @@ -7,10 +6,33 @@ on: - dev tags: - '*' + pull_request: + branch: + - stable + - dev jobs: - build: - name: Build, test and push container image + build-amd64: + name: Build and test amd64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build amd64 + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + load: true + tags: aflplusplus/aflplusplus:test + # cache-from: type=gha # Ensure we always build a fresh image. We just use the cache for the subsequent push job. + cache-to: type=gha,mode=max + - name: Test linux/amd64 image + run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests + continue-on-error: true + build-arm64: + name: Build and test arm64 runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -20,36 +42,47 @@ jobs: platforms: arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build amd64 image - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64 - load: true - tags: aflplusplus/aflplusplus:test - - name: Build arm64 image + - name: Build arm64 uses: docker/build-push-action@v3 with: context: . platforms: linux/arm64 load: true tags: aflplusplus/aflplusplus:test - - name: Test linux/amd64 image - run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests + # cache-from: type=gha # Ensure we always build a fresh image. We just use the cache for the subsequent push job. + cache-to: type=gha,mode=max - name: Test linux/arm64 image run: docker run --rm --platform linux/arm64 aflplusplus/aflplusplus:test make tests + continue-on-error: true + push: + needs: + - build-amd64 + - build-arm64 + if: ${{ github.event_name == 'push' }} + name: Push image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Login to Dockerhub uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - - name: Publish ${{ github.ref_name }} to docker.io registry + - name: Publish ${{ github.ref_name }} for amd64 and arm64 to docker.io registry uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: aflplusplus/aflplusplus:${{ github.ref_name }} + cache-from: type=gha + # cache-to: type=gha,mode=max # No need to add to cache as we'll never use this if: ${{ github.ref_name != 'stable' }} - name: Publish stable and latest to docker.io registry uses: docker/build-push-action@v3 @@ -58,4 +91,6 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest + cache-from: type=gha + # cache-to: type=gha,mode=max # No need to add to cache as we'll never use this if: ${{ github.ref_name == 'stable' }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d5dce05f..55d477fc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -21,12 +21,12 @@ jobs: uses: actions/checkout@v3 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 diff --git a/Dockerfile b/Dockerfile index e9b1bc7e..b8b45ac5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,9 +33,9 @@ ENV PATH=$PATH:/etc/cargo/bin ARG LLVM_VERSION=14 ARG GCC_VERSION=12 -RUN mkdir -p /usr/local/share/keyrings && \ - echo "deb [signed-by=/usr/local/share/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ - wget -qO /usr/local/share/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key +RUN mkdir -p /etc/apt/keyrings && \ + echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ + wget -qO /etc/apt/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key RUN apt-get update && \ apt-get -y install --no-install-recommends \ @@ -45,9 +45,11 @@ RUN apt-get update && \ libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \ libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ - $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) && \ + $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) \ + $([ "$(dpkg --print-architecture)" = "arm64" ] && echo libcapstone-dev) && \ rm -rf /var/lib/apt/lists/* - # arm64 doesn't have gcc-multilib, and it's only used for -m32 support on x86 + # gcc-multilib is only used for -m32 support on x86 + # libcapstone-dev is used for coresight_mode on arm64 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 @@ -66,6 +68,10 @@ COPY . . # Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029 ENV NO_NYX 1 +# Build currently broken +ENV NO_CORESIGHT 1 +ENV NO_UNICORN_ARM64 1 + RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && make clean && \ make distrib && make install && make clean diff --git a/GNUmakefile b/GNUmakefile index fad619cf..9982ad75 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -623,15 +623,23 @@ endif $(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT $(MAKE) -C coresight_mode + endif endif ifeq "$(SYS)" "Linux" -ifndef NO_NYX + ifndef NO_NYX cd nyx_mode && ./build_nyx_support.sh -endif + endif endif cd qemu_mode && sh ./build_qemu_support.sh + ifeq "$(ARCH)" "aarch64" + ifndef NO_UNICORN_ARM64 cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif + else + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif endif .PHONY: binary-only @@ -645,15 +653,23 @@ binary-only: test_shm test_python ready $(PROGS) $(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT $(MAKE) -C coresight_mode + endif endif ifeq "$(SYS)" "Linux" ifndef NO_NYX - -cd nyx_mode && ./build_nyx_support.sh + cd nyx_mode && ./build_nyx_support.sh endif endif - -cd qemu_mode && sh ./build_qemu_support.sh - -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + cd qemu_mode && sh ./build_qemu_support.sh + ifeq "$(ARCH)" "aarch64" + ifndef NO_UNICORN_ARM64 + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif + else + cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + endif endif .PHONY: source-only From 1dac69b1eb16dff00d7a625138da5555c3acccb8 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Fri, 24 Jun 2022 09:26:09 -0400 Subject: [PATCH 09/24] use container; more is built + tested; use make -i (#2) --- .../workflows/build-test-push-container.yaml | 96 ------------- .github/workflows/codeql-analysis.yml | 32 ----- .github/workflows/container.yaml | 136 ++++++++++++++++++ Dockerfile | 66 +++++---- GNUmakefile | 34 ++--- Makefile | 28 ++-- docs/INSTALL.md | 18 +-- qemu_mode/build_qemu_support.sh | 2 +- 8 files changed, 210 insertions(+), 202 deletions(-) delete mode 100644 .github/workflows/build-test-push-container.yaml delete mode 100644 .github/workflows/codeql-analysis.yml create mode 100644 .github/workflows/container.yaml diff --git a/.github/workflows/build-test-push-container.yaml b/.github/workflows/build-test-push-container.yaml deleted file mode 100644 index 5d9e69e4..00000000 --- a/.github/workflows/build-test-push-container.yaml +++ /dev/null @@ -1,96 +0,0 @@ -name: Build and test container image -on: - push: - branches: - - stable - - dev - tags: - - '*' - pull_request: - branch: - - stable - - dev - -jobs: - build-amd64: - name: Build and test amd64 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build amd64 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64 - load: true - tags: aflplusplus/aflplusplus:test - # cache-from: type=gha # Ensure we always build a fresh image. We just use the cache for the subsequent push job. - cache-to: type=gha,mode=max - - name: Test linux/amd64 image - run: docker run --rm --platform linux/amd64 aflplusplus/aflplusplus:test make tests - continue-on-error: true - build-arm64: - name: Build and test arm64 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Build arm64 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/arm64 - load: true - tags: aflplusplus/aflplusplus:test - # cache-from: type=gha # Ensure we always build a fresh image. We just use the cache for the subsequent push job. - cache-to: type=gha,mode=max - - name: Test linux/arm64 image - run: docker run --rm --platform linux/arm64 aflplusplus/aflplusplus:test make tests - continue-on-error: true - push: - needs: - - build-amd64 - - build-arm64 - if: ${{ github.event_name == 'push' }} - name: Push image - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Publish ${{ github.ref_name }} for amd64 and arm64 to docker.io registry - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: aflplusplus/aflplusplus:${{ github.ref_name }} - cache-from: type=gha - # cache-to: type=gha,mode=max # No need to add to cache as we'll never use this - if: ${{ github.ref_name != 'stable' }} - - name: Publish stable and latest to docker.io registry - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: aflplusplus/aflplusplus:${{ github.ref_name }},aflplusplus/aflplusplus:latest - cache-from: type=gha - # cache-to: type=gha,mode=max # No need to add to cache as we'll never use this - if: ${{ github.ref_name == 'stable' }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 55d477fc..00000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [ stable, dev ] - pull_request: - branches: [ stable, dev ] - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - language: [ 'cpp' ] - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml new file mode 100644 index 00000000..504eadfa --- /dev/null +++ b/.github/workflows/container.yaml @@ -0,0 +1,136 @@ +name: Build, test, CodeQL and push container image +on: + push: + branches: + - stable + - dev + tags: + - "*" + pull_request: + branches: + - dev # No need for stable-pull-request, as that equals dev-push + +jobs: + build-amd64: + name: Build amd64 image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry # Container cache registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build amd64 + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64 + tags: ghcr.io/${{ github.actor }}/aflplusplus:amd64 + push: true + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:amd64 + + build-arm64: + name: Build arm64 image + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry # Container cache registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Build arm64 + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/arm64 + tags: ghcr.io/${{ github.actor }}/aflplusplus:arm64 + push: true + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:arm64 + + test-amd64: + name: Test amd64 image + runs-on: ubuntu-latest + needs: build-amd64 + steps: + - name: Test amd64 + run: docker run --rm ghcr.io/${{ github.actor }}/aflplusplus:amd64 make tests + continue-on-error: true + + codeql-amd64: + name: CodeQL Analyze amd64 compiled code + runs-on: ubuntu-latest + needs: + - build-amd64 + container: + image: ghcr.io/${{ github.actor }}/aflplusplus:amd64 + steps: + - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760 + run: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus + - name: Checkout + uses: actions/checkout@v3 + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: cpp + - name: Build AFLplusplus # Rebuild because CodeQL needs to monitor the build process + env: + CC: gcc # These are symlinked to the version used in the container build + CXX: g++ + run: make -i distrib # Best effort using -i + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + + push: + name: Push amd64 and arm64 image + runs-on: ubuntu-latest + needs: + - test-amd64 + - build-arm64 + if: ${{ github.event_name == 'push' }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to docker.io + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Set tags to push + id: push-tags + run: | + PUSH_TAGS=docker.io/aflplusplus/aflplusplus:${GITHUB_REF_NAME} + if [ "${GITHUB_REF_NAME}" = "stable" ]; then + PUSH_TAGS=${PUSH_TAGS},docker.io/aflplusplus/aflplusplus:latest + fi + export PUSH_TAGS + echo "::set-output name=PUSH_TAGS::${PUSH_TAGS}" + - name: Push to docker.io registry + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 # TODO: Fix: arm64 image doesn't use cached layer from COPY line + push: true + tags: ${{ steps.push-tags.outputs.PUSH_TAGS }} + cache-from: | + type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:amd64 + type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:arm64 diff --git a/Dockerfile b/Dockerfile index b8b45ac5..24bf0dda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,46 +5,41 @@ FROM ubuntu:22.04 AS aflplusplus LABEL "maintainer"="afl++ team " -LABEL "about"="AFLplusplus docker image" +LABEL "about"="AFLplusplus container image" ARG DEBIAN_FRONTEND=noninteractive -ENV NO_ARCH_OPT 1 -ENV IS_DOCKER="1" +ENV NO_ARCH_OPT=1 +ENV IS_DOCKER=1 RUN apt-get update && apt-get full-upgrade -y && \ - apt-get -y install --no-install-recommends \ - make cmake automake \ - meson ninja-build bison flex \ - xz-utils bzip2 \ - git \ - python3 python3-dev python3-setuptools python-is-python3 \ - libtool libtool-bin \ - libglib2.0-dev \ - wget vim jupp nano bash-completion less \ - apt-utils apt-transport-https ca-certificates gnupg dialog \ - libpixman-1-dev \ - gnuplot-nox && \ + apt-get install -y --no-install-recommends wget ca-certificates && \ rm -rf /var/lib/apt/lists/* -RUN wget -qO- https://sh.rustup.rs | CARGO_HOME=/etc/cargo sh -s -- -y -q --no-modify-path -ENV PATH=$PATH:/etc/cargo/bin - ARG LLVM_VERSION=14 ARG GCC_VERSION=12 -RUN mkdir -p /etc/apt/keyrings && \ - echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ +RUN echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ wget -qO /etc/apt/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key RUN apt-get update && \ apt-get -y install --no-install-recommends \ + make cmake automake meson ninja-build bison flex \ + git xz-utils bzip2 wget vim jupp nano bash-completion less \ + python3 python3-dev python3-setuptools python-is-python3 \ + libtool libtool-bin libglib2.0-dev \ + apt-utils apt-transport-https gnupg dialog \ + gnuplot-nox libpixman-1-dev \ gcc-${GCC_VERSION} g++-${GCC_VERSION} gcc-${GCC_VERSION}-plugin-dev gdb lcov \ - clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} libc++-${LLVM_VERSION}-dev \ - libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \ - libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ - liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev \ - libomp5-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ + clang-${LLVM_VERSION} clang-tools-${LLVM_VERSION} libc++1-${LLVM_VERSION} \ + libc++-${LLVM_VERSION}-dev libc++abi1-${LLVM_VERSION} libc++abi-${LLVM_VERSION}-dev \ + libclang1-${LLVM_VERSION} libclang-${LLVM_VERSION}-dev \ + libclang-common-${LLVM_VERSION}-dev libclang-cpp${LLVM_VERSION} \ + libclang-cpp${LLVM_VERSION}-dev liblld-${LLVM_VERSION} \ + liblld-${LLVM_VERSION}-dev liblldb-${LLVM_VERSION} liblldb-${LLVM_VERSION}-dev \ + libllvm${LLVM_VERSION} libomp-${LLVM_VERSION}-dev libomp5-${LLVM_VERSION} \ + lld-${LLVM_VERSION} lldb-${LLVM_VERSION} llvm-${LLVM_VERSION} \ + llvm-${LLVM_VERSION}-dev llvm-${LLVM_VERSION}-runtime llvm-${LLVM_VERSION}-tools \ $([ "$(dpkg --print-architecture)" = "amd64" ] && echo gcc-${GCC_VERSION}-multilib gcc-multilib) \ $([ "$(dpkg --print-architecture)" = "arm64" ] && echo libcapstone-dev) && \ rm -rf /var/lib/apt/lists/* @@ -54,6 +49,9 @@ RUN apt-get update && \ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 +RUN wget -qO- https://sh.rustup.rs | CARGO_HOME=/etc/cargo sh -s -- -y -q --no-modify-path +ENV PATH=$PATH:/etc/cargo/bin + ENV LLVM_CONFIG=llvm-config-${LLVM_VERSION} ENV AFL_SKIP_CPUFREQ=1 ENV AFL_TRY_AFFINITY=1 @@ -62,18 +60,18 @@ ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov && \ (cd afl-cov && make install) && rm -rf afl-cov +# Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029 +ARG NO_NYX=1 + +# Build currently broken +ARG NO_CORESIGHT=1 +ARG NO_UNICORN_ARM64=1 + WORKDIR /AFLplusplus COPY . . -# Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029 -ENV NO_NYX 1 - -# Build currently broken -ENV NO_CORESIGHT 1 -ENV NO_UNICORN_ARM64 1 - -RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && make clean && \ - make distrib && make install && make clean +RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && gmake clean && \ + gmake distrib && gmake install && gmake clean RUN echo "set encoding=utf-8" > /root/.vimrc && \ echo ". /etc/bash_completion" >> ~/.bashrc && \ diff --git a/GNUmakefile b/GNUmakefile index 9982ad75..477518fa 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -308,17 +308,17 @@ endif .PHONY: all all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_build all_done - -$(MAKE) -C utils/aflpp_driver + $(MAKE) -C utils/aflpp_driver .PHONY: llvm llvm: - -$(MAKE) -j4 -f GNUmakefile.llvm + $(MAKE) -j$(nproc) -f GNUmakefile.llvm @test -e afl-cc || { echo "[-] Compiling afl-cc failed. You seem not to have a working compiler." ; exit 1; } .PHONY: gcc_plugin gcc_plugin: ifneq "$(SYS)" "Darwin" - -$(MAKE) -f GNUmakefile.gcc_plugin + $(MAKE) -f GNUmakefile.gcc_plugin endif .PHONY: man @@ -568,19 +568,19 @@ all_done: test_build .PHONY: clean clean: rm -rf $(PROGS) afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM lib*.a - -$(MAKE) -f GNUmakefile.llvm clean - -$(MAKE) -f GNUmakefile.gcc_plugin clean - -$(MAKE) -C utils/libdislocator clean - -$(MAKE) -C utils/libtokencap clean + $(MAKE) -f GNUmakefile.llvm clean + $(MAKE) -f GNUmakefile.gcc_plugin clean + $(MAKE) -C utils/libdislocator clean + $(MAKE) -C utils/libtokencap clean $(MAKE) -C utils/aflpp_driver clean - -$(MAKE) -C utils/afl_network_proxy clean - -$(MAKE) -C utils/socket_fuzzing clean - -$(MAKE) -C utils/argv_fuzzing clean - -$(MAKE) -C utils/plot_ui clean - -$(MAKE) -C qemu_mode/unsigaction clean - -$(MAKE) -C qemu_mode/libcompcov clean - -$(MAKE) -C qemu_mode/libqasan clean - -$(MAKE) -C frida_mode clean + $(MAKE) -C utils/afl_network_proxy clean + $(MAKE) -C utils/socket_fuzzing clean + $(MAKE) -C utils/argv_fuzzing clean + $(MAKE) -C utils/plot_ui clean + $(MAKE) -C qemu_mode/unsigaction clean + $(MAKE) -C qemu_mode/libcompcov clean + $(MAKE) -C qemu_mode/libqasan clean + $(MAKE) -C frida_mode clean rm -rf nyx_mode/packer/linux_initramfs/init.cpio.gz nyx_mode/libnyx/libnyx/target/release/* nyx_mode/QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64 ifeq "$(IN_REPO)" "1" -test -e coresight_mode/coresight-trace/Makefile && $(MAKE) -C coresight_mode/coresight-trace clean || true @@ -610,7 +610,7 @@ endif .PHONY: distrib distrib: all - $(MAKE) -j4 -f GNUmakefile.llvm + $(MAKE) -j$(nproc) -f GNUmakefile.llvm ifneq "$(SYS)" "Darwin" $(MAKE) -f GNUmakefile.gcc_plugin endif @@ -674,7 +674,7 @@ endif .PHONY: source-only source-only: all - $(MAKE) -j4 -f GNUmakefile.llvm + $(MAKE) -j$(nproc) -f GNUmakefile.llvm ifneq "$(SYS)" "Darwin" $(MAKE) -f GNUmakefile.gcc_plugin endif diff --git a/Makefile b/Makefile index 96f67991..972ef84a 100644 --- a/Makefile +++ b/Makefile @@ -1,42 +1,42 @@ all: @echo trying to use GNU make... - @gmake all || echo please install GNUmake + -@gmake all source-only: - @gmake source-only + -@gmake source-only binary-only: - @gmake binary-only + -@gmake binary-only distrib: - @gmake distrib + -@gmake distrib man: - @gmake man + -@gmake man install: - @gmake install + -@gmake install document: - @gmake document + -@gmake document deepclean: - @gmake deepclean + -@gmake deepclean code-format: - @gmake code-format + -@gmake code-format help: - @gmake help + -@gmake help tests: - @gmake tests + -@gmake tests unit: - @gmake unit + -@gmake unit unit_clean: - @gmake unit_clean + -@gmake unit_clean clean: - @gmake clean + -@gmake clean diff --git a/docs/INSTALL.md b/docs/INSTALL.md index e29fca96..98fb024a 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -30,20 +30,22 @@ sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* // sudo apt-get install -y ninja-build # for QEMU mode git clone https://github.com/AFLplusplus/AFLplusplus cd AFLplusplus -make distrib +make -i distrib sudo make install ``` +> We add the `-i` to make so it will skip any features that fail to install. + It is recommended to install the newest available gcc, clang and llvm-dev possible in your distribution! -Note that `make distrib` also builds FRIDA mode, QEMU mode, unicorn_mode, and +Note that `make -i distrib` also builds FRIDA mode, QEMU mode, unicorn_mode, and more. If you just want plain AFL++, then do `make all`. If you want some assisting tooling compiled but are not interested in binary-only targets, then instead choose: ```shell -make source-only +make -i source-only ``` These build targets exist: @@ -70,7 +72,7 @@ you can also build statically linked versions of the AFL++ binaries by passing the `STATIC=1` argument to make: ```shell -make STATIC=1 +make -i STATIC=1 ``` These build options exist: @@ -88,7 +90,7 @@ These build options exist: * LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian) -e.g.: `make ASAN_BUILD=1` +e.g.: `make -i ASAN_BUILD=1` ## MacOS X on x86 and arm64 (M1) @@ -114,11 +116,11 @@ export PATH="/usr/local/opt/llvm/bin:$PATH" export PATH="/usr/local/opt/coreutils/libexec/gnubin:/usr/local/bin:$PATH" export CC=clang export CXX=clang++ -gmake +gmake -i cd frida_mode -gmake +gmake -i cd .. -sudo gmake install +sudo gmake -i install ``` `afl-gcc` will fail unless you have GCC installed, but that is using outdated diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index 5dbd9d44..277a6323 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -273,7 +273,7 @@ echo "[+] Configuration complete." echo "[*] Attempting to build QEMU (fingers crossed!)..." -make -j `nproc` || exit 1 +make -j$(nproc) || exit 1 echo "[+] Build process successful!" From 22c3b519fa92e02add77a0cf30600b5034d179d9 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Mon, 27 Jun 2022 11:05:17 -0400 Subject: [PATCH 10/24] reverts, fixes and even smaller image (#3) --- .dockerignore | 96 +++++++++++++++++--------------- .github/workflows/container.yaml | 37 +++++++----- Dockerfile | 11 +++- GNUmakefile | 72 ++++++++++++------------ 4 files changed, 118 insertions(+), 98 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9d3d434b..271d338c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,67 +1,75 @@ -/.github +!/coresight_mode +*.dSYM +*.o +*.pyc +*.so +.sync_tmp .test .test2 -.sync_tmp -*.o -*.so -*.pyc -*.dSYM -as -ld -in -out -core* -!/coresight_mode +.git +.dockerignore +.github +CITATION.cff +CONTRIBUTING.md +Changelog.md +Dockerfile +LICENSE +TODO.md afl-analyze -afl-as -afl-clang -afl-clang\+\+ -afl-clang-fast -afl-clang-fast\+\+ -afl-clang-lto -afl-clang-lto\+\+ -afl-fuzz -afl-g\+\+ -afl-gcc -afl-gcc-fast -afl-g\+\+-fast -afl-gotcpu -afl-ld -afl-ld-lto -afl-qemu-trace -afl-showmap -afl-tmin afl-analyze.8 +afl-as afl-as.8 -afl-clang-fast\+\+.8 +afl-clang +afl-clang-fast afl-clang-fast.8 +afl-clang-fast\+\+ +afl-clang-fast\+\+.8 +afl-clang-lto afl-clang-lto.8 +afl-clang-lto\+\+ afl-clang-lto\+\+.8 +afl-clang\+\+ afl-cmin.8 afl-cmin.bash.8 +afl-fuzz afl-fuzz.8 -afl-gcc.8 -afl-gcc-fast.8 +afl-g\+\+ +afl-g\+\+-fast afl-g\+\+-fast.8 +afl-gcc +afl-gcc-fast +afl-gcc-fast.8 +afl-gcc.8 +afl-gotcpu afl-gotcpu.8 +afl-ld +afl-ld-lto afl-plot.8 +afl-qemu-trace +afl-showmap afl-showmap.8 afl-system-config.8 +afl-tmin afl-tmin.8 afl-whatsup.8 +as +core* +examples/afl_frida/afl-frida +examples/afl_frida/frida-gum-example.c +examples/afl_frida/frida-gum.h +examples/afl_frida/libtestinstr.so +examples/afl_network_proxy/afl-network-client +examples/afl_network_proxy/afl-network-server +in +ld +out qemu_mode/libcompcov/compcovtest qemu_mode/qemu-* +test/unittests/unit_hash +test/unittests/unit_list +test/unittests/unit_maybe_alloc +test/unittests/unit_preallocable +test/unittests/unit_rand unicorn_mode/samples/*/\.test-* unicorn_mode/samples/*/output unicorn_mode/unicornafl -test/unittests/unit_maybe_alloc -test/unittests/unit_preallocable -test/unittests/unit_list -test/unittests/unit_rand -test/unittests/unit_hash -examples/afl_network_proxy/afl-network-server -examples/afl_network_proxy/afl-network-client -examples/afl_frida/afl-frida -examples/afl_frida/libtestinstr.so -examples/afl_frida/frida-gum-example.c -examples/afl_frida/frida-gum.h diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml index 504eadfa..534e737a 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yaml @@ -19,20 +19,21 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry # Container cache registry + - name: Login to GitHub Container Registry # Container cache registry uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build amd64 uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64 - tags: ghcr.io/${{ github.actor }}/aflplusplus:amd64 + tags: ghcr.io/${{ github.repository_owner }}/aflplusplus:amd64 push: true - cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:amd64 + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:amd64,mode=max + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:amd64 build-arm64: name: Build arm64 image @@ -46,7 +47,7 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -57,18 +58,24 @@ jobs: with: context: . platforms: linux/arm64 - tags: ghcr.io/${{ github.actor }}/aflplusplus:arm64 + tags: ghcr.io/${{ github.repository_owner }}/aflplusplus:arm64 push: true - cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:arm64 + cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:arm64,mode=max + cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:arm64 test-amd64: name: Test amd64 image runs-on: ubuntu-latest needs: build-amd64 + container: + image: ghcr.io/${{ github.repository_owner }}/aflplusplus:amd64 steps: - name: Test amd64 - run: docker run --rm ghcr.io/${{ github.actor }}/aflplusplus:amd64 make tests - continue-on-error: true + working-directory: /AFLplusplus + run: | + apt-get update + apt-get install -y libcmocka-dev + make -i tests codeql-amd64: name: CodeQL Analyze amd64 compiled code @@ -76,7 +83,7 @@ jobs: needs: - build-amd64 container: - image: ghcr.io/${{ github.actor }}/aflplusplus:amd64 + image: ghcr.io/${{ github.repository_owner }}/aflplusplus:amd64 steps: - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760 run: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus @@ -100,7 +107,7 @@ jobs: needs: - test-amd64 - build-arm64 - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'push' && github.repository == 'AFLplusplus/AFLplusplus' }} steps: - name: Checkout uses: actions/checkout@v3 @@ -128,9 +135,9 @@ jobs: uses: docker/build-push-action@v3 with: context: . - platforms: linux/amd64,linux/arm64 # TODO: Fix: arm64 image doesn't use cached layer from COPY line + platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.push-tags.outputs.PUSH_TAGS }} - cache-from: | - type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:amd64 - type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus:arm64 + cache-from: | # TODO: From COPY line on is not cached + type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:amd64 + type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:arm64 diff --git a/Dockerfile b/Dockerfile index 24bf0dda..82854f4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,9 @@ RUN apt-get update && \ # libcapstone-dev is used for coresight_mode on arm64 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 0 && \ - update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 0 && \ + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${LLVM_VERSION} 0 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${LLVM_VERSION} 0 RUN wget -qO- https://sh.rustup.rs | CARGO_HOME=/etc/cargo sh -s -- -y -q --no-modify-path ENV PATH=$PATH:/etc/cargo/bin @@ -70,8 +72,11 @@ ARG NO_UNICORN_ARM64=1 WORKDIR /AFLplusplus COPY . . -RUN export CC=gcc-${GCC_VERSION} && export CXX=g++-${GCC_VERSION} && gmake clean && \ - gmake distrib && gmake install && gmake clean +ARG CC=gcc-$GCC_VERSION +ARG CXX=g++-$GCC_VERSION + +RUN sed -i 's/-$(MAKE)/$(MAKE)/g' GNUmakefile && make clean && \ + make distrib && make install && make clean RUN echo "set encoding=utf-8" > /root/.vimrc && \ echo ". /etc/bash_completion" >> ~/.bashrc && \ diff --git a/GNUmakefile b/GNUmakefile index 477518fa..625530bc 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -312,7 +312,7 @@ all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu .PHONY: llvm llvm: - $(MAKE) -j$(nproc) -f GNUmakefile.llvm + -$(MAKE) -j$(nproc) -f GNUmakefile.llvm @test -e afl-cc || { echo "[-] Compiling afl-cc failed. You seem not to have a working compiler." ; exit 1; } .PHONY: gcc_plugin @@ -568,19 +568,19 @@ all_done: test_build .PHONY: clean clean: rm -rf $(PROGS) afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM lib*.a - $(MAKE) -f GNUmakefile.llvm clean - $(MAKE) -f GNUmakefile.gcc_plugin clean - $(MAKE) -C utils/libdislocator clean - $(MAKE) -C utils/libtokencap clean - $(MAKE) -C utils/aflpp_driver clean - $(MAKE) -C utils/afl_network_proxy clean - $(MAKE) -C utils/socket_fuzzing clean - $(MAKE) -C utils/argv_fuzzing clean - $(MAKE) -C utils/plot_ui clean - $(MAKE) -C qemu_mode/unsigaction clean - $(MAKE) -C qemu_mode/libcompcov clean - $(MAKE) -C qemu_mode/libqasan clean - $(MAKE) -C frida_mode clean + -$(MAKE) -f GNUmakefile.llvm clean + -$(MAKE) -f GNUmakefile.gcc_plugin clean + -$(MAKE) -C utils/libdislocator clean + -$(MAKE) -C utils/libtokencap clean + -$(MAKE) -C utils/aflpp_driver clean + -$(MAKE) -C utils/afl_network_proxy clean + -$(MAKE) -C utils/socket_fuzzing clean + -$(MAKE) -C utils/argv_fuzzing clean + -$(MAKE) -C utils/plot_ui clean + -$(MAKE) -C qemu_mode/unsigaction clean + -$(MAKE) -C qemu_mode/libcompcov clean + -$(MAKE) -C qemu_mode/libqasan clean + -$(MAKE) -C frida_mode clean rm -rf nyx_mode/packer/linux_initramfs/init.cpio.gz nyx_mode/libnyx/libnyx/target/release/* nyx_mode/QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64 ifeq "$(IN_REPO)" "1" -test -e coresight_mode/coresight-trace/Makefile && $(MAKE) -C coresight_mode/coresight-trace clean || true @@ -610,21 +610,21 @@ endif .PHONY: distrib distrib: all - $(MAKE) -j$(nproc) -f GNUmakefile.llvm + -$(MAKE) -j$(nproc) -f GNUmakefile.llvm ifneq "$(SYS)" "Darwin" - $(MAKE) -f GNUmakefile.gcc_plugin + -$(MAKE) -f GNUmakefile.gcc_plugin endif - $(MAKE) -C utils/libdislocator - $(MAKE) -C utils/libtokencap - $(MAKE) -C utils/afl_network_proxy - $(MAKE) -C utils/socket_fuzzing - $(MAKE) -C utils/argv_fuzzing + -$(MAKE) -C utils/libdislocator + -$(MAKE) -C utils/libtokencap + -$(MAKE) -C utils/afl_network_proxy + -$(MAKE) -C utils/socket_fuzzing + -$(MAKE) -C utils/argv_fuzzing # $(MAKE) -C utils/plot_ui - $(MAKE) -C frida_mode + -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" ifndef NO_CORESIGHT - $(MAKE) -C coresight_mode + -$(MAKE) -C coresight_mode endif endif ifeq "$(SYS)" "Linux" @@ -644,17 +644,17 @@ endif .PHONY: binary-only binary-only: test_shm test_python ready $(PROGS) - $(MAKE) -C utils/libdislocator - $(MAKE) -C utils/libtokencap - $(MAKE) -C utils/afl_network_proxy - $(MAKE) -C utils/socket_fuzzing - $(MAKE) -C utils/argv_fuzzing + -$(MAKE) -C utils/libdislocator + -$(MAKE) -C utils/libtokencap + -$(MAKE) -C utils/afl_network_proxy + -$(MAKE) -C utils/socket_fuzzing + -$(MAKE) -C utils/argv_fuzzing # $(MAKE) -C utils/plot_ui - $(MAKE) -C frida_mode + -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" ifndef NO_CORESIGHT - $(MAKE) -C coresight_mode + -$(MAKE) -C coresight_mode endif endif ifeq "$(SYS)" "Linux" @@ -674,12 +674,12 @@ endif .PHONY: source-only source-only: all - $(MAKE) -j$(nproc) -f GNUmakefile.llvm + -$(MAKE) -j$(nproc) -f GNUmakefile.llvm ifneq "$(SYS)" "Darwin" - $(MAKE) -f GNUmakefile.gcc_plugin + -$(MAKE) -f GNUmakefile.gcc_plugin endif - $(MAKE) -C utils/libdislocator - $(MAKE) -C utils/libtokencap + -$(MAKE) -C utils/libdislocator + -$(MAKE) -C utils/libtokencap # $(MAKE) -C utils/plot_ui ifeq "$(SYS)" "Linux" ifndef NO_NYX @@ -728,9 +728,9 @@ install: all $(MANPAGES) @if [ -f utils/afl_network_proxy/afl-network-server ]; then $(MAKE) -C utils/afl_network_proxy install; fi @if [ -f utils/aflpp_driver/libAFLDriver.a ]; then set -e; install -m 644 utils/aflpp_driver/libAFLDriver.a $${DESTDIR}$(HELPER_PATH); fi @if [ -f utils/aflpp_driver/libAFLQemuDriver.a ]; then set -e; install -m 644 utils/aflpp_driver/libAFLQemuDriver.a $${DESTDIR}$(HELPER_PATH); fi - $(MAKE) -f GNUmakefile.llvm install + -$(MAKE) -f GNUmakefile.llvm install ifneq "$(SYS)" "Darwin" - $(MAKE) -f GNUmakefile.gcc_plugin install + -$(MAKE) -f GNUmakefile.gcc_plugin install endif ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-gcc ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-g++ From 73db4fe9939482f4c2f38c24d903869bb0d4a7eb Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 27 Jun 2022 15:13:16 +0000 Subject: [PATCH 11/24] use actor --- .github/workflows/container.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml index 534e737a..a6d31668 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yaml @@ -23,17 +23,17 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build amd64 uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64 - tags: ghcr.io/${{ github.repository_owner }}/aflplusplus:amd64 + tags: ghcr.io/${{ github.actor }}/aflplusplus:amd64 push: true - cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:amd64,mode=max - cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:amd64 + cache-to: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64,mode=max + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64 build-arm64: name: Build arm64 image @@ -47,7 +47,7 @@ jobs: uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -58,17 +58,17 @@ jobs: with: context: . platforms: linux/arm64 - tags: ghcr.io/${{ github.repository_owner }}/aflplusplus:arm64 + tags: ghcr.io/${{ github.actor }}/aflplusplus:arm64 push: true - cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:arm64,mode=max - cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:arm64 + cache-to: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64,mode=max + cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64 test-amd64: name: Test amd64 image runs-on: ubuntu-latest needs: build-amd64 container: - image: ghcr.io/${{ github.repository_owner }}/aflplusplus:amd64 + image: ghcr.io/${{ github.actor }}/aflplusplus:amd64 steps: - name: Test amd64 working-directory: /AFLplusplus @@ -83,7 +83,7 @@ jobs: needs: - build-amd64 container: - image: ghcr.io/${{ github.repository_owner }}/aflplusplus:amd64 + image: ghcr.io/${{ github.actor }}/aflplusplus:amd64 steps: - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760 run: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus @@ -139,5 +139,5 @@ jobs: push: true tags: ${{ steps.push-tags.outputs.PUSH_TAGS }} cache-from: | # TODO: From COPY line on is not cached - type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:amd64 - type=registry,ref=ghcr.io/${{ github.repository_owner }}/aflplusplus/cache:arm64 + type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64 + type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64 From 80e829f79d4d70d9b0b73bf0c8b1b391c710fbb5 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 27 Jun 2022 15:30:35 +0000 Subject: [PATCH 12/24] revert more and global replace --- Dockerfile | 5 +++-- GNUmakefile | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 82854f4d..a9be22de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,8 +75,9 @@ COPY . . ARG CC=gcc-$GCC_VERSION ARG CXX=g++-$GCC_VERSION -RUN sed -i 's/-$(MAKE)/$(MAKE)/g' GNUmakefile && make clean && \ - make distrib && make install && make clean +RUN sed -i.bak 's/^ -/ /g' GNUmakefile && \ + make clean && make distrib && make install && make clean && \ + mv GNUmakefile.bak GNUmakefile RUN echo "set encoding=utf-8" > /root/.vimrc && \ echo ". /etc/bash_completion" >> ~/.bashrc && \ diff --git a/GNUmakefile b/GNUmakefile index 625530bc..6e977467 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -308,7 +308,7 @@ endif .PHONY: all all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_build all_done - $(MAKE) -C utils/aflpp_driver + -$(MAKE) -C utils/aflpp_driver .PHONY: llvm llvm: @@ -318,7 +318,7 @@ llvm: .PHONY: gcc_plugin gcc_plugin: ifneq "$(SYS)" "Darwin" - $(MAKE) -f GNUmakefile.gcc_plugin + -$(MAKE) -f GNUmakefile.gcc_plugin endif .PHONY: man @@ -619,7 +619,7 @@ endif -$(MAKE) -C utils/afl_network_proxy -$(MAKE) -C utils/socket_fuzzing -$(MAKE) -C utils/argv_fuzzing - # $(MAKE) -C utils/plot_ui + # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" @@ -629,16 +629,16 @@ ifeq "$(ARCH)" "aarch64" endif ifeq "$(SYS)" "Linux" ifndef NO_NYX - cd nyx_mode && ./build_nyx_support.sh + -cd nyx_mode && ./build_nyx_support.sh endif endif - cd qemu_mode && sh ./build_qemu_support.sh + -cd qemu_mode && sh ./build_qemu_support.sh ifeq "$(ARCH)" "aarch64" ifndef NO_UNICORN_ARM64 - cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif else - cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif endif @@ -649,7 +649,7 @@ binary-only: test_shm test_python ready $(PROGS) -$(MAKE) -C utils/afl_network_proxy -$(MAKE) -C utils/socket_fuzzing -$(MAKE) -C utils/argv_fuzzing - # $(MAKE) -C utils/plot_ui + # -$(MAKE) -C utils/plot_ui -$(MAKE) -C frida_mode ifneq "$(SYS)" "Darwin" ifeq "$(ARCH)" "aarch64" @@ -659,16 +659,16 @@ ifeq "$(ARCH)" "aarch64" endif ifeq "$(SYS)" "Linux" ifndef NO_NYX - cd nyx_mode && ./build_nyx_support.sh + -cd nyx_mode && ./build_nyx_support.sh endif endif - cd qemu_mode && sh ./build_qemu_support.sh + -cd qemu_mode && sh ./build_qemu_support.sh ifeq "$(ARCH)" "aarch64" ifndef NO_UNICORN_ARM64 - cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif else - cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh + -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif endif From e969972c9799788df0f4af07b9112bb198024f21 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 27 Jun 2022 15:35:41 +0000 Subject: [PATCH 13/24] gcta --- .github/workflows/ci.yml | 7 +++---- .github/workflows/container.yaml | 4 ++-- GNUmakefile | 2 +- GNUmakefile.gcc_plugin | 2 +- Makefile | 28 ++++++++++++++-------------- docs/INSTALL.md | 16 ++++++++-------- 6 files changed, 29 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f0cacca..906267a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,13 +2,13 @@ name: CI on: push: - branches: [ stable, dev ] + branches: [stable, dev] pull_request: - branches: [ stable, dev ] + branches: [stable, dev] jobs: linux: - runs-on: '${{ matrix.os }}' + runs-on: "${{ matrix.os }}" strategy: matrix: os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04] @@ -51,4 +51,3 @@ jobs: run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests - name: force frida test for MacOS run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr - \ No newline at end of file diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml index a6d31668..d4731775 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yaml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry # Container cache registry + - name: Login to GitHub Container Registry # Container cache registry uses: docker/login-action@v2 with: registry: ghcr.io @@ -138,6 +138,6 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.push-tags.outputs.PUSH_TAGS }} - cache-from: | # TODO: From COPY line on is not cached + cache-from: | # TODO: From COPY line on is not cached type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64 type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64 diff --git a/GNUmakefile b/GNUmakefile index 6e977467..b6865f0c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -680,7 +680,7 @@ ifneq "$(SYS)" "Darwin" endif -$(MAKE) -C utils/libdislocator -$(MAKE) -C utils/libtokencap - # $(MAKE) -C utils/plot_ui + # -$(MAKE) -C utils/plot_ui ifeq "$(SYS)" "Linux" ifndef NO_NYX -cd nyx_mode && ./build_nyx_support.sh diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index 5e651688..e21203ae 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -122,7 +122,7 @@ test_deps: # @echo "[*] Checking for gcc for plugin support..." # @$(CC) -v 2>&1 | grep -q -- --enable-plugin || ( echo "[-] Oops, this gcc has not been configured with plugin support."; exit 1 ) @echo "[*] Checking for gcc plugin development header files..." - -@test -d `$(CC) -print-file-name=plugin`/include || ( echo "[-] Oops, can't find gcc header files. Be sure to install 'gcc-X-plugin-dev'."; exit 1 ) + @test -d `$(CC) -print-file-name=plugin`/include || ( echo "[-] Oops, can't find gcc header files. Be sure to install 'gcc-X-plugin-dev'."; exit 1 ) @echo "[*] Checking for './afl-showmap'..." @test -f ./afl-showmap || ( echo "[-] Oops, can't find './afl-showmap'. Be sure to compile AFL first."; exit 1 ) @echo "[+] All set and ready to build." diff --git a/Makefile b/Makefile index 972ef84a..96f67991 100644 --- a/Makefile +++ b/Makefile @@ -1,42 +1,42 @@ all: @echo trying to use GNU make... - -@gmake all + @gmake all || echo please install GNUmake source-only: - -@gmake source-only + @gmake source-only binary-only: - -@gmake binary-only + @gmake binary-only distrib: - -@gmake distrib + @gmake distrib man: - -@gmake man + @gmake man install: - -@gmake install + @gmake install document: - -@gmake document + @gmake document deepclean: - -@gmake deepclean + @gmake deepclean code-format: - -@gmake code-format + @gmake code-format help: - -@gmake help + @gmake help tests: - -@gmake tests + @gmake tests unit: - -@gmake unit + @gmake unit unit_clean: - -@gmake unit_clean + @gmake unit_clean clean: - -@gmake clean + @gmake clean diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 98fb024a..05ec6693 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -30,7 +30,7 @@ sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* // sudo apt-get install -y ninja-build # for QEMU mode git clone https://github.com/AFLplusplus/AFLplusplus cd AFLplusplus -make -i distrib +make distrib sudo make install ``` @@ -39,13 +39,13 @@ sudo make install It is recommended to install the newest available gcc, clang and llvm-dev possible in your distribution! -Note that `make -i distrib` also builds FRIDA mode, QEMU mode, unicorn_mode, and +Note that `make distrib` also builds FRIDA mode, QEMU mode, unicorn_mode, and more. If you just want plain AFL++, then do `make all`. If you want some assisting tooling compiled but are not interested in binary-only targets, then instead choose: ```shell -make -i source-only +make source-only ``` These build targets exist: @@ -72,7 +72,7 @@ you can also build statically linked versions of the AFL++ binaries by passing the `STATIC=1` argument to make: ```shell -make -i STATIC=1 +make STATIC=1 ``` These build options exist: @@ -90,7 +90,7 @@ These build options exist: * LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian) -e.g.: `make -i ASAN_BUILD=1` +e.g.: `make ASAN_BUILD=1` ## MacOS X on x86 and arm64 (M1) @@ -116,11 +116,11 @@ export PATH="/usr/local/opt/llvm/bin:$PATH" export PATH="/usr/local/opt/coreutils/libexec/gnubin:/usr/local/bin:$PATH" export CC=clang export CXX=clang++ -gmake -i +gmake cd frida_mode -gmake -i +gmake cd .. -sudo gmake -i install +sudo gmake install ``` `afl-gcc` will fail unless you have GCC installed, but that is using outdated From 5fa7861149174fcd16fb11527a3340756e730f5c Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 27 Jun 2022 15:37:15 +0000 Subject: [PATCH 14/24] no need for that line anymore --- docs/INSTALL.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 05ec6693..e29fca96 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -34,8 +34,6 @@ make distrib sudo make install ``` -> We add the `-i` to make so it will skip any features that fail to install. - It is recommended to install the newest available gcc, clang and llvm-dev possible in your distribution! From a1889db9bf1e83054e7978d22715c3e197882428 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Mon, 27 Jun 2022 16:17:02 -0400 Subject: [PATCH 15/24] simpler jobs --- .github/workflows/codeql.yaml | 30 ++++++++++ .github/workflows/container.yaml | 95 ++++---------------------------- 2 files changed, 40 insertions(+), 85 deletions(-) create mode 100644 .github/workflows/codeql.yaml diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 00000000..9398df75 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,30 @@ +name: "CodeQL" + +on: + push: + branches: [stable, dev] + pull_request: + branches: [stable, dev] + +jobs: + codeql: + name: Analyze + runs-on: ubuntu-latest + container: # We use a previous image as it's expected to have all the dependencies + image: docker.io/aflplusplus/aflplusplus:dev + steps: + - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760 + run: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus + - name: Checkout + uses: actions/checkout@v3 + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: cpp + - name: Build AFLplusplus # Rebuild because CodeQL needs to monitor the build process + env: + CC: gcc # These are symlinked to the version used in the container build + CXX: g++ + run: make -i distrib # Best effort using -i + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml index d4731775..0fd20dd7 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yaml @@ -1,4 +1,4 @@ -name: Build, test, CodeQL and push container image +name: Container on: push: branches: @@ -11,102 +11,29 @@ on: - dev # No need for stable-pull-request, as that equals dev-push jobs: - build-amd64: - name: Build amd64 image + build-and-test-amd64: + name: Build and test amd64 image runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry # Container cache registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - name: Build amd64 uses: docker/build-push-action@v3 with: context: . - platforms: linux/amd64 - tags: ghcr.io/${{ github.actor }}/aflplusplus:amd64 - push: true - cache-to: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64,mode=max - cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64 - - build-arm64: - name: Build arm64 image - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry # Container cache registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - name: Build arm64 - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/arm64 - tags: ghcr.io/${{ github.actor }}/aflplusplus:arm64 - push: true - cache-to: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64,mode=max - cache-from: type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64 - - test-amd64: - name: Test amd64 image - runs-on: ubuntu-latest - needs: build-amd64 - container: - image: ghcr.io/${{ github.actor }}/aflplusplus:amd64 - steps: + tags: aflplusplus:amd64 + load: true + cache-to: type=gha,mode=max - name: Test amd64 - working-directory: /AFLplusplus - run: | - apt-get update - apt-get install -y libcmocka-dev - make -i tests - - codeql-amd64: - name: CodeQL Analyze amd64 compiled code - runs-on: ubuntu-latest - needs: - - build-amd64 - container: - image: ghcr.io/${{ github.actor }}/aflplusplus:amd64 - steps: - - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760 - run: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus - - name: Checkout - uses: actions/checkout@v3 - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: cpp - - name: Build AFLplusplus # Rebuild because CodeQL needs to monitor the build process - env: - CC: gcc # These are symlinked to the version used in the container build - CXX: g++ - run: make -i distrib # Best effort using -i - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + run: docker run --rm aflplusplus:amd64 bash -c "apt-get update && apt-get install -y libcmocka-dev && make -i tests" push: - name: Push amd64 and arm64 image + name: Build and push amd64 and arm64 image runs-on: ubuntu-latest needs: - - test-amd64 - - build-arm64 + - build-and-test-amd64 if: ${{ github.event_name == 'push' && github.repository == 'AFLplusplus/AFLplusplus' }} steps: - name: Checkout @@ -138,6 +65,4 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.push-tags.outputs.PUSH_TAGS }} - cache-from: | # TODO: From COPY line on is not cached - type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:amd64 - type=registry,ref=ghcr.io/${{ github.actor }}/aflplusplus/cache:arm64 + cache-from: type=gha From b3edb657c2221f7d292931c8df04388c552737dd Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 28 Jun 2022 01:50:39 +0000 Subject: [PATCH 16/24] check python too --- .github/workflows/{codeql.yaml => codeql-analysis.yaml} | 4 ++-- .github/workflows/container.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{codeql.yaml => codeql-analysis.yaml} (88%) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql-analysis.yaml similarity index 88% rename from .github/workflows/codeql.yaml rename to .github/workflows/codeql-analysis.yaml index 9398df75..c9efb296 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -10,7 +10,7 @@ jobs: codeql: name: Analyze runs-on: ubuntu-latest - container: # We use a previous image as it's expected to have all the dependencies + container: # We use a previous image as it's expected to have all the dependencies image: docker.io/aflplusplus/aflplusplus:dev steps: - name: Fix for using external repo in container build # https://github.com/actions/checkout/issues/760 @@ -20,7 +20,7 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: - languages: cpp + languages: cpp, python - name: Build AFLplusplus # Rebuild because CodeQL needs to monitor the build process env: CC: gcc # These are symlinked to the version used in the container build diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml index 0fd20dd7..81e2a3f8 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yaml @@ -12,7 +12,7 @@ on: jobs: build-and-test-amd64: - name: Build and test amd64 image + name: Test amd64 image runs-on: ubuntu-latest steps: - name: Checkout @@ -30,7 +30,7 @@ jobs: run: docker run --rm aflplusplus:amd64 bash -c "apt-get update && apt-get install -y libcmocka-dev && make -i tests" push: - name: Build and push amd64 and arm64 image + name: Push amd64 and arm64 images runs-on: ubuntu-latest needs: - build-and-test-amd64 From 3e65e1a029c44b33a77ca04724335ee3ecd30e67 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Tue, 28 Jun 2022 15:59:56 +0000 Subject: [PATCH 17/24] fix test using test specific build --- .../{codeql-analysis.yaml => codeql-analysis.yml} | 2 +- .github/workflows/{container.yaml => container.yml} | 11 +++++++++-- Dockerfile | 12 ++++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) rename .github/workflows/{codeql-analysis.yaml => codeql-analysis.yml} (98%) rename .github/workflows/{container.yaml => container.yml} (87%) diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yml similarity index 98% rename from .github/workflows/codeql-analysis.yaml rename to .github/workflows/codeql-analysis.yml index c9efb296..ba6792b1 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yml @@ -7,7 +7,7 @@ on: branches: [stable, dev] jobs: - codeql: + analyze: name: Analyze runs-on: ubuntu-latest container: # We use a previous image as it's expected to have all the dependencies diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yml similarity index 87% rename from .github/workflows/container.yaml rename to .github/workflows/container.yml index 81e2a3f8..8836997d 100644 --- a/.github/workflows/container.yaml +++ b/.github/workflows/container.yml @@ -23,11 +23,18 @@ jobs: uses: docker/build-push-action@v3 with: context: . - tags: aflplusplus:amd64 + tags: aflplusplus:test-amd64 load: true cache-to: type=gha,mode=max + build-args: | + TEST_BUILD=1 - name: Test amd64 - run: docker run --rm aflplusplus:amd64 bash -c "apt-get update && apt-get install -y libcmocka-dev && make -i tests" + run: > + docker run --rm aflplusplus:test-amd64 bash -c " + apt-get update && + apt-get install -y libcmocka-dev && + make -i tests + " push: name: Push amd64 and arm64 images diff --git a/Dockerfile b/Dockerfile index a9be22de..29ca2a5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,11 +63,11 @@ RUN git clone --depth=1 https://github.com/vanhauser-thc/afl-cov && \ (cd afl-cov && make install) && rm -rf afl-cov # Until gcc v12.1 is released for ubuntu https://bugs.launchpad.net/ubuntu/+source/gcc-11/+bug/1940029 -ARG NO_NYX=1 +ENV NO_NYX=1 # Build currently broken -ARG NO_CORESIGHT=1 -ARG NO_UNICORN_ARM64=1 +ENV NO_CORESIGHT=1 +ENV NO_UNICORN_ARM64=1 WORKDIR /AFLplusplus COPY . . @@ -75,8 +75,12 @@ COPY . . ARG CC=gcc-$GCC_VERSION ARG CXX=g++-$GCC_VERSION +# Used in CI to prevent a 'make clean' which would remove the binaries to be tested +ARG TEST_BUILD + RUN sed -i.bak 's/^ -/ /g' GNUmakefile && \ - make clean && make distrib && make install && make clean && \ + make clean && make distrib && \ + ([ "${TEST_BUILD}" ] || (make install && make clean)) && \ mv GNUmakefile.bak GNUmakefile RUN echo "set encoding=utf-8" > /root/.vimrc && \ From 6c9c8fdc651e3c3376f3ba6590c49424be1844d7 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Mon, 4 Jul 2022 14:32:48 -0400 Subject: [PATCH 18/24] check code formatting (#5) --- .custom-format.py | 5 ++++ .github/workflows/ci.yml | 7 +++-- .github/workflows/code-format.yml | 36 +++++++++++++++++++++++ .github/workflows/codeql-analysis.yml | 7 +++-- .github/workflows/rust_custom_mutator.yml | 7 +++-- Dockerfile | 4 +-- 6 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/code-format.yml diff --git a/.custom-format.py b/.custom-format.py index 7ac63396..8c7ea404 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -19,6 +19,7 @@ import subprocess import sys import os import re +import shutil # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # future use @@ -26,6 +27,10 @@ with open(".clang-format") as f: fmt = f.read() CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN") + +if CLANG_FORMAT_BIN is None: + CLANG_FORMAT_BIN = shutil.which("clang-format") + if CLANG_FORMAT_BIN is None: o = 0 try: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 906267a0..4ca76f2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,12 @@ name: CI on: push: - branches: [stable, dev] + branches: + - stable + - dev pull_request: - branches: [stable, dev] + branches: + - dev # No need for stable-pull-request, as that equals dev-push jobs: linux: diff --git a/.github/workflows/code-format.yml b/.github/workflows/code-format.yml new file mode 100644 index 00000000..f24e652b --- /dev/null +++ b/.github/workflows/code-format.yml @@ -0,0 +1,36 @@ +name: Formatting + +on: + push: + branches: + - stable + - dev + pull_request: + branches: + - dev # No need for stable-pull-request, as that equals dev-push + +jobs: + code-format-check: + name: Check code format + runs-on: ubuntu-22.04 + container: + image: docker.io/aflplusplus/aflplusplus:dev + env: + LLVM_VERSION: 14 # Remove _after_ this branch's image is built and pushed. The new image will have this var in ENV. + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Format + run: | + git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus + apt-get update + apt-get install -y clang-format-${LLVM_VERSION} + update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_VERSION} 0 + make code-format + - name: Check if code needed formatting + run: | + git --no-pager -c color.ui=always diff HEAD + if ! git diff HEAD --quiet; then + echo "[!] Please run 'make code-format' and push its changes." + exit 1 + fi diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ba6792b1..54934395 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,9 +2,12 @@ name: "CodeQL" on: push: - branches: [stable, dev] + branches: + - stable + - dev pull_request: - branches: [stable, dev] + branches: + - dev # No need for stable-pull-request, as that equals dev-push jobs: analyze: diff --git a/.github/workflows/rust_custom_mutator.yml b/.github/workflows/rust_custom_mutator.yml index eb3e2581..7c2f0c12 100644 --- a/.github/workflows/rust_custom_mutator.yml +++ b/.github/workflows/rust_custom_mutator.yml @@ -2,9 +2,12 @@ name: Rust Custom Mutators on: push: - branches: [ stable, dev ] + branches: + - stable + - dev pull_request: - branches: [ stable, dev ] + branches: + - dev # No need for stable-pull-request, as that equals dev-push jobs: test: diff --git a/Dockerfile b/Dockerfile index 29ca2a5f..a82fffdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,8 @@ RUN apt-get update && apt-get full-upgrade -y && \ apt-get install -y --no-install-recommends wget ca-certificates && \ rm -rf /var/lib/apt/lists/* -ARG LLVM_VERSION=14 -ARG GCC_VERSION=12 +ENV LLVM_VERSION=14 +ENV GCC_VERSION=12 RUN echo "deb [signed-by=/etc/apt/keyrings/llvm-snapshot.gpg.key] http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list && \ wget -qO /etc/apt/keyrings/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key From 250000ee79c87476d81d0869e5d2c3435f224ed9 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 4 Jul 2022 22:27:55 +0000 Subject: [PATCH 19/24] simpler format bin selector --- .custom-format.py | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/.custom-format.py b/.custom-format.py index 8c7ea404..75dbda79 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -26,38 +26,15 @@ import shutil with open(".clang-format") as f: fmt = f.read() +CURRENT_LLVM = 14 CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN") -if CLANG_FORMAT_BIN is None: - CLANG_FORMAT_BIN = shutil.which("clang-format") +if shutil.which(CLANG_FORMAT_BIN) is None: + CLANG_FORMAT_BIN = f"clang-format-{os.getenv('LLVM_VERSION', CURRENT_LLVM)}" -if CLANG_FORMAT_BIN is None: - o = 0 - try: - p = subprocess.Popen(["clang-format-11", "--version"], stdout=subprocess.PIPE) - o, _ = p.communicate() - o = str(o, "utf-8") - o = re.sub(r".*ersion ", "", o) - # o = o[len("clang-format version "):].strip() - o = o[: o.find(".")] - o = int(o) - except: - print("clang-format-11 is needed. Aborted.") - exit(1) - # if o < 7: - # if subprocess.call(['which', 'clang-format-7'], stdout=subprocess.PIPE) == 0: - # CLANG_FORMAT_BIN = 'clang-format-7' - # elif subprocess.call(['which', 'clang-format-8'], stdout=subprocess.PIPE) == 0: - # CLANG_FORMAT_BIN = 'clang-format-8' - # elif subprocess.call(['which', 'clang-format-9'], stdout=subprocess.PIPE) == 0: - # CLANG_FORMAT_BIN = 'clang-format-9' - # elif subprocess.call(['which', 'clang-format-11'], stdout=subprocess.PIPE) == 0: - # CLANG_FORMAT_BIN = 'clang-format-11' - # else: - # print ("clang-format 7 or above is needed. Aborted.") - # exit(1) - else: - CLANG_FORMAT_BIN = "clang-format-11" +if shutil.which(CLANG_FORMAT_BIN) is None: + print(f"[!] clang-format-{CURRENT_LLVM} is needed. Aborted.") + exit(1) COLUMN_LIMIT = 80 for line in fmt.split("\n"): From 4848c8d24cb3d790838bebb06450962939dd082b Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 4 Jul 2022 22:30:03 +0000 Subject: [PATCH 20/24] use env as base --- .custom-format.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.custom-format.py b/.custom-format.py index 75dbda79..f9301585 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -26,11 +26,11 @@ import shutil with open(".clang-format") as f: fmt = f.read() -CURRENT_LLVM = 14 +CURRENT_LLVM = os.getenv('LLVM_VERSION', 14) CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN") if shutil.which(CLANG_FORMAT_BIN) is None: - CLANG_FORMAT_BIN = f"clang-format-{os.getenv('LLVM_VERSION', CURRENT_LLVM)}" + CLANG_FORMAT_BIN = f"clang-format-{CURRENT_LLVM}" if shutil.which(CLANG_FORMAT_BIN) is None: print(f"[!] clang-format-{CURRENT_LLVM} is needed. Aborted.") From 0da935183fcc33ba81b9153247069465a1681fe2 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 4 Jul 2022 22:36:06 +0000 Subject: [PATCH 21/24] allow unset env var --- .custom-format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.custom-format.py b/.custom-format.py index f9301585..428d7b0d 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -27,7 +27,7 @@ with open(".clang-format") as f: fmt = f.read() CURRENT_LLVM = os.getenv('LLVM_VERSION', 14) -CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN") +CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "") if shutil.which(CLANG_FORMAT_BIN) is None: CLANG_FORMAT_BIN = f"clang-format-{CURRENT_LLVM}" From c90dd00fc977170048271ca8f4f614b8b625e67a Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Mon, 4 Jul 2022 22:38:28 +0000 Subject: [PATCH 22/24] no longer need this line --- .github/workflows/code-format.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/code-format.yml b/.github/workflows/code-format.yml index f24e652b..088793e7 100644 --- a/.github/workflows/code-format.yml +++ b/.github/workflows/code-format.yml @@ -25,7 +25,6 @@ jobs: git config --global --add safe.directory /__w/AFLplusplus/AFLplusplus apt-get update apt-get install -y clang-format-${LLVM_VERSION} - update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_VERSION} 0 make code-format - name: Check if code needed formatting run: | From c7c74764531beb280c4baf0f51499f494fbaf990 Mon Sep 17 00:00:00 2001 From: Ruben ten Hove Date: Thu, 7 Jul 2022 17:10:42 +0000 Subject: [PATCH 23/24] disable formatter --- .github/workflows/code-format.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/code-format.yml b/.github/workflows/code-format.yml index 088793e7..314137ff 100644 --- a/.github/workflows/code-format.yml +++ b/.github/workflows/code-format.yml @@ -12,11 +12,9 @@ on: jobs: code-format-check: name: Check code format + if: ${{ 'false' == 'true' }} # Disable the job runs-on: ubuntu-22.04 - container: - image: docker.io/aflplusplus/aflplusplus:dev - env: - LLVM_VERSION: 14 # Remove _after_ this branch's image is built and pushed. The new image will have this var in ENV. + container: docker.io/aflplusplus/aflplusplus:dev steps: - name: Checkout uses: actions/checkout@v3 From 1f17aa082d4ee75c5a034a1ec32025fb828357d2 Mon Sep 17 00:00:00 2001 From: rhtenhove Date: Fri, 8 Jul 2022 08:32:17 -0400 Subject: [PATCH 24/24] Dont codeql submodules --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 54934395..75935123 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,6 +28,6 @@ jobs: env: CC: gcc # These are symlinked to the version used in the container build CXX: g++ - run: make -i distrib # Best effort using -i + run: make -i all # Best effort using -i - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2