Test standard and persistent modes separately

benchmark/benchmark.py

@@ -3,9 +3,9 @@
 # Author: Chris Ball <chris@printf.net>
 # Ported from Marc "van Hauser" Heuse's "benchmark.sh".
 import os
-import subprocess
-import shutil
 import re
+import shutil
+import subprocess
 import sys
 
 def colon_value_or_none(filename: str, searchKey: str) -> str | None:
@@ -20,6 +20,16 @@ def colon_value_or_none(filename: str, searchKey: str) -> str | None:
                     return value
         return None
 
+def compile_target(source: str, binary: str) -> None:
+    with open("afl.log", "w") as f:
+        process = subprocess.run(
+            ["afl-cc", "-o", binary, source],
+            stdout=f,
+            stderr=subprocess.STDOUT,
+            env={"AFL_INSTRUMENT": "PCGUARD", "PATH": os.environ["PATH"]}
+        )
+        if process.returncode != 0:
+            sys.exit("Error: afl-cc is unable to compile")
 
 # Check if the necessary files exist and are executable
 if not (
@@ -27,13 +37,15 @@ if not (
     and os.access("../afl-cc", os.X_OK)
     and os.path.exists("../SanitizerCoveragePCGUARD.so")
 ):
-    print(
-        "Error: you need to compile AFL++ first, we need afl-fuzz, afl-clang-fast and SanitizerCoveragePCGUARD.so built."
-    )
-    exit(1)
+    sys.exit("Error: you need to compile AFL++ first, we need afl-fuzz, afl-clang-fast and SanitizerCoveragePCGUARD.so built.")
 
 print("Preparing environment")
 
+targets = [
+    {"source": "../test-instr.c", "binary": "test-instr"},
+    {"source": "../utils/persistent_mode/test-instr.c", "binary": "test-instr-persistent"}
+]
+
 # Unset AFL_* environment variables
 for e in list(os.environ.keys()):
     if e.startswith("AFL_"):
@@ -42,17 +54,8 @@ for e in list(os.environ.keys()):
 AFL_PATH = os.path.abspath("../")
 os.environ["PATH"] = AFL_PATH + ":" + os.environ["PATH"]
 
-# Compile test-instr.c
-with open("afl.log", "w") as f:
-    process = subprocess.run(
-        ["../afl-cc", "-o", "test-instr", "../test-instr.c"],
-        stdout=f,
-        stderr=subprocess.STDOUT,
-        env={"AFL_INSTRUMENT": "PCGUARD"}
-    )
-    if process.returncode != 0:
-        print("Error: afl-cc is unable to compile")
-        exit(1)
+for target in targets:
+    compile_target(target["source"], target["binary"])
 
 # Create input directory and file
 os.makedirs("in", exist_ok=True)
@@ -69,33 +72,34 @@ env_vars = {
     "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES": "1",
     "AFL_BENCH_JUST_ONE": "1",
 }
-with open("afl.log", "a") as f:
-    process = subprocess.run(
-        [
-            "afl-fuzz",
-            "-i",
-            "in",
-            "-o",
-            "out",
-            "-s",
-            "123",
-            "-D",
-            "./test-instr",
-        ],
-        stdout=f,
-        stderr=subprocess.STDOUT,
-        env={**os.environ, **env_vars},
-    )
+
+for target in targets:
+    with open(f"afl-{target['binary']}.log", "a") as f:
+        process = subprocess.run(
+            [
+                "afl-fuzz",
+                "-i",
+                "in",
+                "-o",
+                f"out-{target['binary']}",
+                "-s",
+                "123",
+                "-D",
+                f"./{target['binary']}",
+            ],
+            stdout=f,
+            stderr=subprocess.STDOUT,
+            env={**os.environ, **env_vars},
+        )
 
 print("Analysis:")
 
 # Extract CPUID from afl.log
-with open("afl.log", "r") as f:
+with open(f"afl-test-instr.log", "r") as f:
     match = re.search(r".*try binding to.*#(\d+)", f.read())
     if not match:
         sys.exit("Couldn't see which CPU# was used in afl.log", 1)
     cpuid = match.group(1)
-    print(cpuid)
 
 # Print CPU model
 model = colon_value_or_none("/proc/cpuinfo", "model name")
@@ -120,16 +124,19 @@ if cpu_speed:
     print(" Mhz:", cpu_speed)
 
 # Print execs_per_sec from fuzzer_stats
-execs = colon_value_or_none("out/default/fuzzer_stats", "execs_per_sec")
-if execs:
-    print(" execs/s:", execs)
+for target in targets:
+    execs = colon_value_or_none(f"out-{target['binary']}/default/fuzzer_stats", "execs_per_sec")
+    if execs:
+        print(f" {target['binary']} single-core execs/s:", execs)
 
 print("\nComparison: (note that values can change by 10-15% per run)")
 with open("COMPARISON", "r") as f:
     print(f.read())
 
 # Clean up
-shutil.rmtree("in")
-shutil.rmtree("out")
-os.remove("test-instr")
 os.remove("afl.log")
+shutil.rmtree("in")
+for target in targets:
+    shutil.rmtree(f"out-{target['binary']}")
+    os.remove(target["binary"])
+