OSX-specific improvements (#912)

* Fix afl-cc to work correctly by default on OSX using xcode

- CLANG_ENV_VAR must be set for afl-as to work
- Use clang mode by default if no specific compiler selected

* Add OSX-specific documentation for configuring shared memory

docs/INSTALL.md

@@ -103,6 +103,41 @@ The llvm instrumentation requires a fully-operational installation of clang. The
 comes with Xcode is missing some of the essential headers and helper tools.
 See README.llvm.md for advice on how to build the compiler from scratch.
 
+MacOS X supports SYSV shared memory used by AFL's instrumentation, but the
+default settings aren't usable with AFL++. The default settings on 10.14 seem
+to be:
+
+```bash
+$ ipcs -M
+IPC status from <running system> as of XXX
+shminfo:
+        shmmax: 4194304 (max shared memory segment size)
+        shmmin:       1 (min shared memory segment size)
+        shmmni:      32 (max number of shared memory identifiers)
+        shmseg:       8 (max shared memory segments per process)
+        shmall:    1024 (max amount of shared memory in pages)
+```
+
+To temporarily change your settings to something minimally usable with AFL++,
+run these commands as root:
+
+```bash
+sysctl kern.sysv.shmmax=8388608
+sysctl kern.sysv.shmall=4096
+```
+
+If you're running more than one instance of AFL you likely want to make `shmall`
+bigger and increase `shmseg` as well:
+
+```bash
+sysctl kern.sysv.shmmax=8388608
+sysctl kern.sysv.shmseg=48
+sysctl kern.sysv.shmall=98304
+```
+
+See http://www.spy-hill.com/help/apple/SharedMemory.html for documentation for
+these settings and how to make them permanent.
+
 ## 4. Linux or *BSD on non-x86 systems
 
 Standard build will fail on non-x86 systems, but you should be able to

src/afl-cc.c

@@ -1574,7 +1574,12 @@ int main(int argc, char **argv, char **envp) {
     else if (have_gcc_plugin)
       compiler_mode = GCC_PLUGIN;
     else if (have_gcc)
-      compiler_mode = GCC;
+      #ifdef __APPLE__
+        // on OSX clang masquerades as GCC
+        compiler_mode = CLANG;
+      #else
+        compiler_mode = GCC;
+      #endif
     else if (have_lto)
       compiler_mode = LTO;
     else
@@ -1596,7 +1601,10 @@ int main(int argc, char **argv, char **envp) {
 
   }
 
-  if (compiler_mode == CLANG) { instrument_mode = INSTRUMENT_CLANG; }
+  if (compiler_mode == CLANG) {
+    instrument_mode = INSTRUMENT_CLANG;
+    setenv(CLANG_ENV_VAR, "1", 1); // used by afl-as
+  }
 
   if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) {