Merge pull request #2086 from smoelius/dev

Add `AFL_SHA1_FILENAMES` option

docs/env_variables.md

@@ -550,6 +550,9 @@ checks or alter some of the more exotic semantics of the tool:
     use a custom afl-qemu-trace or if you need to modify the afl-qemu-trace
     arguments.
 
+  - `AFL_SHA1_FILENAMES` causes AFL++ to generate files named by the SHA1 hash
+    of their contents, rather than use the standard `id:000000,...` names.
+
   - `AFL_SHUFFLE_QUEUE` randomly reorders the input queue on startup. Requested
     by some users for unorthodox parallelized fuzzing setups, but not advisable
     otherwise.

include/afl-fuzz.h

@@ -452,7 +452,8 @@ typedef struct afl_env_vars {
       afl_keep_timeouts, afl_no_crash_readme, afl_ignore_timeouts,
       afl_no_startup_calibration, afl_no_warn_instability,
       afl_post_process_keep_original, afl_crashing_seeds_as_new_crash,
-      afl_final_sync, afl_ignore_seed_problems, afl_disable_redundant;
+      afl_final_sync, afl_ignore_seed_problems, afl_disable_redundant,
+      afl_sha1_filenames;
 
   u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path,
       *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload,
@@ -1404,6 +1405,32 @@ void queue_testcase_retake_mem(afl_state_t *afl, struct queue_entry *q, u8 *in,
 
 void queue_testcase_store_mem(afl_state_t *afl, struct queue_entry *q, u8 *mem);
 
+/* Compute the SHA1 hash of `data`, which is of `len` bytes, and return the
+ * result as a `\0`-terminated hex string, which the caller much `ck_free`. */
+char *sha1_hex(const u8 *data, size_t len);
+
+/* Apply `sha1_hex` to the first `len` bytes of data of the file at `fname`. */
+char *sha1_hex_for_file(const char *fname, u32 len);
+
+/* Create file `fn`, but allow it to already exist if `AFL_SHA1_FILENAMES` is
+ * enabled. */
+static inline int permissive_create(afl_state_t *afl, const char *fn) {
+
+  int fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION);
+  if (unlikely(fd < 0)) {
+
+    if (!(afl->afl_env.afl_sha1_filenames && errno == EEXIST)) {
+
+      PFATAL("Unable to create '%s'", fn);
+
+    }
+
+  }
+
+  return fd;
+
+}
+
 #if TESTCASE_CACHE == 1
   #error define of TESTCASE_CACHE must be zero or larger than 1
 #endif

include/envs.h

@@ -108,15 +108,15 @@ static char *afl_environment_variables[] = {
     "AFL_QEMU_PERSISTENT_RETADDR_OFFSET", "AFL_QEMU_PERSISTENT_EXITS",
     "AFL_QEMU_INST_RANGES", "AFL_QEMU_EXCLUDE_RANGES", "AFL_QEMU_SNAPSHOT",
     "AFL_QEMU_TRACK_UNSTABLE", "AFL_QUIET", "AFL_RANDOM_ALLOC_CANARY",
-    "AFL_REAL_PATH", "AFL_SHUFFLE_QUEUE", "AFL_SKIP_BIN_CHECK",
+    "AFL_REAL_PATH", "AFL_SHA1_FILENAMES", "AFL_SHUFFLE_QUEUE",
-    "AFL_SKIP_CPUFREQ", "AFL_SKIP_CRASHES", "AFL_SKIP_OSSFUZZ", "AFL_STATSD",
+    "AFL_SKIP_BIN_CHECK", "AFL_SKIP_CPUFREQ", "AFL_SKIP_CRASHES",
-    "AFL_STATSD_HOST", "AFL_STATSD_PORT", "AFL_STATSD_TAGS_FLAVOR",
+    "AFL_SKIP_OSSFUZZ", "AFL_STATSD", "AFL_STATSD_HOST", "AFL_STATSD_PORT",
-    "AFL_SYNC_TIME", "AFL_TESTCACHE_SIZE", "AFL_TESTCACHE_ENTRIES",
+    "AFL_STATSD_TAGS_FLAVOR", "AFL_SYNC_TIME", "AFL_TESTCACHE_SIZE",
-    "AFL_TMIN_EXACT", "AFL_TMPDIR", "AFL_TOKEN_FILE", "AFL_TRACE_PC",
+    "AFL_TESTCACHE_ENTRIES", "AFL_TMIN_EXACT", "AFL_TMPDIR", "AFL_TOKEN_FILE",
-    "AFL_USE_ASAN", "AFL_USE_MSAN", "AFL_USE_TRACE_PC", "AFL_USE_UBSAN",
+    "AFL_TRACE_PC", "AFL_USE_ASAN", "AFL_USE_MSAN", "AFL_USE_TRACE_PC",
-    "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN", "AFL_WINE_PATH",
+    "AFL_USE_UBSAN", "AFL_USE_TSAN", "AFL_USE_CFISAN", "AFL_USE_LSAN",
-    "AFL_NO_SNAPSHOT", "AFL_EXPAND_HAVOC_NOW", "AFL_USE_FASAN", "AFL_USE_QASAN",
+    "AFL_WINE_PATH", "AFL_NO_SNAPSHOT", "AFL_EXPAND_HAVOC_NOW", "AFL_USE_FASAN",
-    "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", NULL
+    "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", NULL
 
 };
 

src/afl-fuzz-bitmap.c

@@ -527,12 +527,24 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) {
 
 #ifndef SIMPLE_FILES
 
-    queue_fn = alloc_printf(
+    if (!afl->afl_env.afl_sha1_filenames) {
-        "%s/queue/id:%06u,%s%s%s", afl->out_dir, afl->queued_items,
+
-        describe_op(afl, new_bits + is_timeout,
+      queue_fn = alloc_printf(
-                    NAME_MAX - strlen("id:000000,")),
+          "%s/queue/id:%06u,%s%s%s", afl->out_dir, afl->queued_items,
-        afl->file_extension ? "." : "",
+          describe_op(afl, new_bits + is_timeout,
-        afl->file_extension ? (const char *)afl->file_extension : "");
+                      NAME_MAX - strlen("id:000000,")),
+          afl->file_extension ? "." : "",
+          afl->file_extension ? (const char *)afl->file_extension : "");
+
+    } else {
+
+      const char *hex = sha1_hex(mem, len);
+      queue_fn = alloc_printf(
+          "%s/queue/%s%s%s", afl->out_dir, hex, afl->file_extension ? "." : "",
+          afl->file_extension ? (const char *)afl->file_extension : "");
+      ck_free((char *)hex);
+
+    }
 
 #else
 
@@ -542,10 +554,14 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) {
         afl->file_extension ? (const char *)afl->file_extension : "");
 
 #endif                                                    /* ^!SIMPLE_FILES */
-    fd = open(queue_fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION);
+    fd = permissive_create(afl, queue_fn);
-    if (unlikely(fd < 0)) { PFATAL("Unable to create '%s'", queue_fn); }
+    if (likely(fd >= 0)) {
-    ck_write(fd, mem, len, queue_fn);
+
-    close(fd);
+      ck_write(fd, mem, len, queue_fn);
+      close(fd);
+
+    }
+
     add_to_queue(afl, queue_fn, len, 0);
 
     if (unlikely(afl->fuzz_mode) &&
@@ -743,11 +759,23 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) {
 
 #ifndef SIMPLE_FILES
 
-      snprintf(fn, PATH_MAX, "%s/hangs/id:%06llu,%s%s%s", afl->out_dir,
+      if (!afl->afl_env.afl_sha1_filenames) {
-               afl->saved_hangs,
+
-               describe_op(afl, 0, NAME_MAX - strlen("id:000000,")),
+        snprintf(fn, PATH_MAX, "%s/hangs/id:%06llu,%s%s%s", afl->out_dir,
-               afl->file_extension ? "." : "",
+                 afl->saved_hangs,
-               afl->file_extension ? (const char *)afl->file_extension : "");
+                 describe_op(afl, 0, NAME_MAX - strlen("id:000000,")),
+                 afl->file_extension ? "." : "",
+                 afl->file_extension ? (const char *)afl->file_extension : "");
+
+      } else {
+
+        const char *hex = sha1_hex(mem, len);
+        snprintf(fn, PATH_MAX, "%s/hangs/%s%s%s", afl->out_dir, hex,
+                 afl->file_extension ? "." : "",
+                 afl->file_extension ? (const char *)afl->file_extension : "");
+        ck_free((char *)hex);
+
+      }
 
 #else
 
@@ -799,11 +827,23 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) {
 
 #ifndef SIMPLE_FILES
 
-      snprintf(fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s%s%s",
+      if (!afl->afl_env.afl_sha1_filenames) {
-               afl->out_dir, afl->saved_crashes, afl->fsrv.last_kill_signal,
+
-               describe_op(afl, 0, NAME_MAX - strlen("id:000000,sig:00,")),
+        snprintf(fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s%s%s",
-               afl->file_extension ? "." : "",
+                 afl->out_dir, afl->saved_crashes, afl->fsrv.last_kill_signal,
-               afl->file_extension ? (const char *)afl->file_extension : "");
+                 describe_op(afl, 0, NAME_MAX - strlen("id:000000,sig:00,")),
+                 afl->file_extension ? "." : "",
+                 afl->file_extension ? (const char *)afl->file_extension : "");
+
+      } else {
+
+        const char *hex = sha1_hex(mem, len);
+        snprintf(fn, PATH_MAX, "%s/crashes/%s%s%s", afl->out_dir, hex,
+                 afl->file_extension ? "." : "",
+                 afl->file_extension ? (const char *)afl->file_extension : "");
+        ck_free((char *)hex);
+
+      }
 
 #else
 
@@ -873,10 +913,13 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) {
   /* If we're here, we apparently want to save the crash or hang
      test case, too. */
 
-  fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION);
+  fd = permissive_create(afl, fn);
-  if (unlikely(fd < 0)) { PFATAL("Unable to create '%s'", fn); }
+  if (fd >= 0) {
-  ck_write(fd, mem, len, fn);
+
-  close(fd);
+    ck_write(fd, mem, len, fn);
+    close(fd);
+
+  }
 
 #ifdef __linux__
   if (afl->fsrv.nyx_mode && fault == FSRV_RUN_CRASH) {

src/afl-fuzz-init.c

@@ -1190,14 +1190,27 @@ void perform_dry_run(afl_state_t *afl) {
 
 #ifndef SIMPLE_FILES
 
-          snprintf(
+          if (!afl->afl_env.afl_sha1_filenames) {
-              crash_fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s%s%s%s",
+
-              afl->out_dir, afl->saved_crashes, afl->fsrv.last_kill_signal,
+            snprintf(
-              describe_op(
+                crash_fn, PATH_MAX, "%s/crashes/id:%06llu,sig:%02u,%s%s%s%s",
-                  afl, 0,
+                afl->out_dir, afl->saved_crashes, afl->fsrv.last_kill_signal,
-                  NAME_MAX - strlen("id:000000,sig:00,") - strlen(use_name)),
+                describe_op(
-              use_name, afl->file_extension ? "." : "",
+                    afl, 0,
-              afl->file_extension ? (const char *)afl->file_extension : "");
+                    NAME_MAX - strlen("id:000000,sig:00,") - strlen(use_name)),
+                use_name, afl->file_extension ? "." : "",
+                afl->file_extension ? (const char *)afl->file_extension : "");
+
+          } else {
+
+            const char *hex = sha1_hex(use_mem, read_len);
+            snprintf(
+                crash_fn, PATH_MAX, "%s/crashes/%s%s%s", afl->out_dir, hex,
+                afl->file_extension ? "." : "",
+                afl->file_extension ? (const char *)afl->file_extension : "");
+            ck_free((char *)hex);
+
+          }
 
 #else
 
@@ -1518,10 +1531,23 @@ void pivot_inputs(afl_state_t *afl) {
 
       }
 
-      nfn = alloc_printf(
+      if (!afl->afl_env.afl_sha1_filenames) {
-          "%s/queue/id:%06u,time:0,execs:%llu,orig:%s%s%s", afl->out_dir, id,
+
-          afl->fsrv.total_execs, use_name, afl->file_extension ? "." : "",
+        nfn = alloc_printf(
-          afl->file_extension ? (const char *)afl->file_extension : "");
+            "%s/queue/id:%06u,time:0,execs:%llu,orig:%s%s%s", afl->out_dir, id,
+            afl->fsrv.total_execs, use_name, afl->file_extension ? "." : "",
+            afl->file_extension ? (const char *)afl->file_extension : "");
+
+      } else {
+
+        const char *hex = sha1_hex_for_file(q->fname, q->len);
+        nfn = alloc_printf(
+            "%s/queue/%s%s%s", afl->out_dir, hex,
+            afl->file_extension ? "." : "",
+            afl->file_extension ? (const char *)afl->file_extension : "");
+        ck_free((char *)hex);
+
+      }
 
       u8 *pos = strrchr(nfn, '/');
       no_spaces(pos + 30);
@@ -1738,10 +1764,11 @@ double get_runnable_processes(void) {
 
 void nuke_resume_dir(afl_state_t *afl) {
 
-  u8 *fn;
+  u8 *const case_prefix = afl->afl_env.afl_sha1_filenames ? "" : CASE_PREFIX;
+  u8       *fn;
 
   fn = alloc_printf("%s/_resume/.state/deterministic_done", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   fn = alloc_printf("%s/_resume/.state/auto_extras", afl->out_dir);
@@ -1749,11 +1776,11 @@ void nuke_resume_dir(afl_state_t *afl) {
   ck_free(fn);
 
   fn = alloc_printf("%s/_resume/.state/redundant_edges", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   fn = alloc_printf("%s/_resume/.state/variable_behavior", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   fn = alloc_printf("%s/_resume/.state", afl->out_dir);
@@ -1761,7 +1788,7 @@ void nuke_resume_dir(afl_state_t *afl) {
   ck_free(fn);
 
   fn = alloc_printf("%s/_resume", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   return;
@@ -1778,8 +1805,9 @@ dir_cleanup_failed:
 
 static void handle_existing_out_dir(afl_state_t *afl) {
 
-  FILE *f;
+  u8 *const case_prefix = afl->afl_env.afl_sha1_filenames ? "" : CASE_PREFIX;
-  u8   *fn = alloc_printf("%s/fuzzer_stats", afl->out_dir);
+  FILE     *f;
+  u8       *fn = alloc_printf("%s/fuzzer_stats", afl->out_dir);
 
   /* See if the output directory is locked. If yes, bail out. If not,
      create a lock that will persist for the lifetime of the process
@@ -1901,7 +1929,7 @@ static void handle_existing_out_dir(afl_state_t *afl) {
   /* Next, we need to clean up <afl->out_dir>/queue/.state/ subdirectories: */
 
   fn = alloc_printf("%s/queue/.state/deterministic_done", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   fn = alloc_printf("%s/queue/.state/auto_extras", afl->out_dir);
@@ -1909,11 +1937,11 @@ static void handle_existing_out_dir(afl_state_t *afl) {
   ck_free(fn);
 
   fn = alloc_printf("%s/queue/.state/redundant_edges", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   fn = alloc_printf("%s/queue/.state/variable_behavior", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   /* Then, get rid of the .state subdirectory itself (should be empty by now)
@@ -1924,7 +1952,7 @@ static void handle_existing_out_dir(afl_state_t *afl) {
   ck_free(fn);
 
   fn = alloc_printf("%s/queue", afl->out_dir);
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   /* All right, let's do <afl->out_dir>/crashes/id:* and
@@ -1971,7 +1999,7 @@ static void handle_existing_out_dir(afl_state_t *afl) {
 #ifdef AFL_PERSISTENT_RECORD
   delete_files(fn, RECORD_PREFIX);
 #endif
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   fn = alloc_printf("%s/hangs", afl->out_dir);
@@ -2006,7 +2034,7 @@ static void handle_existing_out_dir(afl_state_t *afl) {
 #ifdef AFL_PERSISTENT_RECORD
   delete_files(fn, RECORD_PREFIX);
 #endif
-  if (delete_files(fn, CASE_PREFIX)) { goto dir_cleanup_failed; }
+  if (delete_files(fn, case_prefix)) { goto dir_cleanup_failed; }
   ck_free(fn);
 
   /* And now, for some finishing touches. */

src/afl-fuzz-queue.c

@@ -371,9 +371,8 @@ void mark_as_redundant(afl_state_t *afl, struct queue_entry *q, u8 state) {
     s32 fd;
 
     if (unlikely(afl->afl_env.afl_disable_redundant)) { q->disabled = 1; }
-    fd = open(fn, O_WRONLY | O_CREAT | O_EXCL, DEFAULT_PERMISSION);
+    fd = permissive_create(afl, fn);
-    if (fd < 0) { PFATAL("Unable to create '%s'", fn); }
+    if (fd >= 0) { close(fd); }
-    close(fd);
 
   } else {
 

src/afl-fuzz-state.c

@@ -626,6 +626,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) {
 
             }
 
+          } else if (!strncmp(env, "AFL_SHA1_FILENAMES",
+
+                              afl_environment_variable_len)) {
+
+            afl->afl_env.afl_sha1_filenames =
+                get_afl_env(afl_environment_variables[i]) ? 1 : 0;
+
           }
 
         } else {

src/afl-performance.c

@@ -95,3 +95,313 @@ inline u64 hash64(u8 *key, u32 len, u64 seed) {
 
 }
 
+// Public domain SHA1 implementation copied from:
+// https://github.com/x42/liboauth/blob/7001b8256cd654952ec2515b055d2c5b243be600/src/sha1.c
+
+/* This code is public-domain - it is based on libcrypt
+ * placed in the public domain by Wei Dai and other contributors.
+ */
+// gcc -Wall -DSHA1TEST -o sha1test sha1.c && ./sha1test
+
+#include <stdint.h>
+#include <string.h>
+
+#ifdef __BIG_ENDIAN__
+  #define SHA_BIG_ENDIAN
+#elif defined __LITTLE_ENDIAN__
+/* override */
+#elif defined __BYTE_ORDER
+  #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    #define SHA_BIG_ENDIAN
+  #endif
+#else                  // ! defined __LITTLE_ENDIAN__
+  #include <endian.h>  // machine/endian.h
+  #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    #define SHA_BIG_ENDIAN
+  #endif
+#endif
+
+/* header */
+
+#define HASH_LENGTH 20
+#define BLOCK_LENGTH 64
+
+typedef struct sha1nfo {
+
+  uint32_t buffer[BLOCK_LENGTH / 4];
+  uint32_t state[HASH_LENGTH / 4];
+  uint32_t byteCount;
+  uint8_t  bufferOffset;
+  uint8_t  keyBuffer[BLOCK_LENGTH];
+  uint8_t  innerHash[HASH_LENGTH];
+
+} sha1nfo;
+
+/* public API - prototypes - TODO: doxygen*/
+
+/**
+ */
+void sha1_init(sha1nfo *s);
+/**
+ */
+void sha1_writebyte(sha1nfo *s, uint8_t data);
+/**
+ */
+void sha1_write(sha1nfo *s, const char *data, size_t len);
+/**
+ */
+uint8_t *sha1_result(sha1nfo *s);
+/**
+ */
+void sha1_initHmac(sha1nfo *s, const uint8_t *key, int keyLength);
+/**
+ */
+uint8_t *sha1_resultHmac(sha1nfo *s);
+
+/* code */
+#define SHA1_K0 0x5a827999
+#define SHA1_K20 0x6ed9eba1
+#define SHA1_K40 0x8f1bbcdc
+#define SHA1_K60 0xca62c1d6
+
+void sha1_init(sha1nfo *s) {
+
+  s->state[0] = 0x67452301;
+  s->state[1] = 0xefcdab89;
+  s->state[2] = 0x98badcfe;
+  s->state[3] = 0x10325476;
+  s->state[4] = 0xc3d2e1f0;
+  s->byteCount = 0;
+  s->bufferOffset = 0;
+
+}
+
+uint32_t sha1_rol32(uint32_t number, uint8_t bits) {
+
+  return ((number << bits) | (number >> (32 - bits)));
+
+}
+
+void sha1_hashBlock(sha1nfo *s) {
+
+  uint8_t  i;
+  uint32_t a, b, c, d, e, t;
+
+  a = s->state[0];
+  b = s->state[1];
+  c = s->state[2];
+  d = s->state[3];
+  e = s->state[4];
+  for (i = 0; i < 80; i++) {
+
+    if (i >= 16) {
+
+      t = s->buffer[(i + 13) & 15] ^ s->buffer[(i + 8) & 15] ^
+          s->buffer[(i + 2) & 15] ^ s->buffer[i & 15];
+      s->buffer[i & 15] = sha1_rol32(t, 1);
+
+    }
+
+    if (i < 20) {
+
+      t = (d ^ (b & (c ^ d))) + SHA1_K0;
+
+    } else if (i < 40) {
+
+      t = (b ^ c ^ d) + SHA1_K20;
+
+    } else if (i < 60) {
+
+      t = ((b & c) | (d & (b | c))) + SHA1_K40;
+
+    } else {
+
+      t = (b ^ c ^ d) + SHA1_K60;
+
+    }
+
+    t += sha1_rol32(a, 5) + e + s->buffer[i & 15];
+    e = d;
+    d = c;
+    c = sha1_rol32(b, 30);
+    b = a;
+    a = t;
+
+  }
+
+  s->state[0] += a;
+  s->state[1] += b;
+  s->state[2] += c;
+  s->state[3] += d;
+  s->state[4] += e;
+
+}
+
+void sha1_addUncounted(sha1nfo *s, uint8_t data) {
+
+  uint8_t *const b = (uint8_t *)s->buffer;
+#ifdef SHA_BIG_ENDIAN
+  b[s->bufferOffset] = data;
+#else
+  b[s->bufferOffset ^ 3] = data;
+#endif
+  s->bufferOffset++;
+  if (s->bufferOffset == BLOCK_LENGTH) {
+
+    sha1_hashBlock(s);
+    s->bufferOffset = 0;
+
+  }
+
+}
+
+void sha1_writebyte(sha1nfo *s, uint8_t data) {
+
+  ++s->byteCount;
+  sha1_addUncounted(s, data);
+
+}
+
+void sha1_write(sha1nfo *s, const char *data, size_t len) {
+
+  for (; len--;)
+    sha1_writebyte(s, (uint8_t)*data++);
+
+}
+
+void sha1_pad(sha1nfo *s) {
+
+  // Implement SHA-1 padding (fips180-2 §5.1.1)
+
+  // Pad with 0x80 followed by 0x00 until the end of the block
+  sha1_addUncounted(s, 0x80);
+  while (s->bufferOffset != 56)
+    sha1_addUncounted(s, 0x00);
+
+  // Append length in the last 8 bytes
+  sha1_addUncounted(s, 0);  // We're only using 32 bit lengths
+  sha1_addUncounted(s, 0);  // But SHA-1 supports 64 bit lengths
+  sha1_addUncounted(s, 0);  // So zero pad the top bits
+  sha1_addUncounted(s, s->byteCount >> 29);  // Shifting to multiply by 8
+  sha1_addUncounted(
+      s, s->byteCount >> 21);  // as SHA-1 supports bitstreams as well as
+  sha1_addUncounted(s, s->byteCount >> 13);  // byte.
+  sha1_addUncounted(s, s->byteCount >> 5);
+  sha1_addUncounted(s, s->byteCount << 3);
+
+}
+
+uint8_t *sha1_result(sha1nfo *s) {
+
+  // Pad to complete the last block
+  sha1_pad(s);
+
+#ifndef SHA_BIG_ENDIAN
+  // Swap byte order back
+  int i;
+  for (i = 0; i < 5; i++) {
+
+    s->state[i] = (((s->state[i]) << 24) & 0xff000000) |
+                  (((s->state[i]) << 8) & 0x00ff0000) |
+                  (((s->state[i]) >> 8) & 0x0000ff00) |
+                  (((s->state[i]) >> 24) & 0x000000ff);
+
+  }
+
+#endif
+
+  // Return pointer to hash (20 characters)
+  return (uint8_t *)s->state;
+
+}
+
+#define HMAC_IPAD 0x36
+#define HMAC_OPAD 0x5c
+
+void sha1_initHmac(sha1nfo *s, const uint8_t *key, int keyLength) {
+
+  uint8_t i;
+  memset(s->keyBuffer, 0, BLOCK_LENGTH);
+  if (keyLength > BLOCK_LENGTH) {
+
+    // Hash long keys
+    sha1_init(s);
+    for (; keyLength--;)
+      sha1_writebyte(s, *key++);
+    memcpy(s->keyBuffer, sha1_result(s), HASH_LENGTH);
+
+  } else {
+
+    // Block length keys are used as is
+    memcpy(s->keyBuffer, key, keyLength);
+
+  }
+
+  // Start inner hash
+  sha1_init(s);
+  for (i = 0; i < BLOCK_LENGTH; i++) {
+
+    sha1_writebyte(s, s->keyBuffer[i] ^ HMAC_IPAD);
+
+  }
+
+}
+
+uint8_t *sha1_resultHmac(sha1nfo *s) {
+
+  uint8_t i;
+  // Complete inner hash
+  memcpy(s->innerHash, sha1_result(s), HASH_LENGTH);
+  // Calculate outer hash
+  sha1_init(s);
+  for (i = 0; i < BLOCK_LENGTH; i++)
+    sha1_writebyte(s, s->keyBuffer[i] ^ HMAC_OPAD);
+  for (i = 0; i < HASH_LENGTH; i++)
+    sha1_writebyte(s, s->innerHash[i]);
+  return sha1_result(s);
+
+}
+
+// End public domain SHA1 implementation
+
+void sha1(const u8 *data, size_t len, u8 *out) {
+
+  sha1nfo s;
+  sha1_init(&s);
+  sha1_write(&s, (const char *)data, len);
+  memcpy(out, sha1_result(&s), HASH_LENGTH);
+
+}
+
+char *sha1_hex(const u8 *data, size_t len) {
+
+  u8 digest[HASH_LENGTH];
+  sha1(data, len, digest);
+  u8 *hex = ck_alloc(HASH_LENGTH * 2 + 1);
+  for (size_t i = 0; i < HASH_LENGTH; ++i) {
+
+    sprintf((char *)(hex + i * 2), "%02x", digest[i]);
+
+  }
+
+  return hex;
+
+}
+
+char *sha1_hex_for_file(const char *fname, u32 len) {
+
+  int fd = open(fname, O_RDONLY);
+  if (fd < 0) { PFATAL("Unable to open '%s'", fname); }
+
+  u32 read_len = MIN(len, (u32)MAX_FILE);
+  u8 *tmp = ck_alloc(read_len);
+  ck_read(fd, tmp, read_len, fname);
+
+  close(fd);
+
+  char *hex = sha1_hex(tmp, read_len);
+  ck_free(tmp);
+  return hex;
+
+}
+