From ef2cc1404cc69c082e20a1679e0e7ee946af2c39 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 14 Oct 2013 16:45:56 +1030
Subject: [PATCH 01/24] Improve test framework: tfw_cmp_version()

Also add unit tests for test framework itself
---
 testframework.sh | 14 ++++++++++++++
 tests/framework  | 47 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100755 tests/framework

diff --git a/testframework.sh b/testframework.sh
index 083f77b7..9c183ef4 100644
--- a/testframework.sh
+++ b/testframework.sh
@@ -840,6 +840,20 @@ tfw_quietly() {
    fi
 }
 
+# Compare the two arguments as dotted ascii decimal version strings.
+# Return 0 if they are equal, 1 if arg1 < arg2, 2 if arg1 > arg2
+tfw_cmp_version() {
+   local IFS=.
+   local i=0 a=($1) b=($2)
+   for (( i=0; i < ${#a[@]} || i < ${#b[@]}; ++i )); do
+      local ai="${a[i]:-0}"
+      local bi="${b[i]:-0}"
+      (( 10#$ai < 10#$bi )) && return 1
+      (( 10#$ai > 10#$bi )) && return 2
+   done
+   return 0
+}
+
 # Append the contents of a file to the test case's stdout log.  A normal 'cat'
 # to stdout would also do this, but tfw_cat echoes header and footer delimiter
 # lines around to content to help distinguish it, and also works even in a
diff --git a/tests/framework b/tests/framework
new file mode 100755
index 00000000..2343ceb2
--- /dev/null
+++ b/tests/framework
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Tests for Serval rhizome operations.
+#
+# Copyright 2012 Serval Project, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+source "${0%/*}/../testframework.sh"
+
+shopt -s extglob
+
+test_tfw_cmp_version() {
+   execute --exit-status=1 tfw_cmp_version 1 2
+   execute --exit-status=2 tfw_cmp_version 1.0.1 1.0.0
+   execute --exit-status=1 tfw_cmp_version 1.0 1.1
+   execute --exit-status=0 tfw_cmp_version 1            1
+   execute --exit-status=1 tfw_cmp_version 2.1          2.2
+   execute --exit-status=2 tfw_cmp_version 3.0.4.10     3.0.4.2
+   execute --exit-status=1 tfw_cmp_version 4.08         4.08.01
+   execute --exit-status=2 tfw_cmp_version 3.2.1.9.8144 3.2
+   execute --exit-status=1 tfw_cmp_version 3.2          3.2.1.9.8144
+   execute --exit-status=1 tfw_cmp_version 1.2          2.1
+   execute --exit-status=2 tfw_cmp_version 2.1          1.2
+   execute --exit-status=0 tfw_cmp_version 5.6.7        5.6.7
+   execute --exit-status=0 tfw_cmp_version 1.01.1       1.1.1
+   execute --exit-status=0 tfw_cmp_version 1.1.1        1.01.1
+   execute --exit-status=0 tfw_cmp_version 1            1.0
+   execute --exit-status=0 tfw_cmp_version 1.0          1
+   execute --exit-status=0 tfw_cmp_version 1.0.2.0      1.0.2
+   execute --exit-status=0 tfw_cmp_version 1..0         1.0
+   execute --exit-status=0 tfw_cmp_version 1.0          1..0
+}
+
+runTests "$@"

From eb46cc99b38f678f9a5d0f3d1f3ee65a5db58dc7 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 14 Oct 2013 16:48:20 +1030
Subject: [PATCH 02/24] Improve test defs: setup_curl()

Replace setup_curl_7() function in separate test scripts with a single,
general setup_curl() function in testdefs.sh which takes the minimum
version number as its argument.
---
 testdefs.sh           | 28 ++++++++++++++++++++++++++++
 tests/rhizomechannels | 12 ------------
 tests/rhizomeprotocol | 18 +++---------------
 3 files changed, 31 insertions(+), 27 deletions(-)

diff --git a/testdefs.sh b/testdefs.sh
index 791c7fb3..4196e03e 100644
--- a/testdefs.sh
+++ b/testdefs.sh
@@ -732,3 +732,31 @@ has_seen_instances() {
 instances_see_each_other() {
    foreach_instance "$@" has_seen_instances "$@"
 }
+
+# Setup function:
+# - ensure that the given version of the curl(1) utility is available
+# - remove all proxy settings
+setup_curl() {
+   local minversion="${1?}"
+   local ver="$(curl --version | tr '\n' ' ')" 
+   case "$ver" in
+   '')
+      fail "curl(1) command is not present"
+      ;;
+   curl\ *\ Protocols:*\ http\ *)
+      set -- $ver
+      tfw_cmp_version "$2" 7
+      case $? in
+      0|2)
+         unset http_proxy
+         unset HTTP_PROXY
+         unset HTTPS_PROXY
+         unset ALL_PROXY
+         return 0
+         ;;
+      esac
+      fail "curl(1) version $2 is not adequate (expecting $minversion or higher)"
+      ;;
+   esac
+   fail "cannot parse output of curl --version: $ver"
+}
diff --git a/tests/rhizomechannels b/tests/rhizomechannels
index 48c40a07..330e3afb 100755
--- a/tests/rhizomechannels
+++ b/tests/rhizomechannels
@@ -46,18 +46,6 @@ configure_servald_server() {
       set mdp.iftype.wifi.tick_ms 500
 }
 
-setup_curl_7() {
-   case "$(curl --version | tr '\n' ' ')" in
-   curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;;
-   '') fail "curl(1) command is not present";;
-   *) fail "curl(1) version is not adequate (expecting 7 or higher)";;
-   esac
-   unset http_proxy
-   unset HTTP_PROXY
-   unset HTTPS_PROXY
-   unset ALL_PROXY
-}
-
 setup_common() {
    setup_servald
    assert_no_servald_processes
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 0f4663b3..6f58a43b 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -47,18 +47,6 @@ configure_servald_server() {
       set mdp.iftype.wifi.tick_ms 500
 }
 
-setup_curl_7() {
-   case "$(curl --version | tr '\n' ' ')" in
-   curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;;
-   '') fail "curl(1) command is not present";;
-   *) fail "curl(1) version is not adequate (expecting 7 or higher)";;
-   esac
-   unset http_proxy
-   unset HTTP_PROXY
-   unset HTTPS_PROXY
-   unset ALL_PROXY
-}
-
 setup_common() {
    setup_servald
    assert_no_servald_processes
@@ -526,7 +514,7 @@ test_CorruptPayload() {
 
 doc_HttpFetchRange="Fetch a file range using HTTP GET."
 setup_HttpFetchRange() {
-   setup_curl_7
+   setup_curl 7
    setup_common
    set_instance +A
    rhizome_add_file file1
@@ -549,7 +537,7 @@ test_HttpFetchRange() {
 
 doc_HttpImport="Import bundle using HTTP POST multi-part form."
 setup_HttpImport() {
-   setup_curl_7
+   setup_curl 7
    setup_common
    cat >README.WHYNOTSIPS <<'EOF'
 When we were looking at implementing secure calls for OpenBTS it was suggested
@@ -599,7 +587,7 @@ test_HttpImport() {
 
 doc_HttpAddLocal="Add file locally using HTTP, returns manifest"
 setup_HttpAddLocal() {
-   setup_curl_7
+   setup_curl 7
    setup_common
    set_instance +A
    executeOk_servald config \

From 70af3289f642228758b688602ddda64003f16a3f Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 14 Oct 2013 16:49:06 +1030
Subject: [PATCH 03/24] Refactor setup for 'rhizomeops' test script

---
 tests/rhizomeops | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/tests/rhizomeops b/tests/rhizomeops
index 29f5edd0..c8af2b35 100755
--- a/tests/rhizomeops
+++ b/tests/rhizomeops
@@ -26,21 +26,24 @@ shopt -s extglob
 
 setup_rhizome() {
    set_instance +A
-   executeOk_servald config \
-      set debug.rhizome on \
-      set debug.verbose on \
-      set log.console.level debug
+   set_rhizome_config
    create_single_identity
+   echo "$SIDA1" >sids
    set_instance +B
+   set_rhizome_config
+   create_identities 4
+   echo "$SIDB1" >>sids
+   echo "$SIDB2" >>sids
+   echo "$SIDB3" >>sids
+   echo "$SIDB4" >>sids
+   assert [ $(sort sids | uniq | wc -l) -eq 5 ]
+}
+
+set_rhizome_config() {
    executeOk_servald config \
       set debug.rhizome on \
       set debug.verbose on \
       set log.console.level debug
-   create_identities 4
-   assert [ $SIDB1 != $SIDA1 ]
-   assert [ $SIDB2 != $SIDA1 ]
-   assert [ $SIDB3 != $SIDA1 ]
-   assert [ $SIDB4 != $SIDA1 ]
 }
 
 doc_InitialEmptyList="Initial list is empty"

From f3f39faf843cc19ce1f34805e91b1081f7268ad3 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 14 Oct 2013 16:50:02 +1030
Subject: [PATCH 04/24] Improve test defs: setup_jq()

For test fixtures that need the jq(1) JSON filter utility
---
 testdefs.sh | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/testdefs.sh b/testdefs.sh
index 4196e03e..2e083640 100644
--- a/testdefs.sh
+++ b/testdefs.sh
@@ -760,3 +760,24 @@ setup_curl() {
    esac
    fail "cannot parse output of curl --version: $ver"
 }
+
+# Setup function:
+# - ensure that version 1.2 or later of the jq(1) utility is available
+setup_jq() {
+   local minversion="${1?}"
+   local ver="$(jq --version 2>&1)"
+   case "$ver" in
+   '')
+      fail "jq(1) command is not present"
+      ;;
+   jq\ version\ *)
+      set -- $ver
+      tfw_cmp_version "$3" "$minversion"
+      case $? in
+      0|2) return 0;;
+      esac
+      fail "jq(1) version $3 is not adequate (need $minversion or higher)"
+      ;;
+   esac
+   fail "cannot parse output of jq --version: $ver"
+}

From 3d3e900e727fac20f1984ced519de948d073b9ca Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 14 Oct 2013 17:46:09 +1030
Subject: [PATCH 05/24] First HTTP RESTful interface tests

---
 conf_schema.h     |  14 +++++
 tests/rhizomehttp | 140 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+)
 create mode 100755 tests/rhizomehttp

diff --git a/conf_schema.h b/conf_schema.h
index 5de82ff9..ad83bc20 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -375,6 +375,15 @@ STRUCT(rhizome_direct)
 SUB_STRUCT(peerlist,        peer,)
 END_STRUCT
 
+STRUCT(user)
+STRING(50,                  password,   "", str,, "Authentication password")
+END_STRUCT
+
+ARRAY(userlist,)
+KEY_STRING(25, str)
+VALUE_SUB_STRUCT(user)
+END_ARRAY(10)
+
 STRUCT(rhizome_api_addfile)
 STRING(64,                  uri_path,               "", absolute_path,, "URI path for HTTP add-file request")
 ATOM(struct in_addr,        allow_host,             hton_in_addr(INADDR_LOOPBACK), in_addr,, "IP address of host allowed to make HTTP add-file request")
@@ -383,8 +392,13 @@ ATOM(sid_t,                 default_author,         SID_ANY, sid,, "Author of ad
 ATOM(rhizome_bk_t,          bundle_secret_key,      RHIZOME_BK_NONE, rhizome_bk,, "Secret key of add-file bundle to try if sender not given")
 END_STRUCT
 
+STRUCT(rhizome_api_restful)
+SUB_STRUCT(userlist,        users,)
+END_STRUCT
+
 STRUCT(rhizome_api)
 SUB_STRUCT(rhizome_api_addfile, addfile,)
+SUB_STRUCT(rhizome_api_restful, restful,)
 END_STRUCT
 
 STRUCT(rhizome_http)
diff --git a/tests/rhizomehttp b/tests/rhizomehttp
new file mode 100755
index 00000000..8fa74c77
--- /dev/null
+++ b/tests/rhizomehttp
@@ -0,0 +1,140 @@
+#!/bin/bash
+
+# Tests for Serval rhizome operations.
+#
+# Copyright 2013 Serval Project, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+source "${0%/*}/../testframework.sh"
+source "${0%/*}/../testdefs.sh"
+source "${0%/*}/../testdefs_rhizome.sh"
+
+shopt -s extglob
+
+setup() {
+   setup_curl 7
+   setup_jq 1.2
+   setup_servald
+   set_instance +A
+   set_rhizome_config
+   executeOk_servald config \
+      set rhizome.api.restful.users.harry.password potter \
+      set rhizome.api.restful.users.ron.password weasley \
+      set rhizome.api.restful.users.hermione.password grainger
+   create_single_identity
+   echo "$SIDA1" >sids
+   start_servald_instances +A
+   wait_until rhizome_http_server_started +A
+   get_rhizome_server_port PORTA +A
+}
+
+set_rhizome_config() {
+   executeOk_servald config \
+      set debug.rhizome on \
+      set debug.verbose on \
+      set log.console.level debug
+}
+
+doc_AuthBasicMissing="Basic Authentication credentials are required"
+test_AuthBasicMissing() {
+   execute --exit-status=67 curl \
+         --silent --fail --show-error \
+         --output http.output \
+         --dump-header http.headers \
+         "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
+}
+teardown_AuthBasicMissing() {
+   tfw_cat http.headers http.output
+}
+
+doc_AuthBasicWrong="Basic Authentication credentials must be correct"
+test_AuthBasicWrong() {
+   execute --exit-status=67 curl \
+         --silent --fail --show-error \
+         --output http.output \
+         --dump-header http.headers \
+         --basic --user fred:nurks \
+         "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
+   executeOk curl \
+         --silent --fail --show-error \
+         --output http.output \
+         --dump-header http.headers \
+         --basic --user ron:weasley \
+         "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
+}
+
+doc_RhizomeList="Fetch full Rhizome bundle list in JSON format"
+setup_RhizomeList() {
+   for n in 1 2 3 4; do
+      create_file file$n ${n}k
+      executeOk_servald rhizome add file $SIDA file$n file$n.manifest
+   done
+}
+test_RhizomeList() {
+   executeOk curl \
+         --silent --fail --show-error \
+         --output http.output \
+         --dump-header http.headers \
+         --basic --user harry:potter \
+         "http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
+}
+
+doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format"
+test_RhizomeListSinceJSON() {
+   :
+}
+
+doc_RhizomeManifest="Fetch Rhizome bundle manifest"
+test_RhizomeManifest() {
+   :
+}
+
+doc_RhizomePayloadRaw="Fetch Rhizome raw payload"
+test_RhizomePayloadRaw() {
+   :
+}
+
+doc_RhizomePayloadDecrypted="Fetch Rhizome decrypted payload"
+test_RhizomePayloadDecrypted() {
+   :
+}
+
+doc_RhizomeInsert="Insert new Rhizome bundle"
+test_RhizomeInsert() {
+   :
+}
+
+doc_MeshmsListConversations="List MeshMS conversations"
+test_MeshmsListConversations() {
+   :
+}
+
+doc_MeshmsListMessages="List all MeshMS messages in a single conversation"
+test_MeshmsListMessages() {
+   :
+}
+
+doc_MeshmsListMessagesSince="List MeshMS messages in a single conversation since token"
+test_MeshmsListMessagesSince() {
+   :
+}
+
+doc_MeshmsSend="Send MeshMS message"
+test_MeshmsSend() {
+   :
+}
+
+runTests "$@"

From 609e47faba825505cf9fa572591d551fabce1959 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 16 Oct 2013 11:16:04 +1030
Subject: [PATCH 06/24] Add str_to_int() and str_to_uint()

---
 str.c | 43 +++++++++++++++++++++++++++++++++++++++++--
 str.h |  2 ++
 2 files changed, 43 insertions(+), 2 deletions(-)

diff --git a/str.c b/str.c
index f1e7ef3d..11a69c38 100644
--- a/str.c
+++ b/str.c
@@ -28,6 +28,7 @@
 #include <ctype.h>
 #include <assert.h>
 #include <limits.h>
+#include <errno.h>
 
 const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
 
@@ -227,13 +228,50 @@ char *str_str(char *haystack, const char *needle, int haystack_len)
   return NULL;
 }
 
+int str_to_int(const char *str, int base, int *result, const char **afterp)
+{
+  if (isspace(*str))
+    return 0;
+  const char *end = str;
+  errno = 0;
+  long value = strtol(str, (char**)&end, base);
+  if (errno == ERANGE || end == str || value > INT_MAX || value < INT_MIN)
+    return 0;
+  if (afterp)
+    *afterp = end;
+  else if (*end)
+    return 0;
+  if (result)
+    *result = value;
+  return 1;
+}
+
+int str_to_uint(const char *str, int base, unsigned *result, const char **afterp)
+{
+  if (isspace(*str))
+    return 0;
+  const char *end = str;
+  errno = 0;
+  unsigned long value = strtoul(str, (char**)&end, base);
+  if (errno == ERANGE || end == str || value > UINT_MAX)
+    return 0;
+  if (afterp)
+    *afterp = end;
+  else if (*end)
+    return 0;
+  if (result)
+    *result = value;
+  return 1;
+}
+
 int str_to_int64(const char *str, int base, int64_t *result, const char **afterp)
 {
   if (isspace(*str))
     return 0;
   const char *end = str;
+  errno = 0;
   long long value = strtoll(str, (char**)&end, base);
-  if (end == str)
+  if (errno == ERANGE || end == str)
     return 0;
   if (afterp)
     *afterp = end;
@@ -249,8 +287,9 @@ int str_to_uint64(const char *str, int base, uint64_t *result, const char **afte
   if (isspace(*str))
     return 0;
   const char *end = str;
+  errno = 0;
   unsigned long long value = strtoull(str, (char**)&end, base);
-  if (end == str)
+  if (errno == ERANGE || end == str)
     return 0;
   if (afterp)
     *afterp = end;
diff --git a/str.h b/str.h
index 811ffc52..555897ac 100644
--- a/str.h
+++ b/str.h
@@ -206,6 +206,8 @@ char *str_str(char *haystack, const char *needle, int haystack_len);
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
+int str_to_int(const char *str, int base, int *result, const char **afterp);
+int str_to_uint(const char *str, int base, unsigned *result, const char **afterp);
 int str_to_int64(const char *str, int base, int64_t *result, const char **afterp);
 int str_to_uint64(const char *str, int base, uint64_t *result, const char **afterp);
 

From 00cf61721d661f649705d93873722192f588e8a0 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 16 Oct 2013 11:22:02 +1030
Subject: [PATCH 07/24] Rename http_header_complete() to
 is_http_header_complete()

---
 rhizome.h             | 2 +-
 rhizome_direct_http.c | 2 +-
 rhizome_fetch.c       | 4 ++--
 rhizome_http.c        | 4 ++--
 str.h                 | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/rhizome.h b/rhizome.h
index fa64ee98..9b73ff68 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -269,7 +269,7 @@ int rhizome_str_is_bundle_crypt_key(const char *text);
 int rhizome_strn_is_file_hash(const char *text);
 int rhizome_str_is_file_hash(const char *text);
 
-int http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
+int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
 
 typedef struct sqlite_retry_state {
   unsigned int limit; // do not retry once elapsed >= limit
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index ef6a9430..3e9e9db7 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -688,7 +688,7 @@ static int receive_http_response(int sock, char *buffer, size_t buffer_len, stru
       if ((count = read(sock, &buffer[len], buffer_len - len)) == -1)
 	return WHYF_perror("read(%d, %p, %d)", sock, &buffer[len], (int)buffer_len - len);
       len += count;
-  } while (len < buffer_len && count != 0 && !http_header_complete(buffer, len, len));
+  } while (len < buffer_len && count != 0 && !is_http_header_complete(buffer, len, len));
   if (config.debug.rhizome_rx)
     DEBUGF("Received HTTP response %s", alloca_toprint(-1, buffer, len));
   if (unpack_http_response(buffer, parts) == -1)
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 20a51f26..abdc4cf2 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -1483,7 +1483,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
 	slot->alarm.deadline = slot->alarm.alarm + config.rhizome.idle_timeout;
 	schedule(&slot->alarm);
 	slot->request_len += bytes;
-	if (http_header_complete(slot->request, slot->request_len, bytes)) {
+	if (is_http_header_complete(slot->request, slot->request_len, bytes)) {
 	  if (config.debug.rhizome_rx)
 	    DEBUGF("Got HTTP reply: %s", alloca_toprint(160, slot->request, slot->request_len));
 	  /* We have all the reply headers, so parse them, taking care of any following bytes of
@@ -1564,7 +1564,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
    This function takes a pointer to a buffer into which the entire HTTP response header has been
    read.  The caller must have ensured that the buffer contains at least one consecutive pair of
    newlines '\n', optionally with carriage returns '\r' preceding and optionally interspersed with
-   nul characters '\0' (which can originate from telnet).  The http_header_complete() function
+   nul characters '\0' (which can originate from telnet).  The is_http_header_complete() function
    is useful for this.
    This returns pointers to within the supplied buffer, and may overwrite some characters in the
    buffer, for example to nul-terminate a string that was terminated by space ' ' or newline '\r'
diff --git a/rhizome_http.c b/rhizome_http.c
index 3ae5c495..dff920f4 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -232,7 +232,7 @@ void rhizome_client_poll(struct sched_ent *alarm)
 	  unschedule(&r->alarm);
 	  schedule(&r->alarm);
 	  r->request_length += bytes;
-	  r->header_length = http_header_complete(r->request, r->request_length, bytes);
+	  r->header_length = is_http_header_complete(r->request, r->request_length, bytes);
 	  if (r->header_length){
 	    /* We have the request. Now parse it to see if we can respond to it */
 	    if (rhizome_http_parse_func!=NULL) 
@@ -334,7 +334,7 @@ int rhizome_server_free_http_request(rhizome_http_request *r)
   return 0;
 }
 
-int http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
+int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
 {
   IN();
   const char *bufend = buf + len;
diff --git a/str.h b/str.h
index 555897ac..276cee29 100644
--- a/str.h
+++ b/str.h
@@ -148,7 +148,7 @@ __STR_INLINE ssize_t str_rindex(const char *s, char c)
  * nul-terminated, but are held in a buffer which has an associated length.  To avoid this function
  * running past the end of the buffer, the caller must ensure that the buffer contains a sub-string
  * that is not part of the sub-string being sought, eg, "\r\n\r\n" as detected by
- * http_header_complete().  This guarantees that this function will return nonzero before running
+ * is_http_header_complete().  This guarantees that this function will return nonzero before running
  * past the end of the buffer.
  *
  * @author Andrew Bettison <andrew@servalproject.com>

From 640a61cbe596f16de96b237737a34104b86149c7 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 16 Oct 2013 11:23:45 +1030
Subject: [PATCH 08/24] Add 'debug.rhizome_httpd' config option

---
 conf_schema.h         |  1 +
 rhizome_direct_http.c |  4 ++--
 rhizome_http.c        | 12 ++++++------
 tests/rhizomechannels |  1 +
 tests/rhizomeprotocol |  3 +++
 tests/rhizomestress   |  2 ++
 6 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/conf_schema.h b/conf_schema.h
index ad83bc20..d32f966d 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -261,6 +261,7 @@ ATOM(bool_t, slipbytestream,            0, boolean,, "")
 ATOM(bool_t, packetconstruction,        0, boolean,, "")
 ATOM(bool_t, rhizome,                   0, boolean,, "")
 ATOM(bool_t, rhizome_bind,              0, boolean,, "")
+ATOM(bool_t, rhizome_httpd,             0, boolean,, "")
 ATOM(bool_t, rhizome_tx,                0, boolean,, "")
 ATOM(bool_t, rhizome_rx,                0, boolean,, "")
 ATOM(bool_t, rhizome_ads,               0, boolean,, "")
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index 3e9e9db7..c8b075cc 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -600,12 +600,12 @@ int rhizome_direct_parse_http_request(rhizome_http_request *r)
     }
   }
   if (content == NULL) {
-    if (config.debug.rhizome_tx)
+    if (config.debug.rhizome_httpd)
       DEBUGF("Received malformed HTTP request %s", alloca_toprint(160, (const char *)r->request, r->request_length));
     return rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
   }
   INFOF("RHIZOME HTTP SERVER, %s %s %s", verb, alloca_toprint(-1, path, pathlen), proto);
-  if (config.debug.rhizome_tx)
+  if (config.debug.rhizome_httpd)
     DEBUGF("headers %s", alloca_toprint(-1, headers, headerlen));
   if (strcmp(verb, "POST") == 0
       && (   strcmp(path, "/rhizome/import") == 0 
diff --git a/rhizome_http.c b/rhizome_http.c
index dff920f4..fe9b848d 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -100,7 +100,7 @@ int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *),
   if (now < rhizome_server_last_start_attempt + 5000)
     return 2;
   rhizome_server_last_start_attempt  = now;
-  if (config.debug.rhizome_tx)
+  if (config.debug.rhizome_httpd)
     DEBUGF("Starting rhizome HTTP server");
 
   uint16_t port;
@@ -245,8 +245,8 @@ void rhizome_client_poll(struct sched_ent *alarm)
 	  return;
 	}
 	if (sigPipeFlag) {
-	  if (config.debug.rhizome_tx)
-	    DEBUG("Received SIGPIPE, closing connection");
+	  if (config.debug.rhizome_httpd)
+	    DEBUG("Received SIGPIPE, closing HTTP connection");
 	  rhizome_server_free_http_request(r);
 	  return;
 	}
@@ -595,7 +595,7 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
   }
   
   if (!path) {
-    if (config.debug.rhizome_tx)
+    if (config.debug.rhizome_httpd)
       DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length));
     rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
     return 0;
@@ -696,7 +696,7 @@ int rhizome_server_set_response(rhizome_http_request *r, const struct http_respo
     r->buffer_length+=h->content_length;
   }
   r->buffer_offset = 0;
-  if (config.debug.rhizome_tx)
+  if (config.debug.rhizome_httpd)
     DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->buffer, r->buffer_length));
   return 0;
 }
@@ -774,7 +774,7 @@ int rhizome_server_http_send_bytes(rhizome_http_request *r)
   
   // once we've written the whole buffer, and nothing new has been generated, close the connection
   if (!r->buffer_length){
-    if (config.debug.rhizome_tx)
+    if (config.debug.rhizome_httpd)
       DEBUG("Closing connection, done");
     return rhizome_server_free_http_request(r);
   }
diff --git a/tests/rhizomechannels b/tests/rhizomechannels
index 330e3afb..ee67bd7d 100755
--- a/tests/rhizomechannels
+++ b/tests/rhizomechannels
@@ -40,6 +40,7 @@ configure_servald_server() {
       set log.show_pid on \
       set log.show_time on \
       set debug.rhizome on \
+      set debug.rhizome_httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \
       set server.respawn_on_crash off \
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 6f58a43b..67d2bbd0 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -41,6 +41,7 @@ configure_servald_server() {
       set log.console.show_pid on \
       set log.console.show_time on \
       set debug.rhizome on \
+      set debug.rhizome_httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \
       set server.respawn_on_crash off \
@@ -216,6 +217,7 @@ start_radio_instance() {
    executeOk_servald config \
       set debug.rhizome on \
       set debug.rhizome_ads on \
+      set debug.rhizome_httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \
       set debug.throttling on \
@@ -627,6 +629,7 @@ setup_direct() {
       set log.console.level debug \
       set log.console.show_time on \
       set debug.rhizome on \
+      set debug.rhizome_httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on
    rhizome_add_file fileB1 2000
diff --git a/tests/rhizomestress b/tests/rhizomestress
index a99d7c6d..e5fa9757 100755
--- a/tests/rhizomestress
+++ b/tests/rhizomestress
@@ -40,6 +40,7 @@ configure_servald_server() {
       set log.file.show_pid on \
       set log.file.show_time on \
       set debug.rhizome off \
+      set debug.rhizome_httpd off \
       set debug.rhizome_tx off \
       set debug.rhizome_rx off \
       set server.respawn_on_crash off \
@@ -147,6 +148,7 @@ setup_StressRhizomeDirect() {
       executeOk_servald config \
          set log.file.show_time on \
          set debug.rhizome off \
+         set debug.rhizome_httpd off \
          set debug.rhizome_tx off \
          set debug.rhizome_rx off \
          set server.respawn_on_crash off \

From fa21bec8804f404d7e67fb9bf295633caaeeaf7a Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 16 Oct 2013 18:25:58 +1030
Subject: [PATCH 09/24] Rewrite HTTP server

---
 conf.h                |    2 -
 constants.h           |    3 +
 headerfiles.mk        |    1 +
 http_server.c         | 1536 +++++++++++++++++++++++++++++++++++++++++
 http_server.h         |  156 +++++
 overlay.c             |    8 +-
 rhizome.h             |   84 +--
 rhizome_direct.c      |    3 +-
 rhizome_direct_http.c | 1092 +++++++++++------------------
 rhizome_http.c        |  649 ++++++-----------
 rhizome_store.c       |  121 ++--
 rhizome_sync.c        |    4 +-
 serval.h              |    1 -
 sourcefiles.mk        |    1 +
 strbuf_helpers.c      |   27 +
 strbuf_helpers.h      |    7 +
 16 files changed, 2434 insertions(+), 1261 deletions(-)
 create mode 100644 http_server.c
 create mode 100644 http_server.h

diff --git a/conf.h b/conf.h
index 2d7b40fb..2a3fbb7d 100644
--- a/conf.h
+++ b/conf.h
@@ -233,8 +233,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "rhizome.h"
 
-typedef char bool_t;
-
 #define CONFIG_FILE_MAX_SIZE        (32 * 1024)
 #define INTERFACE_NAME_STRLEN       40
 
diff --git a/constants.h b/constants.h
index f99cc307..cc8b4ffb 100644
--- a/constants.h
+++ b/constants.h
@@ -220,5 +220,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #define UNLOCK_CHALLENGE (0xF1)
 #define UNLOCK_RESPONSE (0xF2)
 
+// should there be a types.h to hold this?
+typedef char bool_t;
+
 
 #endif // __SERVALD_CONSTANTS_H
diff --git a/headerfiles.mk b/headerfiles.mk
index 45528bd9..a5f7fca7 100644
--- a/headerfiles.mk
+++ b/headerfiles.mk
@@ -18,6 +18,7 @@ HDRS=	fifo.h \
 	crypto.h \
 	log.h \
 	net.h \
+	http_server.h \
 	xprintf.h \
 	constants.h \
 	monitor-client.h \
diff --git a/http_server.c b/http_server.c
new file mode 100644
index 00000000..7a58282a
--- /dev/null
+++ b/http_server.c
@@ -0,0 +1,1536 @@
+/*
+Serval DNA - HTTP Server
+Copyright (C) 2013 Serval Project, Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#define __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
+#include <assert.h>
+#include <inttypes.h>
+#include <time.h>
+#include "serval.h"
+#include "http_server.h"
+#include "log.h"
+#include "str.h"
+#include "strbuf.h"
+#include "strbuf_helpers.h"
+#include "net.h"
+#include "mem.h"
+
+#define BOUNDARY_STRING_MAXLEN  70 // legislated limit from RFC-1341
+
+/* The (struct http_request).verb field points to one of these static strings, so that a simple
+ * equality test can be used, eg, (r->verb == HTTP_VERB_GET) instead of a strcmp().
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+const char HTTP_VERB_GET[] = "GET";
+const char HTTP_VERB_POST[] = "POST";
+const char HTTP_VERB_PUT[] = "PUT";
+const char HTTP_VERB_HEAD[] = "HEAD";
+const char HTTP_VERB_DELETE[] = "DELETE";
+const char HTTP_VERB_TRACE[] = "TRACE";
+const char HTTP_VERB_OPTIONS[] = "OPTIONS";
+const char HTTP_VERB_CONNECT[] = "CONNECT";
+const char HTTP_VERB_PATCH[] = "PATCH";
+
+static struct {
+  const char *word;
+  size_t wordlen;
+} http_verbs[] = {
+#define VERB_ENTRY(NAME) { HTTP_VERB_##NAME, sizeof HTTP_VERB_##NAME - 1 }
+  VERB_ENTRY(GET),
+  VERB_ENTRY(POST),
+  VERB_ENTRY(PUT),
+  VERB_ENTRY(HEAD),
+  VERB_ENTRY(DELETE),
+  VERB_ENTRY(TRACE),
+  VERB_ENTRY(OPTIONS),
+  VERB_ENTRY(CONNECT),
+  VERB_ENTRY(PATCH)
+#undef VERB_ENTRY
+};
+
+static struct profile_total http_server_stats = {
+  .name = "http_server_poll",
+};
+
+static void http_server_poll(struct sched_ent *);
+static int http_request_parse_verb(struct http_request *r);
+static int http_request_parse_path(struct http_request *r);
+static int http_request_parse_http_version(struct http_request *r);
+static int http_request_start_parsing_headers(struct http_request *r);
+static int http_request_parse_header(struct http_request *r);
+static int http_request_start_body(struct http_request *r);
+static int http_request_parse_body_form_data(struct http_request *r);
+static void http_request_start_response(struct http_request *r);
+
+void http_request_init(struct http_request *r, int sockfd)
+{
+  bzero(r, sizeof *r);
+  assert(sockfd != -1);
+  r->request_header.content_length = CONTENT_LENGTH_UNKNOWN;
+  r->request_content_remaining = CONTENT_LENGTH_UNKNOWN;
+  r->alarm.stats = &http_server_stats;
+  r->alarm.function = http_server_poll;
+  if (r->idle_timeout == 0)
+    r->idle_timeout = 10000; // 10 seconds
+  r->alarm.alarm = gettime_ms() + r->idle_timeout;
+  r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
+  r->alarm.poll.fd = sockfd;
+  r->alarm.poll.events = POLLIN;
+  r->phase = RECEIVE;
+  r->received = r->end = r->parsed = r->cursor = r->buffer;
+  r->limit = r->buffer + sizeof r->buffer;
+  r->parser = http_request_parse_verb;
+  watch(&r->alarm);
+  schedule(&r->alarm);
+}
+
+void http_request_free_response_buffer(struct http_request *r)
+{
+  if (r->response_free_buffer) {
+    r->response_free_buffer(r->response_buffer);
+    r->response_free_buffer = NULL;
+  }
+  r->response_buffer = NULL;
+  r->response_buffer_size = 0;
+}
+
+int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz)
+{
+  const char *const bufe = r->buffer + sizeof r->buffer;
+  assert(r->received < bufe);
+  size_t rbufsiz = bufe - r->received;
+  if (bufsiz <= rbufsiz) {
+    http_request_free_response_buffer(r);
+    r->response_buffer = (char *) r->received;
+    r->response_buffer_size = rbufsiz;
+    return 0;
+  }
+  if (bufsiz != r->response_buffer_size) {
+    http_request_free_response_buffer(r);
+    if ((r->response_buffer = emalloc(bufsiz)) == NULL)
+      return -1;
+    r->response_free_buffer = free;
+    r->response_buffer_size = bufsiz;
+  }
+  assert(r->response_buffer_size >= bufsiz);
+  assert(r->response_buffer != NULL);
+  return 0;
+}
+
+void http_request_finalise(struct http_request *r)
+{
+  if (r->phase == DONE)
+    return;
+  assert(r->phase == RECEIVE || r->phase == TRANSMIT);
+  unschedule(&r->alarm);
+  unwatch(&r->alarm);
+  close(r->alarm.poll.fd);
+  r->alarm.poll.fd = -1;
+  if (r->finalise)
+    r->finalise(r);
+  r->finalise = NULL;
+  http_request_free_response_buffer(r);
+  r->phase = DONE;
+}
+
+#define _SEP (1 << 0)
+#define _BND (1 << 1)
+
+uint8_t http_ctype[256] = {
+  ['0'] = _BND, ['1'] = _BND, ['2'] = _BND, ['3'] = _BND, ['4'] = _BND,
+  ['5'] = _BND, ['6'] = _BND, ['7'] = _BND, ['8'] = _BND, ['9'] = _BND,
+  ['A'] = _BND, ['B'] = _BND, ['C'] = _BND, ['D'] = _BND, ['E'] = _BND,
+  ['F'] = _BND, ['G'] = _BND, ['H'] = _BND, ['I'] = _BND, ['J'] = _BND,
+  ['K'] = _BND, ['L'] = _BND, ['M'] = _BND, ['N'] = _BND, ['O'] = _BND,
+  ['P'] = _BND, ['Q'] = _BND, ['R'] = _BND, ['S'] = _BND, ['T'] = _BND,
+  ['U'] = _BND, ['V'] = _BND, ['W'] = _BND, ['X'] = _BND, ['Y'] = _BND,
+  ['Z'] = _BND,
+  ['a'] = _BND, ['b'] = _BND, ['c'] = _BND, ['d'] = _BND, ['e'] = _BND,
+  ['f'] = _BND, ['g'] = _BND, ['h'] = _BND, ['i'] = _BND, ['j'] = _BND,
+  ['k'] = _BND, ['l'] = _BND, ['m'] = _BND, ['n'] = _BND, ['o'] = _BND,
+  ['p'] = _BND, ['q'] = _BND, ['r'] = _BND, ['s'] = _BND, ['t'] = _BND,
+  ['u'] = _BND, ['v'] = _BND, ['w'] = _BND, ['x'] = _BND, ['y'] = _BND,
+  ['z'] = _BND,
+  ['+'] = _BND, ['-'] = _BND, ['.'] = _BND, ['/'] = _BND, [':'] = _BND,
+  ['_'] = _BND,
+  ['('] = _SEP | _BND,
+  [')'] = _SEP | _BND,
+  [','] = _SEP | _BND,
+  ['?'] = _SEP | _BND,
+  ['='] = _SEP | _BND,
+  [' '] = _SEP | _BND,
+  ['\t'] = _SEP,
+  ['<'] = _SEP,
+  ['>'] = _SEP,
+  ['@'] = _SEP,
+  [';'] = _SEP,
+  [':'] = _SEP,
+  ['\\'] = _SEP,
+  ['"'] = _SEP,
+  ['/'] = _SEP,
+  ['['] = _SEP,
+  [']'] = _SEP,
+  ['{'] = _SEP,
+  ['}'] = _SEP,
+};
+
+inline int is_http_char(char c)
+{
+  return c >= 0;
+}
+
+inline int is_http_ctl(char c)
+{
+  return iscntrl(c);
+}
+
+inline int is_http_separator(char c)
+{
+  return (http_ctype[(unsigned char) c] & _SEP) != 0;
+}
+
+inline int is_http_boundary(char c)
+{
+  return (http_ctype[(unsigned char) c] & _BND) != 0;
+}
+
+inline int is_http_token(char c)
+{
+  return is_http_char(c) && !is_http_ctl(c) && !is_http_separator(c);
+}
+
+inline int is_valid_http_boundary_string(const char *s)
+{
+  if (s[0] == '\0')
+    return 0;
+  for (; *s; ++s)
+    if (!is_http_boundary(*s))
+      return 0;
+  return s[-1] != ' ';
+}
+
+struct substring {
+  const char *start;
+  const char *end;
+};
+
+#define alloca_substring_toprint(sub) alloca_toprint(-1, (sub).start, (sub).end - (sub).start)
+
+const struct substring substring_NULL = { NULL, NULL };
+
+#if 0
+static int _matches(struct substring str, const char *text)
+{
+  return strlen(text) == str.end - str.start && memcmp(str.start, text, str.end - str.start) == 0;
+}
+#endif
+
+static const char * _reserve(struct http_request *r, struct substring str)
+{
+  char *reslim = r->buffer + sizeof r->buffer - 1024; // always leave this much unreserved space
+  assert(r->received <= reslim);
+  size_t len = str.end - str.start;
+  size_t siz = len + 1;
+  if (r->received + siz > reslim) {
+    r->response.result_code = 414;
+    return NULL;
+  }
+  char *ret = (char *) r->received;
+  if (r->received + siz > r->parsed) {
+    WARNF("Error during HTTP parsing, unparsed content %s would be overwritten by reserving %s",
+	alloca_toprint(30, r->parsed, r->end - r->parsed),
+	alloca_substring_toprint(str)
+      );
+    r->response.result_code = 500;
+    return NULL;
+  }
+  if (ret != str.start)
+    memmove(ret, str.start, len);
+  ret[len] = '\0';
+  r->received += siz;
+  assert(r->received <= r->parsed);
+  return ret;
+}
+
+static const char * _reserve_str(struct http_request *r, const char *str)
+{
+  struct substring sub = { .start = str, .end = str + strlen(str) };
+  return _reserve(r, sub);
+}
+
+static inline int _buffer_full(struct http_request *r)
+{
+  return r->parsed == r->received && r->end == r->limit;
+}
+
+static inline int _run_out(struct http_request *r)
+{
+  assert(r->cursor <= r->end);
+  return r->cursor == r->end;
+}
+
+static inline void _rewind(struct http_request *r)
+{
+  assert(r->parsed >= r->received);
+  r->cursor = r->parsed;
+}
+
+static inline void _commit(struct http_request *r)
+{
+  assert(r->cursor <= r->end);
+  r->parsed = r->cursor;
+}
+
+static inline int _skip_crlf(struct http_request *r)
+{
+  return !_run_out(r) && *r->cursor == '\r' && ++r->cursor && !_run_out(r) && *r->cursor == '\n' && ++r->cursor;
+}
+
+static inline int _skip_to_crlf(struct http_request *r)
+{
+  const char *const start = r->cursor;
+  for (; !_run_out(r); ++r->cursor)
+    if (*r->cursor == '\n' && r->cursor > start + 1 && r->cursor[-2] == '\r') {
+      --r->cursor;
+      return 1;
+    }
+  return 0;
+}
+
+static inline void _rewind_crlf(struct http_request *r)
+{
+  assert(r->cursor >= r->parsed + 2);
+  assert(r->cursor[-2] == '\r');
+  assert(r->cursor[-1] == '\n');
+  r->cursor -= 2;
+}
+
+/* More permissive than _skip_crlf(), this counts NUL characters preceding and between the CR and LF
+ * as part of the end-of-line sequence and treats the CR as optional.  This allows simple manual
+ * testing using telnet(1).
+ */
+static inline int _skip_eol(struct http_request *r)
+{
+  unsigned crcount = 0;
+  for (; !_run_out(r); ++r->cursor) {
+    switch (*r->cursor) {
+      case '\0': // ignore any leading NULs (telnet inserts them)
+	break;
+      case '\r': // ignore up to one leading CR
+	if (++crcount > 1)
+	  return 0;
+	break;
+      case '\n':
+	++r->cursor;
+	return 1;
+      default:
+	return 0;
+    }
+  }
+  return 0;
+}
+
+/* More permissive than _skip_crlf(), this counts NUL characters preceding and between the CR and LF
+ * as part of the end-of-line sequence and treats the CR as optional.  This allows simple manual
+ * testing using telnet(1).
+ */
+static int _skip_to_eol(struct http_request *r)
+{
+  const char *const start = r->cursor;
+  while (!_run_out(r) && *r->cursor != '\n')
+    ++r->cursor;
+  if (_run_out(r))
+    return 0;
+  // consume preceding NULs (telnet inserts them)
+  while (r->cursor > start && r->cursor[-1] == '\0')
+    --r->cursor;
+  // consume a single preceding CR
+  if (r->cursor > start && r->cursor[-1] == '\r')
+    --r->cursor;
+  // consume any more preceding NULs
+  while (r->cursor > start && r->cursor[-1] == '\0')
+    --r->cursor;
+  return 1;
+}
+
+static int _skip_literal(struct http_request *r, const char *literal)
+{
+  while (!_run_out(r) && *literal && *r->cursor == *literal)
+    ++literal, ++r->cursor;
+  return *literal == '\0';
+}
+
+static int _skip_literal_nocase(struct http_request *r, const char *literal)
+{
+  while (!_run_out(r) && *literal && toupper(*r->cursor) == toupper(*literal))
+    ++literal, ++r->cursor;
+  return *literal == '\0';
+}
+
+static int _skip_optional_space(struct http_request *r)
+{
+  while (!_run_out(r) && (*r->cursor == ' ' || *r->cursor == '\t'))
+    ++r->cursor;
+  return 1;
+}
+
+static inline int _skip_space(struct http_request *r)
+{
+  const char *const start = r->cursor;
+  _skip_optional_space(r);
+  return r->cursor > start;
+}
+
+static size_t _skip_word_printable(struct http_request *r, struct substring *str)
+{
+  if (_run_out(r) || isspace(*r->cursor) || !isprint(*r->cursor))
+    return 0;
+  const char *start = r->cursor;
+  for (++r->cursor; !_run_out(r) && !isspace(*r->cursor) && isprint(*r->cursor); ++r->cursor)
+    ;
+  if (_run_out(r))
+    return 0;
+  assert(r->cursor > start);
+  assert(isspace(*r->cursor));
+  if (str) {
+    str->start = start;
+    str->end = r->cursor;
+  }
+  return r->cursor - start;
+}
+
+static size_t _skip_token(struct http_request *r, struct substring *str)
+{
+  if (_run_out(r) || !is_http_token(*r->cursor))
+    return 0;
+  const char *start = r->cursor;
+  for (++r->cursor; !_run_out(r) && is_http_token(*r->cursor); ++r->cursor)
+    ;
+  if (_run_out(r))
+    return 0;
+  assert(r->cursor > start);
+  assert(!is_http_token(*r->cursor));
+  if (str) {
+    str->start = start;
+    str->end = r->cursor;
+  }
+  return r->cursor - start;
+}
+
+static size_t _parse_token(struct http_request *r, char *dst, size_t dstsiz)
+{
+  struct substring str;
+  size_t len = _skip_token(r, &str);
+  if (len && dst) {
+    size_t cpy = len < dstsiz - 1 ? len : dstsiz - 1;
+    strncpy(dst, str.start, cpy)[cpy] = '\0';
+  }
+  return len;
+}
+
+static size_t _parse_quoted_string(struct http_request *r, char *dst, size_t dstsiz)
+{
+  assert(r->cursor <= r->end);
+  if (_run_out(r) || *r->cursor != '"')
+    return 0;
+  int slosh = 0;
+  size_t len = 0;
+  for (++r->cursor; !_run_out(r); ++r->cursor) {
+    if (!isprint(*r->cursor))
+      return 0;
+    if (slosh) {
+      if (dst && len < dstsiz - 1)
+	dst[len] = *r->cursor;
+      ++len;
+      slosh = 0;
+    } else if (*r->cursor == '"')
+      break;
+    else if (*r->cursor == '\\')
+      slosh = 1;
+    else {
+      if (dst && len < dstsiz - 1)
+	dst[len] = *r->cursor;
+      ++len;
+    }
+  }
+  if (dst)
+    dst[len < dstsiz - 1 ? len : dstsiz - 1] = '\0';
+  if (_run_out(r))
+    return 0;
+  assert(*r->cursor == '"');
+  ++r->cursor;
+  return len;
+}
+
+static size_t _parse_token_or_quoted_string(struct http_request *r, char *dst, size_t dstsiz)
+{
+  assert(dstsiz > 0);
+  if (!_run_out(r) && *r->cursor == '"')
+    return _parse_quoted_string(r, dst, dstsiz);
+  return _parse_token(r, dst, dstsiz);
+}
+
+static inline int _parse_http_size_t(struct http_request *r, http_size_t *szp)
+{
+  return !_run_out(r) && isdigit(*r->cursor) && str_to_uint64(r->cursor, 10, szp, &r->cursor);
+}
+
+static inline int _parse_uint(struct http_request *r, unsigned int *uintp)
+{
+  return !_run_out(r) && isdigit(*r->cursor) && str_to_uint(r->cursor, 10, uintp, &r->cursor);
+}
+
+static unsigned _parse_ranges(struct http_request *r, struct http_range *range, unsigned nrange)
+{
+  unsigned i;
+  for (i = 0; 1; ++i) {
+    enum http_range_type type;
+    http_size_t first = 0, last = 0;
+    if (_skip_literal(r, "-") && _parse_http_size_t(r, &last))
+      type = SUFFIX;
+    else if (_parse_http_size_t(r, &first) && _skip_literal(r, "-"))
+      type = (_parse_http_size_t(r, &last)) ? CLOSED : OPEN;
+    else
+      return 0;
+    if (i < nrange) {
+      range[i].type = type;
+      range[i].first = first;
+      range[i].last = last;
+    }
+    if (!_skip_literal(r, ","))
+      break;
+    _skip_optional_space(r);
+  }
+  return i;
+}
+
+static int _parse_quoted_rfc822_time(struct http_request *r, time_t *timep)
+{
+  char datestr[40];
+  size_t n = _parse_quoted_string(r, datestr, sizeof datestr);
+  if (n == 0 || n >= sizeof datestr)
+    return 0;
+  // TODO: Move the following code into its own function in str.c
+  struct tm tm;
+  bzero(&tm, sizeof tm);
+  // TODO: Ensure this works in non-English locales, ie, "%a" still accepts "Mon", "Tue" etc. and
+  // "%b" still accepts "Jan", "Feb" etc.
+  // TODO: Support symbolic time zones, eg, "UT", "GMT", "UTC", "EST"...
+  const char *c = strptime(datestr, "%a, %d %b %Y %T ", &tm);
+  if ((c[0] == '-' || c[0] == '+') && isdigit(c[1]) && isdigit(c[2]) && isdigit(c[3]) && isdigit(c[4]) && c[5] == '\0') {
+    time_t zone = (c[0] == '-' ? -1 : 1) * ((c[1] - '0') * 600 + (c[2] - '0') * 60 + (c[3] - '0') * 10 + (c[4] - '0'));
+    const char *tz = getenv("TZ");
+    if (tz)
+      tz = alloca_strdup(tz);
+    setenv("TZ", "", 1);
+    tzset();
+    *timep = mktime(&tm) - zone;
+    if (tz)
+      setenv("TZ", tz, 1);
+    else
+      unsetenv("TZ");
+    tzset();
+    return 1;
+  }
+  return 0;
+}
+
+/* Return 100 if more received data is needed.  Returns 0 if parsing completes or fails.  Returns a
+ * 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_parse_verb(struct http_request *r)
+{
+  _rewind(r);
+  assert(r->cursor >= r->received);
+  assert(!_run_out(r));
+  // Parse verb: GET, PUT, POST, etc.
+  assert(r->verb == NULL);
+  unsigned i;
+  for (i = 0; i < NELS(http_verbs); ++i) {
+    _rewind(r);
+    if (_skip_literal(r, http_verbs[i].word) && _skip_literal(r, " ")) {
+      r->verb = http_verbs[i].word;
+      break;
+    }
+    if (_run_out(r))
+      return 100; // read more and try again
+  }
+  if (r->verb == NULL) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Malformed HTTP request: invalid verb: %s", alloca_toprint(20, r->cursor, r->end - r->cursor));
+    return 400;
+  }
+  _commit(r);
+  r->parser = http_request_parse_path;
+  return 0;
+}
+
+/* Return 100 if more received data is needed.  Returns 0 if parsing completes or fails.  Returns a
+ * 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_parse_path(struct http_request *r)
+{
+  // Parse path: word immediately following verb, delimited by spaces.
+  assert(r->path == NULL);
+  struct substring path;
+  if (!(_skip_word_printable(r, &path) && _skip_literal(r, " "))) {
+    if (_run_out(r))
+      return 100; // read more and try again
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Malformed HTTP %s request at path: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
+    return 400;
+  }
+  if ((r->path = _reserve(r, path)) == NULL)
+    return 0; // error
+  _commit(r);
+  r->parser = http_request_parse_http_version;
+  return 0;
+}
+
+/* Return 100 if more received data is needed.  Returns 0 if parsing completes or fails.  Returns a
+ * 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_parse_http_version(struct http_request *r)
+{
+  // Parse HTTP version: HTTP/m.n followed by CRLF.
+  assert(r->version_major == 0);
+  assert(r->version_minor == 0);
+  unsigned major, minor;
+  if (!(   _skip_literal(r, "HTTP/")
+	&& _parse_uint(r, &major)
+	&& major > 0 && major < UINT8_MAX
+	&& _skip_literal(r, ".")
+	&& _parse_uint(r, &minor)
+	&& minor > 0 && minor < UINT8_MAX
+	&& _skip_eol(r)
+	)
+  ) {
+    if (_run_out(r))
+      return 100; // read more and try again
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("HTTP %s malformed request: malformed version: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
+    return 400;
+  }
+  _commit(r);
+  r->version_major = major;
+  r->version_minor = minor;
+  r->parser = http_request_start_parsing_headers;
+  if (r->handle_first_line)
+    return r->handle_first_line(r);
+  return 0; // parsing complete
+}
+
+/* Select the header parser.  Returns 0 after setting the new parser function.  Returns a 4nn or 5nn
+ * HTTP result code if parsing fails.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_start_parsing_headers(struct http_request *r)
+{
+  assert(r->verb != NULL);
+  assert(r->path != NULL);
+  assert(r->version_major != 0);
+  assert(r->version_minor != 0);
+  if (r->version_major != 1) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Unsupported HTTP version: %u.%u", r->version_major, r->version_minor);
+    return 400;
+  }
+  r->parser = http_request_parse_header;
+  return 0;
+}
+
+/* Parse one request header line.  Returns 100 if more received data is needed.  Returns 0 if there
+ * are no more headers or parsing fails.  Returns a 4nn or 5nn HTTP result code if parsing fails.
+ * Returns -1 if an unexpected error occurs.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_parse_header(struct http_request *r)
+{
+  _skip_to_eol(r);
+  const char *const eol = r->cursor;
+  _skip_eol(r);
+  if (eol == r->parsed) { // if EOL is at start of line (ie, blank line)...
+    _commit(r);
+    r->parser = http_request_start_body;
+    if (r->handle_headers)
+      return r->handle_headers(r);
+    return 0;
+  }
+  const char *const nextline = r->cursor;
+  _rewind(r);
+  const char *const sol = r->cursor;
+  if (_skip_literal_nocase(r, "Content-Length:")) {
+    _skip_optional_space(r);
+    http_size_t length;
+    if (_parse_http_size_t(r, &length) && _skip_optional_space(r) && r->cursor == eol) {
+      r->cursor = nextline;
+      _commit(r);
+      r->request_header.content_length = length;
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Parsed HTTP request Content-Length: %"PRIhttp_size_t, r->request_header.content_length);
+      return 0;
+    }
+    goto malformed;
+  }
+  _rewind(r);
+  if (_skip_literal_nocase(r, "Content-Type:")) {
+    _skip_optional_space(r);
+    struct substring type = substring_NULL;
+    struct substring subtype = substring_NULL;
+    char boundary[BOUNDARY_STRING_MAXLEN + 1];
+    boundary[0] = '\0';
+    if (_skip_token(r, &type) && _skip_literal(r, "/") && _skip_token(r, &subtype)) {
+      // Parse zero or more content-type parameters.
+      for (_skip_optional_space(r); r->cursor < eol && _skip_literal(r, ";"); _skip_optional_space(r)) {
+	_skip_optional_space(r);
+	const char *startparam = r->cursor;
+	if (_skip_literal(r, "boundary=")) {
+	  size_t n = _parse_token_or_quoted_string(r, boundary, sizeof boundary);
+	  if (n == 0 || n >= sizeof boundary || !is_valid_http_boundary_string(boundary))
+	    goto malformed;
+	  continue;
+	}
+	// Silently ignore unrecognised parameters (eg, charset=) if they are well formed.
+	r->cursor = startparam; // partial rewind
+	if (_skip_token(r, NULL) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0))
+	  continue;
+	break;
+      }
+      if (r->cursor == eol) {
+	r->cursor = nextline;
+	_commit(r);
+	if (   (r->request_header.content_type = _reserve(r, type)) == NULL
+	    || (r->request_header.content_subtype = _reserve(r, subtype)) == NULL
+	    || (boundary[0] && (r->request_header.content_subtype = _reserve_str(r, boundary)) == NULL)
+	)
+	  return 0; // error
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("Parsed HTTP request Content-type: %s/%s%s%s",
+	      r->request_header.content_type,
+	      r->request_header.content_subtype,
+	      r->request_header.boundary ? "; boundary=" : "",
+	      r->request_header.boundary ? alloca_str_toprint(r->request_header.boundary) : ""
+	    );
+	return 0;
+      }
+    }
+    goto malformed;
+  }
+  _rewind(r);
+  if (_skip_literal_nocase(r, "Range:")) {
+    _skip_optional_space(r);
+    unsigned int n;
+    if (   _skip_literal(r, "bytes=")
+	&& (n = _parse_ranges(r, r->request_header.content_ranges, NELS(r->request_header.content_ranges)))
+	&& _skip_optional_space(r)
+	&& (r->cursor == eol)
+    ) {
+      r->cursor = nextline;
+      _commit(r);
+      if (n > NELS(r->request_header.content_ranges)) {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("HTTP request Range header overflow (%u ranges in set, can only handle %u): %s",
+	      n, NELS(r->request_header.content_ranges), alloca_toprint(-1, sol, eol - sol));
+	// In this case ignore the Range: header -- respond with the entire resource.
+	r->request_header.content_range_count = 0;
+      } else {
+	r->request_header.content_range_count = n;
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("Parsed HTTP request Range: bytes=%s", alloca_http_ranges(r->request_header.content_ranges));
+      }
+      return 0;
+    }
+    goto malformed;
+  }
+  _rewind(r);
+  if (r->debug_flag && *r->debug_flag)
+    DEBUGF("Skipped HTTP request header: %s", alloca_toprint(-1, sol, eol - sol));
+  r->cursor = nextline;
+  _commit(r);
+  return 0;
+malformed:
+  if (r->debug_flag && *r->debug_flag)
+    DEBUGF("Malformed HTTP request header: %s", alloca_toprint(-1, sol, eol - sol));
+  return 400;
+}
+
+/* Select the header parser.  Returns 0 after setting the new parser function.  Returns a 4nn or 5nn
+ * HTTP result code if parsing fails.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_start_body(struct http_request *r)
+{
+  assert(r->verb != NULL);
+  assert(r->path != NULL);
+  assert(r->version_major != 0);
+  assert(r->version_minor != 0);
+  assert(r->parsed <= r->end);
+  if (r->verb == HTTP_VERB_GET) {
+    // TODO: Implement HEAD requests
+    if (r->request_header.content_length != 0 && r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Malformed HTTP %s request: Content-Length not allowed", r->verb);
+      return 400;
+    }
+    if (r->request_header.content_type) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Malformed HTTP %s request: Content-Type not allowed", r->verb);
+      return 400;
+    }
+    r->parser = NULL;
+  }
+  else if (r->verb == HTTP_VERB_POST) {
+    if (r->request_header.content_length == CONTENT_LENGTH_UNKNOWN) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Malformed HTTP %s request: missing Content-Length header", r->verb);
+      return 400;
+    }
+    if (r->request_header.content_type == NULL) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Malformed HTTP %s request: missing Content-Type header", r->verb);
+      return 400;
+    }
+    if (   strcmp(r->request_header.content_type, "multipart") == 0
+	&& strcmp(r->request_header.content_subtype, "form-data") == 0
+    ) {
+      if (r->request_header.boundary == NULL || r->request_header.boundary[0] == '\0') {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("Malformed HTTP %s request: Content-Type %s/%s missing boundary parameter",
+	      r->verb, r->request_header.content_type, r->request_header.content_subtype);
+	return 400;
+      }
+      r->parser = http_request_parse_body_form_data;
+      r->form_data_state = START;
+    } else {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Unsupported HTTP %s request: Content-Type %s/%s not supported",
+	    r->verb, r->request_header.content_type, r->request_header.content_subtype);
+      return 415;
+    }
+    size_t unparsed = r->end - r->parsed;
+    if (unparsed > r->request_header.content_length) {
+      WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length));
+      r->request_content_remaining = 0;
+    }
+    else
+      r->request_content_remaining = r->request_header.content_length - unparsed;
+  }
+  else {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Unsupported HTTP %s request", r->verb);
+    r->parser = NULL;
+    return 501;
+  }
+  return 0;
+}
+
+/* Returns 1 if a MIME delimiter is skipped, 2 if a MIME close-delimiter is skipped.
+ */
+static int _skip_mime_boundary(struct http_request *r)
+{
+  if (!_skip_literal(r, "--") || !_skip_literal(r, r->request_header.boundary))
+    return 0;
+  if (_skip_literal(r, "--") && _skip_optional_space(r) && _skip_crlf(r))
+    return 2;
+  if (_skip_optional_space(r) && _skip_crlf(r))
+    return 1;
+  return 0;
+}
+
+static int _parse_content_disposition(struct http_request *r, struct mime_content_disposition *cd)
+{
+  size_t n = _parse_token(r, cd->type, sizeof cd->type);
+  if (n == 0)
+    return 0;
+  if (n >= sizeof cd->type) {
+    WARNF("HTTP Content-Disposition type truncated: %s", alloca_str_toprint(cd->type));
+    return 0;
+  }
+  while (_skip_optional_space(r) && _skip_literal(r, ";") && _skip_optional_space(r)) {
+    const char *start = r->cursor;
+    if (_skip_literal(r, "filename=")) {
+      size_t n = _parse_token_or_quoted_string(r, cd->filename, sizeof cd->filename);
+      if (n == 0)
+	return 0;
+      if (n >= sizeof cd->filename) {
+	WARNF("HTTP Content-Disposition filename truncated: %s", alloca_str_toprint(cd->filename));
+	return 0;
+      }
+      continue;
+    }
+    r->cursor = start;
+    if (_skip_literal(r, "name=")) {
+      size_t n = _parse_token_or_quoted_string(r, cd->name, sizeof cd->name);
+      if (n == 0)
+	return 0;
+      if (n >= sizeof cd->name) {
+	WARNF("HTTP Content-Disposition name truncated: %s", alloca_str_toprint(cd->name));
+	return 0;
+      }
+      continue;
+    }
+    r->cursor = start;
+    if (_skip_literal(r, "size=")) {
+      if (!_parse_http_size_t(r, &cd->size))
+	goto malformed;
+      continue;
+    }
+    r->cursor = start;
+    if (_skip_literal(r, "creation-date=")) {
+      if (!_parse_quoted_rfc822_time(r, &cd->creation_date))
+	goto malformed;
+      continue;
+    }
+    r->cursor = start;
+    if (_skip_literal(r, "modification-date=")) {
+      if (!_parse_quoted_rfc822_time(r, &cd->modification_date))
+	goto malformed;
+      continue;
+    }
+    r->cursor = start;
+    if (_skip_literal(r, "read-date=")) {
+      if (!_parse_quoted_rfc822_time(r, &cd->read_date))
+	goto malformed;
+      continue;
+    }
+    r->cursor = start;
+    struct substring param;
+    if (_skip_token(r, &param) && _skip_literal(r, "=") && _parse_token_or_quoted_string(r, NULL, 0)) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Skipping HTTP Content-Disposition parameter: %s", alloca_substring_toprint(param));
+      continue;
+    }
+malformed:
+    WARNF("Malformed HTTP Content-Disposition: %s", alloca_toprint(50, r->cursor, r->end - r->cursor));
+    return 0;
+  }
+  return 1;
+}
+
+/* Return zero if more received data is needed.  The first call that completes parsing of the body
+ * returns 200.  All subsequent calls return 100.  Returns a 4nn or 5nn HTTP result code if parsing
+ * fails.
+ *
+ * NOTE: No support for nested/mixed parts, as that would considerably complicate the parser.  If
+ * the need arises in future, we will deal with it then.  In the meantime, we will have something
+ * that meets our immediate needs for Rhizome Direct and a variety of use cases.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int http_request_parse_body_form_data(struct http_request *r)
+{
+  int at_start = 0;
+  switch (r->form_data_state) {
+    case START:
+      // The logic here allows for a missing initial CRLF before the first boundary line.
+      at_start = 1;
+      r->form_data_state = PREAMBLE;
+      // fall through
+    case PREAMBLE:
+      while (!_run_out(r)) {
+	const char *end_preamble = r->cursor;
+	int b;
+	if ((_skip_crlf(r) || at_start) && (b = _skip_mime_boundary(r))) {
+	  assert(end_preamble >= r->parsed);
+	  if (r->form_data.handle_mime_preamble && end_preamble != r->parsed)
+	    r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed);
+	  _rewind_crlf(r);
+	  _commit(r);
+	  r->form_data_state = EPILOGUE;
+	  if (b == 1) {
+	    r->form_data_state = HEADER;
+	    if (r->form_data.handle_mime_part_start)
+	      r->form_data.handle_mime_part_start(r);
+	  }
+	  return 0;
+	}
+	_skip_to_crlf(r);
+	at_start = 0;
+      }
+      if (r->cursor > r->parsed && r->form_data.handle_mime_preamble)
+	r->form_data.handle_mime_preamble(r, r->parsed, r->parsed - r->cursor);
+      _commit(r);
+      return 0;
+    case HEADER: {
+	if (_skip_crlf(r) && _skip_crlf(r)) {
+	  _commit(r);
+	  r->form_data_state = BODY;
+	  return 0;
+	}
+	if (_run_out(r))
+	  return 0; // read more and try again
+	_rewind(r);
+	int b;
+	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
+	  _rewind_crlf(r);
+	  _commit(r);
+	  if (r->form_data.handle_mime_part_end)
+	    r->form_data.handle_mime_part_end(r);
+	  r->form_data_state = EPILOGUE;
+	  // Boundary in the middle of headers starts a new part.
+	  if (b == 1) {
+	    r->form_data_state = HEADER;
+	    if (r->form_data.handle_mime_part_start)
+	      r->form_data.handle_mime_part_start(r);
+	  }
+	  return 0;
+	}
+	if (_run_out(r))
+	  return 0; // read more and try again
+	_rewind(r);
+	struct substring label;
+	if (_skip_crlf(r) && _skip_token(r, &label) && _skip_literal(r, ":") && _skip_optional_space(r)) {
+	  size_t labellen = label.end - label.start;
+	  char labelstr[labellen + 1];
+	  strncpy(labelstr, label.start, labellen)[labellen] = '\0';
+	  str_tolower_inplace(labelstr);
+	  const char *value = r->cursor;
+	  if (strcmp(labelstr, "content-disposition") == 0) {
+	    struct mime_content_disposition cd;
+	    bzero(&cd, sizeof cd);
+	    if (_parse_content_disposition(r, &cd) && _skip_optional_space(r) && _skip_crlf(r)) {
+	      if (r->form_data.handle_mime_content_disposition)
+		r->form_data.handle_mime_content_disposition(r, &cd);
+	      _commit(r);
+	      return 0;
+	    }
+	  } else if (_skip_to_crlf(r)) {
+	    if (r->form_data.handle_mime_header)
+	      r->form_data.handle_mime_header(r, labelstr, value, value - r->cursor); // excluding CRLF at end
+	    _skip_crlf(r);
+	    _commit(r);
+	    return 0;
+	  }
+	}
+	if (_run_out(r))
+	  return 0; // read more and try again
+	_rewind(r);
+      }
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
+      return 400;
+    case BODY:
+      while (!_run_out(r)) {
+	int b;
+	const char *eol = r->cursor;
+	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
+	  _rewind_crlf(r);
+	  _commit(r);
+	  if (r->form_data.handle_mime_body)
+	    r->form_data.handle_mime_body(r, r->parsed, r->parsed - eol); // excluding CRLF at end
+	  if (r->form_data.handle_mime_part_end)
+	    r->form_data.handle_mime_part_end(r);
+	  r->form_data_state = EPILOGUE;
+	  if (b == 1) {
+	    r->form_data_state = HEADER;
+	    if (r->form_data.handle_mime_part_start)
+	      r->form_data.handle_mime_part_start(r);
+	  }
+	  return 0;
+	}
+	_skip_to_crlf(r);
+      }
+      if (r->cursor > r->parsed && r->form_data.handle_mime_body)
+	r->form_data.handle_mime_body(r, r->parsed, r->parsed - r->cursor);
+      _commit(r);
+      return 0;
+  case EPILOGUE:
+    r->cursor = r->end;
+    if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed)
+      r->form_data.handle_mime_epilogue(r, r->parsed, r->cursor - r->parsed);
+    _commit(r);
+    return 0;
+  }
+  assert(0); // not reached
+}
+
+static ssize_t http_request_read(struct http_request *r, char *buf, size_t len)
+{
+  sigPipeFlag = 0;
+  ssize_t bytes = read_nonblock(r->alarm.poll.fd, buf, len);
+  if (bytes == -1) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUG("HTTP socket read error, closing connection");
+    http_request_finalise(r);
+    return -1;
+  }
+  if (sigPipeFlag) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUG("Received SIGPIPE on HTTP socket read, closing connection");
+    http_request_finalise(r);
+    return -1;
+  }
+  return bytes;
+}
+
+static void http_request_receive(struct http_request *r)
+{
+  assert(r->phase == RECEIVE);
+  r->limit = r->buffer + sizeof r->buffer;
+  assert(r->end < r->limit);
+  size_t room = r->limit - r->end;
+  assert(r->parsed >= r->received);
+  assert(r->parsed <= r->end);
+  // If buffer is running short on unused space, shift existing content in buffer down to make more
+  // room if possible.
+  if (   (room < 128 || (room < 1024 && r->parsed - r->received >= 32))
+      && (r->request_content_remaining == CONTENT_LENGTH_UNKNOWN || room < r->request_content_remaining)
+  ) {
+    size_t unparsed = r->end - r->parsed;
+    memmove((char *)r->received, r->parsed, unparsed); // memcpy() does not handle overlapping src and dst
+    r->parsed = r->received;
+    r->end = r->received + unparsed;
+    room = r->limit - r->end;
+  }
+  // If there is no more buffer space, fail the request.
+  if (room == 0) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUG("Buffer size reached, reporting overflow");
+    http_request_simple_response(r, 431, NULL);
+    return;
+  }
+  // Read up to the end of available buffer space or the end of content, whichever is first.
+  size_t read_size = room;
+  if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && read_size > r->request_content_remaining) {
+    r->limit = r->end + r->request_content_remaining;
+    read_size = r->request_content_remaining;
+  }
+  if (read_size != 0) {
+    // Read as many bytes as possible into the unused buffer space.  Any read error
+    // closes the connection without any response.
+    ssize_t bytes = http_request_read(r, (char *)r->end, read_size);
+    if (bytes == -1)
+      return;
+    // If no data was read, then just return to polling.  Don't drop the connection on an empty read,
+    // because that drops connections when they shouldn't, including during testing.  The inactivity
+    // timeout will drop the connections instead.
+    if (bytes == 0)
+      return;
+    r->end += (size_t) bytes;
+    // We got some data, so reset the inactivity timer and invoke the parsing state machine to process
+    // it.  The state machine invokes the caller-supplied callback functions.
+    r->alarm.alarm = gettime_ms() + r->idle_timeout;
+    r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
+    unschedule(&r->alarm);
+    schedule(&r->alarm);
+  }
+  // Parse the unparsed and received data.
+  while (r->phase == RECEIVE) {
+    int result;
+    if (_run_out(r) && r->request_content_remaining == 0) {
+      if (r->handle_content_end)
+	result = r->handle_content_end(r);
+      else {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUG("Internal failure parsing HTTP request: no end-of-content function set");
+	result = 500;
+      }
+    } else {
+      const char *oldparsed = r->parsed;
+      if (r->parser == NULL) {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUG("Internal failure parsing HTTP request: no parser function set");
+	result = 500;
+      } else {
+	_rewind(r);
+	result = r->parser(r);
+	assert(r->parsed >= oldparsed);
+      }
+      if (result == 100) {
+	if (_run_out(r))
+	  return; // needs more data; poll again
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUG("Internal failure parsing HTTP request: parser function returned 100 but not run out");
+	result = 500;
+      }
+      if (result == 0 && r->parsed == oldparsed) {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUG("Internal failure parsing HTTP request: parser function did not advance");
+	result = 500;
+      }
+    }
+    if (result >= 300)
+      r->response.result_code = result;
+    else if (result) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("Internal failure parsing HTTP request: invalid result=%d", result);
+      r->response.result_code = 500;
+    }
+    if (r->response.result_code) {
+      http_request_start_response(r);
+      return;
+    }
+    if (result == -1) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUG("Unrecoverable error parsing HTTP request, closing connection");
+      http_request_finalise(r);
+      return;
+    }
+  }
+  if (r->phase != RECEIVE)
+    return;
+  if (r->response.result_code == 0) {
+    WHY("No HTTP response set, using 500 Server Error");
+    r->response.result_code = 500;
+  }
+  http_request_start_response(r);
+}
+
+/* Write the current contents of the response buffer to the HTTP socket.  When no more bytes can be
+ * written, return so that socket polling can continue.  Once all bytes are sent, if there is a
+ * content generator function, invoke it to put more content in the response buffer, and write that
+ * content.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static void http_request_send_response(struct http_request *r)
+{
+  assert(r->response_sent <= r->response_length);
+  while (r->response_sent < r->response_length) {
+    assert(r->response_buffer_sent <= r->response_buffer_length);
+    if (r->response_buffer_sent == r->response_buffer_length) {
+      if (r->response.content_generator) {
+	// Content generator must fill response_buffer[] and set response_buffer_length.
+	r->response_buffer_sent = r->response_buffer_length = 0;
+	if (r->response.content_generator(r) == -1) {
+	  if (r->debug_flag && *r->debug_flag)
+	    DEBUG("Content generation error, closing connection");
+	  http_request_finalise(r);
+	  return;
+	}
+	assert(r->response_buffer_sent <= r->response_buffer_length);
+	if (r->response_buffer_sent == r->response_buffer_length) {
+	  WHYF("HTTP response generator produced no content at offset %"PRIhttp_size_t"/%"PRIhttp_size_t" (%"PRIhttp_size_t" bytes remaining)",
+	      r->response_sent, r->response_length, r->response_length - r->response_sent);
+	  http_request_finalise(r);
+	  return;
+	}
+      } else {
+	WHYF("HTTP response is short of total length (%"PRIhttp_size_t") by %"PRIhttp_size_t" bytes",
+	    r->response_length, r->response_length - r->response_sent);
+	http_request_finalise(r);
+	return;
+      }
+    }
+    assert(r->response_buffer_sent < r->response_buffer_length);
+    size_t bytes = r->response_buffer_length - r->response_buffer_sent;
+    if (r->response_sent + bytes > r->response_length) {
+      WHYF("HTTP response overruns total length (%"PRIhttp_size_t") by %"PRIhttp_size_t"  bytes -- truncating",
+	  r->response_length,
+	  r->response_sent + bytes - r->response_length);
+      bytes = r->response_length - r->response_sent;
+    }
+    sigPipeFlag = 0;
+    ssize_t written = write_nonblock(r->alarm.poll.fd, r->response_buffer + r->response_buffer_sent, bytes);
+    if (written == -1) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUG("HTTP socket write error, closing connection");
+      http_request_finalise(r);
+      return;
+    }
+    if (sigPipeFlag) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUG("Received SIGPIPE on HTTP socket write, closing connection");
+      http_request_finalise(r);
+      return;
+    }
+    // If we wrote nothing, go back to polling.
+    if (written == 0)
+      return;
+    r->response_sent += (size_t) written;
+    r->response_buffer_sent += (size_t) written;
+    assert(r->response_sent <= r->response_length);
+    assert(r->response_buffer_sent <= r->response_buffer_length);
+    // Reset inactivity timer.
+    r->alarm.alarm = gettime_ms() + r->idle_timeout;
+    r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
+    unschedule(&r->alarm);
+    schedule(&r->alarm);
+    // If we wrote less than we tried, then go back to polling.
+    if (written < (size_t) bytes)
+      return;
+  }
+  if (r->debug_flag && *r->debug_flag)
+    DEBUG("Done, closing connection");
+  http_request_finalise(r);
+}
+
+static void http_server_poll(struct sched_ent *alarm)
+{
+  struct http_request *r = (struct http_request *) alarm;
+  if (alarm->poll.revents == 0) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Timeout, closing connection");
+    http_request_finalise(r);
+  }
+  else if (alarm->poll.revents & (POLLHUP | POLLERR)) {
+    if (r->debug_flag && *r->debug_flag)
+      DEBUGF("Poll error (%s), closing connection", alloca_poll_events(alarm->poll.revents));
+    http_request_finalise(r);
+  }
+  else {
+    if (r->phase == RECEIVE && (alarm->poll.revents & POLLIN))
+      http_request_receive(r); // this could change the phase to TRANSMIT
+    if (r->phase == TRANSMIT && (alarm->poll.revents & POLLOUT))
+      http_request_send_response(r);
+  }
+  // Any of the above calls could change the phase to DONE.
+  if (r->phase == DONE && r->free)
+    r->free(r); // after this, *r is no longer valid
+}
+
+/* Count the number of bytes in a given set of ranges, given the length of the content to which the
+ * ranges will be applied.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+http_size_t http_range_bytes(const struct http_range *range, unsigned nranges, http_size_t resource_length)
+{
+  return http_range_close(NULL, range, nranges, resource_length);
+}
+
+/* Copy the array of byte ranges, closing it (converting all ranges to CLOSED) using the supplied
+ * resource length.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+http_size_t http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t resource_length)
+{
+  http_size_t bytes = 0;
+  unsigned i;
+  for (i = 0; i != nranges; ++i) {
+    http_size_t first = 0;
+    http_size_t last = resource_length;
+    const struct http_range *range = &src[i];
+    switch (range->type) {
+      case CLOSED:
+	last = range->last < resource_length ? range->last : resource_length;
+      case OPEN:
+	first = range->first < resource_length ? range->first : resource_length;
+	break;
+      case SUFFIX:
+	first = range->last < resource_length ? resource_length - range->last : 0;
+	break;
+      default:
+	abort(); // not reached
+    }
+    assert(first <= last);
+    if (dst)
+      *dst++ = (struct http_range){ .type = CLOSED, .first=first, .last=last };
+    bytes += last - first;
+  }
+  return bytes;
+}
+
+/* Return appropriate message for HTTP response codes, both known and unknown.
+ */
+static const char *httpResultString(int response_code)
+{
+  switch (response_code) {
+  case 200: return "OK";
+  case 201: return "Created";
+  case 206: return "Partial Content";
+  case 400: return "Bad Request";
+  case 401: return "Unauthorized";
+  case 403: return "Forbidden";
+  case 404: return "Not Found";
+  case 405: return "Method Not Allowed";
+  case 414: return "Request-URI Too Long";
+  case 415: return "Unsupported Media Type";
+  case 416: return "Requested Range Not Satisfiable";
+  case 431: return "Request Header Fields Too Large";
+  case 500: return "Internal Server Error";
+  case 501: return "Not Implemented";
+  default:  return (response_code <= 4) ? "Unknown status code" : "A suffusion of yellow";
+  }
+}
+
+static strbuf strbuf_append_quoted_string(strbuf sb, const char *str)
+{
+  strbuf_putc(sb, '"');
+  for (; *str; ++str) {
+    if (*str == '"' || *str == '\\')
+      strbuf_putc(sb, '\\');
+    strbuf_putc(sb, *str);
+  }
+  strbuf_putc(sb, '"');
+  return sb;
+}
+
+/* Render the HTTP response into the current response buffer.  Return 1 if it fits, 0 if it does
+ * not.  The buffer response_pointer may be NULL, in which case no response is rendered, but the
+ * content_length is still computed
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int _render_response(struct http_request *r)
+{
+  strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size);
+  struct http_response hr = r->response;
+  assert(hr.result_code != 0);
+  const char *result_string = httpResultString(hr.result_code);
+  if (hr.content == NULL) {
+    strbuf cb = strbuf_alloca(100 + strlen(result_string));
+    strbuf_puts(cb, "<html><h1>");
+    strbuf_puts(cb, result_string);
+    strbuf_puts(cb, "</h1></html>\r\n");
+    hr.content = strbuf_str(cb);
+    hr.header.resource_length = hr.header.content_length = strbuf_len(cb);
+    hr.header.content_type = "text/html";
+    hr.header.content_range_start = 0;
+  }
+  assert(hr.header.content_type != NULL);
+  strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", hr.result_code, result_string);
+  strbuf_sprintf(sb, "Content-Type: %s", hr.header.content_type);
+  if (hr.header.boundary) {
+    strbuf_puts(sb, "; boundary=");
+    if (strchr(hr.header.boundary, '"') || strchr(hr.header.boundary, '\\'))
+      strbuf_append_quoted_string(sb, hr.header.boundary);
+    else
+      strbuf_puts(sb, hr.header.boundary);
+  }
+  strbuf_puts(sb, "\r\n");
+  assert(hr.header.content_range_start <= hr.header.resource_length);
+  assert(hr.header.content_length <= hr.header.resource_length);
+  if (hr.header.content_range_start && hr.header.content_length)
+    strbuf_sprintf(sb,
+	  "Content-Range: bytes %"PRIhttp_size_t"-%"PRIhttp_size_t"/%"PRIhttp_size_t"\r\n",
+	  hr.header.content_range_start,
+	  hr.header.content_range_start + hr.header.content_length - 1,
+	  hr.header.resource_length
+	);
+  strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length);
+  strbuf_puts(sb, "\r\n");
+  r->response_length = strbuf_len(sb) + hr.header.content_length;
+  if (strbuf_overrun(sb) || (r->response_buffer_size < r->response_length))
+    return 0;
+  bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
+  r->response_buffer_sent = 0;
+  r->response_buffer_length = r->response_length;
+  return 1;
+}
+
+/* Returns with the length of the rendered response in r->response_length.  If the rendered response
+ * did not fit into any available buffer, then returns with r->response_buffer == NULL, otherwise
+ * r->response_buffer points to the rendered response.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static void http_request_render_response(struct http_request *r)
+{
+  // If there is no response buffer allocated yet, use the available part of the in-struct buffer.
+  http_request_set_response_bufsize(r, 1);
+  // Try rendering the response into the existing buffer.  This will discover the length of the
+  // rendered headers, so after this step, whether or not the buffer was overrun, we know the total
+  // length of the response.
+  if (!_render_response(r)) {
+    // If the response did not fit into the existing buffer, then allocate a large buffer from the
+    // heap and try rendering again.
+    if (http_request_set_response_bufsize(r, r->response_length + 1) == -1)
+      WHY("Cannot render HTTP response, out of memory");
+    else if (!_render_response(r))
+      FATAL("Re-render of HTTP response overflowed buffer");
+  }
+}
+
+static void http_request_start_response(struct http_request *r)
+{
+  // If HTTP responses are disabled (eg, for testing purposes) then skip all response construction
+  // and close the connection.
+  if (r->disable_tx_flag && *r->disable_tx_flag) {
+    INFO("HTTP transmit disabled, closing connection");
+    http_request_finalise(r);
+    return;
+  }
+  // Drain the rest of the request that has not been received yet (eg, if sending an error response
+  // provoked while parsing the early part of a partially-received request).  If a read error
+  // occurs, the connection is closed.
+  ssize_t bytes;
+  while ((bytes = http_request_read(r, (char *) r->received, (r->buffer + sizeof r->buffer) - r->received)) > 0)
+    ;
+  if (bytes == -1)
+    return;
+  // If the response cannot be rendered, then render a 500 Server Error instead.  If that fails,
+  // then just close the connection.
+  http_request_render_response(r);
+  if (r->response_buffer == NULL) {
+    WARN("Cannot render HTTP response, sending 500 Server Error instead");
+    r->response.result_code = 500;
+    r->response.content = NULL;
+    http_request_render_response(r);
+    if (r->response_buffer == NULL) {
+      WHY("Cannot render HTTP 500 Server Error response, closing connection");
+      http_request_finalise(r);
+      return;
+    }
+  }
+  r->response_sent = 0;
+  if (r->debug_flag && *r->debug_flag)
+    DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_length));
+  r->phase = TRANSMIT;
+  r->alarm.poll.events = POLLOUT;
+  watch(&r->alarm);
+}
+
+void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes)
+{
+  assert(result != 0);
+  r->response.result_code = result;
+  r->response.header.content_type = mime_type;
+  r->response.header.content_range_start = 0;
+  r->response.header.content_length = r->response.header.resource_length = bytes;
+  r->response.content = body;
+  r->response.content_generator = NULL;
+  http_request_start_response(r);
+}
+
+void http_request_simple_response(struct http_request *r, uint16_t result, const char *body)
+{
+  strbuf h = NULL;
+  if (body) {
+    size_t html_len = strlen(body) + 40;
+    char html[html_len];
+    h = strbuf_local(html, html_len);
+    strbuf_sprintf(h, "<html><h1>%s</h1></html>", body);
+  }
+  r->response.result_code = result;
+  r->response.header.content_type = body ? "text/html" : NULL;
+  r->response.header.content_range_start = 0;
+  r->response.header.resource_length = r->response.header.content_length = h ? strbuf_len(h) : 0;
+  r->response.content = h ? strbuf_str(h) : NULL;
+  r->response.content_generator = NULL;
+  http_request_start_response(r);
+}
+
+void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes)
+{
+  r->response.result_code = result;
+  r->response.content = NULL;
+  r->response.content_generator = NULL;
+  http_request_start_response(r);
+}
diff --git a/http_server.h b/http_server.h
new file mode 100644
index 00000000..7b851dbd
--- /dev/null
+++ b/http_server.h
@@ -0,0 +1,156 @@
+/*
+Serval DNA - HTTP Server API
+Copyright (C) 2013 Serval Project, Inc.
+ 
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+ 
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#ifndef __SERVALDNA__HTTP_SERVER_H
+#define __SERVALDNA__HTTP_SERVER_H
+
+#include <limits.h>
+#include "constants.h"
+#include "strbuf.h"
+
+/* Generic HTTP request handling.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+
+extern const char HTTP_VERB_GET[];
+extern const char HTTP_VERB_POST[];
+extern const char HTTP_VERB_PUT[];
+extern const char HTTP_VERB_HEAD[];
+extern const char HTTP_VERB_DELETE[];
+extern const char HTTP_VERB_TRACE[];
+extern const char HTTP_VERB_OPTIONS[];
+extern const char HTTP_VERB_CONNECT[];
+extern const char HTTP_VERB_PATCH[];
+
+typedef uint64_t http_size_t;
+#define PRIhttp_size_t  PRIu64
+
+struct http_request;
+
+struct http_range {
+  enum http_range_type { NIL = 0, CLOSED, OPEN, SUFFIX } type;
+  http_size_t first; // only for CLOSED or OPEN
+  http_size_t last; // only for CLOSED or SUFFIX
+};
+
+http_size_t http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t content_length);
+http_size_t http_range_bytes(const struct http_range *range, unsigned nranges, http_size_t content_length);
+
+#define CONTENT_LENGTH_UNKNOWN   UINT64_MAX
+
+struct http_request_headers {
+  http_size_t content_length;
+  const char *content_type;
+  const char *content_subtype;
+  const char *boundary;
+  unsigned short content_range_count;
+  struct http_range content_ranges[5];
+};
+
+struct http_response_headers {
+  http_size_t content_length;
+  http_size_t content_range_start; // range_end = range_start + content_length - 1
+  http_size_t resource_length; // size of entire resource
+  const char *content_type; // "type/subtype"
+  const char *boundary;
+};
+
+struct http_response {
+  uint16_t result_code;
+  struct http_response_headers header;
+  const char *content;
+  int (*content_generator)(struct http_request *); // callback to produce more content
+};
+
+#define MIME_FILENAME_MAXLEN 127
+
+struct mime_content_disposition {
+  char type[64];
+  char name[64];
+  char filename[MIME_FILENAME_MAXLEN + 1];
+  http_size_t size;
+  time_t creation_date;
+  time_t modification_date;
+  time_t read_date;
+};
+
+struct http_mime_handler {
+  void (*handle_mime_preamble)(struct http_request *, const char *, size_t);
+  void (*handle_mime_part_start)(struct http_request *);
+  void (*handle_mime_content_disposition)(struct http_request *, const struct mime_content_disposition *);
+  void (*handle_mime_header)(struct http_request *, const char *label, const char *, size_t);
+  void (*handle_mime_body)(struct http_request *, const char *, size_t);
+  void (*handle_mime_part_end)(struct http_request *);
+  void (*handle_mime_epilogue)(struct http_request *, const char *, size_t);
+};
+
+struct http_request;
+
+void http_request_init(struct http_request *r, int sockfd);
+void http_request_free_response_buffer(struct http_request *r);
+int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
+void http_request_finalise(struct http_request *r);
+void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
+void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
+void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes);
+
+#ifdef __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
+
+struct http_request {
+  struct sched_ent alarm; // MUST BE FIRST ELEMENT
+  enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase;
+  bool_t *debug_flag;
+  bool_t *disable_tx_flag;
+  time_ms_t initiate_time; // time connection was initiated
+  time_ms_t idle_timeout; // disconnect if no bytes received for this long
+  struct sockaddr_in client_in_addr;
+  int (*parser)(struct http_request *); // current parser function
+  int (*handle_first_line)(struct http_request *); // called after first line is parsed
+  int (*handle_headers)(struct http_request *); // called after all headers are parsed
+  int (*handle_content_end)(struct http_request *); // called after all content is received
+  enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state;
+  struct http_mime_handler form_data; // called to parse multipart/form-data body
+  void (*finalise)(struct http_request *);
+  void (*free)(void*);
+  const char *verb; // points to nul terminated static string, "GET", "PUT", etc.
+  const char *path; // points into buffer; nul terminated
+  uint8_t version_major; // m from from HTTP/m.n
+  uint8_t version_minor; // n from HTTP/m.n
+  struct http_request_headers request_header;
+  const char *received; // start of received data in buffer[]
+  const char *end; // end of received data in buffer[]
+  const char *limit; // end of content in buffer[]
+  const char *parsed; // start of unparsed data in buffer[]
+  const char *cursor; // for parsing
+  http_size_t request_content_remaining;
+  struct http_response response;
+  http_size_t response_length; // total response bytes (header + content)
+  http_size_t response_sent; // for counting up to response_length
+  char *response_buffer;
+  size_t response_buffer_size;
+  size_t response_buffer_length;
+  size_t response_buffer_sent;
+  void (*response_free_buffer)(void*);
+  char buffer[8 * 1024];
+};
+
+#endif // __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
+
+#endif // __SERVALDNA__HTTP_SERVER_H
diff --git a/overlay.c b/overlay.c
index 6c70d943..d16b0ef9 100644
--- a/overlay.c
+++ b/overlay.c
@@ -133,12 +133,8 @@ schedule(&_sched_##X); }
       rhizome_cleanup(NULL);
   }
 
-  /* Rhizome http server needs to know which callback to attach
-	 to client sockets, so provide it here, along with the name to
-	 appear in time accounting statistics. */
-  rhizome_http_server_start(rhizome_server_parse_http_request,
-			    "rhizome_server_parse_http_request",
-			    RHIZOME_HTTP_PORT,RHIZOME_HTTP_PORT_MAX);    
+  // start the HTTP server if enabled
+  rhizome_http_server_start(RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX);    
 
   // start the dna helper if configured
   dna_helper_start();
diff --git a/rhizome.h b/rhizome.h
index 9b73ff68..54201f44 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "sha2.h"
 #include "str.h"
 #include "strbuf.h"
+#include "http_server.h"
 #include "nacl.h"
 #include <sys/stat.h>
 
@@ -525,71 +526,27 @@ struct rhizome_read
   int64_t length;
 };
 
-typedef struct rhizome_http_request {
-  struct sched_ent alarm;
-  time_ms_t initiate_time; /* time connection was initiated */
-  
-  struct sockaddr_in requestor;
+/* Rhizome-specific HTTP request handling.
+ */
+typedef struct rhizome_http_request
+{
+  struct http_request http; // MUST BE FIRST ELEMENT
 
-  /* identify request from others being run.
+  /* Identify request from others being run.
      Monotonic counter feeds it.  Only used for debugging when we write
      post-<uuid>.log files for multi-part form requests. */
   unsigned int uuid;
 
-  /* The HTTP request as currently received */
-  int request_length;
-  int header_length;
-  char request[1024];
-  
-  /* Nature of the request */
-  int request_type;
-  /* All of the below are receiving data */
-#define RHIZOME_HTTP_REQUEST_RECEIVING -1
-#define RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART -2
-  
-  // callback function to fill the response buffer
-  int (*generator)(struct rhizome_http_request *r);
-  
-  /* Local buffer of data to be sent.
-   If a RHIZOME_HTTP_REQUEST_FROMBUFFER, then the buffer is sent, and when empty
-   the request is closed.
-   Else emptying the buffer triggers a request to fetch more data.  Only if no
-   more data is provided do we then close the request. */
-  unsigned char *buffer;
-  int buffer_size; // size
-  int buffer_length; // number of bytes loaded into buffer
-  int buffer_offset; // where we are between [0,buffer_length)
-  
   struct rhizome_read read_state;
   
-  /* Path of request (used by POST multipart form requests where
-     the actual processing of the request does not occur while the
-     request headers are still available. */
-  char path[1024];
-  /* Boundary string for POST multipart form requests */
-  char boundary_string[1024];
-  int boundary_string_length;
   /* File currently being written to while decoding POST multipart form */
-  FILE *field_file;
+  enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part;
+  int part_fd;
+  /* Which parts have been received in POST multipart form */
+  bool_t received_manifest;
+  bool_t received_data;
   /* Name of data file supplied */
-  char data_file_name[1024];
-  /* Which fields have been seen in POST multipart form */
-  int fields_seen;
-  /* The seen fields bitmap above shares values with the actual Rhizome Direct
-     state machine.  The state numbers (and thus bitmap values for the various
-     fields) are listed here.
-     
-     To avoid confusion, we should not use single bit values for states that do
-     not correspond directly to a particular field.
-     Doesn't really matter what they are apart from not having exactly one bit set.
-     In fact, the only reason to not have exactly one bit set is so that we keep as
-     many bits available for field types as possible.
-  */
-#define RD_MIME_STATE_MANIFESTHEADERS (1<<0)
-#define RD_MIME_STATE_DATAHEADERS (1<<1)
-#define RD_MIME_STATE_INITIAL 0
-#define RD_MIME_STATE_PARTHEADERS 0xffff0000
-#define RD_MIME_STATE_BODY 0xffff0001
+  char data_file_name[MIME_FILENAME_MAXLEN + 1];
 
   /* The source specification data which are used in different ways by different 
    request types */
@@ -607,15 +564,6 @@ typedef struct rhizome_http_request {
   
 } rhizome_http_request;
 
-struct http_response {
-  unsigned int result_code;
-  const char * content_type;
-  uint64_t content_start;
-  uint64_t content_end;
-  uint64_t content_length;
-  const char * body;
-};
-
 int rhizome_received_content(const unsigned char *bidprefix,uint64_t version, 
 			     uint64_t offset,int count,unsigned char *bytes,
 			     int type);
@@ -629,9 +577,7 @@ int rhizome_server_simple_http_response(rhizome_http_request *r, int result, con
 int rhizome_server_http_response(rhizome_http_request *r, int result, 
     const char *mime_type, const char *body, uint64_t bytes);
 int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes);
-int rhizome_http_server_start(int (*http_parse_func)(rhizome_http_request *),
-			      const char *http_parse_func_description,
-			      uint16_t port_low, uint16_t port_high);
+int rhizome_http_server_start(uint16_t port_low, uint16_t port_high);
 
 int is_rhizome_enabled();
 int is_rhizome_mdp_enabled();
@@ -787,7 +733,7 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath);
 int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
 
 int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size);
-int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len);
+int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len);
 int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
 int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
 
diff --git a/rhizome_direct.c b/rhizome_direct.c
index 518b02d0..b4ce9631 100644
--- a/rhizome_direct.c
+++ b/rhizome_direct.c
@@ -260,8 +260,7 @@ int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r)
   multiple versions of a given bundle introduces only a slight complication. 
 */
 
-rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
-(unsigned char *buffer,int size, int max_response_bytes)
+rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response(unsigned char *buffer,int size, int max_response_bytes)
 {
   if (size<10) return NULL;
   if (size>65536) return NULL;
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index c8b075cc..b8d6125a 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -28,658 +28,390 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <sys/wait.h>
 #include <arpa/inet.h>
 
-int rhizome_direct_clear_temporary_files(rhizome_http_request *r)
+static int _form_temporary_file_path(struct __sourceloc __whence, rhizome_http_request *r, char *pathbuf, size_t bufsiz, const char *field)
 {
-  char filename[1024];
-  char *fields[]={"manifest","data","unknown",NULL};
-  int i;
-
-  for(i=0;fields[i];i++) {
-    snprintf(filename,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,fields[i]);
-    filename[1023]=0;
-    unlink(filename);
+  strbuf b = strbuf_local(pathbuf, bufsiz);
+  // TODO: use a temporary directory
+  strbuf_sprintf(b, "rhizomedirect.%d.%s", r->http.alarm.poll.fd, field);
+  if (strbuf_overrun(b)) {
+    WHYF("Rhizome Direct pathname overflow: %s", alloca_str_toprint(pathbuf));
+    return -1;
   }
   return 0;
 }
 
-int rhizome_direct_form_received(rhizome_http_request *r)
+#define form_temporary_file_path(r,buf,field) _form_temporary_file_path(__WHENCE__, (r), (buf), sizeof(buf), (field))
+
+static void rhizome_direct_clear_temporary_files(rhizome_http_request *r)
 {
-  /* Process completed form based on the set of fields seen */
-  if (!strcmp(r->path,"/rhizome/import")) {
-    switch(r->fields_seen) {
-    case RD_MIME_STATE_MANIFESTHEADERS | RD_MIME_STATE_DATAHEADERS: {
-	/* Got a bundle to import */
-	DEBUGF("Call bundle import for rhizomedata.%d.{data,file}",
-	       r->alarm.poll.fd);
-	strbuf manifest_path = strbuf_alloca(50);
-	strbuf payload_path = strbuf_alloca(50);
-	strbuf_sprintf(manifest_path, "rhizomedirect.%d.manifest", r->alarm.poll.fd);
-	strbuf_sprintf(payload_path, "rhizomedirect.%d.data", r->alarm.poll.fd);
-      
-	int ret=0;
-	rhizome_manifest *m = rhizome_new_manifest();
-	
-	if (!m)
-	  ret=WHY("Out of manifests.");
-	else{
-	  ret=rhizome_bundle_import_files(m, strbuf_str(manifest_path), strbuf_str(payload_path));
-	  rhizome_manifest_free(m);
-	}
-      
-	rhizome_direct_clear_temporary_files(r);
-	/* report back to caller.
-	  200 = ok, which is probably appropriate for when we already had the bundle.
-	  201 = content created, which is probably appropriate for when we successfully
-	  import a bundle (or if we already have it).
-	  403 = forbidden, which might be appropriate if we refuse to accept it, e.g.,
-	  the import fails due to malformed data etc.
-	  (should probably also indicate if we have a newer version if possible)
-	*/
-	switch (ret) {
-	case 0:
-	  return rhizome_server_simple_http_response(r, 201, "Bundle succesfully imported.");
-	case 2:
-	  return rhizome_server_simple_http_response(r, 200, "Bundle already imported.");
-	}
-	return rhizome_server_simple_http_response(r, 500, "Server error: Rhizome import command failed.");
-      }
-      break;     
-    default:
-      /* Clean up after ourselves */
-      rhizome_direct_clear_temporary_files(r);	     
-    }
-  } else if (!strcmp(r->path,"/rhizome/enquiry")) {
-    int fd=-1;
-    char file[1024];
-    switch(r->fields_seen) {
-    case RD_MIME_STATE_DATAHEADERS:
-      /* Read data buffer in, pass to rhizome direct for comparison with local
-	 rhizome database, and send back responses. */
-      snprintf(file,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,"data");
-      fd=open(file,O_RDONLY);
-      if (fd == -1) {
-	WHYF_perror("open(%s, O_RDONLY)", alloca_str_toprint(file));
-	/* Clean up after ourselves */
-	rhizome_direct_clear_temporary_files(r);	     
-	return rhizome_server_simple_http_response(r,500,"Couldn't read a file");
-      }
-      struct stat stat;
-      if (fstat(fd, &stat) == -1) {
-	WHYF_perror("stat(%d)", fd);
-	/* Clean up after ourselves */
-	close(fd);
-	rhizome_direct_clear_temporary_files(r);	     
-	return rhizome_server_simple_http_response(r,500,"Couldn't stat a file");
-      }
-      unsigned char *addr = mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0);
-      if (addr==MAP_FAILED) {
-	WHYF_perror("mmap(NULL, %"PRId64", PROT_READ, MAP_SHARED, %d, 0)", (int64_t) stat.st_size, fd);
-	/* Clean up after ourselves */
-	close(fd);
-	rhizome_direct_clear_temporary_files(r);	     
-	return rhizome_server_simple_http_response(r,500,"Couldn't mmap() a file");
-      }
-      /* Ask for a fill response.  Regardless of the size of the set of BARs passed
-	 to us, we will allow up to 64KB of response. */
-      rhizome_direct_bundle_cursor 
-	*c=rhizome_direct_get_fill_response(addr,stat.st_size,65536);
-      munmap(addr,stat.st_size);
-      close(fd);
+  const char *fields[] = { "manifest", "data" };
+  int i;
+  for (i = 0; i != NELS(fields); ++i) {
+    char path[1024];
+    if (form_temporary_file_path(r, path, fields[i]) != -1)
+      if (unlink(path) == -1 && errno != ENOENT)
+	WARNF_perror("unlink(%s)", alloca_str_toprint(path));
+  }
+}
 
-      if (c)
-	{
-	  /* TODO: Write out_buffer as the body of the response.
-	     We should be able to do this using the async framework fairly easily.
-	  */
-	  
-	  int bytes=c->buffer_offset_bytes+c->buffer_used;
-	  r->buffer=malloc(bytes+1024);
-	  r->buffer_size=bytes+1024;
-	  r->buffer_offset=0;
-	  assert(r->buffer);
-
-	  /* Write HTTP response header */
-	  struct http_response hr;
-	  bzero(&hr, sizeof hr);
-	  hr.result_code=200;
-	  hr.content_type="binary/octet-stream";
-	  hr.content_length=bytes;
-	  hr.body=NULL;
-	  r->request_type=0;
-	  rhizome_server_set_response(r,&hr);
-	  assert(r->buffer_offset<1024);
-
-	  /* Now append body and send it back. */
-	  bcopy(c->buffer,&r->buffer[r->buffer_length],bytes);
-	  r->buffer_length+=bytes;
-	  r->buffer_offset=0;
-
-	  /* Clean up cursor after sending response */
-	  rhizome_direct_bundle_iterator_free(&c);
-	  /* Clean up after ourselves */
-	  rhizome_direct_clear_temporary_files(r);	     
-
-	  return 0;
-	}
-      else
-	{
-	  return rhizome_server_simple_http_response(r,500,"Could not get response to enquiry");
-	}
-
-      /* Clean up after ourselves */
-      rhizome_direct_clear_temporary_files(r);	     
-      break;
-    default:
-      /* Clean up after ourselves */
-      rhizome_direct_clear_temporary_files(r);	     
-
-      return rhizome_server_simple_http_response(r, 404, "/rhizome/enquiry requires 'data' field");
-    }
-  }  
-  /* Allow servald to be configured to accept files without manifests via HTTP
-     from localhost, so that rhizome bundles can be created programatically.
-     There are probably still some security loop-holes here, which is part of
-     why we leave it disabled by default, but it will be sufficient for testing
-     possible uses, including integration with OpenDataKit.
+int rhizome_direct_import_end(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  if (!r->received_manifest) {
+    http_request_simple_response(&r->http, 400, "Missing 'manifest' part");
+    return 0;
+  }
+  if (!r->received_data) {
+    http_request_simple_response(&r->http, 400, "Missing 'data' part");
+    return 0;
+  }
+  /* Got a bundle to import */
+  char manifest_path[512];
+  char payload_path[512];
+  if (   form_temporary_file_path(r, manifest_path, "manifest") == -1
+      || form_temporary_file_path(r, payload_path, "data") == -1
+  ) {
+    http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
+    return 0;
+  }
+  if (config.debug.rhizome)
+    DEBUGF("Call rhizome_bundle_import_files(%s, %s)",
+	alloca_str_toprint(manifest_path),
+	alloca_str_toprint(payload_path)
+      );
+  int ret = 0;
+  rhizome_manifest *m = rhizome_new_manifest();
+  if (!m)
+    ret = WHY("Out of manifests");
+  else {
+    ret = rhizome_bundle_import_files(m, manifest_path, payload_path);
+    rhizome_manifest_free(m);
+  }
+  rhizome_direct_clear_temporary_files(r);
+  /* report back to caller.
+    200 = ok, which is probably appropriate for when we already had the bundle.
+    201 = content created, which is probably appropriate for when we successfully
+    import a bundle (or if we already have it).
+    403 = forbidden, which might be appropriate if we refuse to accept it, e.g.,
+    the import fails due to malformed data etc.
+    (should probably also indicate if we have a newer version if possible)
   */
-  else if (config.rhizome.api.addfile.uri_path[0] && strcmp(r->path, config.rhizome.api.addfile.uri_path) == 0) {
-    if (r->requestor.sin_addr.s_addr != config.rhizome.api.addfile.allow_host.s_addr) {
-      DEBUGF("rhizome.api.addfile request received from %s, but is only allowed from %s",
-	  inet_ntoa(r->requestor.sin_addr),
-	  inet_ntoa(config.rhizome.api.addfile.allow_host)
-	);
+  switch (ret) {
+  case 0:
+    http_request_simple_response(&r->http, 201, "Bundle succesfully imported");
+    return 0;
+  case 2:
+    http_request_simple_response(&r->http, 200, "Bundle already imported");
+    return 0;
+  }
+  http_request_simple_response(&r->http, 500, "Internal Error: Rhizome import failed");
+  return 0;
+}
 
-      rhizome_direct_clear_temporary_files(r);	     
-      return rhizome_server_simple_http_response(r,404,"Not available from here.");
+int rhizome_direct_enquiry_end(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  if (!r->received_data) {
+    http_request_simple_response(&r->http, 400, "Missing 'data' part");
+    return 0;
+  }
+  char data_path[512];
+  if (form_temporary_file_path(r, data_path, "data") == -1) {
+    http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
+    return 0;
+  }
+  if (config.debug.rhizome)
+    DEBUGF("Call rhizome_direct_fill_response(%s)", alloca_str_toprint(data_path));
+  /* Read data buffer in, pass to rhizome direct for comparison with local
+      rhizome database, and send back responses. */
+  int fd = open(data_path, O_RDONLY);
+  if (fd == -1) {
+    WHYF_perror("open(%s, O_RDONLY)", alloca_str_toprint(data_path));
+    /* Clean up after ourselves */
+    rhizome_direct_clear_temporary_files(r);
+    http_request_simple_response(&r->http, 500, "Internal Error: Couldn't read file");
+    return 0;
+  }
+  struct stat stat;
+  if (fstat(fd, &stat) == -1) {
+    WHYF_perror("stat(%d)", fd);
+    /* Clean up after ourselves */
+    close(fd);
+    rhizome_direct_clear_temporary_files(r);
+    http_request_simple_response(&r->http, 500, "Internal Error: Couldn't stat file");
+    return 0;
+  }
+  unsigned char *addr = mmap(NULL, stat.st_size, PROT_READ, MAP_SHARED, fd, 0);
+  if (addr==MAP_FAILED) {
+    WHYF_perror("mmap(NULL,%"PRId64",PROT_READ,MAP_SHARED,%d,0)", (int64_t) stat.st_size, fd);
+    /* Clean up after ourselves */
+    close(fd);
+    rhizome_direct_clear_temporary_files(r);
+    http_request_simple_response(&r->http, 500, "Internal Error: Couldn't mmap file");
+    return 0;
+  }
+  /* Ask for a fill response.  Regardless of the size of the set of BARs passed
+      to us, we will allow up to 64KB of response. */
+  rhizome_direct_bundle_cursor *c = rhizome_direct_get_fill_response(addr, stat.st_size, 65536);
+  munmap(addr,stat.st_size);
+  close(fd);
+  if (c) {
+    size_t bytes = c->buffer_offset_bytes + c->buffer_used;
+    if (http_request_set_response_bufsize(&r->http, bytes) == -1)
+      http_request_simple_response(&r->http, 500, "Internal Error: Out of memory");
+    else
+      http_request_response(&r->http, 200, "binary/octet-stream", (const char *)c->buffer, bytes);
+    rhizome_direct_bundle_iterator_free(&c);
+  } else
+    http_request_simple_response(&r->http, 500, "Internal Error: No response to enquiry");
+  rhizome_direct_clear_temporary_files(r);
+  return 0;
+}
+
+int rhizome_direct_addfile_end(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  // If given a file without a manifest, we should only accept if it we are configured to do so, and
+  // the connection is from localhost.  Otherwise people could cause your servald to create
+  // arbitrary bundles, which would be bad.
+  if (!r->received_manifest) {
+    char payload_path[512];
+    if (form_temporary_file_path(r, payload_path, "data") == -1) {
+      http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
+      return 0;
     }
-
-    switch(r->fields_seen) {
-    case RD_MIME_STATE_DATAHEADERS:
-      /* We have been given a file without a manifest, we should only
-	 accept if it we are configured to do so, and the connection is from
-	 localhost.  Otherwise people could cause your servald to create
-	 arbitrary bundles, which would be bad.
-      */
-      /* A bundle to import */
-      DEBUGF("Call bundle import sans-manifest for rhizomedata.%d.{data,file}",
-	     r->alarm.poll.fd);
-      
-      char filepath[1024];
-      snprintf(filepath,1024,"rhizomedirect.%d.data",r->alarm.poll.fd);
-
-      char manifestTemplate[1024];
-      manifestTemplate[0] = '\0';
-      if (config.rhizome.api.addfile.manifest_template_file[0]) {
-	strbuf b = strbuf_local(manifestTemplate, sizeof manifestTemplate);
-	strbuf_path_join(b, serval_instancepath(), config.rhizome.api.addfile.manifest_template_file, NULL);
-	if (access(manifestTemplate, R_OK) != 0) {
-	  rhizome_direct_clear_temporary_files(r);
-	  return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifest_template_file points to a file I could not read.");
-	}
+    if (config.debug.rhizome)
+      DEBUGF("Call rhizome_add_file(%s)", alloca_str_toprint(payload_path));
+    char manifestTemplate[1024];
+    manifestTemplate[0] = '\0';
+    if (config.rhizome.api.addfile.manifest_template_file[0]) {
+      strbuf b = strbuf_local(manifestTemplate, sizeof manifestTemplate);
+      strbuf_path_join(b, serval_instancepath(), config.rhizome.api.addfile.manifest_template_file, NULL);
+      if (strbuf_overrun(b)) {
+	rhizome_direct_clear_temporary_files(r);
+	http_request_simple_response(&r->http, 500, "Internal Error: Template path too long");
+	return 0;
       }
-
-      rhizome_manifest *m = rhizome_new_manifest();
-      if (!m)
-	{
-	  rhizome_server_simple_http_response(r,500,"No free manifest slots. Try again later.");
-	  rhizome_direct_clear_temporary_files(r);	     
-	  return WHY("Manifest struct could not be allocated -- not added to rhizome");
-	}
-
-      if (manifestTemplate[0])
-	if (rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) {
-	  rhizome_manifest_free(m);
-	  rhizome_direct_clear_temporary_files(r);	     
-	  return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifest_template_file can't be read as a manifest.");
-	}
-	
-      if (rhizome_stat_file(m, filepath)){
+      if (access(manifestTemplate, R_OK) != 0) {
+	rhizome_direct_clear_temporary_files(r);
+	http_request_simple_response(&r->http, 500, "Internal Error: Cannot read template");
+	return 0;
+      }
+      if (config.debug.rhizome)
+	DEBUGF("Using manifest template %s", alloca_str_toprint(manifestTemplate));
+    }
+    rhizome_manifest *m = rhizome_new_manifest();
+    if (!m) {
+      WHY("Manifest struct could not be allocated -- not added to rhizome");
+      http_request_simple_response(&r->http, 500, "Internal Error: No free manifest slots");
+      rhizome_direct_clear_temporary_files(r);
+      return 0;
+    }
+    if (manifestTemplate[0] && rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) {
+      WHY("Manifest template read failed");
+      rhizome_manifest_free(m);
+      rhizome_direct_clear_temporary_files(r);
+      http_request_simple_response(&r->http, 500, "Internal Error: Malformed manifest template");
+      return 0;
+    }
+    if (rhizome_stat_file(m, payload_path)) {
+      WHY("Payload file stat failed");
+      rhizome_manifest_free(m);
+      rhizome_direct_clear_temporary_files(r);
+      http_request_simple_response(&r->http, 500, "Internal Error: Could not store file");
+      return 0;
+    }
+    // If manifest template did not specify a service field, then by default it is "file".
+    if (rhizome_manifest_get(m, "service", NULL, 0) == NULL)
+      rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
+    sid_t *author = NULL;
+    if (!is_sid_t_any(config.rhizome.api.addfile.default_author))
+      author = &config.rhizome.api.addfile.default_author;
+    rhizome_bk_t bsk = config.rhizome.api.addfile.bundle_secret_key;
+    if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)) {
+      rhizome_manifest_free(m);
+      rhizome_direct_clear_temporary_files(r);
+      http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest");
+      return 0;
+    }
+    m->payloadEncryption=0;
+    rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0);
+    // import file contents
+    // TODO, stream file into database
+    if (m->fileLength) {
+      if (rhizome_add_file(m, payload_path)) {
 	rhizome_manifest_free(m);
 	rhizome_direct_clear_temporary_files(r);
-	return rhizome_server_simple_http_response(r,500,"Could not store file");
+	http_request_simple_response(&r->http, 500, "Internal Error: Could not store file");
+	return 0;
       }
-
-      if (rhizome_manifest_get(m, "service", NULL, 0) == NULL)
-	rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
-
-      sid_t *author=NULL;
-      if (!is_sid_t_any(config.rhizome.api.addfile.default_author))
-	author = &config.rhizome.api.addfile.default_author;
-      
-      rhizome_bk_t bsk;
-      memcpy(bsk.binary, config.rhizome.api.addfile.bundle_secret_key.binary, RHIZOME_BUNDLE_KEY_BYTES);
-
-      if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)){
-	rhizome_manifest_free(m);
-	m = NULL;
-	rhizome_direct_clear_temporary_files(r);
-	return rhizome_server_simple_http_response(r,500,"Could not fill manifest default values");
-      }
-      
-      m->payloadEncryption=0;
-      rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0);
-	
-      // import file contents
-      // TODO, stream file into database
-      if (m->fileLength){
-	if (rhizome_add_file(m, filepath)){
-	  rhizome_manifest_free(m);
-	  rhizome_direct_clear_temporary_files(r);
-	  return rhizome_server_simple_http_response(r,500,"Could not store file");
-	}
-      }
-
-      rhizome_manifest *mout = NULL;
-      if (rhizome_manifest_finalise(m, &mout, 1)) {
-	if (mout && mout!=m)
-	  rhizome_manifest_free(mout);
-	rhizome_manifest_free(m);
-	rhizome_direct_clear_temporary_files(r);
-	return rhizome_server_simple_http_response(r,500,
-						   "Could not finalise manifest");
-      }
-            
-      DEBUGF("Import sans-manifest appeared to succeed");
-      
-      /* Respond with the manifest that was added. */
-      rhizome_server_simple_http_response(r, 200, (char *)m->manifestdata);
-
-      /* clean up after ourselves */
-      if (mout && mout!=m)
+    }
+    rhizome_manifest *mout = NULL;
+    if (rhizome_manifest_finalise(m, &mout, 1)) {
+      if (mout && mout != m)
 	rhizome_manifest_free(mout);
       rhizome_manifest_free(m);
       rhizome_direct_clear_temporary_files(r);
-
+      http_request_simple_response(&r->http, 500, "Internal Error: Could not finalise manifest");
       return 0;
-      break;
-    default:
-      /* Clean up after ourselves */
-      rhizome_direct_clear_temporary_files(r);	     
-      
-      return rhizome_server_simple_http_response(r, 400, "Rhizome create bundle from file API requires 'data' field");     
     }
-  }  
-
-  /* Clean up after ourselves */
-  rhizome_direct_clear_temporary_files(r);	     
-  /* Report error */
-  return rhizome_server_simple_http_response(r, 500, "Something went wrong.  Probably a missing data or manifest part, or invalid combination of URI and data/manifest provision.");
-
-}
-
-
-int rhizome_direct_process_mime_line(rhizome_http_request *r,char *buffer,int count)
-{
-  /* Check for boundary line at start of buffer.
-     Boundary line = CRLF + "--" + boundary_string + optional whitespace + CRLF
-     EXCEPT end of form boundary, which is:
-     CRLF + "--" + boundary_string + "--" + CRLF
-
-     NOTE: We attach the "--" to boundary_string when setting things up so that
-     we don't have to keep manually checking for it here.
-
-     NOTE: The parser eats the CRLF from the front, and attaches it to the end
-     of the previous line.  This means we need to rewind 2 bytes from whatever
-     file we were writing to whenever we encounter a boundary line, at least
-     if those last two bytes were CRLF. That can be safely assumed if we
-     assume that the boundary string has been chosen to be a string never appearing
-     anywhere in the contents of the form.  In practice, that is only "almost
-     certain" (according to the mathematical meaning of that phrase) if boundary
-     strings are randomly selected and are of sufficient length.
-   
-     NOTE: We are not supporting nested/mixed parts, as that would considerably
-     complicate the parser.  If the need arises in future, we will deal with it
-     then.  In the meantime, we will have something that meets our immediate
-     needs for Rhizome Direct and a variety of use cases.
-  */
-
-  /* Regardless of the state of the parser, the presence of boundary lines
-     is significant, so lets just check once, and remember the result.
-     Similarly check a few other conditions. */
-  int boundaryLine=0;
-  if (!memcmp(buffer,r->boundary_string,r->boundary_string_length))
-    boundaryLine=1;
-
-  int endOfForm=0;
-  if (boundaryLine&&
-      buffer[r->boundary_string_length]=='-'&&
-      buffer[r->boundary_string_length+1]=='-')
-    endOfForm=1;
-  int blankLine=0;
-  if (!strcmp(buffer,"\r\n")) blankLine=1;
-
-  switch(r->source_flags) {
-  case RD_MIME_STATE_INITIAL:
-    if (boundaryLine) r->source_flags=RD_MIME_STATE_PARTHEADERS;
-    break;
-  case RD_MIME_STATE_PARTHEADERS:
-  case RD_MIME_STATE_MANIFESTHEADERS:
-  case RD_MIME_STATE_DATAHEADERS:
-    if (blankLine) {
-      /* End of headers */
-      if (r->source_flags==RD_MIME_STATE_PARTHEADERS)
-	{
-	  /* Multiple content-disposition lines.  This is very naughty. */
-	  rhizome_server_simple_http_response
-	    (r, 400, "<html><h1>Malformed multi-part form POST: Missing content-disposition lines in MIME encoded part.</h1></html>\r\n");
-	  return -1;
-	}
-      
-      /* Prepare to write to file for field.
-	 We may have multiple rhizome direct transactions running at the same
-	 time on different TCP connections.  So serialise using file descriptor.
-	 We could use the boundary string or some other random thing, but using
-	 the file descriptor places a reasonable upper limit on the clutter that
-	 is possible, while still preventing collisions -- provided that we don't
-	 close the file descriptor until we have completed processing the 
-	 request. */
-      r->field_file=NULL;
-      char filename[1024];
-      char *field="unknown";
-      switch(r->source_flags) {
-      case RD_MIME_STATE_DATAHEADERS: field="data"; break;
-      case RD_MIME_STATE_MANIFESTHEADERS: field="manifest"; break;
-      }
-      snprintf(filename,1024,"rhizomedirect.%d.%s",r->alarm.poll.fd,field);
-      filename[1023]=0;
-      r->field_file=fopen(filename,"w");
-      if (!r->field_file) {
-	WHYF_perror("fopen(%s, \"w\")", alloca_str_toprint(filename));
-	goto scram;
-      }
-      r->source_flags=RD_MIME_STATE_BODY;
-    } else {
-      char name[1024];
-      char field[1024];
-      if (sscanf(buffer,
-		 "Content-Disposition: form-data; name=\"%[^\"]\";"
-		 " filename=\"%[^\"]\"",field,name)==2)
-	{
-	  if (r->source_flags!=RD_MIME_STATE_PARTHEADERS)
-	    {
-	      /* Multiple content-disposition lines.  This is very naughty. */
-	      rhizome_server_simple_http_response
-		(r, 400, "<html><h1>Malformed multi-part form POST: Multiple content-disposition lines in single MIME encoded part.</h1></html>\r\n");
-	      return -1;
-	    }
-	  if (!strcasecmp(field,"manifest")) 
-	    r->source_flags=RD_MIME_STATE_MANIFESTHEADERS;
-	  if (!strcasecmp(field,"data")) {
-	    r->source_flags=RD_MIME_STATE_DATAHEADERS;
-	    /* record file name of data field for HTTP manifest-less import */
-	    strncpy(r->data_file_name,name,1023);
-	    r->data_file_name[1023]=0;
-	  }
-	  if (r->source_flags!=RD_MIME_STATE_PARTHEADERS)
-	    r->fields_seen|=r->source_flags;
-	} 
-    }
-    break;
-  case RD_MIME_STATE_BODY:
-    if (boundaryLine) {
-      r->source_flags=RD_MIME_STATE_PARTHEADERS;
-      /* We will have written an extra CRLF to the end of the file, so prune that off. */
-      if (fflush(r->field_file) == EOF) {
-	WHYF_perror("fflush()");
-	goto scram;
-      }
-      int fd = fileno(r->field_file);
-      off_t correct_size = ftell(r->field_file) - 2;
-      if (ftruncate(fd,correct_size) == -1) {
-	WHYF_perror("ftruncate()");
-	goto scram;
-      }
-      if (fclose(r->field_file) == EOF) {
-	WHYF_perror("fclose()");
-	r->field_file = NULL;
-	goto scram;
-      }
-      r->field_file = NULL;
-    }
-    else {
-      int written=fwrite(r->request,count,1,r->field_file);
-      if (written<1) 
-	DEBUGF("Short write for multi-part form file -- %d bytes may be missing",
-	       count);
-    }
-    break;
-  }
-
-  if (endOfForm) {
-    /* End of form marker found. 
-       Pass it to function that deals with what has been received,
-       and will also send response or close the http request if required. */
-
-    /* XXX Rewind last two bytes from file if open, and close file */
-
-    return rhizome_direct_form_received(r);
-  }
-  return 0;
-
-scram:
-  if (r->field_file) {
-    if (fclose(r->field_file) == EOF)
-      WARNF_perror("fclose()");
-  }
-  rhizome_direct_clear_temporary_files(r);
-  rhizome_server_simple_http_response(r, 500,
-      "<html><h1>Sorry, couldn't complete your request, reasonable as it was.  Perhaps try again later.</h1></html>\r\n");
-  return -1;
-}
-
-int rhizome_direct_process_post_multipart_bytes(rhizome_http_request *r,const char *bytes,int count)
-{
-  {
-    char logname[128];
-    snprintf(logname,128,"post-%08x.log",r->uuid);
-    FILE *f=fopen(logname,"a"); 
-    if (f) fwrite(bytes,count,1,f);
-    if (f) fclose(f);
-  }
-
-  /* This function looks for multi-part form separators and descriptor lines,
-     and streams any "manifest" or "data" blocks to respectively named files.
-
-     The challenge is that we might only get a partial boundary string passed
-     to us.  So we need to remember the last KB or so of data and glue it to
-     the front of the current set of bytes.
-
-     In multi-part form parsing we don't need r->request for anything, so if
-     we are not in a form part already, then we can stow the bytes there
-     for reexamination when more bytes arrive.
-     
-     Side effect will be that the entire boundary string and associated bits will
-     need to be <=1KB, the size of r->request.  This seems quite reasonable.
-
-     Example of such a block is:
-
-     ------WebKitFormBoundaryEoJwSoSVW4qsrBZW
-     Content-Disposition: form-data; name="manifest"; filename="spleen"
-     Content-Type: application/octet-stream     
-  */
-
-  int o;
-
-  /* Split into lines and process each line separately using a
-     simple state machine. 
-     Lines containing binary are truncated into arbitrarily length pieces, but
-     a newline will ALWAYS break the line.
-  */
-
-  for(o=0;o<count;o++)
-    {
-      int newline=0;
-      if (bytes[o]=='\n')
-	if (r->request_length>0&&r->request[r->request_length-1]=='\r')
-	  { newline=1; r->request_length--; }
-      if (r->request_length>1020) newline=2;
-      if (newline) {	
-	/* Found end of line, so process it */
-	if (newline==1) {
-	  /* Put the real new line onto the end if it was present, so that
-	     we don't go doing anything silly, like joining lines in files
-	     that really were separated by CRLF, or similarly inserting CRLF
-	     in the middle of slabs of bytes that were not CRLF terminated.
-	  */
-	  r->request[r->request_length++]='\r';
-	  r->request[r->request_length++]='\n';
-	}
-	r->request[r->request_length]=0;
-	if (rhizome_direct_process_mime_line(r,r->request,r->request_length)) 
-	  return -1;
-	r->request_length=0;
-	/* If a real new line was detected, then
-	   don't include the \n as part of the next line.
-	   But if it wasn't a real new line, then make sure we
-	   don't loose the byte. */
-	if (newline==1) continue;
-      }
-
-      r->request[r->request_length++]=bytes[o];
-    }
-
-  r->source_count-=count;
-  if (r->source_count<=0) {
-    /* Got to end of multi-part form data */
-
-    /* If the form is still being processed, then flush things through */
-    if (r->request_type<0) {
-      /* Flush out any remaining data */
-      if (r->request_length) {
-	DEBUGF("Flushing last %d bytes",r->request_length);
-	r->request[r->request_length]=0;
-	rhizome_direct_process_mime_line(r,r->request,r->request_length);
-      }      
-      return rhizome_direct_form_received(r);
-    } else {
-      /* Form has already been processed, so do nothing */
-    }
-  }
-  return 0;
-}
-
-struct http_request_parts {
-};
-
-int rhizome_direct_parse_http_request(rhizome_http_request *r)
-{
-  DEBUGF("uri=%s", alloca_str_toprint(config.rhizome.api.addfile.uri_path));
-  
-  // Parse the HTTP request into verb, path, protocol, headers and content.
-  char *const request_end = r->request + r->request_length;
-  char *verb = r->request;
-  char *path = NULL;
-  char *proto = NULL;
-  size_t pathlen = 0;
-  char *headers = NULL;
-  int headerlen = 0;
-  char *content = NULL;
-  int contentlen = 0;
-  char *p;
-  if ((str_startswith(verb, "GET", (const char **)&p) || str_startswith(verb, "POST", (const char **)&p)) && isspace(*p)) {
-    *p++ = '\0';
-    path = p;
-    while (p < request_end && !isspace(*p))
-      ++p;
-    if (p < request_end) {
-      pathlen = p - path;
-      *p++ = '\0';
-      proto = p;
-      if ( str_startswith(p, "HTTP/1.", (const char **)&p)
-	&& (str_startswith(p, "0", (const char **)&p) || str_startswith(p, "1", (const char **)&p))
-	&& (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers))
-      ) {
-	*p = '\0';
-	char *eoh = str_str(headers, "\r\n\r\n", request_end - p);
-	if (eoh) {
-	  content = eoh + 4;
-	  headerlen = content - headers;
-	  contentlen = request_end - content;
-	}
-      }
-    }
-  }
-  if (content == NULL) {
-    if (config.debug.rhizome_httpd)
-      DEBUGF("Received malformed HTTP request %s", alloca_toprint(160, (const char *)r->request, r->request_length));
-    return rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
-  }
-  INFOF("RHIZOME HTTP SERVER, %s %s %s", verb, alloca_toprint(-1, path, pathlen), proto);
-  if (config.debug.rhizome_httpd)
-    DEBUGF("headers %s", alloca_toprint(-1, headers, headerlen));
-  if (strcmp(verb, "POST") == 0
-      && (   strcmp(path, "/rhizome/import") == 0 
-	  || strcmp(path, "/rhizome/enquiry") == 0
-	  || (config.rhizome.api.addfile.uri_path[0] && strcmp(path, config.rhizome.api.addfile.uri_path) == 0)
-	 )
-  ) {
-    const char *cl_str=str_str(headers,"Content-Length: ",headerlen);
-    const char *ct_str=str_str(headers,"Content-Type: multipart/form-data; boundary=",headerlen);
-    if (!cl_str)
-      return rhizome_server_simple_http_response(r,400,"<html><h1>Missing Content-Length header</h1></html>\r\n");
-    if (!ct_str)
-      return rhizome_server_simple_http_response(r,400,"<html><h1>Missing or unsupported Content-Type header</h1></html>\r\n");
-    /* ok, we have content-type and content-length, now make sure they are well formed. */
-    int64_t content_length;
-    if (sscanf(cl_str,"Content-Length: %"PRId64,&content_length)!=1)
-      return rhizome_server_simple_http_response(r,400,"<html><h1>Malformed Content-Length header</h1></html>\r\n");
-    char boundary_string[1024];
-    int i;
-    ct_str+=strlen("Content-Type: multipart/form-data; boundary=");
-    for(i=0;i<1023&&*ct_str&&*ct_str!='\n'&&*ct_str!='\r';i++,ct_str++)
-      boundary_string[i]=*ct_str;
-    boundary_string[i] = '\0';
-    if (i<4||i>128)
-      return rhizome_server_simple_http_response(r,400,"<html><h1>Malformed Content-Type header</h1></html>\r\n");
-
-    DEBUGF("content_length=%"PRId64", boundary_string=%s contentlen=%d", (int64_t) content_length, alloca_str_toprint(boundary_string), contentlen);
-
-    /* Now start receiving and parsing multi-part data.  If we already received some of the
-	post-header data, process that first.  Tell the HTTP request that it has moved to multipart
-	form data parsing, and what the actual requested action is.
-    */
-
-    /* Remember boundary string and source path.
-	Put the preceeding -- on the front to make our life easier when
-	parsing the rest later. */
-    strbuf bs = strbuf_local(r->boundary_string, sizeof r->boundary_string);
-    strbuf_puts(bs, "--");
-    strbuf_puts(bs, boundary_string);
-    if (strbuf_overrun(bs))
-      return rhizome_server_simple_http_response(r,500,"<html><h1>Internal server error: Multipart boundary string too long</h1></html>\r\n");
-    strbuf ps = strbuf_local(r->path, sizeof r->path);
-    strbuf_puts(ps, path);
-    if (strbuf_overrun(ps))
-      return rhizome_server_simple_http_response(r,500,"<html><h1>Internal server error: Path too long</h1></html>\r\n");
-    r->boundary_string_length = strbuf_len(bs);
-    r->source_index = 0;
-    r->source_count = content_length;
-    r->request_type = RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART;
-    r->request_length = 0;
-    r->source_flags = 0;
-
-    /* Find the end of the headers and start of any body bytes that we have read
-	so far. Copy the bytes to a separate buffer, because r->request and 
-	r->request_length get used internally in the parser.
-      */
-    if (contentlen) {
-      char buffer[contentlen];
-      bcopy(content, buffer, contentlen);
-      rhizome_direct_process_post_multipart_bytes(r, buffer, contentlen);
-    }
-
-    /* Handle the rest of the transfer asynchronously. */
+    if (config.debug.rhizome)
+      DEBUGF("Import sans-manifest appeared to succeed");
+    /* Respond with the manifest that was added. */
+    http_request_response(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes);
+    /* clean up after ourselves */
+    if (mout && mout != m)
+      rhizome_manifest_free(mout);
+    rhizome_manifest_free(m);
+    rhizome_direct_clear_temporary_files(r);
     return 0;
   } else {
-    rhizome_server_simple_http_response(r, 404, "<html><h1>Not found (OTHER)</h1></html>\r\n");
+    http_request_simple_response(&r->http, 501, "Not Implemented: Rhizome add with manifest");
+    return 0;
   }
-  
-  /* Try sending data immediately. */
-  rhizome_server_http_send_bytes(r);
+}
 
+void rhizome_direct_process_mime_start(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  assert(r->current_part == NONE);
+  assert(r->part_fd == -1);
+}
+
+void rhizome_direct_process_mime_end(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  if (r->part_fd != -1) {
+    if (close(r->part_fd) == -1) {
+      WHYF_perror("close(%d)", r->part_fd);
+      http_request_simple_response(&r->http, 500, "Internal Error: Close temporary file failed");
+      return;
+    }
+    r->part_fd = -1;
+  }
+  switch (r->current_part) {
+    case MANIFEST:
+      r->received_manifest = 1;
+      break;
+    case DATA:
+      r->received_data = 1;
+      break;
+    case NONE:
+      break;
+  }
+}
+
+void rhizome_direct_process_mime_content_disposition(struct http_request *hr, const struct mime_content_disposition *cd)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  if (strcmp(cd->name, "data") == 0) {
+    r->current_part = DATA;
+    strncpy(r->data_file_name, cd->filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0';
+  }
+  else if (strcmp(cd->name, "manifest") == 0) {
+    r->current_part = MANIFEST;
+  } else
+    return;
+  char path[512];
+  if (form_temporary_file_path(r, path, cd->name) == -1) {
+    http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
+    return;
+  }
+  if ((r->part_fd = open(path, O_WRONLY | O_CREAT, 0666)) == -1) {
+    WHYF_perror("open(%s,O_WRONLY|O_CREAT,0666)", alloca_str_toprint(path));
+    http_request_simple_response(&r->http, 500, "Internal Error: Create temporary file failed");
+    return;
+  }
+}
+
+void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  if (r->part_fd != -1) {
+    if (write_all(r->part_fd, buf, len) == -1) {
+      http_request_simple_response(&r->http, 500, "Internal Error: Write temporary file failed");
+      return;
+    }
+  }
+}
+
+int rhizome_direct_import(rhizome_http_request *r, const char *remainder)
+{
+  if (r->http.verb != HTTP_VERB_POST) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
+  r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
+  r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
+  r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition;
+  r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
+  r->http.handle_content_end = rhizome_direct_import_end;
+  r->current_part = NONE;
+  r->part_fd = -1;
+  r->data_file_name[0] = '\0';
   return 0;
 }
 
+int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder)
+{
+  if (r->http.verb != HTTP_VERB_POST) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
+  r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
+  r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
+  r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition;
+  r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
+  r->http.handle_content_end = rhizome_direct_enquiry_end;
+  r->current_part = NONE;
+  r->part_fd = -1;
+  r->data_file_name[0] = '\0';
+  return 0;
+}
+
+/* Servald can be configured to accept files without manifests via HTTP from localhost, so that
+ * rhizome bundles can be created programatically.  There are probably still some security
+ * loop-holes here, which is part of why we leave it disabled by default, but it will be sufficient
+ * for testing possible uses, including integration with OpenDataKit.
+ */
+int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
+{
+  if (r->http.verb != HTTP_VERB_POST) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
+  if (cmp_sockaddr((struct sockaddr *)&r->http.client_in_addr, sizeof r->http.client_in_addr,
+		   (struct sockaddr *)&config.rhizome.api.addfile.allow_host, sizeof config.rhizome.api.addfile.allow_host
+		  ) != 0
+  ) {
+    INFOF("rhizome.api.addfile request received from %s, but is only allowed from %s",
+	alloca_sockaddr(&r->http.client_in_addr, sizeof r->http.client_in_addr),
+	alloca_sockaddr(&config.rhizome.api.addfile.allow_host, sizeof config.rhizome.api.addfile.allow_host)
+      );
+    rhizome_direct_clear_temporary_files(r);
+    http_request_simple_response(&r->http, 404, "<html><h1>Not available from here</h1></html>");
+    return 0;
+  }
+  r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
+  r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
+  r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition;
+  r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
+  r->http.handle_content_end = rhizome_direct_addfile_end;
+  r->current_part = NONE;
+  r->part_fd = -1;
+  r->data_file_name[0] = '\0';
+  return 0;
+}
+
+int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder)
+{
+  if (   config.rhizome.api.addfile.uri_path[0]
+      && strcmp(r->http.path, config.rhizome.api.addfile.uri_path) == 0
+  )
+    return rhizome_direct_addfile(r, remainder);
+  return 1;
+}
+
 static int receive_http_response(int sock, char *buffer, size_t buffer_len, struct http_response_parts *parts)
 {
   int len = 0;
@@ -702,7 +434,8 @@ static int receive_http_response(int sock, char *buffer, size_t buffer_len, stru
       DEBUGF("Invalid HTTP reply: missing Content-Length header");
     return -1;
   }
-  DEBUGF("content_length=%"PRId64, parts->content_length);
+  if (config.debug.rhizome_rx)
+    DEBUGF("content_length=%"PRId64, parts->content_length);
   return len - (parts->content_start - buffer);
 }
 
@@ -718,36 +451,38 @@ static int fill_buffer(int sock, unsigned char *buffer, int len, int buffer_size
 
 void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 {
-  DEBUGF("Dispatch size_high=%"PRId64,r->cursor->size_high);
+  if (config.debug.rhizome_tx)
+    DEBUGF("Dispatch size_high=%"PRId64,r->cursor->size_high);
   rhizome_direct_transport_state_http *state = r->transport_specific_state;
 
   sid_t zerosid = SID_ANY;
 
   int sock=socket(AF_INET, SOCK_STREAM, 0);
   if (sock==-1) {
-    WHY_perror("socket");    
+    WHY_perror("socket");
     goto end;
-  } 
+  }
 
   struct hostent *hostent;
   hostent = gethostbyname(state->host);
   if (!hostent) {
-    DEBUGF("could not resolve hostname");
+    if (config.debug.rhizome_tx)
+      DEBUGF("could not resolve hostname");
     goto end;
   }
 
-  struct sockaddr_in addr;  
-  addr.sin_family = AF_INET;     
-  addr.sin_port = htons(state->port);   
+  struct sockaddr_in addr;
+  addr.sin_family = AF_INET;
+  addr.sin_port = htons(state->port);
   addr.sin_addr = *((struct in_addr *)hostent->h_addr);
-  bzero(&(addr.sin_zero),8);     
+  bzero(&(addr.sin_zero),8);
 
-  if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) {
-    WHY_perror("connect");
+  if (connect(sock, (struct sockaddr *)&addr, sizeof addr) == -1) {
+    WHYF_perror("connect(%s)", alloca_sockaddr(&addr, sizeof addr));
     close(sock);
     goto end;
   }
- 
+
   char boundary[20];
   char buffer[8192];
 
@@ -785,7 +520,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
   int len = strbuf_len(request);
   int sent=0;
   while(sent<len) {
-    DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
+    if (config.debug.rhizome_tx)
+      DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
     int count=write(sock,&buffer[sent],len-sent);
     if (count == -1) {
       if (errno==EPIPE) goto rx;
@@ -815,7 +551,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
   len = strbuf_len(request);
   sent=0;
   while(sent<len) {
-    DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
+    if (config.debug.rhizome_tx)
+      DEBUGF("write(%d, %s, %d)", sock, alloca_toprint(-1, &buffer[sent], len-sent), len-sent);
     int count=write(sock,&buffer[sent],len-sent);
     if (count == -1) {
       if (errno==EPIPE) goto rx;
@@ -849,10 +586,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
     goto end;
   }
   close(sock);
-  
+
   /* We now have the list of (1+RHIZOME_BAR_PREFIX_BYTES)-byte records that indicate
      the list of BAR prefixes that differ between the two nodes.  We can now action
-     those which are relevant, i.e., based on whether we are pushing, pulling or 
+     those which are relevant, i.e., based on whether we are pushing, pulling or
      synchronising (both).
 
      I am currently undecided as to whether it is cleaner to have some general
@@ -867,7 +604,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
       int type=actionlist[i];
       if (type==2&&r->pullP) {
 	/* Need to fetch manifest.  Once we have the manifest, then we can
-	   use our normal bundle fetch routines from rhizome_fetch.c	 
+	   use our normal bundle fetch routines from rhizome_fetch.c
 
 	   Generate a request like: GET /rhizome/manifestbybar/<hex of bar>
 	   and add it to our list of HTTP fetch requests, then watch
@@ -875,16 +612,17 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	   Then as noted above, we can use that to pull the file down using
 	   existing routines.
 	*/
-	DEBUGF("Fetching manifest %s* @ 0x%x",alloca_tohex(&actionlist[i], 1+RHIZOME_BAR_PREFIX_BYTES),i);
+	if (config.debug.rhizome_tx)
+	  DEBUGF("Fetching manifest %s* @ 0x%x",alloca_tohex(&actionlist[i], 1+RHIZOME_BAR_PREFIX_BYTES),i);
 	if (!rhizome_fetch_request_manifest_by_prefix(&addr, &zerosid, &actionlist[i+1], RHIZOME_BAR_PREFIX_BYTES))
 	  {
-	    /* Fetching the manifest, and then using it to see if we want to 
+	    /* Fetching the manifest, and then using it to see if we want to
 	       fetch the file for import is all handled asynchronously, so just
 	       wait for it to finish. */
 	    while (rhizome_any_fetch_active() || rhizome_any_fetch_queued())
 	      fd_poll();
 	  }
-	
+
       } else if (type==1&&r->pushP) {
 	/* Form up the POST request to submit the appropriate bundle. */
 
@@ -897,11 +635,14 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	}
 
 	/* Get filehash and size from manifest if present */
-	DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
-	DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash));
-	DEBUGF("file size = %"PRId64, m->fileLength);
+	if (config.debug.rhizome_tx) {
+	  DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
+	  DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash));
+	  DEBUGF("file size = %"PRId64, m->fileLength);
+	}
 	int64_t version = rhizome_manifest_get_ll(m, "version");
-	DEBUGF("version = %"PRId64,version);
+	if (config.debug.rhizome_tx)
+	  DEBUGF("version = %"PRId64,version);
 
 	/* We now have everything we need to compose the POST request and send it.
 	 */
@@ -919,8 +660,8 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  "Content-Type: application/octet-stream\r\n"
 	  "\r\n";
 	/* Work out what the content length should be */
-	DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d",
-	       m->manifest_all_bytes,m->manifest_bytes);
+	if (config.debug.rhizome_tx)
+	  DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", m->manifest_all_bytes,m->manifest_bytes);
 	int content_length
 	  =strlen(template2)-2 /* minus 2 for the "%s" that gets replaced */
 	  +strlen(boundary)
@@ -939,21 +680,22 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	len+=m->manifest_all_bytes;
 	len+=snprintf(&buffer[len],8192-len,template3,boundary);
 
-	addr.sin_family = AF_INET;     
-	addr.sin_port = htons(state->port);   
+	addr.sin_family = AF_INET;
+	addr.sin_port = htons(state->port);
 	addr.sin_addr = *((struct in_addr *)hostent->h_addr);
-	bzero(&(addr.sin_zero),8);     
-	
+	bzero(&(addr.sin_zero),8);
+
 	sock=socket(AF_INET, SOCK_STREAM, 0);
 	if (sock==-1) {
-	  DEBUGF("could not open socket");    
+	  if (config.debug.rhizome_tx)
+	    DEBUGF("could not open socket");
 	  goto closeit;
-	} 
-	if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1)
-	  {
+	}
+	if (connect(sock,(struct sockaddr *)&addr,sizeof(struct sockaddr)) == -1) {
+	  if (config.debug.rhizome_tx)
 	    DEBUGF("Could not connect to remote");
-	    goto closeit;
-	  }
+	  goto closeit;
+	}
 
 	int sent=0;
 	/* Send buffer now */
@@ -968,12 +710,12 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  rhizome_filehash_t filehash;
 	  if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1)
 	    goto closeit;
-	  
+
 	  struct rhizome_read read;
 	  bzero(&read, sizeof read);
 	  if (rhizome_open_read(&read, &filehash))
 	    goto closeit;
-	
+
 	  int64_t read_ofs;
 	  for(read_ofs=0;read_ofs<m->fileLength;){
 	    unsigned char buffer[4096];
@@ -994,7 +736,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	      }
 	      write_ofs+=written;
 	    }
-	    
+
 	    read_ofs+=bytes_read;
 	  }
 	  rhizome_read_close(&read);
@@ -1006,7 +748,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  int r=write(sock,&buffer[sent],len-sent);
 	  if (r>0) sent+=r;
 	  if (r<0) goto closeit;
-	}	
+	}
 
 	/* get response back. */
 	if (receive_http_response(sock, buffer, sizeof buffer, &parts) == -1)
@@ -1023,7 +765,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
     }
 
   free(actionlist);
-  
+
   /* now update cursor according to what range was covered in the response.
      We set our current position to just past the high limit of the returned
      cursor.
@@ -1033,18 +775,20 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
      end up in an infinite loop.  We could also end up in a very long finite loop
      if the cursor doesn't advance far.  A simple solution is to not adjust the
      cursor position, and simply re-attempt the sync until no actions result.
-     That will do for now.     
+     That will do for now.
  */
 #ifdef FANCY_CURSOR_POSITION_HANDLING
   rhizome_direct_bundle_cursor *c=rhizome_direct_bundle_iterator(10);
   assert(c!=NULL);
-  if (rhizome_direct_bundle_iterator_unpickle_range(c,(unsigned char *)&p[0],10))
-    {
+  if (rhizome_direct_bundle_iterator_unpickle_range(c,(unsigned char *)&p[0],10)) {
+    if (config.debug.rhizome_tx)
       DEBUGF("Couldn't unpickle range. This should never happen.  Assuming near and far cursor ranges match.");
-    }
+  }
   else {
-    DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high, c->limit_size_high);
-    DEBUGF("c->buffer_size=%d",c->buffer_size);
+    if (config.debug.rhizome_tx) {
+      DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64, c->size_high, c->limit_size_high);
+      DEBUGF("c->buffer_size=%d",c->buffer_size);
+    }
     r->cursor->size_low=c->limit_size_high;
     /* Set tail of BID to all high, as we assume the far end has returned all
        BIDs with the specified prefix. */
@@ -1055,7 +799,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 #endif
 
   end:
-  /* Warning: tail recursion when done this way. 
+  /* Warning: tail recursion when done this way.
      Should be triggered by an asynchronous event.
      But this will do for now. */
   rhizome_direct_continue_sync_request(r);
diff --git a/rhizome_http.c b/rhizome_http.c
index fe9b848d..418e653f 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -23,18 +23,74 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #ifdef HAVE_SYS_FILIO_H
 #include <sys/filio.h>
 #endif
+#include <assert.h>
 
+#define __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
 #include "serval.h"
 #include "overlay_address.h"
 #include "conf.h"
 #include "str.h"
 #include "rhizome.h"
+#include "http_server.h"
+
 #define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32
 
-struct sched_ent server_alarm;
-struct profile_total server_stats;
+struct http_handler{
+  const char *path;
+  int (*parser)(rhizome_http_request *r, const char *remainder);
+};
 
-struct profile_total connection_stats;
+static int rhizome_status_page(rhizome_http_request *r, const char *remainder);
+static int rhizome_file_page(rhizome_http_request *r, const char *remainder);
+static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder);
+static int interface_page(rhizome_http_request *r, const char *remainder);
+static int neighbour_page(rhizome_http_request *r, const char *remainder);
+static int fav_icon_header(rhizome_http_request *r, const char *remainder);
+static int root_page(rhizome_http_request *r, const char *remainder);
+
+extern int rhizome_direct_import(rhizome_http_request *r, const char *remainder);
+extern int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder);
+extern int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder);
+
+struct http_handler paths[]={
+  {"/rhizome/status", rhizome_status_page},
+  {"/rhizome/file/", rhizome_file_page},
+  {"/rhizome/import", rhizome_direct_import},
+  {"/rhizome/enquiry", rhizome_direct_enquiry},
+  {"/rhizome/manifestbyprefix/", manifest_by_prefix_page},
+  {"/rhizome/", rhizome_direct_dispatch},
+  {"/interface/", interface_page},
+  {"/neighbour/", neighbour_page},
+  {"/favicon.ico", fav_icon_header},
+  {"/", root_page},
+};
+
+static int rhizome_dispatch(struct http_request *hr)
+{
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  INFOF("RHIZOME HTTP SERVER, %s %s", r->http.verb, r->http.path);
+  r->http.response.content_generator = NULL;
+  unsigned i;
+  for (i = 0; i < NELS(paths); ++i) {
+    const char *remainder;
+    if (str_startswith(r->http.path, paths[i].path, &remainder)){
+      int ret = paths[i].parser(r, remainder);
+      if (ret < 0) {
+	http_request_simple_response(&r->http, 500, NULL);
+	return 0;
+      }
+      if (ret == 0)
+	return 0;
+    }
+  }
+  http_request_simple_response(&r->http, 404, NULL);
+  return 0;
+}
+
+struct sched_ent server_alarm;
+struct profile_total server_stats = {
+  .name = "rhizome_server_poll",
+};
 
 /*
   HTTP server and client code for rhizome transfers and rhizome direct.
@@ -47,9 +103,6 @@ static int rhizome_server_socket = -1;
 static int request_count=0;
 static time_ms_t rhizome_server_last_start_attempt = -1;
 
-int (*rhizome_http_parse_func)(rhizome_http_request *)=NULL;
-const char *rhizome_http_parse_func_description="(null)";
-
 // Format icon data using:
 //   od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/  */,/g'
 unsigned char favicon_bytes[]={
@@ -88,9 +141,7 @@ int is_rhizome_http_server_running()
    Return 1 if the server is already started successfully.
    Return 2 if the server was not started because it is too soon since last failed attempt.
  */
-int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *),
-			      const char *parse_func_desc,
-			      uint16_t port_low, uint16_t port_high)
+int rhizome_http_server_start(uint16_t port_low, uint16_t port_high)
 {
   if (rhizome_server_socket != -1)
     return 1;
@@ -162,15 +213,10 @@ success:
   else
     INFOF("HTTP SERVER (LIMITED SERVICE), START port=%"PRIu16" fd=%d", port, rhizome_server_socket);
 
-  /* Remember which function to call when handling client connections */
-  rhizome_http_parse_func=parse_func;
-  rhizome_http_parse_func_description=parse_func_desc;
-
   rhizome_http_server_port = port;
   /* Add Rhizome HTTPd server to list of file descriptors to watch */
   server_alarm.function = rhizome_server_poll;
-  server_stats.name="rhizome_server_poll";
-  server_alarm.stats=&server_stats;
+  server_alarm.stats = &server_stats;
   server_alarm.poll.fd = rhizome_server_socket;
   server_alarm.poll.events = POLLIN;
   watch(&server_alarm);
@@ -178,90 +224,16 @@ success:
 
 }
 
-void rhizome_client_poll(struct sched_ent *alarm)
+static void rhizome_server_finalise_http_request(struct http_request *_r)
 {
-  rhizome_http_request *r = (rhizome_http_request *)alarm;
-  if (alarm->poll.revents == 0 || alarm->poll.revents & (POLLHUP | POLLERR)){
-    if (config.debug.rhizome_tx)
-      DEBUGF("Closing connection due to timeout or error %d", alarm->poll.revents);
-    rhizome_server_free_http_request(r);
-    return;
-  }
-  
-  if (alarm->poll.revents & POLLIN){
-    switch(r->request_type)
-      {
-      case RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART:
-	{
-	  /* Reading multi-part form data. Read some bytes and proces them. */
-	  char buffer[16384];
-	  sigPipeFlag=0;
-	  int bytes = read_nonblock(r->alarm.poll.fd, buffer, 16384);
-	  /* If we got some data, see if we have found the end of the HTTP request */
-	  if (bytes > 0) {
-	    // reset inactivity timer
-	    r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT;
-	    r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
-	    unschedule(&r->alarm);
-	    schedule(&r->alarm);
-	    rhizome_direct_process_post_multipart_bytes(r,buffer,bytes);
-	  }
-	  /* We don't drop the connection on an empty read, because that results
-	     in connections dropping when they shouldn't, including during testing.
-	     The idle timeout should drop the connections instead.
-	  */
-	  if (sigPipeFlag) {
-	    if (config.debug.rhizome_tx)
-	      DEBUG("Received SIGPIPE, closing connection");
-	    rhizome_server_free_http_request(r);
-	    return;
-	  }
-	}
-	break;
-	
-      case RHIZOME_HTTP_REQUEST_RECEIVING:
-	/* Keep reading until we have two CR/LFs in a row */
-	r->request[r->request_length] = '\0';
-	sigPipeFlag=0;
-	int bytes = read_nonblock(r->alarm.poll.fd, &r->request[r->request_length], sizeof r->request - r->request_length);
-	/* If we got some data, see if we have found the end of the HTTP request */
-	if (bytes > 0) {
-	  // reset inactivity timer
-	  r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT;
-	  r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
-	  unschedule(&r->alarm);
-	  schedule(&r->alarm);
-	  r->request_length += bytes;
-	  r->header_length = is_http_header_complete(r->request, r->request_length, bytes);
-	  if (r->header_length){
-	    /* We have the request. Now parse it to see if we can respond to it */
-	    if (rhizome_http_parse_func!=NULL) 
-	      rhizome_http_parse_func(r);
-	  }
-	} else {
-	  if (config.debug.rhizome_tx)
-	    DEBUG("Empty read, closing connection");
-	  rhizome_server_free_http_request(r);
-	  return;
-	}
-	if (sigPipeFlag) {
-	  if (config.debug.rhizome_httpd)
-	    DEBUG("Received SIGPIPE, closing HTTP connection");
-	  rhizome_server_free_http_request(r);
-	  return;
-	}
-	break;
-      }
-  }
-  
-  if (alarm->poll.revents & POLLOUT){
-    /* Socket already has request -- so just try to send some data. */
-    rhizome_server_http_send_bytes(r);
-  }
-  return;
+  rhizome_http_request *r = (rhizome_http_request *) _r;
+  rhizome_read_close(&r->read_state);
+  request_count--;
 }
 
-static unsigned int rhizome_http_request_uuid_counter=0;
+static int rhizome_dispatch(struct http_request *);
+
+static unsigned int rhizome_http_request_uuid_counter = 0;
 
 void rhizome_server_poll(struct sched_ent *alarm)
 {
@@ -269,7 +241,10 @@ void rhizome_server_poll(struct sched_ent *alarm)
     struct sockaddr addr;
     unsigned int addr_len = sizeof addr;
     int sock;
-    if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) != -1) {
+    if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) == -1) {
+      if (errno && errno != EAGAIN)
+	WARN_perror("accept");
+    } else {
       struct sockaddr_in *peerip=NULL;
       if (addr.sa_family == AF_INET) {
 	peerip = (struct sockaddr_in *)&addr;
@@ -285,55 +260,32 @@ void rhizome_server_poll(struct sched_ent *alarm)
 	    addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data)
 	  );
       }
-      rhizome_http_request *request = calloc(sizeof(rhizome_http_request), 1);
+      rhizome_http_request *request = emalloc_zero(sizeof(rhizome_http_request));
       if (request == NULL) {
-	WHYF_perror("calloc(%u, 1)", (int)sizeof(rhizome_http_request));
-	WHY("Cannot respond to request, out of memory");
+	WHY("Cannot respond to HTTP request, out of memory");
 	close(sock);
       } else {
 	request_count++;
-	request->uuid=rhizome_http_request_uuid_counter++;
-	if (peerip) request->requestor=*peerip; 
-	else bzero(&request->requestor,sizeof(request->requestor));
-	request->data_file_name[0]=0;
-	/* We are now trying to read the HTTP request */
-	request->request_type=RHIZOME_HTTP_REQUEST_RECEIVING;
-	request->alarm.function = rhizome_client_poll;
-	request->read_state.blob_fd=-1;
-	request->read_state.blob_rowid=-1;
-	connection_stats.name="rhizome_client_poll";
-	request->alarm.stats=&connection_stats;
-	request->alarm.poll.fd=sock;
-	request->alarm.poll.events=POLLIN;
-	request->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT;
-	request->alarm.deadline = request->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
-	// watch for the incoming http request
-	watch(&request->alarm);
-	// set an inactivity timeout to close the connection
-	schedule(&request->alarm);
+	request->uuid = rhizome_http_request_uuid_counter++;
+	request->data_file_name[0] = '\0';
+	request->read_state.blob_fd = -1;
+	request->read_state.blob_rowid = -1;
+	if (peerip)
+	  request->http.client_in_addr = *peerip;
+	request->http.handle_headers = rhizome_dispatch;
+	request->http.debug_flag = &config.debug.rhizome_httpd;
+	request->http.finalise = rhizome_server_finalise_http_request;
+	request->http.free = free;
+	request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT;
+	http_request_init(&request->http, sock);
       }
     }
-    if (errno && errno != EAGAIN)
-      WARN_perror("accept");
   }
   if (alarm->poll.revents & (POLLHUP | POLLERR)) {
     INFO("Error on tcp listen socket");
   }
 }
 
-int rhizome_server_free_http_request(rhizome_http_request *r)
-{
-  unwatch(&r->alarm);
-  unschedule(&r->alarm);
-  close(r->alarm.poll.fd);
-  if (r->buffer)
-    free(r->buffer);
-  rhizome_read_close(&r->read_state);
-  free(r);
-  request_count--;
-  return 0;
-}
-
 int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
 {
   IN();
@@ -360,53 +312,59 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_
   OUT();
 }
 
-static int neighbour_page(rhizome_http_request *r, const char *remainder, const char *headers)
+static int neighbour_page(rhizome_http_request *r, const char *remainder)
 {
+  if (r->http.verb != HTTP_VERB_GET) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
   char buf[8*1024];
-  strbuf b=strbuf_local(buf, sizeof buf);
-  
+  strbuf b = strbuf_local(buf, sizeof buf);
   sid_t neighbour_sid;
   if (str_to_sid_t(&neighbour_sid, remainder) == -1)
-    return -1;
-    
+    return 1;
   struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0);
   if (!neighbour)
     return 1;
-    
   strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
   link_neighbour_status_html(b, neighbour);
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-  rhizome_server_simple_http_response(r, 200, buf);
+  http_request_simple_response(&r->http, 200, buf);
   return 0;
 }
 
-static int interface_page(rhizome_http_request *r, const char *remainder, const char *headers)
+static int interface_page(rhizome_http_request *r, const char *remainder)
 {
+  if (r->http.verb != HTTP_VERB_GET) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
   char buf[8*1024];
   strbuf b=strbuf_local(buf, sizeof buf);
   int index=atoi(remainder);
   if (index<0 || index>=OVERLAY_MAX_INTERFACES)
     return 1;
-    
   strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
   interface_state_html(b, &overlay_interfaces[index]);
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-    
-  rhizome_server_simple_http_response(r, 200, buf);
+  http_request_simple_response(&r->http, 200, buf);
   return 0;
 }
 
-static int rhizome_status_page(rhizome_http_request *r, const char *remainder, const char *headers)
+static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
 {
   if (!is_rhizome_http_enabled())
     return 1;
   if (*remainder)
     return 1;
-    
+  if (r->http.verb != HTTP_VERB_GET) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
   char buf[32*1024];
   struct strbuf b;
   strbuf_init(&b, buf, sizeof buf);
@@ -417,87 +375,100 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder, c
   strbuf_puts(&b, "</body></html>");
   if (strbuf_overrun(&b))
     return -1;
-  rhizome_server_simple_http_response(r, 200, buf);
+  http_request_simple_response(&r->http, 200, buf);
   return 0;
 }
 
-static int rhizome_file_content(rhizome_http_request *r)
+static int rhizome_file_content(struct http_request *hr)
 {
-  int suggested_size=65536;
-  if (suggested_size > r->read_state.length - r->read_state.offset)
-    suggested_size = r->read_state.length - r->read_state.offset;
-  if (suggested_size<=0)
+  rhizome_http_request *r = (rhizome_http_request *) hr;
+  assert(r->http.response_length < r->http.response_buffer_size);
+  assert(r->read_state.offset <= r->read_state.length);
+  uint64_t readlen = r->read_state.length - r->read_state.offset;
+  if (readlen == 0)
     return 0;
-  
-  if (r->buffer_size < suggested_size){
-    r->buffer_size = suggested_size;
-    if (r->buffer)
-      free(r->buffer);
-    r->buffer = malloc(r->buffer_size);
-  }
-  
-  if (!r->buffer)
+  size_t suggested_size = 64 * 1024;
+  if (suggested_size > readlen)
+    suggested_size = readlen;
+  if (r->http.response_buffer_size < suggested_size)
+    http_request_set_response_bufsize(&r->http, suggested_size);
+  if (r->http.response_buffer == NULL)
+    http_request_set_response_bufsize(&r->http, 1);
+  if (r->http.response_buffer == NULL)
     return -1;
-  
-  r->buffer_length = rhizome_read(&r->read_state, r->buffer, r->buffer_size);
+  size_t space = r->http.response_buffer_size - r->http.response_length;
+  int len = rhizome_read(&r->read_state,
+		         (unsigned char *)r->http.response_buffer + r->http.response_length,
+			 space);
+  if (len == -1)
+    return -1;
+  assert(len <= space);
+  r->http.response_length += len;
   return 0;
 }
 
-static int rhizome_file_page(rhizome_http_request *r, const char *remainder, const char *headers)
+static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
 {
   /* Stream the specified payload */
   if (!is_rhizome_http_enabled())
     return 1;
-  
-  rhizome_filehash_t filehash;
-  if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
-    return -1;
-  
-  bzero(&r->read_state, sizeof(r->read_state));
-
-  /* Refuse to honour HTTP request if required (used for debugging and 
-     testing transition from HTTP to MDP) */
-  if (rhizome_open_read(&r->read_state, &filehash))
-    return 1;
-    
-  if (r->read_state.length==-1){
-    if (rhizome_read(&r->read_state, NULL, 0)){
-      rhizome_read_close(&r->read_state);
-      return 1;
-    }
-  }
-  
-  const char *range=str_str((char*)headers,"Range: bytes=",-1);
-  r->read_state.offset = r->source_index = 0;
-  
-  if (range){
-    sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset);
-    if (0)
-      DEBUGF("Found range header %"PRId64,r->read_state.offset);
-  }
-  
-  if (r->read_state.length - r->read_state.offset<=0){
-    rhizome_server_simple_http_response(r, 200, "");
+  if (r->http.verb != HTTP_VERB_GET) {
+    http_request_simple_response(&r->http, 405, NULL);
     return 0;
   }
-  
-  struct http_response hr;
-  bzero(&hr, sizeof hr);
-  hr.result_code = 200;
-  hr.content_type = "application/binary";
-  hr.content_start = r->read_state.offset;
-  hr.content_end = r->read_state.length;
-  hr.content_length = r->read_state.length;
-  hr.body = NULL;
-  r->generator = rhizome_file_content;
-  rhizome_server_set_response(r, &hr);
+  if (r->http.request_header.content_range_count > 1) {
+    // To support byte range sets, eg, Range: bytes=0-100,200-300,400- we would have
+    // to reply with a multipart/byteranges MIME content.
+    http_request_simple_response(&r->http, 501, "Not Implemented: Byte range sets");
+    return 0;
+  }
+  rhizome_filehash_t filehash;
+  if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
+    return 1;
+  bzero(&r->read_state, sizeof r->read_state);
+  int n = rhizome_open_read(&r->read_state, &filehash);
+  if (n == -1) {
+    http_request_simple_response(&r->http, 500, NULL);
+    return 0;
+  }
+  if (n != 0)
+    return 1;
+  if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) {
+    rhizome_read_close(&r->read_state);
+    return 1;
+  }
+  assert(r->read_state.length != -1);
+  int result_code = 200;
+  struct http_range closed = (struct http_range){ .first = 0, .last = r->read_state.length };
+  if (r->http.request_header.content_range_count > 0) {
+    if (http_range_bytes(r->http.request_header.content_ranges,
+			 r->http.request_header.content_range_count,
+			 r->read_state.length
+			) == 0
+    ) {
+      http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
+      return 0;
+    }
+    result_code = 206; // Partial Content
+    http_range_close(&closed, &r->http.request_header.content_ranges[0], 1, r->read_state.length);
+  }
+  r->http.response.header.content_range_start = closed.first;
+  r->http.response.header.resource_length = closed.last;
+  r->http.response.header.content_length = closed.last - closed.first;
+  r->read_state.offset = closed.first;
+  r->http.response.content_generator = rhizome_file_content;
+  http_request_response(&r->http, result_code, "application/binary", NULL, 0);
   return 0;
 }
 
-static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder, const char *headers)
+static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder)
 {
   if (!is_rhizome_http_enabled())
     return 1;
+  if (r->http.verb != HTTP_VERB_GET) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
   rhizome_bid_t prefix;
   const char *endp = NULL;
   unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp);
@@ -505,27 +476,32 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde
     return 1; // not found
   rhizome_manifest *m = rhizome_new_manifest();
   int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m);
-  if (ret==0)
-    rhizome_server_http_response(r, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
+  if (ret == -1)
+    http_request_simple_response(&r->http, 500, NULL);
+  else if (ret == 0)
+    http_request_response(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
   rhizome_manifest_free(m);
-  return ret;
+  return ret <= 0 ? 0 : 1;
 }
 
-static int fav_icon_header(rhizome_http_request *r, const char *remainder, const char *headers)
+static int fav_icon_header(rhizome_http_request *r, const char *remainder)
 {
   if (*remainder)
     return 1;
-  rhizome_server_http_response(r, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
+  http_request_response(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
   return 0;
 }
 
-static int root_page(rhizome_http_request *r, const char *remainder, const char *headers)
+static int root_page(rhizome_http_request *r, const char *remainder)
 {
   if (*remainder)
     return 1;
-  
+  if (r->http.verb != HTTP_VERB_GET) {
+    http_request_simple_response(&r->http, 405, NULL);
+    return 0;
+  }
   char temp[8192];
-  strbuf b=strbuf_local(temp, sizeof(temp));
+  strbuf b = strbuf_local(temp, sizeof temp);
   strbuf_sprintf(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>"
 	   "<h1>Hello, I'm %s*</h1><br>"
 	   "Interfaces;<br>",
@@ -533,250 +509,19 @@ static int root_page(rhizome_http_request *r, const char *remainder, const char
   int i;
   for (i=0;i<OVERLAY_MAX_INTERFACES;i++){
     if (overlay_interfaces[i].state==INTERFACE_STATE_UP)
-      strbuf_sprintf(b, "<a href=\"/interface/%d\">%d: %s, TX: %d, RX: %d</a><br>", 
+      strbuf_sprintf(b, "<a href=\"/interface/%d\">%d: %s, TX: %d, RX: %d</a><br>",
 	i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count);
   }
-  
   strbuf_puts(b, "Neighbours;<br>");
   link_neighbour_short_status_html(b, "/neighbour");
-  
   if (is_rhizome_http_enabled()){
     strbuf_puts(b, "<a href=\"/rhizome/status\">Rhizome Status</a><br>");
   }
   strbuf_puts(b, "</body></html>");
-  if (strbuf_overrun(b))
-    return -1;
-  rhizome_server_simple_http_response(r, 200, temp);
+  if (strbuf_overrun(b)) {
+    WHY("HTTP Root page buffer overrun");
+    http_request_simple_response(&r->http, 500, NULL);
+  } else
+    http_request_simple_response(&r->http, 200, temp);
   return 0;
 }
-
-struct http_handler{
-  const char *path;
-  int (*parser)(rhizome_http_request *r, const char *remainder, const char *headers);
-};
-
-struct http_handler paths[]={
-  {"/rhizome/status", rhizome_status_page},
-  {"/rhizome/file/", rhizome_file_page},
-  {"/rhizome/manifestbyprefix/", manifest_by_prefix_page},
-  {"/interface/", interface_page},
-  {"/neighbour/", neighbour_page},
-  {"/favicon.ico", fav_icon_header},
-  {"/", root_page},
-};
-
-int rhizome_direct_parse_http_request(rhizome_http_request *r);
-int rhizome_server_parse_http_request(rhizome_http_request *r)
-{
-  // Start building up a response.
-  // Parse the HTTP "GET" line.
-  char *path = NULL;
-  char *headers = NULL;
-  if (str_startswith(r->request, "POST ", (const char **)&path)) {
-    return rhizome_direct_parse_http_request(r);
-  } else if (str_startswith(r->request, "GET ", (const char **)&path)) {
-    const char *p;
-    size_t header_length = 0;
-    size_t pathlen = 0;
-    // This loop is guaranteed to terminate before the end of the buffer, because we know that the
-    // buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block.
-    for (p = path; !isspace(*p); ++p)
-      ;
-    pathlen = p - path;
-    if ( str_startswith(p, " HTTP/1.", &p)
-      && (str_startswith(p, "0", &p) || str_startswith(p, "1", &p))
-      && (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers))
-    ){
-      path[pathlen] = '\0';
-      header_length = r->header_length - (headers - r->request);
-      headers[header_length] = '\0';
-    }else
-      path = NULL;
-  }
-  
-  if (!path) {
-    if (config.debug.rhizome_httpd)
-      DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length));
-    rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
-    return 0;
-  }
-  
-  char *id = NULL;
-  INFOF("RHIZOME HTTP SERVER, GET %s", path);
-  
-  int i;
-  r->generator=NULL;
-  
-  for (i=0;i<sizeof(paths)/sizeof(struct http_handler);i++){
-    if (str_startswith(path, paths[i].path, (const char **)&id)){
-      int ret=paths[i].parser(r, id, headers);
-      if (ret<0)
-	rhizome_server_simple_http_response(r, 500, "<html><h1>Internal Error</h1></html>\r\n");
-      if (ret>0)
-	rhizome_server_simple_http_response(r, 404, "<html><h1>Not Found</h1></html>\r\n");
-      
-      /* Try sending data immediately. */
-      rhizome_server_http_send_bytes(r);
-
-      return 0;
-    }
-  }
-  
-  rhizome_server_simple_http_response(r, 404, "<html><h1>Not Found</h1></html>\r\n");
-  return 0;
-}
-
-
-/* Return appropriate message for HTTP response codes, both known and unknown. */
-static const char *httpResultString(int response_code) {
-  switch (response_code) {
-  case 200: return "OK";
-  case 201: return "Created";
-  case 206: return "Partial Content";
-  case 404: return "Not found";
-  case 500: return "Internal server error";
-  default:  
-    if (response_code<=4)
-      return "Unknown status code";
-    else
-      return "A suffusion of yellow";
-  }
-}
-
-static strbuf strbuf_build_http_response(strbuf sb, const struct http_response *h)
-{
-  strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", h->result_code, httpResultString(h->result_code));
-  strbuf_sprintf(sb, "Content-type: %s\r\n", h->content_type);
-  if (h->content_end && h->content_length && (h->content_start!=0 || h->content_end!=h->content_length))
-    strbuf_sprintf(sb, 
-	  "Content-range: bytes %"PRIu64"-%"PRIu64"/%"PRIu64"\r\n"
-	  "Content-length: %"PRIu64"\r\n", 
-      h->content_start, h->content_end, h->content_length, h->content_end - h->content_start);
-  else if (h->content_length)
-    strbuf_sprintf(sb, "Content-length: %"PRIu64"\r\n", h->content_length);
-  strbuf_puts(sb, "\r\n");
-  return sb;
-}
-
-int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h)
-{
-  r->request_type=0;
-  
-  if (config.debug.rhizome_nohttptx)
-    unwatch(&r->alarm);
-  else{
-    /* Switching to writing, so update the call-back */
-    r->alarm.poll.events=POLLOUT;
-    watch(&r->alarm);
-  }  
-  
-  strbuf b = strbuf_local((char *) r->buffer, r->buffer_size);
-  strbuf_build_http_response(b, h);
-  if (r->buffer == NULL || strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length)) {
-    // Need a bigger buffer
-    if (r->buffer)
-      free(r->buffer);
-    r->buffer_size = strbuf_count(b) + 1;
-    if (h->body)
-      r->buffer_size += h->content_length;
-    r->buffer = malloc(r->buffer_size);
-    if (r->buffer == NULL) {
-      WHYF_perror("malloc(%u)", r->buffer_size);
-      r->buffer_size = 0;
-      return WHY("Cannot send response, out of memory");
-    }
-    strbuf_init(b, (char *) r->buffer, r->buffer_size);
-    strbuf_build_http_response(b, h);
-    if (strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length))
-      return WHYF("Bug! Cannot send response, buffer not big enough");
-  }
-  r->buffer_length = strbuf_len(b);
-  if (h->body){
-    bcopy(h->body, strbuf_end(b), h->content_length);
-    r->buffer_length+=h->content_length;
-  }
-  r->buffer_offset = 0;
-  if (config.debug.rhizome_httpd)
-    DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->buffer, r->buffer_length));
-  return 0;
-}
-
-int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response)
-{
-  struct http_response hr;
-  bzero(&hr, sizeof hr);
-  hr.result_code = result;
-  hr.content_type = "text/html";
-  hr.content_length = strlen(response);
-  hr.body = response;
-  if (result==400) {
-    DEBUGF("Rejecting http request as malformed due to: %s",
-	   response);
-  }
-  return rhizome_server_set_response(r, &hr);
-}
-
-int rhizome_server_http_response(rhizome_http_request *r, int result, 
-    const char *mime_type, const char *body, uint64_t bytes)
-{
-  struct http_response hr;
-  bzero(&hr, sizeof hr);
-  hr.result_code = result;
-  hr.content_type = mime_type;
-  hr.content_length = bytes;
-  hr.body = body;
-  return rhizome_server_set_response(r, &hr);
-}
-
-int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes)
-{
-  return rhizome_server_http_response(r, result, mime_type, NULL, bytes);
-}
-
-/*
-  return codes:
-  1: connection still open.
-  0: connection finished.
-  <0: an error occurred.
-*/
-int rhizome_server_http_send_bytes(rhizome_http_request *r)
-{
-  // Don't send anything if disabled for testing HTTP->MDP Rhizome failover
-  if (config.debug.rhizome_nohttptx)
-    return 1;
-
-  // write one block of buffered data
-  if(r->buffer_offset < r->buffer_length){
-    int bytes=r->buffer_length - r->buffer_offset;
-    bytes=write(r->alarm.poll.fd,&r->buffer[r->buffer_offset],bytes);
-    if (bytes<0){
-      // stop writing when the tcp buffer is full
-      // TODO errors?
-      return 1;
-    }
-    r->buffer_offset+=bytes;
-    
-    // reset inactivity timer
-    r->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT;
-    r->alarm.deadline = r->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
-    unschedule(&r->alarm);
-    schedule(&r->alarm);
-    
-    // allow other alarms to fire and wait for the next POLLOUT
-    return 1;
-  }
-  
-  r->buffer_offset=r->buffer_length=0;
-  
-  if (r->generator){
-    r->generator(r);
-  }
-  
-  // once we've written the whole buffer, and nothing new has been generated, close the connection
-  if (!r->buffer_length){
-    if (config.debug.rhizome_httpd)
-      DEBUG("Closing connection, done");
-    return rhizome_server_free_http_request(r);
-  }
-  return 1;
-}
diff --git a/rhizome_store.c b/rhizome_store.c
index 5e761852..b9d8fa51 100644
--- a/rhizome_store.c
+++ b/rhizome_store.c
@@ -1,3 +1,4 @@
+#include <assert.h>
 #include "serval.h"
 #include "rhizome.h"
 #include "conf.h"
@@ -71,7 +72,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex
       DEBUGF("Attempting to put blob for id='%"PRId64"' in %s", write->temp_id, blob_path);
     
     write->blob_fd=open(blob_path, O_CREAT | O_TRUNC | O_WRONLY, 0664);
-    if (write->blob_fd<0)
+    if (write->blob_fd == -1)
       goto insert_row_fail;
     
     if (config.debug.externalblobs)
@@ -111,7 +112,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex
   }
   
   if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1){
-    if (write->blob_fd>=0){
+    if (write->blob_fd != -1){
       if (config.debug.externalblobs)
          DEBUGF("Cancel write to fd %d", write->blob_fd);
       close(write->blob_fd);
@@ -162,7 +163,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer
 
 // open database locks
 static int write_get_lock(struct rhizome_write *write_state){
-  if (write_state->blob_fd>=0 || write_state->sql_blob)
+  if (write_state->blob_fd != -1 || write_state->sql_blob)
     return 0;
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   
@@ -193,7 +194,7 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
   if (file_offset != write_state->written_offset)
     WARNF("Writing file data out of order! [%"PRId64",%"PRId64"]", file_offset, write_state->written_offset);
     
-  if (write_state->blob_fd>=0) {
+  if (write_state->blob_fd != -1) {
     int ofs=0;
     // keep trying until all of the data is written.
     lseek(write_state->blob_fd, file_offset, SEEK_SET);
@@ -233,7 +234,7 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
 // close database locks
 static int write_release_lock(struct rhizome_write *write_state){
   int ret=0;
-  if (write_state->blob_fd>=0)
+  if (write_state->blob_fd != -1)
     return 0;
     
   if (write_state->sql_blob){
@@ -260,7 +261,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
   int ret=0;
   int should_write = 0;
   // if we are writing to a file, or already have the sql blob open, write as much as we can.
-  if (write_state->blob_fd>=0 || write_state->sql_blob){
+  if (write_state->blob_fd != -1 || write_state->sql_blob){
     should_write = 1;
   }else{
     // cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go.
@@ -417,7 +418,7 @@ end:
 
 int rhizome_fail_write(struct rhizome_write *write)
 {
-  if (write->blob_fd>=0){
+  if (write->blob_fd != -1){
     if (config.debug.externalblobs)
       DEBUGF("Closing and removing fd %d", write->blob_fd);
     close(write->blob_fd);
@@ -723,6 +724,58 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp
   return 0; // file opened
 }
 
+static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read *read_state, unsigned char *buffer, size_t bufsz)
+{
+  IN();
+  if (read_state->blob_fd != -1) {
+    if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1)
+      RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset));
+    if (bufsz == 0)
+      RETURN(0);
+    ssize_t rd = read(read_state->blob_fd, buffer, bufsz);
+    if (rd == -1)
+      RETURN(WHYF_perror("read(%d,%p,%zu)", read_state->blob_fd, buffer, bufsz));
+    if (config.debug.externalblobs)
+      DEBUGF("Read %zu bytes from fd=%d @%"PRIx64, (size_t) rd, read_state->blob_fd, read_state->offset);
+    RETURN(rd);
+  }
+  if (read_state->blob_rowid == -1)
+    RETURN(WHY("file not open"));
+  sqlite3_blob *blob = NULL;
+  int ret;
+  do {
+    assert(blob == NULL);
+    ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob);
+  } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_open"));
+  if (ret != SQLITE_OK) {
+    assert(blob == NULL);
+    RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db)));
+  }
+  assert(blob != NULL);
+  if (read_state->length == -1)
+    read_state->length = sqlite3_blob_bytes(blob);
+  // A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out
+  // the length.
+  size_t bytes_read = 0;
+  if (buffer && bufsz && read_state->offset < read_state->length) {
+    bytes_read = read_state->length - read_state->offset;
+    if (bytes_read > bufsz)
+      bytes_read = bufsz;
+    assert(bytes_read > 0);
+    do {
+      ret = sqlite3_blob_read(blob, buffer, (int) bytes_read, read_state->offset);
+    } while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_read"));
+    if (ret != SQLITE_OK) {
+      WHYF("sqlite3_blob_read() failed: %s", sqlite3_errmsg(rhizome_db));
+      sqlite3_blob_close(blob);
+      RETURN(-1);
+    }
+  }
+  sqlite3_blob_close(blob);
+  RETURN(bytes_read);
+  OUT();
+}
+
 /* Read content from the store, hashing and decrypting as we go. 
  Random access is supported, but hashing requires all payload contents to be read sequentially. */
 // returns the number of bytes read
@@ -732,53 +785,13 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
   // hash check failed, just return an error
   if (read_state->invalid)
     RETURN(-1);
-    
-  int bytes_read = 0;
-  if (read_state->blob_fd >= 0) {
-    if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1)
-      RETURN(WHYF_perror("lseek(%d,%ld,SEEK_SET)", read_state->blob_fd, (long)read_state->offset));
-    bytes_read = read(read_state->blob_fd, buffer, buffer_length);
-    if (bytes_read == -1)
-      RETURN(WHYF_perror("read(%d,%p,%ld)", read_state->blob_fd, buffer, (long)buffer_length));
-    if (config.debug.externalblobs)
-      DEBUGF("Read %d bytes from fd %d @%"PRIx64, bytes_read, read_state->blob_fd, read_state->offset);
-  } else if (read_state->blob_rowid != -1) {
-    sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
-    do{
-      sqlite3_blob *blob = NULL;
-      int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob);
-      if (sqlite_code_busy(ret))
-	goto again;
-      else if(ret!=SQLITE_OK)
-	RETURN(WHYF("sqlite3_blob_open failed: %s",sqlite3_errmsg(rhizome_db)));
-      if (read_state->length==-1)
-	read_state->length=sqlite3_blob_bytes(blob);
-      bytes_read = read_state->length - read_state->offset;
-      if (bytes_read>buffer_length)
-	bytes_read=buffer_length;
-      // allow the caller to do a dummy read, just to work out the length
-      if (!buffer)
-	bytes_read=0;
-      if (bytes_read>0){
-	ret = sqlite3_blob_read(blob, buffer, bytes_read, read_state->offset);
-	if (sqlite_code_busy(ret))
-	  goto again;
-	else if(ret!=SQLITE_OK){
-	  WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db));
-	  sqlite3_blob_close(blob);
-	  RETURN(-1);
-	}
-      }
-      sqlite3_blob_close(blob);
-      break;
-    again:
-      if (blob) sqlite3_blob_close(blob);
-      if (!sqlite_retry(&retry, "sqlite3_blob_open"))
-	RETURN(-1);
-    } while (1);
-  } else
-    RETURN(WHY("file not open"));
-  
+
+  sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
+  ssize_t n = rhizome_read_retry(&retry, read_state, buffer, buffer_length);
+  if (n == -1)
+    RETURN(-1);
+  size_t bytes_read = (size_t) n;
+
   // hash the payload as we go, but only if we happen to read the payload data in order
   if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){
     SHA512_Update(&read_state->sha512_context, buffer, bytes_read);
diff --git a/rhizome_sync.c b/rhizome_sync.c
index ae6db73f..415b0944 100644
--- a/rhizome_sync.c
+++ b/rhizome_sync.c
@@ -439,7 +439,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
 
   if (count){
     mdp.out.payload_length = ob_position(b);
-    if (config.debug.rhizome)
+    if (config.debug.rhizome_ads)
       DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last);
     overlay_mdp_dispatch(&mdp,0,NULL,0);
   }
@@ -449,7 +449,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
 
 int rhizome_sync_announce()
 {
+  int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);
   sync_send_response(NULL, 0, HEAD_FLAG, 5);
+  sqlite_set_tracefunc(oldfunc);
   return 0;
 }
 
diff --git a/serval.h b/serval.h
index c990ebc8..8f23d4b4 100644
--- a/serval.h
+++ b/serval.h
@@ -846,7 +846,6 @@ int overlay_queue_init();
 
 void monitor_client_poll(struct sched_ent *alarm);
 void monitor_poll(struct sched_ent *alarm);
-void rhizome_client_poll(struct sched_ent *alarm);
 void rhizome_fetch_poll(struct sched_ent *alarm);
 void rhizome_server_poll(struct sched_ent *alarm);
 
diff --git a/sourcefiles.mk b/sourcefiles.mk
index 00ee579b..1b4516ac 100644
--- a/sourcefiles.mk
+++ b/sourcefiles.mk
@@ -14,6 +14,7 @@ SERVAL_SOURCES = \
 	$(SERVAL_BASE)fdqueue.c \
 	$(SERVAL_BASE)fifo.c \
 	$(SERVAL_BASE)golay.c \
+	$(SERVAL_BASE)http_server.c \
 	$(SERVAL_BASE)keyring.c \
 	$(SERVAL_BASE)log.c \
 	$(SERVAL_BASE)lsif.c \
diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index dd63d473..8ec48516 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <time.h>
 #include <assert.h>
 #include <stdarg.h>
+#include <inttypes.h>
 #include <sys/wait.h>
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
@@ -32,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <arpa/inet.h>
 #endif
 #include <sys/uio.h>
+#include "http_server.h"
 
 static inline strbuf _toprint(strbuf sb, char c)
 {
@@ -391,3 +393,28 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt)
   strbuf_putc(sb, ']');
   return sb;
 }
+
+strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels)
+{
+  unsigned i;
+  int first = 1;
+  for (i = 0; i != nels; ++i) {
+    const struct http_range *r = &ranges[i];
+    switch (r->type) {
+      case NIL: break;
+      case CLOSED:
+	strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-%"PRIhttp_size_t, first ? "" : ",", r->first, r->last);
+	first = 0;
+	break;
+      case OPEN:
+	strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-", first ? "" : ",", r->first);
+	first = 0;
+	break;
+      case SUFFIX:
+	strbuf_sprintf(sb, "%s-%"PRIhttp_size_t, first ? "" : ",", r->last);
+	first = 0;
+	break;
+    }
+  }
+  return sb;
+}
diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index ada181ae..cb545983 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -136,4 +136,11 @@ struct iovec;
 strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt);
 #define alloca_iovec(iov,cnt)    strbuf_str(strbuf_append_iovec(strbuf_alloca(200), (iov), (cnt)))
 
+/* Append a representation of a struct http_range[] array.
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+struct http_range;
+strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels);
+#define alloca_http_ranges(ra)    strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra)))
+
 #endif //__STRBUF_HELPERS_H__

From e82d22d765299b500be2360823e781d15580d416 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 17 Oct 2013 00:01:32 +1030
Subject: [PATCH 10/24] Add alloca_poll_events() macro

---
 strbuf_helpers.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index cb545983..3d3016c8 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -67,6 +67,7 @@ strbuf strbuf_path_join(strbuf sb, ...);
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 strbuf strbuf_append_poll_events(strbuf sb, short events);
+#define alloca_poll_events(ev)    strbuf_str(strbuf_append_poll_events(strbuf_alloca(200), (ev)))
 
 /* Append a nul-terminated string as a single-quoted shell word which, if
  * expanded in a shell command line, would evaluate to the original string.

From 05d4215752190555f70368c99ea0ffee1e030da1 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 17 Oct 2013 23:15:25 +1030
Subject: [PATCH 11/24] Add 'strlen' argument to strn_fromprint()

---
 commandline.c | 2 +-
 keyring.c     | 4 ++--
 str.c         | 7 ++++---
 str.h         | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/commandline.c b/commandline.c
index 74ed1da3..4f4db26c 100644
--- a/commandline.c
+++ b/commandline.c
@@ -483,7 +483,7 @@ int app_echo(const struct cli_parsed *parsed, struct cli_context *context)
       DEBUGF("echo:argv[%d]=\"%s\"", i, arg);
     if (escapes) {
       unsigned char buf[strlen(arg)];
-      size_t len = strn_fromprint(buf, sizeof buf, arg, '\0', NULL);
+      size_t len = strn_fromprint(buf, sizeof buf, arg, 0, '\0', NULL);
       cli_write(context, buf, len);
     } else
       cli_puts(context, arg);
diff --git a/keyring.c b/keyring.c
index f9ce8821..f9c60a6f 100644
--- a/keyring.c
+++ b/keyring.c
@@ -675,7 +675,7 @@ static int load_did_name(keypair *kp, const char *text)
 	return WHY("duplicate DID");
       const char *e = NULL;
       bzero(kp->private_key, kp->private_key_len);
-      strn_fromprint(kp->private_key, kp->private_key_len, t, '"', &e);
+      strn_fromprint(kp->private_key, kp->private_key_len, t, 0, '"', &e);
       if (*e != '"')
 	return WHY("malformed DID quoted string");
       t = e + 1;
@@ -685,7 +685,7 @@ static int load_did_name(keypair *kp, const char *text)
 	return WHY("duplicate Name");
       const char *e = NULL;
       bzero(kp->public_key, kp->public_key_len);
-      strn_fromprint(kp->public_key, kp->public_key_len, t, '"', &e);
+      strn_fromprint(kp->public_key, kp->public_key_len, t, 0, '"', &e);
       if (*e != '"')
 	return WHY("malformed Name quoted string");
       t = e + 1;
diff --git a/str.c b/str.c
index 11a69c38..11d8d906 100644
--- a/str.c
+++ b/str.c
@@ -446,11 +446,12 @@ size_t toprint_str_len(const char *srcStr, const char quotes[2])
   return srcStr ? strbuf_count(strbuf_toprint_quoted(strbuf_local(NULL, 0), quotes, srcStr)) : 4;
 }
 
-size_t strn_fromprint(unsigned char *dst, size_t dstlen, const char *src, char endquote, const char **afterp)
+size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp)
 {
   unsigned char *const odst = dst;
-  unsigned char *const edst = dst + dstlen;
-  while (*src && *src != endquote && dst < edst) {
+  unsigned char *const edst = dst + dstsiz;
+  const char *const esrc = srclen ? src + srclen : NULL;
+  while (src < esrc && *src && *src != endquote && dst < edst) {
     switch (*src) {
     case '\\':
       ++src;
diff --git a/str.h b/str.h
index 276cee29..a2ece4eb 100644
--- a/str.h
+++ b/str.h
@@ -98,7 +98,7 @@ char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcByt
 char *toprint_str(char *dstStr, ssize_t dstBufSiz, const char *srcStr, const char quotes[2]);
 size_t toprint_len(const char *srcBuf, size_t srcBytes, const char quotes[2]);
 size_t toprint_str_len(const char *srcStr, const char quotes[2]);
-size_t strn_fromprint(unsigned char *dst, size_t dstlen, const char *src, char endquote, const char **afterp);
+size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp);
 
 #define alloca_toprint(dstlen,buf,len)  toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), "``") + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), "``")
 #define alloca_str_toprint_quoted(str, quotes)  toprint_str((char *)alloca(toprint_str_len((str), (quotes)) + 1), -1, (str), (quotes))

From 48921802f54f100d46336e85ca5e6712dea43ba7 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 17 Oct 2013 23:17:41 +1030
Subject: [PATCH 12/24] Use size_t in Rhizome store functions

---
 meshms.c         | 54 ++++++++++++++++++++++-----------------------
 rhizome.h        | 32 +++++++++++++--------------
 rhizome_crypto.c | 16 +++++++-------
 rhizome_direct.c |  6 ++---
 rhizome_store.c  | 57 ++++++++++++++++++++++++++----------------------
 5 files changed, 85 insertions(+), 80 deletions(-)

diff --git a/meshms.c b/meshms.c
index 6b5f1d8f..1f51ae26 100644
--- a/meshms.c
+++ b/meshms.c
@@ -49,7 +49,7 @@ struct ply_read{
   // details of the current record
   uint64_t record_end_offset;
   uint16_t record_length;
-  int buffer_size;
+  size_t buffer_size;
   char type;
   // raw record data
   unsigned char *buffer;
@@ -78,8 +78,7 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m)
 	alloca_tohex(keyring->contexts[cn]->identities[in]
 	->keypairs[kp]->private_key, crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES));
 	
-  int ret = rhizome_get_bundle_from_seed(m, seed);
-  if (ret<0)
+  if (rhizome_get_bundle_from_seed(m, seed) == -1)
     return -1;
   
   // always consider the content encrypted, we don't need to rely on the manifest itself.
@@ -225,9 +224,9 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome
   if (rhizome_retrieve_manifest(bid, m))
     return -1;
   int ret = rhizome_open_decrypt_read(m, NULL, &ply->read);
-  if (ret>0)
+  if (ret == 1)
     WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
-  if (ret)
+  if (ret != 0)
     return ret;
   ply->read.offset = ply->read.length = m->fileLength;
   return 0;
@@ -399,11 +398,11 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){
       goto end;
       
     ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_ACK);
-    if (ret<0)
+    if (ret == -1)
       goto end;
       
     if (ret==0){
-      if (unpack_uint(ply.buffer, ply.record_length, &previous_ack)<0)
+      if (unpack_uint(ply.buffer, ply.record_length, &previous_ack) == -1)
 	previous_ack=0;
     }
     if (config.debug.meshms)
@@ -478,7 +477,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
   bzero(&buff, sizeof(buff));
   
   int ret = rhizome_open_decrypt_read(m, NULL, &read);
-  if (ret<0)
+  if (ret == -1)
     goto end;
   
   unsigned char version=0xFF;
@@ -527,8 +526,9 @@ end:
   return 0;
 }
 
-static int write_conversation(struct rhizome_write *write, struct conversations *conv){
-  int len=0;
+static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv)
+{
+  size_t len=0;
   if (!conv)
     return len;
   {
@@ -541,14 +541,14 @@ static int write_conversation(struct rhizome_write *write, struct conversations
       len+=pack_uint(&buffer[len], conv->read_offset);
       len+=pack_uint(&buffer[len], conv->their_size);
       int ret=rhizome_write_buffer(write, buffer, len);
-      if (ret<0)
+      if (ret == -1)
 	return ret;
     }else{
       len+=measure_packed_uint(conv->their_last_message);
       len+=measure_packed_uint(conv->read_offset);
       len+=measure_packed_uint(conv->their_size);
     }
-    DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %d", 
+    DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %zu", 
       alloca_tohex_sid_t(conv->them),
       conv->their_last_message,
       conv->read_offset,
@@ -556,14 +556,14 @@ static int write_conversation(struct rhizome_write *write, struct conversations
       len);
   }
   // write the two child nodes
-  int ret=write_conversation(write, conv->_left);
-  if (ret<0)
+  ssize_t ret = write_conversation(write, conv->_left);
+  if (ret == -1)
     return ret;
-  len+=ret;
-  ret=write_conversation(write, conv->_right);
-  if (ret<0)
+  len += (size_t) ret;
+  ret = write_conversation(write, conv->_right);
+  if (ret == -1)
     return ret;
-  len+=ret;
+  len += (size_t) ret;
   return len;
 }
 
@@ -578,22 +578,22 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
   // TODO rebalance tree...
   
   // measure the final payload first
-  int len=write_conversation(NULL, conv);
-  if (len<0)
+  ssize_t len=write_conversation(NULL, conv);
+  if (len == -1)
     goto end;
   
   // then write it
   m->version++;
   rhizome_manifest_set_ll(m,"version",m->version);
-  m->fileLength = len+1;
+  m->fileLength = (size_t) len + 1;
   rhizome_manifest_set_ll(m,"filesize",m->fileLength);
   
-  if (rhizome_write_open_manifest(&write, m))
+  if (rhizome_write_open_manifest(&write, m) == -1)
     goto end;
   unsigned char version=1;
-  if (rhizome_write_buffer(&write, &version, 1)<0)
+  if (rhizome_write_buffer(&write, &version, 1) == -1)
     goto end;
-  if (write_conversation(&write, conv)<0)
+  if (write_conversation(&write, conv) == -1)
     goto end;
   if (rhizome_finish_write(&write))
     goto end;
@@ -795,7 +795,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
     // find their last ACK so we know if messages have been received
     int r = ply_find_next(&read_theirs, MESHMS_BLOCK_TYPE_ACK);
     if (r==0){
-      if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack)<0)
+      if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack) == -1)
 	their_last_ack=0;
       else
 	their_ack_offset = read_theirs.record_end_offset;
@@ -822,11 +822,11 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
 	// read their message list, and insert all messages that are included in the ack range
 	if (conv->found_their_ply){
 	  int ofs=unpack_uint(read_ours.buffer, read_ours.record_length, (uint64_t*)&read_theirs.read.offset);
-	  if (ofs<0)
+	  if (ofs == -1)
 	    break;
 	  uint64_t end_range;
 	  int x = unpack_uint(read_ours.buffer+ofs, read_ours.record_length - ofs, &end_range);
-	  if (x<0)
+	  if (x == -1)
 	    end_range=0;
 	  else
 	    end_range = read_theirs.read.offset - end_range;
diff --git a/rhizome.h b/rhizome.h
index 54201f44..4188327d 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -472,8 +472,8 @@ struct rhizome_write_buffer
 {
   struct rhizome_write_buffer *_next;
   int64_t offset;
-  int buffer_size;
-  int data_size;
+  size_t buffer_size;
+  size_t data_size;
   unsigned char data[0];
 };
 
@@ -488,7 +488,7 @@ struct rhizome_write
   int64_t written_offset;
   int64_t file_length;
   struct rhizome_write_buffer *buffer_list;
-  int buffer_size;
+  size_t buffer_size;
   
   int crypt;
   unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
@@ -503,7 +503,7 @@ struct rhizome_write
 struct rhizome_read_buffer{
   uint64_t offset;
   unsigned char data[RHIZOME_CRYPT_PAGE_SIZE];
-  int len;
+  size_t len;
 };
 
 struct rhizome_read
@@ -600,12 +600,12 @@ typedef struct rhizome_direct_bundle_cursor {
   rhizome_bid_t bid_low;
   rhizome_bid_t bid_high;
   unsigned char *buffer;
-  int buffer_size;
-  int buffer_used;
-  int buffer_offset_bytes;
+  size_t buffer_size;
+  size_t buffer_used;
+  size_t buffer_offset_bytes;
 } rhizome_direct_bundle_cursor;
 
-rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size);
+rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size);
 void rhizome_direct_bundle_iterator_unlimit(rhizome_direct_bundle_cursor *r);
 int rhizome_direct_bundle_iterator_pickle_range(rhizome_direct_bundle_cursor *r,
 						unsigned char *pickled,
@@ -669,7 +669,7 @@ rhizome_direct_sync_request
 *rhizome_direct_new_sync_request(
 				 void (*transport_specific_dispatch_function)
 				 (struct rhizome_direct_sync_request *),
-				 int buffer_size,int interval, int mode, 
+				 size_t buffer_size, int interval, int mode, 
 				 void *transport_specific_state);
 int rhizome_direct_continue_sync_request(rhizome_direct_sync_request *r);
 int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r);
@@ -720,14 +720,14 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
 
 int rhizome_exists(const rhizome_filehash_t *hashp);
 int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority);
-int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size);
-int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size);
+int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
+int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size);
 int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
 int rhizome_write_file(struct rhizome_write *write, const char *filename);
 int rhizome_fail_write(struct rhizome_write *write);
 int rhizome_finish_write(struct rhizome_write *write);
 int rhizome_import_file(rhizome_manifest *m, const char *filepath);
-int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length);
+int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length);
 int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
 int rhizome_add_file(rhizome_manifest *m, const char *filepath);
 int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
@@ -737,17 +737,17 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64
 int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
 int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
 
-int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, 
+int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, 
 			    const unsigned char *key, const unsigned char *nonce);
 int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
-int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
-int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len);
+int rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
+int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
 int rhizome_read_close(struct rhizome_read *read);
 int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
 int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
 int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length);
 int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout, 
-  uint64_t fileOffset, unsigned char *buffer, int length);
+  uint64_t fileOffset, unsigned char *buffer, size_t length);
 int rhizome_cache_close();
 
 int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);
diff --git a/rhizome_crypto.c b/rhizome_crypto.c
index 7d9dd7e0..3cbb2a32 100644
--- a/rhizome_crypto.c
+++ b/rhizome_crypto.c
@@ -72,13 +72,13 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
     return -1;
   
   int ret=rhizome_retrieve_manifest(&key.Public, m);
-  if (ret<0)
+  if (ret == -1)
     return -1;
   
   m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID;
   m->cryptoSignPublic = key.Public;
   bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret);
-  if (ret>0)
+  if (ret == 1)
     rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
   return ret;
 }
@@ -590,22 +590,22 @@ static void add_nonce(unsigned char *nonce, int64_t value){
 /* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries
  * for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE)
  */
-int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset, 
+int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset, 
 			    const unsigned char *key, const unsigned char *nonce){
   
   if (stream_offset<0)
     return WHY("Invalid stream offset");
   
   int64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
-  int offset=0;
+  size_t offset=0;
   
   unsigned char block_nonce[crypto_stream_xsalsa20_NONCEBYTES];
   bcopy(nonce, block_nonce, sizeof(block_nonce));
   add_nonce(block_nonce, nonce_offset);
   
   if (nonce_offset < stream_offset){
-    int padding = stream_offset & (RHIZOME_CRYPT_PAGE_SIZE -1);
-    int size = RHIZOME_CRYPT_PAGE_SIZE - padding;
+    size_t padding = stream_offset & (RHIZOME_CRYPT_PAGE_SIZE -1);
+    size_t size = RHIZOME_CRYPT_PAGE_SIZE - padding;
     if (size>buffer_size)
       size=buffer_size;
     
@@ -619,11 +619,11 @@ int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stre
   }
   
   while(offset < buffer_size){
-    int size = buffer_size - offset;
+    size_t size = buffer_size - offset;
     if (size>RHIZOME_CRYPT_PAGE_SIZE)
       size=RHIZOME_CRYPT_PAGE_SIZE;
     
-    crypto_stream_xsalsa20_xor(buffer+offset, buffer+offset, size, block_nonce, key);
+    crypto_stream_xsalsa20_xor(buffer+offset, buffer+offset, (unsigned long long) size, block_nonce, key);
     
     add_nonce(block_nonce, RHIZOME_CRYPT_PAGE_SIZE);
     offset+=size;
diff --git a/rhizome_direct.c b/rhizome_direct.c
index b4ce9631..02d91833 100644
--- a/rhizome_direct.c
+++ b/rhizome_direct.c
@@ -135,7 +135,7 @@ rhizome_direct_sync_request
 *rhizome_direct_new_sync_request(
 				 void (*transport_specific_dispatch_function)
 				 (struct rhizome_direct_sync_request *),
-				 int buffer_size,int interval, int mode, void *state)
+				 size_t buffer_size, int interval, int mode, void *state)
 {
   assert(mode&3);
 
@@ -287,7 +287,7 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response(unsigned char *bu
     }
   DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64,
 	 c->size_high,c->limit_size_high);
-  DEBUGF("c->buffer_size=%d",c->buffer_size);
+  DEBUGF("c->buffer_size=%zu",c->buffer_size);
 
   /* Get our list of BARs for the same cursor range */
   int us_count=rhizome_direct_bundle_iterator_fill(c,-1);
@@ -533,7 +533,7 @@ int app_rhizome_direct_sync(const struct cli_parsed *parsed, struct cli_context
   }
 }
 
-rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size)
+rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size)
 {
   rhizome_direct_bundle_cursor *r=calloc(sizeof(rhizome_direct_bundle_cursor),1);
   assert(r!=NULL);
diff --git a/rhizome_store.c b/rhizome_store.c
index b9d8fa51..bbf6a51c 100644
--- a/rhizome_store.c
+++ b/rhizome_store.c
@@ -187,7 +187,8 @@ static int write_get_lock(struct rhizome_write *write_state){
 }
 
 // write data to disk
-static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, int data_size){
+static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, size_t data_size)
+{
   if (data_size<=0)
     return 0;
   
@@ -253,7 +254,8 @@ static int write_release_lock(struct rhizome_write *write_state){
 
 // Write data buffers in any order, the data will be cached and streamed into the database in file order. 
 // Though there is an upper bound on the amount of cached data
-int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size){
+int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size)
+{
   if (offset + data_size > write_state->file_length)
     data_size = write_state->file_length - offset;
   
@@ -266,7 +268,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
   }else{
     // cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go.
     // (Not perfect if the range overlaps)
-    int64_t new_size = write_state->written_offset + write_state->buffer_size + data_size;
+    uint64_t new_size = write_state->written_offset + write_state->buffer_size + data_size;
     if (new_size>=write_state->file_length || new_size>=RHIZOME_BUFFER_MAXIMUM_SIZE)
       should_write = 1;
   }
@@ -325,7 +327,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
     
     if (!*ptr || offset < (*ptr)->offset){
       // found the insert position in the list
-      int64_t size = data_size;
+      size_t size = data_size;
       
       // allow for buffers to overlap, we may need to split the incoming buffer into multiple pieces.
       if (*ptr && offset+size > (*ptr)->offset)
@@ -349,7 +351,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
 	  break;
 	  
 	if (config.debug.rhizome)
-	  DEBUGF("Caching block @%"PRId64", %"PRId64, offset, size);
+	  DEBUGF("Caching block @%"PRId64", %zu", offset, size);
 	struct rhizome_write_buffer *i = emalloc(size + sizeof(struct rhizome_write_buffer));
 	if (!i){
 	  ret=-1;
@@ -378,7 +380,8 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
   return ret;
 }
 
-int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size){
+int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size)
+{
   return rhizome_random_write(write_state, write_state->file_offset, buffer, data_size);
 }
 
@@ -399,8 +402,8 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){
     if (write->file_offset + size > write->file_length)
       size=write->file_length - write->file_offset;
     
-    int r = fread(buffer, 1, size, f);
-    if (r==-1){
+    size_t r = fread(buffer, 1, size, f);
+    if (ferror(f)){
       ret = WHY_perror("fread");
       goto end;
     }
@@ -573,12 +576,12 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
 }
 
 // store a whole payload from a single buffer
-int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length)
+int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length)
 {
   if (m->fileLength<=0)
     return 0;
   if (length!=m->fileLength)
-    return WHYF("Expected %"PRId64" bytes, got %d", m->fileLength, length);
+    return WHYF("Expected %"PRId64" bytes, got %zu", m->fileLength, length);
   
   /* Import the file first, checking the hash as we go */
   struct rhizome_write write;
@@ -779,7 +782,7 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read
 /* Read content from the store, hashing and decrypting as we go. 
  Random access is supported, but hashing requires all payload contents to be read sequentially. */
 // returns the number of bytes read
-int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buffer_length)
+ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, size_t buffer_length)
 {
   IN();
   // hash check failed, just return an error
@@ -823,9 +826,9 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
 }
 
 /* Read len bytes from read->offset into data, using *buffer to cache any reads */
-int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len)
+int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len)
 {
-  int bytes_copied=0;
+  size_t bytes_copied=0;
   
   while (len>0){
     // make sure we only attempt to read data that actually exists
@@ -835,7 +838,7 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer
     // if we can supply either the beginning or end of the data from cache, do that first.
     uint64_t ofs=read->offset - buffer->offset;
     if (ofs>=0 && ofs<=buffer->len){
-      int size=len;
+      size_t size=len;
       if (size > buffer->len - ofs)
 	size = buffer->len - ofs;
       if (size>0){
@@ -851,7 +854,7 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer
     
     ofs = (read->offset+len) - buffer->offset;
     if (ofs>0 && ofs<=buffer->len){
-      int size=len;
+      size_t size=len;
       if (size > ofs)
 	size = ofs;
       if (size>0){
@@ -867,10 +870,12 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer
     // remember the requested read offset so we can put it back
     ofs = read->offset;
     buffer->offset = read->offset = ofs & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
-    buffer->len = rhizome_read(read, buffer->data, sizeof(buffer->data));
+    ssize_t len = rhizome_read(read, buffer->data, sizeof(buffer->data));
     read->offset = ofs;
-    if (buffer->len<=0)
-      return buffer->len;
+    buffer->len = 0;
+    if (len == -1)
+      return -1;
+    buffer->len = (size_t) len;
   }
   return bytes_copied;
 }
@@ -991,7 +996,7 @@ int rhizome_cache_count()
 }
 
 // read a block of data, caching meta data for reuse
-int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length)
+int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, size_t length)
 {
   // look for a cached entry
   struct cache_entry **ptr = find_entry_location(&root, bidp, version);
@@ -1133,17 +1138,17 @@ static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write,
 
   unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
   while(length>0){
-    int size=sizeof(buffer);
+    size_t size=sizeof(buffer);
     if (size > length)
       size=length;
 
-    int r = rhizome_read(read, buffer, size);
-    if (r<0)
+    ssize_t r = rhizome_read(read, buffer, size);
+    if (r == -1)
       return r;
 
-    length -= r;
+    length -= (size_t) r;
     
-    if (rhizome_write_buffer(write, buffer, r))
+    if (rhizome_write_buffer(write, buffer, (size_t) r))
       return -1;
   }
 
@@ -1207,12 +1212,12 @@ failure:
   return ret;
 }
 
-int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len)
+int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len)
 {
   struct rhizome_write write;
   bzero(&write, sizeof write);
 
-  int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, len);
+  int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, (uint64_t) len);
   if (ret)
     return -1;
 

From 291a631095559ebc1eb483b144bf6afdb5272111 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 18 Oct 2013 10:34:43 +1030
Subject: [PATCH 13/24] New header file "fdqueue.h"

So that "http_server.h" does not have to include "serval.h" which
creates a circular dependency.

Remove the __SERVALDNA__HTTP_SERVER_IMPLEMENTATION hack from
"http_server.h"
---
 directory_service.c  |  6 ++-
 fakeradio.c          |  2 +
 fdqueue.c            |  3 +-
 fdqueue.h            | 92 ++++++++++++++++++++++++++++++++++++++++++++
 headerfiles.mk       |  1 +
 http_server.c        |  1 -
 http_server.h        |  5 +--
 monitor-cli.c        |  2 +
 performance_timing.c |  2 +-
 rhizome_http.c       |  1 -
 serval.h             | 71 ++--------------------------------
 strbuf_helpers.c     |  6 ++-
 vomp_console.c       |  4 +-
 13 files changed, 114 insertions(+), 82 deletions(-)
 create mode 100644 fdqueue.h

diff --git a/directory_service.c b/directory_service.c
index c3727f87..6f09e32e 100644
--- a/directory_service.c
+++ b/directory_service.c
@@ -1,9 +1,11 @@
 
-#include "constants.h"
-#include "mdp_client.h"
+#ifdef HAVE_POLL_H
 #include <poll.h>
+#endif
 #include <stdio.h>
 #include <unistd.h>
+#include "constants.h"
+#include "mdp_client.h"
 #include "str.h"
 
 struct item{
diff --git a/fakeradio.c b/fakeradio.c
index b310fbe3..5727ad1b 100644
--- a/fakeradio.c
+++ b/fakeradio.c
@@ -4,7 +4,9 @@
 #include <stdlib.h>
 #include <stdint.h>
 #include <fcntl.h>
+#ifdef HAVE_POLL_H
 #include <poll.h>
+#endif
 #include <errno.h>
 #include <time.h>
 #include <sys/time.h>
diff --git a/fdqueue.c b/fdqueue.c
index 1ef8aea4..83b65b3e 100644
--- a/fdqueue.c
+++ b/fdqueue.c
@@ -17,8 +17,7 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
-#include <poll.h>
-#include "serval.h"
+#include "fdqueue.h"
 #include "conf.h"
 #include "str.h"
 #include "strbuf.h"
diff --git a/fdqueue.h b/fdqueue.h
new file mode 100644
index 00000000..11596cf3
--- /dev/null
+++ b/fdqueue.h
@@ -0,0 +1,92 @@
+/* 
+Serval DNA file descriptor queue
+Copyright (C) 2012-2013 Serval Project, Inc.
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+*/
+
+#ifndef __SERVALDNA__FDQUEUE_H
+#define __SERVALDNA__FDQUEUE_H
+
+#ifdef HAVE_POLL_H
+#include <poll.h>
+#endif
+#include "os.h"
+#include "log.h"
+
+struct profile_total {
+  struct profile_total *_next;
+  int _initialised;
+  const char *name;
+  time_ms_t max_time;
+  time_ms_t total_time;
+  time_ms_t child_time;
+  int calls;
+};
+
+struct call_stats{
+  time_ms_t enter_time;
+  time_ms_t child_time;
+  struct profile_total *totals;
+  struct call_stats *prev;
+};
+
+struct sched_ent;
+
+typedef void (*ALARM_FUNCP) (struct sched_ent *alarm);
+
+struct sched_ent{
+  struct sched_ent *_next;
+  struct sched_ent *_prev;
+  
+  ALARM_FUNCP function;
+  void *context;
+  struct pollfd poll;
+  // when we should first consider the alarm
+  time_ms_t alarm;
+  // the order we will prioritise the alarm
+  time_ms_t deadline;
+  struct profile_total *stats;
+  int _poll_index;
+};
+
+int is_scheduled(const struct sched_ent *alarm);
+int _schedule(struct __sourceloc, struct sched_ent *alarm);
+int _unschedule(struct __sourceloc, struct sched_ent *alarm);
+int _watch(struct __sourceloc, struct sched_ent *alarm);
+int _unwatch(struct __sourceloc, struct sched_ent *alarm);
+#define schedule(alarm)   _schedule(__WHENCE__, alarm)
+#define unschedule(alarm) _unschedule(__WHENCE__, alarm)
+#define watch(alarm)      _watch(__WHENCE__, alarm)
+#define unwatch(alarm)    _unwatch(__WHENCE__, alarm)
+int fd_poll();
+
+/* function timing routines */
+int fd_clearstats();
+int fd_showstats();
+int fd_checkalarms();
+int fd_func_enter(struct __sourceloc, struct call_stats *this_call);
+int fd_func_exit(struct __sourceloc, struct call_stats *this_call);
+void dump_stack(int log_level);
+
+#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \
+    struct call_stats _this_call={.totals=&_aggregate_stats}; \
+    fd_func_enter(__HERE__, &_this_call);
+
+#define OUT() fd_func_exit(__HERE__, &_this_call)
+#define RETURN(X) do { OUT(); return (X); } while (0);
+#define RETURNNULL do { OUT(); return (NULL); } while (0);
+
+#endif // __SERVALDNA__FDQUEUE_H
diff --git a/headerfiles.mk b/headerfiles.mk
index a5f7fca7..20fae731 100644
--- a/headerfiles.mk
+++ b/headerfiles.mk
@@ -18,6 +18,7 @@ HDRS=	fifo.h \
 	crypto.h \
 	log.h \
 	net.h \
+	fdqueue.h \
 	http_server.h \
 	xprintf.h \
 	constants.h \
diff --git a/http_server.c b/http_server.c
index 7a58282a..0e65b878 100644
--- a/http_server.c
+++ b/http_server.c
@@ -17,7 +17,6 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
-#define __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
 #include <assert.h>
 #include <inttypes.h>
 #include <time.h>
diff --git a/http_server.h b/http_server.h
index 7b851dbd..a1b710da 100644
--- a/http_server.h
+++ b/http_server.h
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <limits.h>
 #include "constants.h"
 #include "strbuf.h"
+#include "fdqueue.h"
 
 /* Generic HTTP request handling.
  *
@@ -111,8 +112,6 @@ void http_request_response(struct http_request *r, int result, const char *mime_
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
 void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes);
 
-#ifdef __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
-
 struct http_request {
   struct sched_ent alarm; // MUST BE FIRST ELEMENT
   enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase;
@@ -151,6 +150,4 @@ struct http_request {
   char buffer[8 * 1024];
 };
 
-#endif // __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
-
 #endif // __SERVALDNA__HTTP_SERVER_H
diff --git a/monitor-cli.c b/monitor-cli.c
index cf10da1c..506ece60 100644
--- a/monitor-cli.c
+++ b/monitor-cli.c
@@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#ifdef HAVE_POLL_H
 #include <poll.h>
+#endif
 #include <fcntl.h>
 
 #include "serval.h"
diff --git a/performance_timing.c b/performance_timing.c
index 7c989b18..cb188dc7 100644
--- a/performance_timing.c
+++ b/performance_timing.c
@@ -17,7 +17,7 @@
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-#include "serval.h"
+#include "fdqueue.h"
 #include "conf.h"
 
 struct profile_total *stats_head=NULL;
diff --git a/rhizome_http.c b/rhizome_http.c
index 418e653f..4df3cd46 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -25,7 +25,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #endif
 #include <assert.h>
 
-#define __SERVALDNA__HTTP_SERVER_IMPLEMENTATION
 #include "serval.h"
 #include "overlay_address.h"
 #include "conf.h"
diff --git a/serval.h b/serval.h
index 8f23d4b4..b6f40e62 100644
--- a/serval.h
+++ b/serval.h
@@ -1,7 +1,7 @@
 /* 
-Serval Daemon
+Serval DNA header file
 Copyright (C) 2010-2012 Paul Gardner-Stephen 
-Copyright (C) 2012 Serval Project Inc.
+Copyright (C) 2012-2013 Serval Project, Inc.
 
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
@@ -90,9 +90,6 @@ struct in_addr {
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
 #endif
-#ifdef HAVE_POLL_H
-#include <poll.h>
-#endif
 #ifdef HAVE_NETDB_H
 #include <netdb.h>
 #endif
@@ -109,6 +106,7 @@ struct in_addr {
 #include <ctype.h>
 #include <sys/stat.h>
 
+#include "fdqueue.h"
 #include "cli.h"
 #include "constants.h"
 #include "mem.h"
@@ -341,42 +339,6 @@ int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret);
 
 extern int sock;
 
-struct profile_total {
-  struct profile_total *_next;
-  int _initialised;
-  const char *name;
-  time_ms_t max_time;
-  time_ms_t total_time;
-  time_ms_t child_time;
-  int calls;
-};
-
-struct call_stats{
-  time_ms_t enter_time;
-  time_ms_t child_time;
-  struct profile_total *totals;
-  struct call_stats *prev;
-};
-
-struct sched_ent;
-
-typedef void (*ALARM_FUNCP) (struct sched_ent *alarm);
-
-struct sched_ent{
-  struct sched_ent *_next;
-  struct sched_ent *_prev;
-  
-  ALARM_FUNCP function;
-  void *context;
-  struct pollfd poll;
-  // when we should first consider the alarm
-  time_ms_t alarm;
-  // the order we will prioritise the alarm
-  time_ms_t deadline;
-  struct profile_total *stats;
-  int _poll_index;
-};
-
 struct limit_state{
   // length of time for a burst
   time_ms_t burst_length;
@@ -817,17 +779,6 @@ void sigIoHandler(int signal);
 
 int overlay_mdp_setup_sockets();
 
-int is_scheduled(const struct sched_ent *alarm);
-int _schedule(struct __sourceloc, struct sched_ent *alarm);
-int _unschedule(struct __sourceloc, struct sched_ent *alarm);
-int _watch(struct __sourceloc, struct sched_ent *alarm);
-int _unwatch(struct __sourceloc, struct sched_ent *alarm);
-#define schedule(alarm)   _schedule(__WHENCE__, alarm)
-#define unschedule(alarm) _unschedule(__WHENCE__, alarm)
-#define watch(alarm)      _watch(__WHENCE__, alarm)
-#define unwatch(alarm)    _unwatch(__WHENCE__, alarm)
-int fd_poll();
-
 void overlay_interface_discover(struct sched_ent *alarm);
 void overlay_packetradio_poll(struct sched_ent *alarm);
 int overlay_packetradio_setup_port(overlay_interface *interface);
@@ -857,22 +808,6 @@ time_ms_t limit_next_allowed(struct limit_state *state);
 int limit_is_allowed(struct limit_state *state);
 int limit_init(struct limit_state *state, int rate_micro_seconds);
 
-/* function timing routines */
-int fd_clearstats();
-int fd_showstats();
-int fd_checkalarms();
-int fd_func_enter(struct __sourceloc, struct call_stats *this_call);
-int fd_func_exit(struct __sourceloc, struct call_stats *this_call);
-void dump_stack(int log_level);
-
-#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \
-    struct call_stats _this_call={.totals=&_aggregate_stats}; \
-    fd_func_enter(__HERE__, &_this_call);
-
-#define OUT() fd_func_exit(__HERE__, &_this_call)
-#define RETURN(X) do { OUT(); return (X); } while (0);
-#define RETURNNULL do { OUT(); return (NULL); } while (0);
-
 int olsr_init_socket(void);
 int olsr_send(struct overlay_frame *frame);
 
diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 8ec48516..32745280 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -17,8 +17,6 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
-#include "strbuf_helpers.h"
-#include <poll.h>
 #include <ctype.h>
 #include <string.h>
 #include <time.h>
@@ -26,6 +24,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <stdarg.h>
 #include <inttypes.h>
 #include <sys/wait.h>
+#ifdef HAVE_POLL_H
+#include <poll.h>
+#endif
 #ifdef HAVE_NETINET_IN_H
 #include <netinet/in.h>
 #endif
@@ -34,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #endif
 #include <sys/uio.h>
 #include "http_server.h"
+#include "strbuf_helpers.h"
 
 static inline strbuf _toprint(strbuf sb, char c)
 {
diff --git a/vomp_console.c b/vomp_console.c
index a3ee4c05..4db2ea06 100644
--- a/vomp_console.c
+++ b/vomp_console.c
@@ -21,8 +21,10 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <poll.h>
 #include <fcntl.h>
+#ifdef HAVE_POLL_H
+#include <poll.h>
+#endif
 
 #include "serval.h"
 #include "conf.h"

From 95e45f452e69d9f26d541f272106c47806c85bb7 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 21 Oct 2013 16:17:33 +1030
Subject: [PATCH 14/24] Improve str_to_int() et al

Always set *afterp to point to the first invalid character
even if conversion fails
---
 str.c | 31 ++++++++++++++++---------------
 str.h |  9 +++++----
 2 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/str.c b/str.c
index 11d8d906..7ce43f82 100644
--- a/str.c
+++ b/str.c
@@ -235,11 +235,9 @@ int str_to_int(const char *str, int base, int *result, const char **afterp)
   const char *end = str;
   errno = 0;
   long value = strtol(str, (char**)&end, base);
-  if (errno == ERANGE || end == str || value > INT_MAX || value < INT_MIN)
-    return 0;
   if (afterp)
     *afterp = end;
-  else if (*end)
+  if (errno == ERANGE || end == str || value > INT_MAX || value < INT_MIN || isdigit(*end) || (!afterp && *end))
     return 0;
   if (result)
     *result = value;
@@ -253,11 +251,9 @@ int str_to_uint(const char *str, int base, unsigned *result, const char **afterp
   const char *end = str;
   errno = 0;
   unsigned long value = strtoul(str, (char**)&end, base);
-  if (errno == ERANGE || end == str || value > UINT_MAX)
-    return 0;
   if (afterp)
     *afterp = end;
-  else if (*end)
+  if (errno == ERANGE || end == str || value > UINT_MAX || isdigit(*end) || (!afterp && *end))
     return 0;
   if (result)
     *result = value;
@@ -271,11 +267,9 @@ int str_to_int64(const char *str, int base, int64_t *result, const char **afterp
   const char *end = str;
   errno = 0;
   long long value = strtoll(str, (char**)&end, base);
-  if (errno == ERANGE || end == str)
-    return 0;
   if (afterp)
     *afterp = end;
-  else if (*end)
+  if (errno == ERANGE || end == str || isdigit(*end) || (!afterp && *end))
     return 0;
   if (result)
     *result = value;
@@ -289,11 +283,9 @@ int str_to_uint64(const char *str, int base, uint64_t *result, const char **afte
   const char *end = str;
   errno = 0;
   unsigned long long value = strtoull(str, (char**)&end, base);
-  if (errno == ERANGE || end == str)
-    return 0;
   if (afterp)
     *afterp = end;
-  else if (*end)
+  if (errno == ERANGE || end == str || isdigit(*end) || (!afterp && *end))
     return 0;
   if (result)
     *result = value;
@@ -334,8 +326,11 @@ int str_to_int64_scaled(const char *str, int base, int64_t *result, const char *
 {
   int64_t value;
   const char *end = str;
-  if (!str_to_int64(str, base, &value, &end))
+  if (!str_to_int64(str, base, &value, &end)) {
+    if (afterp)
+      *afterp = end;
     return 0;
+  }
   value *= scale_factor(end, &end);
   if (afterp)
     *afterp = end;
@@ -350,8 +345,11 @@ int str_to_uint64_scaled(const char *str, int base, uint64_t *result, const char
 {
   uint64_t value;
   const char *end = str;
-  if (!str_to_uint64(str, base, &value, &end))
+  if (!str_to_uint64(str, base, &value, &end)) {
+    if (afterp)
+      *afterp = end;
     return 0;
+  }
   value *= scale_factor(end, &end);
   if (afterp)
     *afterp = end;
@@ -386,8 +384,11 @@ int str_to_uint64_interval_ms(const char *str, int64_t *result, const char **aft
     return 0;
   const char *end = str;
   unsigned long long value = strtoull(str, (char**)&end, 10) * precision;
-  if (end == str)
+  if (end == str) {
+    if (afterp)
+      *afterp = end;
     return 0;
+  }
   if (end[0] == '.' && isdigit(end[1])) {
     ++end;
     unsigned factor;
diff --git a/str.h b/str.h
index a2ece4eb..d14b7f47 100644
--- a/str.h
+++ b/str.h
@@ -199,10 +199,11 @@ char *str_str(char *haystack, const char *needle, int haystack_len);
 /* Parse a string as an integer in ASCII radix notation in the given 'base' (eg, base=10 means
  * decimal).
  *
- * Return 1 if a valid integer was parsed, storing the value in *result (unless result is NULL) and
- * storing a pointer to the immediately succeeding character in *afterp (unless afterp is NULL, in
- * which case returns 1 only if the immediately succeeding character is a nul '\0').  Returns 0
- * otherwise, leaving *result and *afterp unchanged.
+ * Returns 1 if a valid integer is parsed, storing the value in *result (unless result is NULL) and
+ * storing a pointer to the immediately succeeding character in *afterp.  If afterp is NULL then
+ * returns 0 unless the immediately succeeding character is a NUL '\0'.  If no integer is parsed or
+ * if the integer overflows (too many digits), then returns 0, leaving *result unchanged and setting
+ * setting *afterp to point to the character where parsing failed.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */

From 1a413b72dbdcb8842134eaa335df743b375e12bd Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 23 Oct 2013 00:57:48 +1030
Subject: [PATCH 15/24] Add str_tolower_inplace() function

---
 str.c | 8 ++++++++
 str.h | 2 ++
 2 files changed, 10 insertions(+)

diff --git a/str.c b/str.c
index 7ce43f82..182a7842 100644
--- a/str.c
+++ b/str.c
@@ -123,6 +123,14 @@ char *str_toupper_inplace(char *str)
   return str;
 }
 
+char *str_tolower_inplace(char *str)
+{
+  register char *s;
+  for (s = str; *s; ++s)
+    *s = tolower(*s);
+  return str;
+}
+
 const char *strnchr(const char *s, size_t n, char c)
 {
   for (; n; --n, ++s) {
diff --git a/str.h b/str.h
index d14b7f47..a02645ce 100644
--- a/str.h
+++ b/str.h
@@ -92,7 +92,9 @@ __STR_INLINE int hexvalue(char c)
 }
 
 int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value);
+
 char *str_toupper_inplace(char *s);
+char *str_tolower_inplace(char *s);
 
 char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcBytes, const char quotes[2]);
 char *toprint_str(char *dstStr, ssize_t dstBufSiz, const char *srcStr, const char quotes[2]);

From 4de6c77d4586ed9ae60ca953d09c84c38bed4637 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Wed, 23 Oct 2013 00:58:09 +1030
Subject: [PATCH 16/24] Add alloca_strdup() macro

---
 str.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/str.h b/str.h
index a02645ce..7f9827be 100644
--- a/str.h
+++ b/str.h
@@ -66,7 +66,9 @@ size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
 int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
 size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, const char **afterp);
 
-#define alloca_tohex(buf,bytes)           tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf))
+#define alloca_tohex(buf,bytes)  tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf))
+
+#define alloca_strdup(str)  strcpy(alloca(strlen(str) + 1), (str))
 
 __STR_INLINE int hexvalue(char c)
 {

From 04efb92ff6c551911e51076c5b09ea17862ab238 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 25 Oct 2013 17:38:28 +1030
Subject: [PATCH 17/24] Make logging D and T macros into expressions

So they can be used in-line in multi-line logical expressions.
---
 log.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/log.h b/log.h
index bba3ec79..1d1b743f 100644
--- a/log.h
+++ b/log.h
@@ -153,8 +153,8 @@ struct strbuf;
 #define DEBUG(X)            DEBUGF("%s", (X))
 #define DEBUGF_perror(F,...) LOGF_perror(LOG_LEVEL_DEBUG, F, ##__VA_ARGS__)
 #define DEBUG_perror(X)     DEBUGF_perror("%s", (X))
-#define D                   DEBUG("D")
-#define T                   { if (config.debug.trace) DEBUG("T"); }
+#define D                   (DEBUG("D"), 1)
+#define T                   (config.debug.trace ? DEBUG("T") : 1)
 #define DEBUG_argv(X,ARGC,ARGV) logArgv(LOG_LEVEL_DEBUG, __WHENCE__, (X), (ARGC), (ARGV))
 
 #define dump(X,A,N)         logDump(LOG_LEVEL_DEBUG, __WHENCE__, (X), (const unsigned char *)(A), (size_t)(N))

From 6488f7ad6542ab6fedcf4d42e8238bb67f568853 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 25 Oct 2013 17:38:51 +1030
Subject: [PATCH 18/24] Fix basic bugs in new HTTP server code

All 'rhizomeprotocol' HTTP tests still fail
---
 conf_schema.h   |   1 +
 http_server.c   | 319 ++++++++++++++++++++++++++++++++----------------
 http_server.h   |  13 +-
 rhizome_fetch.c |   2 +-
 rhizome_http.c  |  24 ++--
 5 files changed, 234 insertions(+), 125 deletions(-)

diff --git a/conf_schema.h b/conf_schema.h
index d32f966d..a44adc67 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -233,6 +233,7 @@ ATOM(bool_t, dnaresponses,              0, boolean,, "")
 ATOM(bool_t, dnahelper,                 0, boolean,, "")
 ATOM(bool_t, queues,                    0, boolean,, "")
 ATOM(bool_t, timing,                    0, boolean,, "")
+ATOM(bool_t, httpd,                     0, boolean,, "")
 ATOM(bool_t, io,                        0, boolean,, "")
 ATOM(bool_t, verbose_io,                0, boolean,, "")
 ATOM(bool_t, interactive_io,            0, boolean,, "")
diff --git a/http_server.c b/http_server.c
index 0e65b878..4786ecd2 100644
--- a/http_server.c
+++ b/http_server.c
@@ -21,6 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <inttypes.h>
 #include <time.h>
 #include "serval.h"
+#include "conf.h"
 #include "http_server.h"
 #include "log.h"
 #include "str.h"
@@ -67,6 +68,18 @@ static struct profile_total http_server_stats = {
   .name = "http_server_poll",
 };
 
+#define DEBUG_DUMP_PARSED(r) do { \
+      if (config.debug.httpd) \
+	DEBUGF("%s %s HTTP/%u.%u", r->verb ? r->verb : "NULL", alloca_str_toprint(r->path), r->version_major, r->version_minor);\
+    } while (0)
+
+#define DEBUG_DUMP_PARSER(r) do { \
+      if (config.debug.httpd) \
+	DEBUGF("parsed %d %s cursor %d %s", \
+	    r->parsed - r->received, alloca_toprint(-1, r->received, r->parsed - r->received), \
+	    r->cursor - r->received, alloca_toprint(50, r->cursor, r->end - r->cursor)); \
+    } while (0)
+
 static void http_server_poll(struct sched_ent *);
 static int http_request_parse_verb(struct http_request *r);
 static int http_request_parse_path(struct http_request *r);
@@ -79,7 +92,6 @@ static void http_request_start_response(struct http_request *r);
 
 void http_request_init(struct http_request *r, int sockfd)
 {
-  bzero(r, sizeof *r);
   assert(sockfd != -1);
   r->request_header.content_length = CONTENT_LENGTH_UNKNOWN;
   r->request_content_remaining = CONTENT_LENGTH_UNKNOWN;
@@ -93,7 +105,7 @@ void http_request_init(struct http_request *r, int sockfd)
   r->alarm.poll.events = POLLIN;
   r->phase = RECEIVE;
   r->received = r->end = r->parsed = r->cursor = r->buffer;
-  r->limit = r->buffer + sizeof r->buffer;
+  r->end_content = NULL;
   r->parser = http_request_parse_verb;
   watch(&r->alarm);
   schedule(&r->alarm);
@@ -273,9 +285,14 @@ static const char * _reserve_str(struct http_request *r, const char *str)
   return _reserve(r, sub);
 }
 
+static inline int _end_of_content(struct http_request *r)
+{
+  return r->cursor == r->end_content;
+}
+
 static inline int _buffer_full(struct http_request *r)
 {
-  return r->parsed == r->received && r->end == r->limit;
+  return r->parsed == r->received && r->end == (r->end_content ? r->end_content : r->buffer + sizeof r->buffer);
 }
 
 static inline int _run_out(struct http_request *r)
@@ -296,6 +313,11 @@ static inline void _commit(struct http_request *r)
   r->parsed = r->cursor;
 }
 
+static inline void _skip_all(struct http_request *r)
+{
+  r->cursor = r->end;
+}
+
 static inline int _skip_crlf(struct http_request *r)
 {
   return !_run_out(r) && *r->cursor == '\r' && ++r->cursor && !_run_out(r) && *r->cursor == '\n' && ++r->cursor;
@@ -550,13 +572,16 @@ static int _parse_quoted_rfc822_time(struct http_request *r, time_t *timep)
   return 0;
 }
 
-/* Return 100 if more received data is needed.  Returns 0 if parsing completes or fails.  Returns a
- * 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
+/* If parsing completes, then sets r->parser to the next parsing function and returns 0.  If parsing
+ * cannot complete due to running out of data, returns 100 without changing r->parser, so this
+ * function will be called again once more data has been read.  Returns a 4nn or 5nn HTTP result
+ * code if parsing fails.  Returns -1 if an unexpected error occurs.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 static int http_request_parse_verb(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   _rewind(r);
   assert(r->cursor >= r->received);
   assert(!_run_out(r));
@@ -574,7 +599,7 @@ static int http_request_parse_verb(struct http_request *r)
   }
   if (r->verb == NULL) {
     if (r->debug_flag && *r->debug_flag)
-      DEBUGF("Malformed HTTP request: invalid verb: %s", alloca_toprint(20, r->cursor, r->end - r->cursor));
+      DEBUGF("Malformed HTTP request, invalid verb: %s", alloca_toprint(20, r->cursor, r->end - r->cursor));
     return 400;
   }
   _commit(r);
@@ -582,13 +607,16 @@ static int http_request_parse_verb(struct http_request *r)
   return 0;
 }
 
-/* Return 100 if more received data is needed.  Returns 0 if parsing completes or fails.  Returns a
- * 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
+/* If parsing completes, then sets r->parser to the next parsing function and returns 0.  If parsing
+ * cannot complete due to running out of data, returns 100 without changing r->parser, so this
+ * function will be called again once more data has been read.  Returns a 4nn or 5nn HTTP result
+ * code if parsing fails.  Returns -1 if an unexpected error occurs.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 static int http_request_parse_path(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   // Parse path: word immediately following verb, delimited by spaces.
   assert(r->path == NULL);
   struct substring path;
@@ -599,20 +627,23 @@ static int http_request_parse_path(struct http_request *r)
       DEBUGF("Malformed HTTP %s request at path: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
     return 400;
   }
+  _commit(r);
   if ((r->path = _reserve(r, path)) == NULL)
     return 0; // error
-  _commit(r);
   r->parser = http_request_parse_http_version;
   return 0;
 }
 
-/* Return 100 if more received data is needed.  Returns 0 if parsing completes or fails.  Returns a
- * 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
+/* If parsing completes, then sets r->parser to the next parsing function and returns 0.  If parsing
+ * cannot complete due to running out of data, returns 100 without changing r->parser, so this
+ * function will be called again once more data has been read.  Returns a 4nn or 5nn HTTP result
+ * code if parsing fails.  Returns -1 if an unexpected error occurs.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 static int http_request_parse_http_version(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   // Parse HTTP version: HTTP/m.n followed by CRLF.
   assert(r->version_major == 0);
   assert(r->version_minor == 0);
@@ -622,14 +653,14 @@ static int http_request_parse_http_version(struct http_request *r)
 	&& major > 0 && major < UINT8_MAX
 	&& _skip_literal(r, ".")
 	&& _parse_uint(r, &minor)
-	&& minor > 0 && minor < UINT8_MAX
+	&& minor < UINT8_MAX
 	&& _skip_eol(r)
-	)
+       )
   ) {
     if (_run_out(r))
       return 100; // read more and try again
     if (r->debug_flag && *r->debug_flag)
-      DEBUGF("HTTP %s malformed request: malformed version: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
+      DEBUGF("Malformed HTTP %s request at version: %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
     return 400;
   }
   _commit(r);
@@ -642,16 +673,16 @@ static int http_request_parse_http_version(struct http_request *r)
 }
 
 /* Select the header parser.  Returns 0 after setting the new parser function.  Returns a 4nn or 5nn
- * HTTP result code if parsing fails.
+ * HTTP result code if the request cannot be handled (eg, unsupported HTTP version or invalid path).
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 static int http_request_start_parsing_headers(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   assert(r->verb != NULL);
   assert(r->path != NULL);
   assert(r->version_major != 0);
-  assert(r->version_minor != 0);
   if (r->version_major != 1) {
     if (r->debug_flag && *r->debug_flag)
       DEBUGF("Unsupported HTTP version: %u.%u", r->version_major, r->version_minor);
@@ -661,14 +692,19 @@ static int http_request_start_parsing_headers(struct http_request *r)
   return 0;
 }
 
-/* Parse one request header line.  Returns 100 if more received data is needed.  Returns 0 if there
- * are no more headers or parsing fails.  Returns a 4nn or 5nn HTTP result code if parsing fails.
- * Returns -1 if an unexpected error occurs.
+/* Parse one request header line.
+ *
+ * If the end of headers is parsed (blank line), then sets r->parser to the next parsing function
+ * and returns 0.  If a single header line is successfully parsed, returns 0 after advancing
+ * r->parsed.  If parsing cannot complete due to running out of data, returns 0 without changing
+ * r->parser, so this function will be called again once more data has been read.  Returns a 4nn or
+ * 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error occurs.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 static int http_request_parse_header(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   _skip_to_eol(r);
   const char *const eol = r->cursor;
   _skip_eol(r);
@@ -777,13 +813,16 @@ malformed:
   return 400;
 }
 
-/* Select the header parser.  Returns 0 after setting the new parser function.  Returns a 4nn or 5nn
- * HTTP result code if parsing fails.
+/* If parsing completes, then sets r->parser to the next parsing function and returns 0.  If parsing
+ * cannot complete due to running out of data, returns 0 without changing r->parser, so this
+ * function will be called again once more data has been read.  Returns a 4nn or 5nn HTTP result
+ * code if parsing fails.  Returns -1 if an unexpected error occurs.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
 static int http_request_start_body(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   assert(r->verb != NULL);
   assert(r->path != NULL);
   assert(r->version_major != 0);
@@ -807,7 +846,7 @@ static int http_request_start_body(struct http_request *r)
     if (r->request_header.content_length == CONTENT_LENGTH_UNKNOWN) {
       if (r->debug_flag && *r->debug_flag)
 	DEBUGF("Malformed HTTP %s request: missing Content-Length header", r->verb);
-      return 400;
+      return 411;
     }
     if (r->request_header.content_type == NULL) {
       if (r->debug_flag && *r->debug_flag)
@@ -931,9 +970,11 @@ malformed:
   return 1;
 }
 
-/* Return zero if more received data is needed.  The first call that completes parsing of the body
- * returns 200.  All subsequent calls return 100.  Returns a 4nn or 5nn HTTP result code if parsing
- * fails.
+/* If parsing completes (ie, parsed to end of epilogue), then sets r->parser to NULL and returns 0,
+ * so this function will not be called again.  If parsing cannot complete due to running out of
+ * data, returns 100, so this function will not be called again until more data has been read.
+ * Returns a 4nn or 5nn HTTP result code if parsing fails.  Returns -1 if an unexpected error
+ * occurs.
  *
  * NOTE: No support for nested/mixed parts, as that would considerably complicate the parser.  If
  * the need arises in future, we will deal with it then.  In the meantime, we will have something
@@ -960,15 +1001,22 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	    r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed);
 	  _rewind_crlf(r);
 	  _commit(r);
-	  r->form_data_state = EPILOGUE;
 	  if (b == 1) {
 	    r->form_data_state = HEADER;
 	    if (r->form_data.handle_mime_part_start)
 	      r->form_data.handle_mime_part_start(r);
-	  }
+	  } else
+	    r->form_data_state = EPILOGUE;
 	  return 0;
 	}
-	_skip_to_crlf(r);
+	if (!_skip_to_crlf(r)) {
+	  if (_end_of_content(r)) {
+	    if (r->debug_flag && *r->debug_flag)
+	      DEBUGF("Malformed HTTP %s form data: missing first boundary", r->verb);
+	    return 400;
+	  }
+	  return 100; // need more data
+	}
 	at_start = 0;
       }
       if (r->cursor > r->parsed && r->form_data.handle_mime_preamble)
@@ -982,7 +1030,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  return 0;
 	}
 	if (_run_out(r))
-	  return 0; // read more and try again
+	  return 100; // read more and try again
 	_rewind(r);
 	int b;
 	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
@@ -990,17 +1038,18 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  _commit(r);
 	  if (r->form_data.handle_mime_part_end)
 	    r->form_data.handle_mime_part_end(r);
-	  r->form_data_state = EPILOGUE;
 	  // Boundary in the middle of headers starts a new part.
 	  if (b == 1) {
 	    r->form_data_state = HEADER;
 	    if (r->form_data.handle_mime_part_start)
 	      r->form_data.handle_mime_part_start(r);
 	  }
+	  else
+	    r->form_data_state = EPILOGUE;
 	  return 0;
 	}
 	if (_run_out(r))
-	  return 0; // read more and try again
+	  return 100; // read more and try again
 	_rewind(r);
 	struct substring label;
 	if (_skip_crlf(r) && _skip_token(r, &label) && _skip_literal(r, ":") && _skip_optional_space(r)) {
@@ -1027,7 +1076,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  }
 	}
 	if (_run_out(r))
-	  return 0; // read more and try again
+	  return 100; // read more and try again
 	_rewind(r);
       }
       if (r->debug_flag && *r->debug_flag)
@@ -1052,7 +1101,14 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  }
 	  return 0;
 	}
-	_skip_to_crlf(r);
+	if (!_skip_to_crlf(r)) {
+	  if (_end_of_content(r)) {
+	    if (r->debug_flag && *r->debug_flag)
+	      DEBUGF("Malformed HTTP %s form data part: missing end boundary", r->verb);
+	    return 400;
+	  }
+	  return 100; // need more data
+	}
       }
       if (r->cursor > r->parsed && r->form_data.handle_mime_body)
 	r->form_data.handle_mime_body(r, r->parsed, r->parsed - r->cursor);
@@ -1063,9 +1119,10 @@ static int http_request_parse_body_form_data(struct http_request *r)
     if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed)
       r->form_data.handle_mime_epilogue(r, r->parsed, r->cursor - r->parsed);
     _commit(r);
+    r->parser = NULL;
     return 0;
   }
-  assert(0); // not reached
+  abort(); // not reached
 }
 
 static ssize_t http_request_read(struct http_request *r, char *buf, size_t len)
@@ -1090,21 +1147,43 @@ static ssize_t http_request_read(struct http_request *r, char *buf, size_t len)
 static void http_request_receive(struct http_request *r)
 {
   assert(r->phase == RECEIVE);
-  r->limit = r->buffer + sizeof r->buffer;
-  assert(r->end < r->limit);
-  size_t room = r->limit - r->end;
+  const char *const bufend = r->buffer + sizeof r->buffer;
+  assert(r->end <= bufend);
   assert(r->parsed >= r->received);
   assert(r->parsed <= r->end);
-  // If buffer is running short on unused space, shift existing content in buffer down to make more
-  // room if possible.
-  if (   (room < 128 || (room < 1024 && r->parsed - r->received >= 32))
-      && (r->request_content_remaining == CONTENT_LENGTH_UNKNOWN || room < r->request_content_remaining)
+  // Work out if the end of content falls within the buffer yet.  If so, set the end_content
+  // pointer (and make sure it doesn't move).
+  if (r->end_content) {
+    assert(r->request_content_remaining != CONTENT_LENGTH_UNKNOWN);
+    assert(r->end < r->end_content);
+    assert(r->end_content - r->end == r->request_content_remaining);
+  } else if (   r->request_content_remaining != CONTENT_LENGTH_UNKNOWN
+	     && r->request_content_remaining < bufend - r->end
   ) {
-    size_t unparsed = r->end - r->parsed;
-    memmove((char *)r->received, r->parsed, unparsed); // memcpy() does not handle overlapping src and dst
-    r->parsed = r->received;
-    r->end = r->received + unparsed;
-    room = r->limit - r->end;
+    r->end_content = r->end + r->request_content_remaining;
+  }
+  //
+  // If the end of content mark is within the buffer, then there is no need to make any more room,
+  // just keep reading up to the end of content.  Otherwise, If buffer is running short on unused
+  // space, shift existing content in buffer down to make more room if possible.
+  size_t room;
+  if (r->end_content) {
+    assert(r->end_content > r->buffer);
+    assert(r->end_content <= bufend);
+    room = r->end_content - r->end;
+  } else {
+    room = bufend - r->end;
+    size_t spare = r->parsed - r->received;
+    if (   spare
+	&& (room < 128 || (room < 1024 && spare >= 32))
+	&& (r->request_content_remaining == CONTENT_LENGTH_UNKNOWN || spare >= r->request_content_remaining)
+    ) {
+      size_t unparsed = r->end - r->parsed;
+      memmove((char *)r->received, r->parsed, unparsed); // memcpy() does not handle overlapping src and dst
+      r->parsed = r->received;
+      r->end = r->received + unparsed;
+      room = bufend - r->end;
+    }
   }
   // If there is no more buffer space, fail the request.
   if (room == 0) {
@@ -1113,35 +1192,34 @@ static void http_request_receive(struct http_request *r)
     http_request_simple_response(r, 431, NULL);
     return;
   }
-  // Read up to the end of available buffer space or the end of content, whichever is first.
-  size_t read_size = room;
-  if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && read_size > r->request_content_remaining) {
-    r->limit = r->end + r->request_content_remaining;
-    read_size = r->request_content_remaining;
-  }
-  if (read_size != 0) {
-    // Read as many bytes as possible into the unused buffer space.  Any read error
-    // closes the connection without any response.
-    ssize_t bytes = http_request_read(r, (char *)r->end, read_size);
-    if (bytes == -1)
-      return;
-    // If no data was read, then just return to polling.  Don't drop the connection on an empty read,
-    // because that drops connections when they shouldn't, including during testing.  The inactivity
-    // timeout will drop the connections instead.
-    if (bytes == 0)
-      return;
-    r->end += (size_t) bytes;
-    // We got some data, so reset the inactivity timer and invoke the parsing state machine to process
-    // it.  The state machine invokes the caller-supplied callback functions.
-    r->alarm.alarm = gettime_ms() + r->idle_timeout;
-    r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
-    unschedule(&r->alarm);
-    schedule(&r->alarm);
-  }
+  // Read up to the end of available buffer space or the end of content, whichever is first.  Read
+  // as many bytes as possible into the unused buffer space.  Any read error closes the connection
+  // without any response.
+  assert(room > 0);
+  if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN)
+    assert(room <= r->request_content_remaining);
+  ssize_t bytes = http_request_read(r, (char *)r->end, room);
+  if (bytes == -1)
+    return;
+  assert((size_t) bytes <= room);
+  // If no data was read, then just return to polling.  Don't drop the connection on an empty read,
+  // because that drops connections when they shouldn't, including during testing.  The inactivity
+  // timeout will drop inactive connections.
+  if (bytes == 0)
+    return;
+  r->end += (size_t) bytes;
+  if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN)
+    r->request_content_remaining -= (size_t) bytes;
+  // We got some data, so reset the inactivity timer and invoke the parsing state machine to process
+  // it.  The state machine invokes the caller-supplied callback functions.
+  r->alarm.alarm = gettime_ms() + r->idle_timeout;
+  r->alarm.deadline = r->alarm.alarm + r->idle_timeout;
+  unschedule(&r->alarm);
+  schedule(&r->alarm);
   // Parse the unparsed and received data.
   while (r->phase == RECEIVE) {
     int result;
-    if (_run_out(r) && r->request_content_remaining == 0) {
+    if (r->parsed == r->end_content) {
       if (r->handle_content_end)
 	result = r->handle_content_end(r);
       else {
@@ -1150,26 +1228,27 @@ static void http_request_receive(struct http_request *r)
 	result = 500;
       }
     } else {
+      HTTP_REQUEST_PARSER oldparser = r->parser;
       const char *oldparsed = r->parsed;
+      _rewind(r);
       if (r->parser == NULL) {
 	if (r->debug_flag && *r->debug_flag)
-	  DEBUG("Internal failure parsing HTTP request: no parser function set");
-	result = 500;
+	  DEBUGF("No HTTP parser function set -- skipping %zu bytes", (size_t)(r->end - r->cursor));
+	_skip_all(r);
+	_commit(r);
+	result = 0;
       } else {
-	_rewind(r);
 	result = r->parser(r);
 	assert(r->parsed >= oldparsed);
       }
-      if (result == 100) {
-	if (_run_out(r))
-	  return; // needs more data; poll again
-	if (r->debug_flag && *r->debug_flag)
-	  DEBUG("Internal failure parsing HTTP request: parser function returned 100 but not run out");
-	result = 500;
-      }
-      if (result == 0 && r->parsed == oldparsed) {
+      if (r->phase != RECEIVE)
+	break;
+      if (result == 100)
+	return; // needs more data; poll again
+      if (result == 0 && r->parsed == oldparsed && r->parser == oldparser) {
 	if (r->debug_flag && *r->debug_flag)
 	  DEBUG("Internal failure parsing HTTP request: parser function did not advance");
+	DEBUG_DUMP_PARSER(r);
 	result = 500;
       }
     }
@@ -1180,10 +1259,8 @@ static void http_request_receive(struct http_request *r)
 	DEBUGF("Internal failure parsing HTTP request: invalid result=%d", result);
       r->response.result_code = 500;
     }
-    if (r->response.result_code) {
-      http_request_start_response(r);
-      return;
-    }
+    if (r->response.result_code)
+      break;
     if (result == -1) {
       if (r->debug_flag && *r->debug_flag)
 	DEBUG("Unrecoverable error parsing HTTP request, closing connection");
@@ -1214,7 +1291,7 @@ static void http_request_send_response(struct http_request *r)
     assert(r->response_buffer_sent <= r->response_buffer_length);
     if (r->response_buffer_sent == r->response_buffer_length) {
       if (r->response.content_generator) {
-	// Content generator must fill response_buffer[] and set response_buffer_length.
+	// Content generator must fill set (or re-set) response_buffer and response_buffer_length.
 	r->response_buffer_sent = r->response_buffer_length = 0;
 	if (r->response.content_generator(r) == -1) {
 	  if (r->debug_flag && *r->debug_flag)
@@ -1359,6 +1436,7 @@ static const char *httpResultString(int response_code)
   case 403: return "Forbidden";
   case 404: return "Not Found";
   case 405: return "Method Not Allowed";
+  case 411: return "Length Required";
   case 414: return "Request-URI Too Long";
   case 415: return "Unsupported Media Type";
   case 416: return "Requested Range Not Satisfiable";
@@ -1404,6 +1482,7 @@ static int _render_response(struct http_request *r)
     hr.header.content_range_start = 0;
   }
   assert(hr.header.content_type != NULL);
+  assert(hr.header.content_type[0]);
   strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", hr.result_code, result_string);
   strbuf_sprintf(sb, "Content-Type: %s", hr.header.content_type);
   if (hr.header.boundary) {
@@ -1457,8 +1536,25 @@ static void http_request_render_response(struct http_request *r)
   }
 }
 
+static size_t http_request_drain(struct http_request *r)
+{
+  assert(r->phase == RECEIVE);
+  char buf[8192];
+  size_t drained = 0;
+  ssize_t bytes;
+  while ((bytes = http_request_read(r, buf, sizeof buf)) != -1 && bytes != 0)
+    drained += (size_t) bytes;
+  return drained;
+}
+
 static void http_request_start_response(struct http_request *r)
 {
+  assert(r->phase == RECEIVE);
+  assert(r->response.result_code != 0);
+  if (r->response.content || r->response.content_generator) {
+    assert(r->response.header.content_type != NULL);
+    assert(r->response.header.content_type[0]);
+  }
   // If HTTP responses are disabled (eg, for testing purposes) then skip all response construction
   // and close the connection.
   if (r->disable_tx_flag && *r->disable_tx_flag) {
@@ -1468,11 +1564,9 @@ static void http_request_start_response(struct http_request *r)
   }
   // Drain the rest of the request that has not been received yet (eg, if sending an error response
   // provoked while parsing the early part of a partially-received request).  If a read error
-  // occurs, the connection is closed.
-  ssize_t bytes;
-  while ((bytes = http_request_read(r, (char *) r->received, (r->buffer + sizeof r->buffer) - r->received)) > 0)
-    ;
-  if (bytes == -1)
+  // occurs, the connection is closed so the phase changes to DONE.
+  http_request_drain(r);
+  if (r->phase != RECEIVE)
     return;
   // If the response cannot be rendered, then render a 500 Server Error instead.  If that fails,
   // then just close the connection.
@@ -1496,9 +1590,20 @@ static void http_request_start_response(struct http_request *r)
   watch(&r->alarm);
 }
 
+/* Start sending a response back to the client.  The response's Content-Type is set by the
+ * 'mime_type' parameter (in the standard format "type/subtype").  The response's content is set
+ * from the 'body' and 'bytes' parameters, which need not point to static data, ie, is no longer
+ * used once this function returns.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
 void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes)
 {
-  assert(result != 0);
+  assert(r->phase == RECEIVE);
+  assert(result >= 100);
+  assert(result < 600);
+  assert(mime_type != NULL);
+  assert(mime_type[0]);
   r->response.result_code = result;
   r->response.header.content_type = mime_type;
   r->response.header.content_range_start = 0;
@@ -1508,28 +1613,30 @@ void http_request_response(struct http_request *r, int result, const char *mime_
   http_request_start_response(r);
 }
 
+/* Start sending a short redirection or error response back to the client.  The result code must be
+ * either a redirection (3xx) or client error (4xx) or server error (5xx) code.  The 'body' argument
+ * may be a bare message which is enclosed in an HTML envelope to form the response content, so it
+ * may contain HTML markup.  If the 'body' argument is NULL, then the response content is generated
+ * automatically from the result code.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body)
 {
+  assert(r->phase == RECEIVE);
+  assert(result >= 300);
+  assert(result < 600);
   strbuf h = NULL;
   if (body) {
     size_t html_len = strlen(body) + 40;
-    char html[html_len];
-    h = strbuf_local(html, html_len);
-    strbuf_sprintf(h, "<html><h1>%s</h1></html>", body);
+    h = strbuf_alloca(html_len);
+    strbuf_sprintf(h, "<html><h1>%03u %s</h1></html>", result, body);
   }
   r->response.result_code = result;
-  r->response.header.content_type = body ? "text/html" : NULL;
+  r->response.header.content_type = "text/html";
   r->response.header.content_range_start = 0;
   r->response.header.resource_length = r->response.header.content_length = h ? strbuf_len(h) : 0;
   r->response.content = h ? strbuf_str(h) : NULL;
   r->response.content_generator = NULL;
   http_request_start_response(r);
 }
-
-void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes)
-{
-  r->response.result_code = result;
-  r->response.content = NULL;
-  r->response.content_generator = NULL;
-  http_request_start_response(r);
-}
diff --git a/http_server.h b/http_server.h
index a1b710da..5a73e6ce 100644
--- a/http_server.h
+++ b/http_server.h
@@ -110,7 +110,8 @@ int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
 void http_request_finalise(struct http_request *r);
 void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
-void http_request_response_header(struct http_request *r, int result, const char *mime_type, uint64_t bytes);
+
+typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
 
 struct http_request {
   struct sched_ent alarm; // MUST BE FIRST ELEMENT
@@ -120,10 +121,10 @@ struct http_request {
   time_ms_t initiate_time; // time connection was initiated
   time_ms_t idle_timeout; // disconnect if no bytes received for this long
   struct sockaddr_in client_in_addr;
-  int (*parser)(struct http_request *); // current parser function
-  int (*handle_first_line)(struct http_request *); // called after first line is parsed
-  int (*handle_headers)(struct http_request *); // called after all headers are parsed
-  int (*handle_content_end)(struct http_request *); // called after all content is received
+  HTTP_REQUEST_PARSER parser; // current parser function
+  HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed
+  HTTP_REQUEST_PARSER handle_headers; // called after all headers are parsed
+  HTTP_REQUEST_PARSER handle_content_end; // called after all content is received
   enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state;
   struct http_mime_handler form_data; // called to parse multipart/form-data body
   void (*finalise)(struct http_request *);
@@ -135,7 +136,7 @@ struct http_request {
   struct http_request_headers request_header;
   const char *received; // start of received data in buffer[]
   const char *end; // end of received data in buffer[]
-  const char *limit; // end of content in buffer[]
+  const char *end_content; // end of content if within buffer[], else NULL
   const char *parsed; // start of unparsed data in buffer[]
   const char *cursor; // for parsing
   http_size_t request_content_remaining;
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index abdc4cf2..6713c193 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -181,7 +181,7 @@ int rhizome_fetch_queue_bytes(){
   return bytes;
 }
 
-int rhizome_fetch_status_html(struct strbuf *b)
+int rhizome_fetch_status_html(strbuf b)
 {
   int i,j;
   for(i=0;i<NQUEUES;i++){
diff --git a/rhizome_http.c b/rhizome_http.c
index 4df3cd46..6807d45b 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -273,6 +273,7 @@ void rhizome_server_poll(struct sched_ent *alarm)
 	  request->http.client_in_addr = *peerip;
 	request->http.handle_headers = rhizome_dispatch;
 	request->http.debug_flag = &config.debug.rhizome_httpd;
+	request->http.disable_tx_flag = &config.debug.rhizome_nohttptx;
 	request->http.finalise = rhizome_server_finalise_http_request;
 	request->http.free = free;
 	request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT;
@@ -330,7 +331,7 @@ static int neighbour_page(rhizome_http_request *r, const char *remainder)
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-  http_request_simple_response(&r->http, 200, buf);
+  http_request_response(&r->http, 200, "text/html", buf, strbuf_len(b));
   return 0;
 }
 
@@ -350,7 +351,7 @@ static int interface_page(rhizome_http_request *r, const char *remainder)
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-  http_request_simple_response(&r->http, 200, buf);
+  http_request_response(&r->http, 200, "text/html", buf, strbuf_len(b));
   return 0;
 }
 
@@ -365,16 +366,15 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
     return 0;
   }
   char buf[32*1024];
-  struct strbuf b;
-  strbuf_init(&b, buf, sizeof buf);
-  strbuf_puts(&b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
-  strbuf_sprintf(&b, "%d HTTP requests<br>", request_count);
-  strbuf_sprintf(&b, "%d Bundles transferring via MDP<br>", rhizome_cache_count());
-  rhizome_fetch_status_html(&b);
-  strbuf_puts(&b, "</body></html>");
-  if (strbuf_overrun(&b))
+  strbuf b = strbuf_local(buf, sizeof buf);
+  strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
+  strbuf_sprintf(b, "%d HTTP requests<br>", request_count);
+  strbuf_sprintf(b, "%d Bundles transferring via MDP<br>", rhizome_cache_count());
+  rhizome_fetch_status_html(b);
+  strbuf_puts(b, "</body></html>");
+  if (strbuf_overrun(b))
     return -1;
-  http_request_simple_response(&r->http, 200, buf);
+  http_request_response(&r->http, 200, "text/html", buf, strbuf_len(b));
   return 0;
 }
 
@@ -521,6 +521,6 @@ static int root_page(rhizome_http_request *r, const char *remainder)
     WHY("HTTP Root page buffer overrun");
     http_request_simple_response(&r->http, 500, NULL);
   } else
-    http_request_simple_response(&r->http, 200, temp);
+    http_request_response(&r->http, 200, "text/html", temp, strbuf_len(b));
   return 0;
 }

From 5ff5a02bb9eedf6a372fe7bc791b5288294e9874 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 25 Oct 2013 23:57:23 +1030
Subject: [PATCH 19/24] Fix a couple more bugs in new HTTP server code

---
 http_server.c         | 26 ++++++++++++++++++++------
 http_server.h         |  7 +++++--
 rhizome_direct_http.c |  4 ++--
 rhizome_http.c        | 15 +++++++--------
 4 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/http_server.c b/http_server.c
index 4786ecd2..60877899 100644
--- a/http_server.c
+++ b/http_server.c
@@ -1590,18 +1590,18 @@ static void http_request_start_response(struct http_request *r)
   watch(&r->alarm);
 }
 
-/* Start sending a response back to the client.  The response's Content-Type is set by the
- * 'mime_type' parameter (in the standard format "type/subtype").  The response's content is set
- * from the 'body' and 'bytes' parameters, which need not point to static data, ie, is no longer
- * used once this function returns.
+/* Start sending a static (pre-computed) response back to the client.  The response's Content-Type
+ * is set by the 'mime_type' parameter (in the standard format "type/subtype").  The response's
+ * content is set from the 'body' and 'bytes' parameters, which need not point to persistent data,
+ * ie, the memory pointed to by 'body' is no longer referenced once this function returns.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes)
+void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes)
 {
   assert(r->phase == RECEIVE);
   assert(result >= 100);
-  assert(result < 600);
+  assert(result < 300);
   assert(mime_type != NULL);
   assert(mime_type[0]);
   r->response.result_code = result;
@@ -1613,6 +1613,20 @@ void http_request_response(struct http_request *r, int result, const char *mime_
   http_request_start_response(r);
 }
 
+void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR generator)
+{
+  assert(r->phase == RECEIVE);
+  assert(result >= 100);
+  assert(result < 300);
+  assert(mime_type != NULL);
+  assert(mime_type[0]);
+  r->response.result_code = result;
+  r->response.header.content_type = mime_type;
+  r->response.content = NULL;
+  r->response.content_generator = generator;
+  http_request_start_response(r);
+}
+
 /* Start sending a short redirection or error response back to the client.  The result code must be
  * either a redirection (3xx) or client error (4xx) or server error (5xx) code.  The 'body' argument
  * may be a bare message which is enclosed in an HTML envelope to form the response content, so it
diff --git a/http_server.h b/http_server.h
index 5a73e6ce..ebedb0ad 100644
--- a/http_server.h
+++ b/http_server.h
@@ -73,11 +73,13 @@ struct http_response_headers {
   const char *boundary;
 };
 
+typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *);
+
 struct http_response {
   uint16_t result_code;
   struct http_response_headers header;
   const char *content;
-  int (*content_generator)(struct http_request *); // callback to produce more content
+  HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content
 };
 
 #define MIME_FILENAME_MAXLEN 127
@@ -108,7 +110,8 @@ void http_request_init(struct http_request *r, int sockfd);
 void http_request_free_response_buffer(struct http_request *r);
 int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
 void http_request_finalise(struct http_request *r);
-void http_request_response(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
+void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
+void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR);
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
 
 typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index b8d6125a..90bce08a 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -160,7 +160,7 @@ int rhizome_direct_enquiry_end(struct http_request *hr)
     if (http_request_set_response_bufsize(&r->http, bytes) == -1)
       http_request_simple_response(&r->http, 500, "Internal Error: Out of memory");
     else
-      http_request_response(&r->http, 200, "binary/octet-stream", (const char *)c->buffer, bytes);
+      http_request_response_static(&r->http, 200, "binary/octet-stream", (const char *)c->buffer, bytes);
     rhizome_direct_bundle_iterator_free(&c);
   } else
     http_request_simple_response(&r->http, 500, "Internal Error: No response to enquiry");
@@ -258,7 +258,7 @@ int rhizome_direct_addfile_end(struct http_request *hr)
     if (config.debug.rhizome)
       DEBUGF("Import sans-manifest appeared to succeed");
     /* Respond with the manifest that was added. */
-    http_request_response(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes);
+    http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes);
     /* clean up after ourselves */
     if (mout && mout != m)
       rhizome_manifest_free(mout);
diff --git a/rhizome_http.c b/rhizome_http.c
index 6807d45b..b0c04668 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -331,7 +331,7 @@ static int neighbour_page(rhizome_http_request *r, const char *remainder)
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-  http_request_response(&r->http, 200, "text/html", buf, strbuf_len(b));
+  http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
   return 0;
 }
 
@@ -351,7 +351,7 @@ static int interface_page(rhizome_http_request *r, const char *remainder)
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-  http_request_response(&r->http, 200, "text/html", buf, strbuf_len(b));
+  http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
   return 0;
 }
 
@@ -374,7 +374,7 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
   strbuf_puts(b, "</body></html>");
   if (strbuf_overrun(b))
     return -1;
-  http_request_response(&r->http, 200, "text/html", buf, strbuf_len(b));
+  http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
   return 0;
 }
 
@@ -455,8 +455,7 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
   r->http.response.header.resource_length = closed.last;
   r->http.response.header.content_length = closed.last - closed.first;
   r->read_state.offset = closed.first;
-  r->http.response.content_generator = rhizome_file_content;
-  http_request_response(&r->http, result_code, "application/binary", NULL, 0);
+  http_request_response_generated(&r->http, result_code, "application/binary", rhizome_file_content);
   return 0;
 }
 
@@ -478,7 +477,7 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde
   if (ret == -1)
     http_request_simple_response(&r->http, 500, NULL);
   else if (ret == 0)
-    http_request_response(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
+    http_request_response_static(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
   rhizome_manifest_free(m);
   return ret <= 0 ? 0 : 1;
 }
@@ -487,7 +486,7 @@ static int fav_icon_header(rhizome_http_request *r, const char *remainder)
 {
   if (*remainder)
     return 1;
-  http_request_response(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
+  http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
   return 0;
 }
 
@@ -521,6 +520,6 @@ static int root_page(rhizome_http_request *r, const char *remainder)
     WHY("HTTP Root page buffer overrun");
     http_request_simple_response(&r->http, 500, NULL);
   } else
-    http_request_response(&r->http, 200, "text/html", temp, strbuf_len(b));
+    http_request_response_static(&r->http, 200, "text/html", temp, strbuf_len(b));
   return 0;
 }

From 0397a47753cd2173064e79e21ffcf8824ea3146a Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Sat, 26 Oct 2013 20:28:47 +1030
Subject: [PATCH 20/24] Fix some test failures in new HTTP server code

---
 http_server.c         | 104 +++++++++++++++++++++++++++---------------
 http_server.h         |   4 +-
 rhizome.h             |   2 +-
 rhizome_fetch.c       |   6 +--
 rhizome_http.c        |  45 +++++++++---------
 tests/rhizomeprotocol |  13 ++++--
 6 files changed, 103 insertions(+), 71 deletions(-)

diff --git a/http_server.c b/http_server.c
index 60877899..69074709 100644
--- a/http_server.c
+++ b/http_server.c
@@ -519,21 +519,30 @@ static inline int _parse_uint(struct http_request *r, unsigned int *uintp)
 
 static unsigned _parse_ranges(struct http_request *r, struct http_range *range, unsigned nrange)
 {
-  unsigned i;
-  for (i = 0; 1; ++i) {
+  unsigned i = 0;
+  while (1) {
     enum http_range_type type;
     http_size_t first = 0, last = 0;
-    if (_skip_literal(r, "-") && _parse_http_size_t(r, &last))
+    if (_skip_literal(r, "-")) {
+      if (!_parse_http_size_t(r, &last))
+	return 0;
       type = SUFFIX;
-    else if (_parse_http_size_t(r, &first) && _skip_literal(r, "-"))
-      type = (_parse_http_size_t(r, &last)) ? CLOSED : OPEN;
-    else
+    }
+    else if (_parse_http_size_t(r, &first) && _skip_literal(r, "-")) {
+      if (_parse_http_size_t(r, &last)) {
+	if (last < first)
+	  return 0;
+	type = CLOSED;
+      } else
+	type = OPEN;
+    } else
       return 0;
     if (i < nrange) {
       range[i].type = type;
       range[i].first = first;
       range[i].last = last;
     }
+    ++i;
     if (!_skip_literal(r, ","))
       break;
     _skip_optional_space(r);
@@ -1291,7 +1300,9 @@ static void http_request_send_response(struct http_request *r)
     assert(r->response_buffer_sent <= r->response_buffer_length);
     if (r->response_buffer_sent == r->response_buffer_length) {
       if (r->response.content_generator) {
-	// Content generator must fill set (or re-set) response_buffer and response_buffer_length.
+	// Content generator must fill or partly fill response_buffer and set response_buffer_sent
+	// and response_buffer_length.  May also malloc() a bigger buffer and set response_buffer to
+	// point to it.
 	r->response_buffer_sent = r->response_buffer_length = 0;
 	if (r->response.content_generator(r) == -1) {
 	  if (r->debug_flag && *r->debug_flag)
@@ -1380,32 +1391,23 @@ static void http_server_poll(struct sched_ent *alarm)
     r->free(r); // after this, *r is no longer valid
 }
 
-/* Count the number of bytes in a given set of ranges, given the length of the content to which the
- * ranges will be applied.
- *
- * @author Andrew Bettison <andrew@servalproject.com>
- */
-http_size_t http_range_bytes(const struct http_range *range, unsigned nranges, http_size_t resource_length)
-{
-  return http_range_close(NULL, range, nranges, resource_length);
-}
-
 /* Copy the array of byte ranges, closing it (converting all ranges to CLOSED) using the supplied
- * resource length.
+ * resource length.  If a range is not satisfiable it is omitted from 'dst'.  Returns the number of
+ * closed ranges written to 'dst'.
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-http_size_t http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t resource_length)
+unsigned http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t resource_length)
 {
-  http_size_t bytes = 0;
   unsigned i;
+  unsigned ndst = 0;
   for (i = 0; i != nranges; ++i) {
     http_size_t first = 0;
-    http_size_t last = resource_length;
+    http_size_t last = resource_length - 1;
     const struct http_range *range = &src[i];
     switch (range->type) {
       case CLOSED:
-	last = range->last < resource_length ? range->last : resource_length;
+	last = range->last < resource_length ? range->last : resource_length - 1;
       case OPEN:
 	first = range->first < resource_length ? range->first : resource_length;
 	break;
@@ -1415,10 +1417,25 @@ http_size_t http_range_close(struct http_range *dst, const struct http_range *sr
       default:
 	abort(); // not reached
     }
-    assert(first <= last);
-    if (dst)
-      *dst++ = (struct http_range){ .type = CLOSED, .first=first, .last=last };
-    bytes += last - first;
+    if (first <= last)
+      dst[ndst++] = (struct http_range){ .type = CLOSED, .first=first, .last=last };
+  }
+  return ndst;
+}
+
+/* Return the total number of bytes represented by the given ranges which must all be CLOSED and
+ * valid.
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+http_size_t http_range_bytes(const struct http_range *range, unsigned nranges)
+{
+  http_size_t bytes = 0;
+  unsigned i;
+  for (i = 0; i != nranges; ++i) {
+    assert(range[i].type == CLOSED);
+    assert(range[i].last >= range[i].first);
+    bytes += range[i].last - range[i].first + 1;
   }
   return bytes;
 }
@@ -1467,11 +1484,17 @@ static strbuf strbuf_append_quoted_string(strbuf sb, const char *str)
  */
 static int _render_response(struct http_request *r)
 {
-  strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size);
   struct http_response hr = r->response;
   assert(hr.result_code != 0);
+  assert(hr.header.content_range_start <= hr.header.resource_length);
+  assert(hr.header.content_length <= hr.header.resource_length);
+  // To save page handlers having to decide between 200 (OK) and 206 (Partial Content), they can
+  // just send 200 and the content range fields, and this logic will detect if it should be 206.
+  if (hr.header.content_length > 0 && hr.header.content_length < hr.header.resource_length && hr.result_code == 200)
+    hr.result_code = 206; // Partial Content
   const char *result_string = httpResultString(hr.result_code);
-  if (hr.content == NULL) {
+  strbuf sb = strbuf_local(r->response_buffer, r->response_buffer_size);
+  if (hr.content == NULL && hr.content_generator == NULL) {
     strbuf cb = strbuf_alloca(100 + strlen(result_string));
     strbuf_puts(cb, "<html><h1>");
     strbuf_puts(cb, result_string);
@@ -1493,23 +1516,32 @@ static int _render_response(struct http_request *r)
       strbuf_puts(sb, hr.header.boundary);
   }
   strbuf_puts(sb, "\r\n");
-  assert(hr.header.content_range_start <= hr.header.resource_length);
-  assert(hr.header.content_length <= hr.header.resource_length);
-  if (hr.header.content_range_start && hr.header.content_length)
+  if (hr.result_code == 206) {
+    // Must only use result code 206 (Partial Content) if the content is in fact less than the whole
+    // resource length.
+    assert(hr.header.content_length > 0);
+    assert(hr.header.content_length < hr.header.resource_length);
     strbuf_sprintf(sb,
 	  "Content-Range: bytes %"PRIhttp_size_t"-%"PRIhttp_size_t"/%"PRIhttp_size_t"\r\n",
 	  hr.header.content_range_start,
 	  hr.header.content_range_start + hr.header.content_length - 1,
 	  hr.header.resource_length
 	);
+  }
   strbuf_sprintf(sb, "Content-Length: %"PRIhttp_size_t"\r\n", hr.header.content_length);
   strbuf_puts(sb, "\r\n");
-  r->response_length = strbuf_len(sb) + hr.header.content_length;
-  if (strbuf_overrun(sb) || (r->response_buffer_size < r->response_length))
+  if (strbuf_overrun(sb))
     return 0;
-  bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
+  r->response_length = strbuf_len(sb) + hr.header.content_length;
+  if (hr.content) {
+    if (r->response_buffer_size < r->response_length)
+      return 0;
+    bcopy(hr.content, strbuf_end(sb), hr.header.content_length);
+    r->response_buffer_length = r->response_length;
+  } else {
+    r->response_buffer_length = strbuf_len(sb);
+  }
   r->response_buffer_sent = 0;
-  r->response_buffer_length = r->response_length;
   return 1;
 }
 
@@ -1584,7 +1616,7 @@ static void http_request_start_response(struct http_request *r)
   }
   r->response_sent = 0;
   if (r->debug_flag && *r->debug_flag)
-    DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_length));
+    DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->response_buffer, r->response_buffer_length));
   r->phase = TRANSMIT;
   r->alarm.poll.events = POLLOUT;
   watch(&r->alarm);
diff --git a/http_server.h b/http_server.h
index ebedb0ad..3cfad0a3 100644
--- a/http_server.h
+++ b/http_server.h
@@ -51,8 +51,8 @@ struct http_range {
   http_size_t last; // only for CLOSED or SUFFIX
 };
 
-http_size_t http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t content_length);
-http_size_t http_range_bytes(const struct http_range *range, unsigned nranges, http_size_t content_length);
+unsigned http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t content_length);
+http_size_t http_range_bytes(const struct http_range *range, unsigned nranges);
 
 #define CONTENT_LENGTH_UNKNOWN   UINT64_MAX
 
diff --git a/rhizome.h b/rhizome.h
index 4188327d..e856ff54 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -707,7 +707,7 @@ int rhizome_fetch_status_html(struct strbuf *b);
 int rhizome_fetch_has_queue_space(unsigned char log2_size);
 
 struct http_response_parts {
-  int code;
+  uint16_t code;
   char *reason;
   int64_t range_start;
   int64_t content_length;
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 6713c193..d0b61e4e 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -1495,9 +1495,9 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
 	    rhizome_fetch_switch_to_mdp(slot);
 	    return;
 	  }
-	  if (parts.code != 200) {
+	  if (parts.code != 200 && parts.code != 206) {
 	    if (config.debug.rhizome_rx)
-	      DEBUGF("Failed HTTP request: rhizome server returned %d != 200 OK", parts.code);
+	      DEBUGF("Failed HTTP request: rhizome server returned %03u", parts.code);
 	    rhizome_fetch_switch_to_mdp(slot);
 	    return;
 	  }
@@ -1576,7 +1576,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
 int unpack_http_response(char *response, struct http_response_parts *parts)
 {
   IN();
-  parts->code = -1;
+  parts->code = 0;
   parts->reason = NULL;
   parts->range_start=0;
   parts->content_length = -1;
diff --git a/rhizome_http.c b/rhizome_http.c
index b0c04668..dc83d203 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -381,11 +381,10 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
 static int rhizome_file_content(struct http_request *hr)
 {
   rhizome_http_request *r = (rhizome_http_request *) hr;
-  assert(r->http.response_length < r->http.response_buffer_size);
-  assert(r->read_state.offset <= r->read_state.length);
+  assert(r->http.response_buffer_sent == 0);
+  assert(r->http.response_buffer_length == 0);
+  assert(r->read_state.offset < r->read_state.length);
   uint64_t readlen = r->read_state.length - r->read_state.offset;
-  if (readlen == 0)
-    return 0;
   size_t suggested_size = 64 * 1024;
   if (suggested_size > readlen)
     suggested_size = readlen;
@@ -395,14 +394,13 @@ static int rhizome_file_content(struct http_request *hr)
     http_request_set_response_bufsize(&r->http, 1);
   if (r->http.response_buffer == NULL)
     return -1;
-  size_t space = r->http.response_buffer_size - r->http.response_length;
-  int len = rhizome_read(&r->read_state,
-		         (unsigned char *)r->http.response_buffer + r->http.response_length,
-			 space);
+  ssize_t len = rhizome_read(&r->read_state,
+		         (unsigned char *)r->http.response_buffer,
+			 r->http.response_buffer_size);
   if (len == -1)
     return -1;
-  assert(len <= space);
-  r->http.response_length += len;
+  assert((size_t) len <= r->http.response_buffer_size);
+  r->http.response_buffer_length += (size_t) len;
   return 0;
 }
 
@@ -437,25 +435,24 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
     return 1;
   }
   assert(r->read_state.length != -1);
-  int result_code = 200;
-  struct http_range closed = (struct http_range){ .first = 0, .last = r->read_state.length };
+  r->http.response.header.resource_length = r->read_state.length;
   if (r->http.request_header.content_range_count > 0) {
-    if (http_range_bytes(r->http.request_header.content_ranges,
-			 r->http.request_header.content_range_count,
-			 r->read_state.length
-			) == 0
-    ) {
+    assert(r->http.request_header.content_range_count == 1);
+    struct http_range closed;
+    unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length);
+    if (n == 0 || http_range_bytes(&closed, 1) == 0) {
       http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
       return 0;
     }
-    result_code = 206; // Partial Content
-    http_range_close(&closed, &r->http.request_header.content_ranges[0], 1, r->read_state.length);
+    r->http.response.header.content_range_start = closed.first;
+    r->http.response.header.content_length = closed.last - closed.first + 1;
+    r->read_state.offset = closed.first;
+  } else {
+    r->http.response.header.content_range_start = 0;
+    r->http.response.header.content_length = r->http.response.header.resource_length;
+    r->read_state.offset = 0;
   }
-  r->http.response.header.content_range_start = closed.first;
-  r->http.response.header.resource_length = closed.last;
-  r->http.response.header.content_length = closed.last - closed.first;
-  r->read_state.offset = closed.first;
-  http_request_response_generated(&r->http, result_code, "application/binary", rhizome_file_content);
+  http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content);
   return 0;
 }
 
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 67d2bbd0..24060cf1 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -514,12 +514,13 @@ test_CorruptPayload() {
    wait_until grep -i "Stored file $FILEHASH" $LOGA
 }
 
-doc_HttpFetchRange="Fetch a file range using HTTP GET."
+doc_HttpFetchRange="Fetch a file range using HTTP GET"
 setup_HttpFetchRange() {
    setup_curl 7
    setup_common
    set_instance +A
-   rhizome_add_file file1
+   rhizome_add_file file1 100
+   tail --bytes +33 file1 >file1.tail
    start_servald_instances +A
    wait_until rhizome_http_server_started +A
    get_rhizome_server_port PORTA +A
@@ -532,9 +533,11 @@ test_HttpFetchRange() {
          --write-out '%{http_code}\n' \
 	 --continue-at 32 \
          "http://$addr_localhost:$PORTA/rhizome/file/$FILEHASH"
-   tfw_cat http.headers http.output
-   assertGrep http.headers "Content-range:"
-   assertGrep http.headers "Content-length:"
+   tfw_cat -v http.headers http.output
+   assertGrep http.headers "^Content-Range: bytes 32-99/100
$"
+   assertGrep http.headers "^Content-Length: 68
$"
+   tfw_cat -v file1.tail http.output
+   assert cmp file1.tail http.output
 }
 
 doc_HttpImport="Import bundle using HTTP POST multi-part form."

From 2a9329c0c82759d5ff1fe2142bd550ccf5386f3e Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Sun, 27 Oct 2013 01:37:07 +1030
Subject: [PATCH 21/24] Fix bugs in new HTTP server MIME body parsing code

Fixes 'rhizomeprotocol' test 24 HttpImport.  Five tests still fail.
---
 http_server.c         | 190 +++++++++++++++++++++++++-----------------
 http_server.h         |   1 -
 rhizome_direct_http.c |   1 +
 rhizome_fetch.c       |   6 +-
 strbuf_helpers.c      |  29 +++++++
 strbuf_helpers.h      |   7 ++
 tests/rhizomeprotocol |   1 +
 7 files changed, 155 insertions(+), 80 deletions(-)

diff --git a/http_server.c b/http_server.c
index 69074709..74f53143 100644
--- a/http_server.c
+++ b/http_server.c
@@ -75,9 +75,12 @@ static struct profile_total http_server_stats = {
 
 #define DEBUG_DUMP_PARSER(r) do { \
       if (config.debug.httpd) \
-	DEBUGF("parsed %d %s cursor %d %s", \
-	    r->parsed - r->received, alloca_toprint(-1, r->received, r->parsed - r->received), \
-	    r->cursor - r->received, alloca_toprint(50, r->cursor, r->end - r->cursor)); \
+	DEBUGF("parsed %d %s cursor %d %s end %d remain %"PRIhttp_size_t, \
+	    r->parsed - r->received, alloca_toprint(-1, r->parsed, r->cursor - r->parsed), \
+	    r->cursor - r->received, alloca_toprint(50, r->cursor, r->end - r->cursor), \
+	    r->end - r->received, \
+	    r->request_content_remaining \
+	  ); \
     } while (0)
 
 static void http_server_poll(struct sched_ent *);
@@ -105,7 +108,6 @@ void http_request_init(struct http_request *r, int sockfd)
   r->alarm.poll.events = POLLIN;
   r->phase = RECEIVE;
   r->received = r->end = r->parsed = r->cursor = r->buffer;
-  r->end_content = NULL;
   r->parser = http_request_parse_verb;
   watch(&r->alarm);
   schedule(&r->alarm);
@@ -262,7 +264,6 @@ static const char * _reserve(struct http_request *r, struct substring str)
     r->response.result_code = 414;
     return NULL;
   }
-  char *ret = (char *) r->received;
   if (r->received + siz > r->parsed) {
     WARNF("Error during HTTP parsing, unparsed content %s would be overwritten by reserving %s",
 	alloca_toprint(30, r->parsed, r->end - r->parsed),
@@ -271,6 +272,7 @@ static const char * _reserve(struct http_request *r, struct substring str)
     r->response.result_code = 500;
     return NULL;
   }
+  char *ret = (char *) r->received;
   if (ret != str.start)
     memmove(ret, str.start, len);
   ret[len] = '\0';
@@ -287,12 +289,7 @@ static const char * _reserve_str(struct http_request *r, const char *str)
 
 static inline int _end_of_content(struct http_request *r)
 {
-  return r->cursor == r->end_content;
-}
-
-static inline int _buffer_full(struct http_request *r)
-{
-  return r->parsed == r->received && r->end == (r->end_content ? r->end_content : r->buffer + sizeof r->buffer);
+  return r->cursor == r->end && r->request_content_remaining == 0;
 }
 
 static inline int _run_out(struct http_request *r)
@@ -325,12 +322,9 @@ static inline int _skip_crlf(struct http_request *r)
 
 static inline int _skip_to_crlf(struct http_request *r)
 {
-  const char *const start = r->cursor;
   for (; !_run_out(r); ++r->cursor)
-    if (*r->cursor == '\n' && r->cursor > start + 1 && r->cursor[-2] == '\r') {
-      --r->cursor;
+    if (r->cursor + 1 < r->end && r->cursor[0] == '\r' && r->cursor[1] == '\n')
       return 1;
-    }
   return 0;
 }
 
@@ -719,6 +713,15 @@ static int http_request_parse_header(struct http_request *r)
   _skip_eol(r);
   if (eol == r->parsed) { // if EOL is at start of line (ie, blank line)...
     _commit(r);
+    if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) {
+      size_t unparsed = r->end - r->parsed;
+      if (unparsed > r->request_header.content_length) {
+	WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length));
+	r->request_content_remaining = 0;
+      }
+      else
+	r->request_content_remaining = r->request_header.content_length - unparsed;
+    }
     r->parser = http_request_start_body;
     if (r->handle_headers)
       return r->handle_headers(r);
@@ -769,7 +772,7 @@ static int http_request_parse_header(struct http_request *r)
 	_commit(r);
 	if (   (r->request_header.content_type = _reserve(r, type)) == NULL
 	    || (r->request_header.content_subtype = _reserve(r, subtype)) == NULL
-	    || (boundary[0] && (r->request_header.content_subtype = _reserve_str(r, boundary)) == NULL)
+	    || (boundary[0] && (r->request_header.boundary = _reserve_str(r, boundary)) == NULL)
 	)
 	  return 0; // error
 	if (r->debug_flag && *r->debug_flag)
@@ -838,10 +841,10 @@ static int http_request_start_body(struct http_request *r)
   assert(r->version_minor != 0);
   assert(r->parsed <= r->end);
   if (r->verb == HTTP_VERB_GET) {
-    // TODO: Implement HEAD requests
+    // TODO: Implement HEAD requests (only send response header, not body)
     if (r->request_header.content_length != 0 && r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) {
       if (r->debug_flag && *r->debug_flag)
-	DEBUGF("Malformed HTTP %s request: Content-Length not allowed", r->verb);
+	DEBUGF("Malformed HTTP %s request: non-zero Content-Length not allowed", r->verb);
       return 400;
     }
     if (r->request_header.content_type) {
@@ -879,13 +882,6 @@ static int http_request_start_body(struct http_request *r)
 	    r->verb, r->request_header.content_type, r->request_header.content_subtype);
       return 415;
     }
-    size_t unparsed = r->end - r->parsed;
-    if (unparsed > r->request_header.content_length) {
-      WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length));
-      r->request_content_remaining = 0;
-    }
-    else
-      r->request_content_remaining = r->request_header.content_length - unparsed;
   }
   else {
     if (r->debug_flag && *r->debug_flag)
@@ -893,6 +889,8 @@ static int http_request_start_body(struct http_request *r)
     r->parser = NULL;
     return 501;
   }
+  if (_run_out(r))
+    return 100;
   return 0;
 }
 
@@ -993,27 +991,39 @@ malformed:
  */
 static int http_request_parse_body_form_data(struct http_request *r)
 {
+  DEBUG_DUMP_PARSER(r);
   int at_start = 0;
   switch (r->form_data_state) {
     case START:
+      if (config.debug.httpd)
+	DEBUGF("START");
       // The logic here allows for a missing initial CRLF before the first boundary line.
       at_start = 1;
       r->form_data_state = PREAMBLE;
       // fall through
     case PREAMBLE:
+      if (config.debug.httpd)
+	DEBUGF("PREAMBLE");
       while (!_run_out(r)) {
 	const char *end_preamble = r->cursor;
 	int b;
 	if ((_skip_crlf(r) || at_start) && (b = _skip_mime_boundary(r))) {
 	  assert(end_preamble >= r->parsed);
-	  if (r->form_data.handle_mime_preamble && end_preamble != r->parsed)
+	  if (r->form_data.handle_mime_preamble && end_preamble != r->parsed) {
+	    if (r->debug_flag && *r->debug_flag)
+	      DEBUGF("handle_mime_preamble(%s length=%zu)",
+		  alloca_toprint(50, r->parsed, end_preamble - r->parsed), end_preamble - r->parsed);
 	    r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed);
+	  }
 	  _rewind_crlf(r);
 	  _commit(r);
 	  if (b == 1) {
 	    r->form_data_state = HEADER;
-	    if (r->form_data.handle_mime_part_start)
+	    if (r->form_data.handle_mime_part_start) {
+	      if (r->debug_flag && *r->debug_flag)
+		DEBUGF("handle_mime_part_start()");
 	      r->form_data.handle_mime_part_start(r);
+	    }
 	  } else
 	    r->form_data_state = EPILOGUE;
 	  return 0;
@@ -1028,11 +1038,17 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	}
 	at_start = 0;
       }
-      if (r->cursor > r->parsed && r->form_data.handle_mime_preamble)
-	r->form_data.handle_mime_preamble(r, r->parsed, r->parsed - r->cursor);
+      if (r->cursor > r->parsed && r->form_data.handle_mime_preamble) {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("handle_mime_preamble(%s length=%zu)",
+	      alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed);
+	r->form_data.handle_mime_preamble(r, r->parsed, r->cursor - r->parsed);
+      }
       _commit(r);
-      return 0;
+      return 100; // need more data
     case HEADER: {
+      if (config.debug.httpd)
+	DEBUGF("HEADER");
 	if (_skip_crlf(r) && _skip_crlf(r)) {
 	  _commit(r);
 	  r->form_data_state = BODY;
@@ -1045,13 +1061,19 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
 	  _rewind_crlf(r);
 	  _commit(r);
-	  if (r->form_data.handle_mime_part_end)
+	  if (r->form_data.handle_mime_part_end) {
+	    if (r->debug_flag && *r->debug_flag)
+	      DEBUGF("handle_mime_part_end()");
 	    r->form_data.handle_mime_part_end(r);
+	  }
 	  // Boundary in the middle of headers starts a new part.
 	  if (b == 1) {
 	    r->form_data_state = HEADER;
-	    if (r->form_data.handle_mime_part_start)
+	    if (r->form_data.handle_mime_part_start) {
+	      if (r->debug_flag && *r->debug_flag)
+		DEBUGF("handle_mime_part_start()");
 	      r->form_data.handle_mime_part_start(r);
+	    }
 	  }
 	  else
 	    r->form_data_state = EPILOGUE;
@@ -1067,19 +1089,26 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  strncpy(labelstr, label.start, labellen)[labellen] = '\0';
 	  str_tolower_inplace(labelstr);
 	  const char *value = r->cursor;
+	  DEBUG_DUMP_PARSER(r);
 	  if (strcmp(labelstr, "content-disposition") == 0) {
 	    struct mime_content_disposition cd;
 	    bzero(&cd, sizeof cd);
 	    if (_parse_content_disposition(r, &cd) && _skip_optional_space(r) && _skip_crlf(r)) {
-	      if (r->form_data.handle_mime_content_disposition)
+	      if (r->form_data.handle_mime_content_disposition) {
+		if (r->debug_flag && *r->debug_flag)
+		  DEBUGF("handle_mime_content_disposition(%s)", alloca_mime_content_disposition(&cd));
 		r->form_data.handle_mime_content_disposition(r, &cd);
+	      }
+	      _rewind_crlf(r);
 	      _commit(r);
 	      return 0;
 	    }
 	  } else if (_skip_to_crlf(r)) {
-	    if (r->form_data.handle_mime_header)
+	    if (r->form_data.handle_mime_header) {
+	      if (r->debug_flag && *r->debug_flag)
+		DEBUGF("handle_mime_header(%s, %s)", alloca_str_toprint(labelstr), alloca_toprint(-1, value, value - r->cursor));
 	      r->form_data.handle_mime_header(r, labelstr, value, value - r->cursor); // excluding CRLF at end
-	    _skip_crlf(r);
+	    }
 	    _commit(r);
 	    return 0;
 	  }
@@ -1089,24 +1118,37 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	_rewind(r);
       }
       if (r->debug_flag && *r->debug_flag)
-	DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(20, r->parsed, r->end - r->parsed));
+	DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(50, r->parsed, r->end - r->parsed));
+      DEBUG_DUMP_PARSER(r);
       return 400;
     case BODY:
+      if (config.debug.httpd)
+	DEBUGF("BODY");
+      const char *start = r->cursor;
       while (!_run_out(r)) {
 	int b;
 	const char *eol = r->cursor;
 	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
 	  _rewind_crlf(r);
 	  _commit(r);
-	  if (r->form_data.handle_mime_body)
-	    r->form_data.handle_mime_body(r, r->parsed, r->parsed - eol); // excluding CRLF at end
-	  if (r->form_data.handle_mime_part_end)
+	  if (r->form_data.handle_mime_body) {
+	    if (r->debug_flag && *r->debug_flag)
+	      DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(50, start, eol - start), eol - start);
+	    r->form_data.handle_mime_body(r, start, eol - start); // excluding CRLF at end
+	  }
+	  if (r->form_data.handle_mime_part_end) {
+	    if (r->debug_flag && *r->debug_flag)
+	      DEBUGF("handle_mime_part_end()");
 	    r->form_data.handle_mime_part_end(r);
+	  }
 	  r->form_data_state = EPILOGUE;
 	  if (b == 1) {
 	    r->form_data_state = HEADER;
-	    if (r->form_data.handle_mime_part_start)
+	    if (r->form_data.handle_mime_part_start) {
+	      if (r->debug_flag && *r->debug_flag)
+		DEBUGF("handle_mime_part_start()");
 	      r->form_data.handle_mime_part_start(r);
+	    }
 	  }
 	  return 0;
 	}
@@ -1119,17 +1161,28 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  return 100; // need more data
 	}
       }
-      if (r->cursor > r->parsed && r->form_data.handle_mime_body)
-	r->form_data.handle_mime_body(r, r->parsed, r->parsed - r->cursor);
+      if (r->cursor > r->parsed && r->form_data.handle_mime_body) {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed);
+	r->form_data.handle_mime_body(r, r->parsed, r->cursor - r->parsed);
+      }
       _commit(r);
-      return 0;
+      return 100; // need more data
   case EPILOGUE:
+      if (config.debug.httpd)
+	DEBUGF("EPILOGUE");
     r->cursor = r->end;
-    if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed)
+    if (r->form_data.handle_mime_epilogue && r->cursor != r->parsed) {
+      if (r->debug_flag && *r->debug_flag)
+	DEBUGF("handle_mime_epilogue(%s length=%zu)",
+	    alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed);
       r->form_data.handle_mime_epilogue(r, r->parsed, r->cursor - r->parsed);
+    }
     _commit(r);
-    r->parser = NULL;
-    return 0;
+    assert(_run_out(r));
+    if (_end_of_content(r))
+      return 0; // done
+    return 100; // need more data
   }
   abort(); // not reached
 }
@@ -1160,33 +1213,15 @@ static void http_request_receive(struct http_request *r)
   assert(r->end <= bufend);
   assert(r->parsed >= r->received);
   assert(r->parsed <= r->end);
-  // Work out if the end of content falls within the buffer yet.  If so, set the end_content
-  // pointer (and make sure it doesn't move).
-  if (r->end_content) {
-    assert(r->request_content_remaining != CONTENT_LENGTH_UNKNOWN);
-    assert(r->end < r->end_content);
-    assert(r->end_content - r->end == r->request_content_remaining);
-  } else if (   r->request_content_remaining != CONTENT_LENGTH_UNKNOWN
-	     && r->request_content_remaining < bufend - r->end
-  ) {
-    r->end_content = r->end + r->request_content_remaining;
-  }
-  //
-  // If the end of content mark is within the buffer, then there is no need to make any more room,
-  // just keep reading up to the end of content.  Otherwise, If buffer is running short on unused
-  // space, shift existing content in buffer down to make more room if possible.
-  size_t room;
-  if (r->end_content) {
-    assert(r->end_content > r->buffer);
-    assert(r->end_content <= bufend);
-    room = r->end_content - r->end;
-  } else {
-    room = bufend - r->end;
+  // If the end of content falls within the buffer, then there is no need to make any more room,
+  // just read up to the end of content.  Otherwise, If buffer is running short on unused space,
+  // shift existing content in buffer down to make more room if possible.
+  size_t room = bufend - r->end;
+  if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && room > r->request_content_remaining)
+    room = r->request_content_remaining;
+  else {
     size_t spare = r->parsed - r->received;
-    if (   spare
-	&& (room < 128 || (room < 1024 && spare >= 32))
-	&& (r->request_content_remaining == CONTENT_LENGTH_UNKNOWN || spare >= r->request_content_remaining)
-    ) {
+    if (spare && (room < 128 || (room < 1024 && spare >= 32))) {
       size_t unparsed = r->end - r->parsed;
       memmove((char *)r->received, r->parsed, unparsed); // memcpy() does not handle overlapping src and dst
       r->parsed = r->received;
@@ -1228,7 +1263,9 @@ static void http_request_receive(struct http_request *r)
   // Parse the unparsed and received data.
   while (r->phase == RECEIVE) {
     int result;
-    if (r->parsed == r->end_content) {
+    _rewind(r);
+    DEBUG_DUMP_PARSER(r);
+    if (_end_of_content(r)) {
       if (r->handle_content_end)
 	result = r->handle_content_end(r);
       else {
@@ -1239,7 +1276,6 @@ static void http_request_receive(struct http_request *r)
     } else {
       HTTP_REQUEST_PARSER oldparser = r->parser;
       const char *oldparsed = r->parsed;
-      _rewind(r);
       if (r->parser == NULL) {
 	if (r->debug_flag && *r->debug_flag)
 	  DEBUGF("No HTTP parser function set -- skipping %zu bytes", (size_t)(r->end - r->cursor));
@@ -1659,8 +1695,8 @@ void http_request_response_generated(struct http_request *r, int result, const c
   http_request_start_response(r);
 }
 
-/* Start sending a short redirection or error response back to the client.  The result code must be
- * either a redirection (3xx) or client error (4xx) or server error (5xx) code.  The 'body' argument
+/* Start sending a short response back to the client.  The result code must be either a success
+ * (2xx), redirection (3xx) or client error (4xx) or server error (5xx) code.  The 'body' argument
  * may be a bare message which is enclosed in an HTML envelope to form the response content, so it
  * may contain HTML markup.  If the 'body' argument is NULL, then the response content is generated
  * automatically from the result code.
@@ -1670,7 +1706,7 @@ void http_request_response_generated(struct http_request *r, int result, const c
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body)
 {
   assert(r->phase == RECEIVE);
-  assert(result >= 300);
+  assert(result >= 200);
   assert(result < 600);
   strbuf h = NULL;
   if (body) {
diff --git a/http_server.h b/http_server.h
index 3cfad0a3..a7bf6a63 100644
--- a/http_server.h
+++ b/http_server.h
@@ -139,7 +139,6 @@ struct http_request {
   struct http_request_headers request_header;
   const char *received; // start of received data in buffer[]
   const char *end; // end of received data in buffer[]
-  const char *end_content; // end of content if within buffer[], else NULL
   const char *parsed; // start of unparsed data in buffer[]
   const char *cursor; // for parsing
   http_size_t request_content_remaining;
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index 90bce08a..9f0875d2 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -299,6 +299,7 @@ void rhizome_direct_process_mime_end(struct http_request *hr)
     case NONE:
       break;
   }
+  r->current_part = NONE;
 }
 
 void rhizome_direct_process_mime_content_disposition(struct http_request *hr, const struct mime_content_disposition *cd)
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index d0b61e4e..7c8dc534 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -503,8 +503,10 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
 	rhizome_manifest_free(slot->previous);
 	slot->previous=NULL;
       }else{
-	strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n", 
-	  slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength);
+	assert(slot->previous->fileLength >= slot->manifest->journalTail);
+	assert(slot->manifest->fileLength > 0);
+	strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n",
+	  slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength - 1);
       }
     }
 
diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 32745280..4bb19474 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -420,3 +420,32 @@ strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, uns
   }
   return sb;
 }
+
+strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd)
+{
+  strbuf_puts(sb, "type=");
+  strbuf_toprint_quoted(sb, "``", cd->type);
+  strbuf_puts(sb, " name=");
+  strbuf_toprint_quoted(sb, "``", cd->name);
+  strbuf_puts(sb, " filename=");
+  strbuf_toprint_quoted(sb, "``", cd->filename);
+  strbuf_puts(sb, " size=");
+  strbuf_sprintf(sb, "%"PRIhttp_size_t, cd->size);
+  struct tm tm;
+  strbuf_puts(sb, " creation_date=");
+  if (cd->creation_date)
+    strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->creation_date, &tm));
+  else
+    strbuf_puts(sb, "0");
+  strbuf_puts(sb, " modification_date=");
+  if (cd->modification_date)
+    strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->modification_date, &tm));
+  else
+    strbuf_puts(sb, "0");
+  strbuf_puts(sb, " read_date=");
+  if (cd->read_date)
+    strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->read_date, &tm));
+  else
+    strbuf_puts(sb, "0");
+  return sb;
+}
diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index 3d3016c8..8bc45de5 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -144,4 +144,11 @@ struct http_range;
 strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels);
 #define alloca_http_ranges(ra)    strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra)))
 
+/* Append a representation of a struct mime_content_disposition struct.
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+struct mime_content_disposition;
+strbuf strbuf_append_mime_content_disposition(strbuf, const struct mime_content_disposition *);
+#define alloca_mime_content_disposition(cd) strbuf_str(strbuf_append_mime_content_disposition(strbuf_alloca(500), (cd)))
+
 #endif //__STRBUF_HELPERS_H__
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 24060cf1..b9eb3545 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -41,6 +41,7 @@ configure_servald_server() {
       set log.console.show_pid on \
       set log.console.show_time on \
       set debug.rhizome on \
+      set debug.httpd on \
       set debug.rhizome_httpd on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \

From 8f60a4ceb52bef3ea5e8b546584f772c074dca58 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 28 Oct 2013 12:08:57 +1030
Subject: [PATCH 22/24] Fix bugs in new HTTP server MIME body parsing code

Fixes 'rhizomeprotocol' test 24 HttpAddLocal.  Four tests still fail.
---
 http_server.h         |  2 +-
 rhizome_direct_http.c | 11 +++++------
 rhizome_http.c        |  2 +-
 strbuf_helpers.c      | 20 +++++++++++++-------
 strbuf_helpers.h      |  8 ++++++++
 5 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/http_server.h b/http_server.h
index a7bf6a63..a647c30e 100644
--- a/http_server.h
+++ b/http_server.h
@@ -123,7 +123,7 @@ struct http_request {
   bool_t *disable_tx_flag;
   time_ms_t initiate_time; // time connection was initiated
   time_ms_t idle_timeout; // disconnect if no bytes received for this long
-  struct sockaddr_in client_in_addr;
+  struct sockaddr_in client_sockaddr_in;
   HTTP_REQUEST_PARSER parser; // current parser function
   HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed
   HTTP_REQUEST_PARSER handle_headers; // called after all headers are parsed
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index 9f0875d2..2ffd9875 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -381,13 +381,12 @@ int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
     http_request_simple_response(&r->http, 405, NULL);
     return 0;
   }
-  if (cmp_sockaddr((struct sockaddr *)&r->http.client_in_addr, sizeof r->http.client_in_addr,
-		   (struct sockaddr *)&config.rhizome.api.addfile.allow_host, sizeof config.rhizome.api.addfile.allow_host
-		  ) != 0
+  if (   r->http.client_sockaddr_in.sin_family != AF_INET
+      || r->http.client_sockaddr_in.sin_addr.s_addr != config.rhizome.api.addfile.allow_host.s_addr
   ) {
-    INFOF("rhizome.api.addfile request received from %s, but is only allowed from %s",
-	alloca_sockaddr(&r->http.client_in_addr, sizeof r->http.client_in_addr),
-	alloca_sockaddr(&config.rhizome.api.addfile.allow_host, sizeof config.rhizome.api.addfile.allow_host)
+    INFOF("rhizome.api.addfile request received from %s, but is only allowed from AF_INET %s",
+	alloca_sockaddr(&r->http.client_sockaddr_in, sizeof r->http.client_sockaddr_in),
+	alloca_in_addr(&config.rhizome.api.addfile.allow_host)
       );
     rhizome_direct_clear_temporary_files(r);
     http_request_simple_response(&r->http, 404, "<html><h1>Not available from here</h1></html>");
diff --git a/rhizome_http.c b/rhizome_http.c
index dc83d203..d08180eb 100644
--- a/rhizome_http.c
+++ b/rhizome_http.c
@@ -270,7 +270,7 @@ void rhizome_server_poll(struct sched_ent *alarm)
 	request->read_state.blob_fd = -1;
 	request->read_state.blob_rowid = -1;
 	if (peerip)
-	  request->http.client_in_addr = *peerip;
+	  request->http.client_sockaddr_in = *peerip;
 	request->http.handle_headers = rhizome_dispatch;
 	request->http.debug_flag = &config.debug.rhizome_httpd;
 	request->http.disable_tx_flag = &config.debug.rhizome_nohttptx;
diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 4bb19474..de8446d6 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -313,6 +313,16 @@ strbuf strbuf_append_socket_type(strbuf sb, int type)
   return sb;
 }
 
+strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr)
+{
+  strbuf_sprintf(sb, " %u.%u.%u.%u",
+      ((unsigned char *) &addr->s_addr)[0],
+      ((unsigned char *) &addr->s_addr)[1],
+      ((unsigned char *) &addr->s_addr)[2],
+      ((unsigned char *) &addr->s_addr)[3]);
+  return sb;
+}
+
 strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen)
 {
   strbuf_append_socket_domain(sb, addr->sa_family);
@@ -336,13 +346,9 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t
     break;
   case AF_INET: {
       const struct sockaddr_in *addr_in = (const struct sockaddr_in *) addr;
-      strbuf_sprintf(sb, " %u.%u.%u.%u:%u",
-	  ((unsigned char *) &addr_in->sin_addr.s_addr)[0],
-	  ((unsigned char *) &addr_in->sin_addr.s_addr)[1],
-	  ((unsigned char *) &addr_in->sin_addr.s_addr)[2],
-	  ((unsigned char *) &addr_in->sin_addr.s_addr)[3],
-	  ntohs(addr_in->sin_port)
-	);
+      strbuf_putc(sb, ' ');
+      strbuf_append_in_addr(sb, &addr_in->sin_addr);
+      strbuf_sprintf(sb, ":%u", ntohs(addr_in->sin_port));
       if (addrlen != sizeof(struct sockaddr_in))
 	strbuf_sprintf(sb, " (addrlen=%d should be %zd)", (int)addrlen, sizeof(struct sockaddr_in));
     }
diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index 8bc45de5..289c7ddf 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -117,6 +117,14 @@ strbuf strbuf_append_socket_domain(strbuf sb, int domain);
 strbuf strbuf_append_socket_type(strbuf sb, int type);
 #define alloca_socket_type(type)    strbuf_str(strbuf_append_socket_type(strbuf_alloca(15), type))
 
+/* Append a textual description of a struct in_addr (in network order) as IPv4
+ * quartet "N.N.N.N".
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+struct in_addr;
+strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr);
+#define alloca_in_addr(addr)    strbuf_str(strbuf_append_in_addr(strbuf_alloca(16), (const struct in_addr *)(addr)))
+
 /* Append a textual description of a struct sockaddr_in.
  * @author Andrew Bettison <andrew@servalproject.com>
  */

From 6ee691b1611ebbd9bb5d600cd35af60d640afc97 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 28 Oct 2013 14:25:16 +1030
Subject: [PATCH 23/24] Fix bugs in new HTTP server MIME body parsing code

Fixes all remaining 'rhizomeprotocol' test failures.
---
 http_server.c         | 119 ++++++++++++++++++++----------------------
 rhizome_direct_http.c |   3 --
 tests/rhizomeprotocol |   2 +
 3 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/http_server.c b/http_server.c
index 74f53143..2fe89968 100644
--- a/http_server.c
+++ b/http_server.c
@@ -838,7 +838,6 @@ static int http_request_start_body(struct http_request *r)
   assert(r->verb != NULL);
   assert(r->path != NULL);
   assert(r->version_major != 0);
-  assert(r->version_minor != 0);
   assert(r->parsed <= r->end);
   if (r->verb == HTTP_VERB_GET) {
     // TODO: Implement HEAD requests (only send response header, not body)
@@ -900,9 +899,9 @@ static int _skip_mime_boundary(struct http_request *r)
 {
   if (!_skip_literal(r, "--") || !_skip_literal(r, r->request_header.boundary))
     return 0;
-  if (_skip_literal(r, "--") && _skip_optional_space(r) && _skip_crlf(r))
+  if (_skip_literal(r, "--") && _skip_crlf(r))
     return 2;
-  if (_skip_optional_space(r) && _skip_crlf(r))
+  if (_skip_crlf(r))
     return 1;
   return 0;
 }
@@ -1001,51 +1000,49 @@ static int http_request_parse_body_form_data(struct http_request *r)
       at_start = 1;
       r->form_data_state = PREAMBLE;
       // fall through
-    case PREAMBLE:
-      if (config.debug.httpd)
-	DEBUGF("PREAMBLE");
-      while (!_run_out(r)) {
-	const char *end_preamble = r->cursor;
-	int b;
-	if ((_skip_crlf(r) || at_start) && (b = _skip_mime_boundary(r))) {
-	  assert(end_preamble >= r->parsed);
-	  if (r->form_data.handle_mime_preamble && end_preamble != r->parsed) {
-	    if (r->debug_flag && *r->debug_flag)
-	      DEBUGF("handle_mime_preamble(%s length=%zu)",
-		  alloca_toprint(50, r->parsed, end_preamble - r->parsed), end_preamble - r->parsed);
-	    r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed);
-	  }
-	  _rewind_crlf(r);
-	  _commit(r);
-	  if (b == 1) {
-	    r->form_data_state = HEADER;
-	    if (r->form_data.handle_mime_part_start) {
+    case PREAMBLE: {
+	if (config.debug.httpd)
+	  DEBUGF("PREAMBLE");
+	const char *start = r->parsed;
+	for (; at_start || _skip_to_crlf(r); at_start = 0) {
+	  const char *end_preamble = r->cursor;
+	  int b;
+	  if ((b = _skip_mime_boundary(r))) {
+	    assert(end_preamble >= r->parsed);
+	    if (r->form_data.handle_mime_preamble && end_preamble != r->parsed) {
 	      if (r->debug_flag && *r->debug_flag)
-		DEBUGF("handle_mime_part_start()");
-	      r->form_data.handle_mime_part_start(r);
+		DEBUGF("handle_mime_preamble(%s length=%zu)",
+		    alloca_toprint(50, r->parsed, end_preamble - r->parsed), end_preamble - r->parsed);
+	      r->form_data.handle_mime_preamble(r, r->parsed, end_preamble - r->parsed);
 	    }
-	  } else
-	    r->form_data_state = EPILOGUE;
-	  return 0;
-	}
-	if (!_skip_to_crlf(r)) {
-	  if (_end_of_content(r)) {
-	    if (r->debug_flag && *r->debug_flag)
-	      DEBUGF("Malformed HTTP %s form data: missing first boundary", r->verb);
-	    return 400;
+	    _rewind_crlf(r);
+	    _commit(r);
+	    if (b == 1) {
+	      r->form_data_state = HEADER;
+	      if (r->form_data.handle_mime_part_start) {
+		if (r->debug_flag && *r->debug_flag)
+		  DEBUGF("handle_mime_part_start()");
+		r->form_data.handle_mime_part_start(r);
+	      }
+	    } else
+	      r->form_data_state = EPILOGUE;
+	    return 0;
 	  }
-	  return 100; // need more data
 	}
-	at_start = 0;
+	if (_end_of_content(r)) {
+	  if (r->debug_flag && *r->debug_flag)
+	    DEBUGF("Malformed HTTP %s form data: missing first boundary", r->verb);
+	  return 400;
+	}
+	_commit(r);
+	if (r->parsed > start && r->form_data.handle_mime_preamble) {
+	  if (r->debug_flag && *r->debug_flag)
+	    DEBUGF("handle_mime_preamble(%s length=%zu)",
+		alloca_toprint(50, start, r->parsed - start), r->parsed - start);
+	  r->form_data.handle_mime_preamble(r, start, r->parsed - start);
+	}
+	return 100; // need more data
       }
-      if (r->cursor > r->parsed && r->form_data.handle_mime_preamble) {
-	if (r->debug_flag && *r->debug_flag)
-	  DEBUGF("handle_mime_preamble(%s length=%zu)",
-	      alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed);
-	r->form_data.handle_mime_preamble(r, r->parsed, r->cursor - r->parsed);
-      }
-      _commit(r);
-      return 100; // need more data
     case HEADER: {
       if (config.debug.httpd)
 	DEBUGF("HEADER");
@@ -1124,17 +1121,18 @@ static int http_request_parse_body_form_data(struct http_request *r)
     case BODY:
       if (config.debug.httpd)
 	DEBUGF("BODY");
-      const char *start = r->cursor;
-      while (!_run_out(r)) {
+      const char *start = r->parsed;
+      while (_skip_to_crlf(r)) {
 	int b;
-	const char *eol = r->cursor;
-	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
+	const char *end_body = r->cursor;
+	_skip_crlf(r);
+	if ((b = _skip_mime_boundary(r))) {
 	  _rewind_crlf(r);
 	  _commit(r);
-	  if (r->form_data.handle_mime_body) {
+	  if (end_body > start && r->form_data.handle_mime_body) {
 	    if (r->debug_flag && *r->debug_flag)
-	      DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(50, start, eol - start), eol - start);
-	    r->form_data.handle_mime_body(r, start, eol - start); // excluding CRLF at end
+	      DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, end_body - start), end_body - start);
+	    r->form_data.handle_mime_body(r, start, end_body - start); // excluding CRLF at end
 	  }
 	  if (r->form_data.handle_mime_part_end) {
 	    if (r->debug_flag && *r->debug_flag)
@@ -1152,21 +1150,18 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  }
 	  return 0;
 	}
-	if (!_skip_to_crlf(r)) {
-	  if (_end_of_content(r)) {
-	    if (r->debug_flag && *r->debug_flag)
-	      DEBUGF("Malformed HTTP %s form data part: missing end boundary", r->verb);
-	    return 400;
-	  }
-	  return 100; // need more data
-	}
       }
-      if (r->cursor > r->parsed && r->form_data.handle_mime_body) {
+      if (_end_of_content(r)) {
 	if (r->debug_flag && *r->debug_flag)
-	  DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(50, r->parsed, r->cursor - r->parsed), r->cursor - r->parsed);
-	r->form_data.handle_mime_body(r, r->parsed, r->cursor - r->parsed);
+	  DEBUGF("Malformed HTTP %s form data part: missing end boundary", r->verb);
+	return 400;
       }
       _commit(r);
+      if (r->parsed > start && r->form_data.handle_mime_body) {
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("handle_mime_body(%s length=%zu)", alloca_toprint(80, start, r->parsed - start), r->parsed - start);
+	r->form_data.handle_mime_body(r, start, r->parsed - start);
+      }
       return 100; // need more data
   case EPILOGUE:
       if (config.debug.httpd)
@@ -1227,6 +1222,8 @@ static void http_request_receive(struct http_request *r)
       r->parsed = r->received;
       r->end = r->received + unparsed;
       room = bufend - r->end;
+      if (r->request_content_remaining != CONTENT_LENGTH_UNKNOWN && room > r->request_content_remaining)
+	room = r->request_content_remaining;
     }
   }
   // If there is no more buffer space, fail the request.
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index 2ffd9875..220fbcbd 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -671,9 +671,6 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  +m->fileLength
 	  +strlen("\r\n--")+strlen(boundary)+strlen("--\r\n");
 
-	/* XXX For some reason the above is four bytes out, so fix that */
-	content_length+=4;
-
 	int len=snprintf(buffer,8192,template,content_length,boundary);
 	len+=snprintf(&buffer[len],8192-len,template2,boundary);
 	memcpy(&buffer[len],m->manifestdata,m->manifest_all_bytes);
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index b9eb3545..2b81620e 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -657,6 +657,8 @@ setup_DirectPush() {
    setup_common
    setup_direct
    setup_direct_peer
+   executeOk ls -l
+   tfw_cat --stdout
 }
 test_DirectPush() {
    set_instance +B

From 3cac1b51a1a506dd9d93fa4c9d1ae3fa11d577ec Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 28 Oct 2013 17:50:49 +1030
Subject: [PATCH 24/24] Fix subtle MIME parsing bug

---
 http_server.c | 73 ++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 57 insertions(+), 16 deletions(-)

diff --git a/http_server.c b/http_server.c
index 2fe89968..87645cd8 100644
--- a/http_server.c
+++ b/http_server.c
@@ -298,6 +298,12 @@ static inline int _run_out(struct http_request *r)
   return r->cursor == r->end;
 }
 
+static inline int _buffer_full(struct http_request *r)
+{
+  const char *const bufend = r->buffer + sizeof r->buffer;
+  return r->parsed == r->received && (r->end == bufend || r->request_content_remaining == 0);
+}
+
 static inline void _rewind(struct http_request *r)
 {
   assert(r->parsed >= r->received);
@@ -328,6 +334,12 @@ static inline int _skip_to_crlf(struct http_request *r)
   return 0;
 }
 
+static inline void _rewind_optional_cr(struct http_request *r)
+{
+  if (r->cursor > r->parsed && r->cursor[-1] == '\r')
+    --r->cursor;
+}
+
 static inline void _rewind_crlf(struct http_request *r)
 {
   assert(r->cursor >= r->parsed + 2);
@@ -1034,6 +1046,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	    DEBUGF("Malformed HTTP %s form data: missing first boundary", r->verb);
 	  return 400;
 	}
+	_rewind_optional_cr(r);
 	_commit(r);
 	if (r->parsed > start && r->form_data.handle_mime_preamble) {
 	  if (r->debug_flag && *r->debug_flag)
@@ -1041,21 +1054,36 @@ static int http_request_parse_body_form_data(struct http_request *r)
 		alloca_toprint(50, start, r->parsed - start), r->parsed - start);
 	  r->form_data.handle_mime_preamble(r, start, r->parsed - start);
 	}
-	return 100; // need more data
       }
+      return 100; // need more data
     case HEADER: {
       if (config.debug.httpd)
 	DEBUGF("HEADER");
-	if (_skip_crlf(r) && _skip_crlf(r)) {
+	// If not at a CRLF, then we are skipping through an over-long header that didn't
+	// fit into the buffer.  Just discard bytes up to the next CRLF.
+	if (!_skip_crlf(r)) {
+	  _skip_to_crlf(r); // advance to next CRLF or end of buffer
+	  _rewind_optional_cr(r); // don't skip a CR at end of buffer (it might be part of a half-received CRLF)
+	  assert(r->cursor > r->parsed);
+	  if (r->debug_flag && *r->debug_flag)
+	    DEBUGF("skipping %zu header bytes", r->cursor - r->parsed);
+	  _commit(r);
+	  return 0;
+	}
+	const char *sol = r->cursor;
+	// A blank line finishes the headers.  The CRLF does not form part of the body.
+	if (_skip_crlf(r)) {
 	  _commit(r);
 	  r->form_data_state = BODY;
 	  return 0;
 	}
 	if (_run_out(r))
 	  return 100; // read more and try again
-	_rewind(r);
+	r->cursor = sol;
+	// A mime boundary technically should not occur in the middle of the headers, but if it
+	// does, treat it as a zero-length body.
 	int b;
-	if (_skip_crlf(r) && (b = _skip_mime_boundary(r))) {
+	if ((b = _skip_mime_boundary(r))) {
 	  _rewind_crlf(r);
 	  _commit(r);
 	  if (r->form_data.handle_mime_part_end) {
@@ -1063,7 +1091,8 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	      DEBUGF("handle_mime_part_end()");
 	    r->form_data.handle_mime_part_end(r);
 	  }
-	  // Boundary in the middle of headers starts a new part.
+	  // A boundary in the middle of headers finishes the current part and starts a new part.
+	  // An end boundary terminates the current part and starts the epilogue.
 	  if (b == 1) {
 	    r->form_data_state = HEADER;
 	    if (r->form_data.handle_mime_part_start) {
@@ -1078,45 +1107,56 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	}
 	if (_run_out(r))
 	  return 100; // read more and try again
-	_rewind(r);
+	r->cursor = sol;
 	struct substring label;
-	if (_skip_crlf(r) && _skip_token(r, &label) && _skip_literal(r, ":") && _skip_optional_space(r)) {
+	if (_skip_token(r, &label) && _skip_literal(r, ":") && _skip_optional_space(r)) {
 	  size_t labellen = label.end - label.start;
 	  char labelstr[labellen + 1];
 	  strncpy(labelstr, label.start, labellen)[labellen] = '\0';
 	  str_tolower_inplace(labelstr);
 	  const char *value = r->cursor;
-	  DEBUG_DUMP_PARSER(r);
 	  if (strcmp(labelstr, "content-disposition") == 0) {
 	    struct mime_content_disposition cd;
 	    bzero(&cd, sizeof cd);
 	    if (_parse_content_disposition(r, &cd) && _skip_optional_space(r) && _skip_crlf(r)) {
+	      _rewind_crlf(r);
+	      _commit(r);
 	      if (r->form_data.handle_mime_content_disposition) {
 		if (r->debug_flag && *r->debug_flag)
 		  DEBUGF("handle_mime_content_disposition(%s)", alloca_mime_content_disposition(&cd));
 		r->form_data.handle_mime_content_disposition(r, &cd);
 	      }
-	      _rewind_crlf(r);
-	      _commit(r);
 	      return 0;
 	    }
-	  } else if (_skip_to_crlf(r)) {
+	  }
+	  else if (_skip_to_crlf(r)) {
+	    _commit(r);
 	    if (r->form_data.handle_mime_header) {
 	      if (r->debug_flag && *r->debug_flag)
 		DEBUGF("handle_mime_header(%s, %s)", alloca_str_toprint(labelstr), alloca_toprint(-1, value, value - r->cursor));
 	      r->form_data.handle_mime_header(r, labelstr, value, value - r->cursor); // excluding CRLF at end
 	    }
-	    _commit(r);
 	    return 0;
 	  }
 	}
+	r->cursor = sol;
+	if (_buffer_full(r)) {
+	  // The line does not start with "Token:" and is too long to fit into the buffer.  Start
+	  // skipping it.
+	  WARNF("Skipping unterminated HTTP MIME header %s", alloca_toprint(50, sol, r->end - sol));
+	  r->cursor = r->end;
+	  _rewind_optional_cr(r);
+	  if (r->debug_flag && *r->debug_flag)
+	    DEBUGF("skipping %zu header bytes", r->cursor - r->parsed);
+	  _commit(r);
+	  return 0;
+	}
 	if (_run_out(r))
 	  return 100; // read more and try again
-	_rewind(r);
+	if (r->debug_flag && *r->debug_flag)
+	  DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(50, sol, r->end - sol));
+	DEBUG_DUMP_PARSER(r);
       }
-      if (r->debug_flag && *r->debug_flag)
-	DEBUGF("Malformed HTTP %s form data part: invalid header %s", r->verb, alloca_toprint(50, r->parsed, r->end - r->parsed));
-      DEBUG_DUMP_PARSER(r);
       return 400;
     case BODY:
       if (config.debug.httpd)
@@ -1156,6 +1196,7 @@ static int http_request_parse_body_form_data(struct http_request *r)
 	  DEBUGF("Malformed HTTP %s form data part: missing end boundary", r->verb);
 	return 400;
       }
+      _rewind_optional_cr(r);
       _commit(r);
       if (r->parsed > start && r->form_data.handle_mime_body) {
 	if (r->debug_flag && *r->debug_flag)