From fd6db4575993fe09cfd716c82882f900d4175e25 Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Tue, 22 Oct 2013 12:23:03 -0700
Subject: [PATCH 01/56] stop failure to get commit id when in a git repository
 but not on a tagged branch (fd 5 was non-existent).  Not tested with a tagged
 branch.

---
 version_string.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/version_string.sh b/version_string.sh
index 0a7cd3df..998f9eed 100755
--- a/version_string.sh
+++ b/version_string.sh
@@ -115,7 +115,14 @@ if [ -n "$dirty" ] && ! $allow_modified; then
 fi
 
 # Use the "git describe" command to form the version string and append $dirty.
-if error="$( (desc="$(git describe --match="$version_tag_glob")" && echo "$desc$dirty") 2>&1 1>&5)" 5>&1; then
+extragitflags=
+set +e
+error=`git describe --match="$version_tag_glob" 2>&1 1>/dev/null`
+desc=`git describe --match="$version_tag_glob" 2>/dev/null`
+set -e
+
+if [ "x$error" == "x" ]; then
+   echo $desc$dirty
    exit 0
 fi
 

From 17d2dda6aeff3ffc5ffcf4c21d4655ce3a330e8e Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Tue, 22 Oct 2013 12:37:10 -0700
Subject: [PATCH 02/56] add simple memory performance test

---
 commandline.c | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/commandline.c b/commandline.c
index b8e1e469..6ea684fb 100644
--- a/commandline.c
+++ b/commandline.c
@@ -2484,6 +2484,40 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont
   return 1;
 }
 
+int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
+{
+  int mem_size;
+  int addr;
+  uint64_t count;
+
+  for(mem_size=1024;mem_size<=(128*1024*1024);mem_size*=2) {
+    uint8_t *mem=malloc(mem_size);
+    if (!mem) {
+      fprintf(stderr,"Could not allocate %dKB memory -- stopping test.\n",mem_size/1024);
+      return -1;
+    }
+
+    // Fill memory with random stuff so that we don't have memory page-in
+    // delays when doing the reads
+    for(addr=0;addr<mem_size;addr++) mem[addr]=random()&0xff;
+    
+    uint64_t end_time=gettime_ms()+100;
+    uint64_t total=0;
+    uint32_t mem_mask=mem_size-1;
+
+    for(count=0;gettime_ms()<end_time;count++) {
+      addr=random()&mem_mask;
+      total+=mem[addr];
+    }
+    printf("Memory size = %8dKB : %lld random reads per second (irrelevant sum is %016llx)\n",mem_size/1024,count,
+	   /* use total so that compiler doesn't optimise away our memory accesses */
+	   total);
+
+    free(mem);
+  }
+  return 0;
+}
+
 int app_network_scan(const struct cli_parsed *parsed, struct cli_context *context)
 {
   int mdp_sockfd;
@@ -2634,6 +2668,8 @@ struct cli_schema command_line_options[]={
    "Run cryptography speed test"},
   {app_nonce_test,{"test","nonce",NULL}, 0,
    "Run nonce generation test"},
+  {app_mem_test,{"test","memory",NULL}, 0,
+   "Run memory speed test"},
   {app_byteorder_test,{"test","byteorder",NULL}, 0,
    "Run byte order handling test"},
   {app_slip_test,{"test","slip","[--seed=<N>]","[--duration=<seconds>|--iterations=<N>]",NULL}, 0,

From 6ff2ae487125e4705e52c80aeb7567886807a4fd Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Tue, 22 Oct 2013 12:53:34 -0700
Subject: [PATCH 03/56] add memory write speed test.

---
 commandline.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/commandline.c b/commandline.c
index 6ea684fb..777d44e8 100644
--- a/commandline.c
+++ b/commandline.c
@@ -2509,10 +2509,20 @@ int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
       addr=random()&mem_mask;
       total+=mem[addr];
     }
-    printf("Memory size = %8dKB : %lld random reads per second (irrelevant sum is %016llx)\n",mem_size/1024,count,
+    printf("Memory size = %8dKB : %lld random  reads per second (irrelevant sum is %016llx)\n",mem_size/1024,count*10,
 	   /* use total so that compiler doesn't optimise away our memory accesses */
 	   total);
 
+    end_time=gettime_ms()+100;
+    for(count=0;gettime_ms()<end_time;count++) {
+      addr=random()&mem_mask;
+      mem[addr]=3;
+    }
+    printf("Memory size = %8dKB : %lld random writes per second (irrelevant sum is %016llx)\n",mem_size/1024,count*10,
+	   /* use total so that compiler doesn't optimise away our memory accesses */
+	   total);
+
+
     free(mem);
   }
   return 0;

From 31b314d2da27bfa2bc5026c09df03df2d3e77a19 Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Tue, 22 Oct 2013 13:12:10 -0700
Subject: [PATCH 04/56] add context switching speed test to "test memory"
 command.

---
 commandline.c  |   6 +++
 context1.c     | 103 +++++++++++++++++++++++++++++++++++++++++++++++++
 sourcefiles.mk |   3 +-
 timeit.c       |  41 ++++++++++++++++++++
 4 files changed, 152 insertions(+), 1 deletion(-)
 create mode 100644 context1.c
 create mode 100644 timeit.c

diff --git a/commandline.c b/commandline.c
index 777d44e8..28e563d1 100644
--- a/commandline.c
+++ b/commandline.c
@@ -2484,12 +2484,17 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont
   return 1;
 }
 
+int context_switch_test(int);
 int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
 {
   int mem_size;
   int addr;
   uint64_t count;
 
+
+  // First test context switch speed
+  context_switch_test(1);
+
   for(mem_size=1024;mem_size<=(128*1024*1024);mem_size*=2) {
     uint8_t *mem=malloc(mem_size);
     if (!mem) {
@@ -2525,6 +2530,7 @@ int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
 
     free(mem);
   }
+
   return 0;
 }
 
diff --git a/context1.c b/context1.c
new file mode 100644
index 00000000..da819447
--- /dev/null
+++ b/context1.c
@@ -0,0 +1,103 @@
+
+/*******************************************************************************
+ *  The BYTE UNIX Benchmarks - Release 3
+ *          Module: context1.c   SID: 3.3 5/15/91 19:30:18
+ *
+ *******************************************************************************
+ * Bug reports, patches, comments, suggestions should be sent to:
+ *
+ *	Ben Smith, Rick Grehan or Tom Yager
+ *	ben@bytepb.byte.com   rick_g@bytepb.byte.com   tyager@bytepb.byte.com
+ *
+ *******************************************************************************
+ *  Modification Log:
+ *  $Header: context1.c,v 3.4 87/06/22 14:22:59 kjmcdonell Beta $
+ *  August 28, 1990 - changed timing routines--now returns total number of
+ *                    iterations in specified time period
+ *  October 22, 1997 - code cleanup to remove ANSI C compiler warnings
+ *                     Andy Kahn <kahn@zk3.dec.com>
+ *
+ ******************************************************************************/
+char SCCSid[] = "@(#) @(#)context1.c:3.3 -- 5/15/91 19:30:18";
+/*
+ *  Context switching via synchronized unbuffered pipe i/o
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include "timeit.c"
+
+unsigned long iter;
+int stop_timing=0;
+
+void report()
+{
+	fprintf(stderr, "%lu context switches per second.\n", iter);	
+	stop_timing=1;
+}
+
+int context_switch_test(int duration)
+{
+	unsigned long	check;
+	int	p1[2], p2[2];
+
+
+	/* set up alarm call */
+	iter = 0;
+	wake_me(duration, report);
+
+	if (pipe(p1) || pipe(p2)) {
+		perror("pipe create failed");
+		exit(1);
+	}
+
+	if (fork()) {	/* parent process */
+		/* master, write p1 & read p2 */
+		close(p1[0]); close(p2[1]);
+		while (!stop_timing) {
+			if (write(p1[1], (char *)&iter, sizeof(iter)) != sizeof(iter)) {
+				if ((errno != 0) && (errno != EINTR))
+					perror("master write failed");
+				exit(1);
+			}
+			if (read(p2[0], (char *)&check, sizeof(check)) != sizeof(check)) {
+				if ((errno != 0) && (errno != EINTR))
+					perror("master read failed");
+				exit(1);
+			}
+			if (check != iter) {
+				fprintf(stderr, "Master sync error: expect %lu, got %lu\n",
+					iter, check);
+				exit(2);
+			}
+			iter++;
+		}
+	}
+	else { /* child process */
+		unsigned long iter1;
+
+		iter1 = 0;
+		/* slave, read p1 & write p2 */
+		close(p1[1]); close(p2[0]);
+		while (!stop_timing) {
+			if (read(p1[0], (char *)&check, sizeof(check)) != sizeof(check)) {
+				if ((errno != 0) && (errno != EINTR))
+					perror("slave read failed");
+				exit(1);
+			}
+			if (check != iter1) {
+				fprintf(stderr, "Slave sync error: expect %lu, got %lu\n",
+					iter, check);
+				exit(2);
+			}
+			if (write(p2[1], (char *)&iter1, sizeof(iter1)) != sizeof(check)) {
+				if ((errno != 0) && (errno != EINTR))
+					perror("slave write failed");
+				exit(1);
+			}
+			iter1++;
+		}
+	}
+}
diff --git a/sourcefiles.mk b/sourcefiles.mk
index 00ee579b..bab84b14 100644
--- a/sourcefiles.mk
+++ b/sourcefiles.mk
@@ -74,4 +74,5 @@ SERVAL_SOURCES = \
         $(SERVAL_BASE)fec-3.0.1/ccsds_tables.c \
 	$(SERVAL_BASE)fec-3.0.1/decode_rs_8.c \
 	$(SERVAL_BASE)fec-3.0.1/encode_rs_8.c \
-	$(SERVAL_BASE)fec-3.0.1/init_rs_char.c
+	$(SERVAL_BASE)fec-3.0.1/init_rs_char.c \
+	$(SERVAL_BASE)context1.c 
diff --git a/timeit.c b/timeit.c
new file mode 100644
index 00000000..00d1cfce
--- /dev/null
+++ b/timeit.c
@@ -0,0 +1,41 @@
+/*******************************************************************************
+ *          
+ *  The BYTE UNIX Benchmarks - Release 3
+ *          Module: timeit.c   SID: 3.3 5/15/91 19:30:21
+ *******************************************************************************
+ * Bug reports, patches, comments, suggestions should be sent to:
+ *
+ *	Ben Smith, Rick Grehan or Tom Yager
+ *	ben@bytepb.byte.com   rick_g@bytepb.byte.com   tyager@bytepb.byte.com
+ *
+ *******************************************************************************
+ *  Modification Log:
+ *  May 12, 1989 - modified empty loops to avoid nullifying by optimizing
+ *                 compilers
+ *  August 28, 1990 - changed timing relationship--now returns total number
+ *	                  of iterations (ty)
+ *  October 22, 1997 - code cleanup to remove ANSI C compiler warnings
+ *                     Andy Kahn <kahn@zk3.dec.com>
+ *
+ ******************************************************************************/
+
+/* this module is #included in other modules--no separate SCCS ID */
+
+/*
+ *  Timing routine
+ *
+ */
+
+#include <signal.h>
+#include <unistd.h>
+
+void wake_me(seconds, func)
+	int seconds;
+	void (*func)();
+{
+	/* set up the signal handler */
+	signal(SIGALRM, func);
+	/* get the clock running */
+	alarm(seconds);
+}
+

From e9e0a1b4a4269a3170eab3dc2ac9ce4cf4e57df9 Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Wed, 23 Oct 2013 19:45:50 -0700
Subject: [PATCH 05/56] allow setting of % of 252 byte RS protected packets
 that we want to have delivered.  Adjust tests accordingly.  Create new test
 for ~50% packet delivery on fakeradio in preparation for network coding.

---
 fakeradio.c           |  38 +++++++++++-
 tests/rhizomeprotocol | 134 ++++++++++++++++++++++++++----------------
 tests/routing         |   2 +-
 3 files changed, 120 insertions(+), 54 deletions(-)

diff --git a/fakeradio.c b/fakeradio.c
index b310fbe3..36621f12 100644
--- a/fakeradio.c
+++ b/fakeradio.c
@@ -364,12 +364,48 @@ int transfer_bytes(struct radio_state *radios)
   return bytes;
 }
 
+int calc_ber(double target_packet_fraction)
+{
+  int byte_count=220+32;
+  int max_error_bytes=16;
+
+  int ber;
+  int p;
+  int byte;
+  int bit;
+
+  // 9,000,000 gives a packet delivery rate of ~99%
+  // so no point starting smaller than that.
+  // Only ~30,000,000 reduces packet delivery rate to
+  // ~1%, so the search range is fairly narrow.
+  ber=9000000;
+  if (target_packet_fraction<=0.9) ber=13000000;
+  if (target_packet_fraction<=0.5) ber=18000000;
+  if (target_packet_fraction<=0.25) ber=21000000;
+  if (target_packet_fraction<=0.1) ber=24000000;
+  if (target_packet_fraction<=0.05) ber=26000000;
+  for(;ber<0x70ffffff;ber+=100000)
+    {
+      int packet_errors=0;
+      for(p=0;p<1000;p++) {
+	int byte_errors=0;
+	for(byte=0;byte<byte_count;byte++) {
+	  for(bit=0;bit<8;bit++) if (random()<ber) { byte_errors++; break; }
+	  if (byte_errors>max_error_bytes) { packet_errors++; break; }
+	}
+      }
+      if (packet_errors>=((1.0-target_packet_fraction)*1000)) break;
+    }
+  fprintf(stderr,"ber magic value=%d\n",ber);
+  return ber;
+}
+
 int main(int argc,char **argv)
 {
   if (argv[1]) {
     chars_per_ms=atol(argv[1]);
     if (argv[2]) 
-      ber=atol(argv[2]);
+      ber=calc_ber(atof(argv[2]));
   }
   fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms);
   fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF);
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 0f4663b3..059d0b6c 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -219,58 +219,6 @@ test_UnicastTransfer() {
    receive_and_update_bundle
 }
 
-doc_SimulatedRadio="MDP Transfer over simulated radio link"
-interface_up() {
-   $GREP "Interface .* is up" $instance_servald_log || return 1
-   return 0
-}
-start_radio_instance() {
-   executeOk_servald config \
-      set debug.rhizome on \
-      set debug.rhizome_ads on \
-      set debug.rhizome_tx on \
-      set debug.rhizome_rx on \
-      set debug.throttling on \
-      set debug.mavlink on \
-      set rhizome.advertise.interval 5000 \
-      set rhizome.rhizome_mdp_block_size 350 \
-      set log.console.level debug \
-      set log.console.show_pid on \
-      set log.console.show_time on \
-      set interfaces.1.type CATEAR \
-      set interfaces.1.mdp.tick_ms 5000 \
-      set interfaces.1.socket_type STREAM \
-      set interfaces.1.encapsulation SINGLE \
-      set interfaces.1.point_to_point on
-   start_servald_server
-   wait_until interface_up
-}
-setup_SimulatedRadio() {
-   setup_common
-   $servald_build_root/fakeradio 6 10000000 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
-   FAKERADIO_PID=$!
-   sleep 1
-   local END1=`head "$SERVALD_VAR/radioout" -n 1`
-   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
-   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
-   set_instance +A
-   rhizome_add_file file1 10000
-   executeOk_servald config \
-      set interfaces.1.file "$END1"
-   set_instance +B
-   executeOk_servald config \
-      set interfaces.1.file "$END2"
-   foreach_instance +A +B start_radio_instance
-}
-test_SimulatedRadio() {
-   receive_and_update_bundle
-}
-teardown_SimulatedRadio() {
-   teardown
-   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
-   kill $FAKERADIO_PID
-   tfw_cat "$SERVALD_VAR/radioerr"
-}
 
 doc_journalMDP="Transfer and update a journal bundle via MDP"
 setup_journalMDP() {
@@ -747,4 +695,86 @@ test_DirectSync() {
    assert_rhizome_received fileA3
 }
 
+interface_up() {
+   $GREP "Interface .* is up" $instance_servald_log || return 1
+   return 0
+}
+start_radio_instance() {
+   executeOk_servald config \
+      set debug.rhizome on \
+      set debug.rhizome_ads on \
+      set debug.rhizome_tx on \
+      set debug.rhizome_rx on \
+      set debug.throttling on \
+      set debug.mavlink on \
+      set rhizome.advertise.interval 5000 \
+      set rhizome.rhizome_mdp_block_size 350 \
+      set log.console.level debug \
+      set log.console.show_pid on \
+      set log.console.show_time on \
+      set interfaces.1.type CATEAR \
+      set interfaces.1.mdp.tick_ms 5000 \
+      set interfaces.1.socket_type STREAM \
+      set interfaces.1.encapsulation SINGLE \
+      set interfaces.1.point_to_point on
+   start_servald_server
+   wait_until interface_up
+}
+
+doc_SimulatedRadio="MDP Transfer over simulated radio link (~90% packet arrival)"
+setup_SimulatedRadio() {
+   setup_common
+   $servald_build_root/fakeradio 6 0.9 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   FAKERADIO_PID=$!
+   sleep 5
+   local END1=`head "$SERVALD_VAR/radioout" -n 1`
+   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
+   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+   set_instance +A
+   rhizome_add_file file1 10000
+   executeOk_servald config \
+      set interfaces.1.file "$END1"
+   set_instance +B
+   executeOk_servald config \
+      set interfaces.1.file "$END2"
+   foreach_instance +A +B start_radio_instance
+}
+test_SimulatedRadio() {
+   receive_and_update_bundle
+}
+teardown_SimulatedRadio() {
+   teardown
+   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
+   kill $FAKERADIO_PID
+   tfw_cat "$SERVALD_VAR/radioerr"
+}
+
+doc_SimulatedRadio2="MDP Transfer over simulated radio link (~50% packet arrival)"
+setup_SimulatedRadio2() {
+   setup_common
+   $servald_build_root/fakeradio 6 0.5 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   FAKERADIO_PID=$!
+   sleep 1
+   local END1=`head "$SERVALD_VAR/radioout" -n 1`
+   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
+   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+   set_instance +A
+   rhizome_add_file file1 10000
+   executeOk_servald config \
+      set interfaces.1.file "$END1"
+   set_instance +B
+   executeOk_servald config \
+      set interfaces.1.file "$END2"
+   foreach_instance +A +B start_radio_instance
+}
+test_SimulatedRadio2() {
+   receive_and_update_bundle
+}
+teardown_SimulatedRadio2() {
+   teardown
+   tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
+   kill $FAKERADIO_PID
+   tfw_cat "$SERVALD_VAR/radioerr"
+}
+
 runTests "$@"
diff --git a/tests/routing b/tests/routing
index 8b15cbbd..24badb53 100755
--- a/tests/routing
+++ b/tests/routing
@@ -265,7 +265,7 @@ setup_simulate_extender() {
    setup_servald
    assert_no_servald_processes
    foreach_instance +A +B create_single_identity
-   $servald_build_root/fakeradio 1 20000000 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   $servald_build_root/fakeradio 1 0.8 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
    FAKERADIO_PID=$!
    sleep 1
    local END1=`head "$SERVALD_VAR/radioout" -n 1`

From 56e91518b860941e2b6862f75b3102eb092cd7cc Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 21 Nov 2013 16:16:12 +1030
Subject: [PATCH 06/56] Fix fetch insert assertion

---
 rhizome_fetch.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index ebb2aa90..7f1d5d74 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -318,7 +318,7 @@ static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch
   if (config.debug.rhizome_rx)
     DEBUGF("insert queue[%d] candidate[%u]", (int)(q - rhizome_fetch_queues), i);
   assert(i >= 0 && i < q->candidate_queue_size);
-  assert(i == 0 || c[i-1].manifest);
+  assert(i == 0 || c[-1].manifest);
   if (e->manifest) // queue is full
     rhizome_manifest_free(e->manifest);
   else

From 8a3ee7979769d747daab4491073d6e1829af36da Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 21 Nov 2013 16:19:14 +1030
Subject: [PATCH 07/56] Fix printf format

---
 rhizome_database.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhizome_database.c b/rhizome_database.c
index 5de4b115..2a02154a 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -743,7 +743,7 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
 	      } else {
 		char hash_hex[RHIZOME_FILEHASH_STRLEN + 1];
 		tohex(hash_hex, RHIZOME_FILEHASH_STRLEN, hashp->binary);
-		BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%zu,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN);
+		BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN);
 		BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT);
 	      }
 	    }

From 36bc5ff390d49c354a3e3f0808f4862df02d9ea7 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 21 Nov 2013 16:29:10 +1030
Subject: [PATCH 08/56] Fix string buffer overflow

---
 rhizome_database.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rhizome_database.c b/rhizome_database.c
index 2a02154a..348e1e77 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -1988,7 +1988,7 @@ static int is_interesting(const char *id_hex, int64_t version)
 int rhizome_is_bar_interesting(unsigned char *bar)
 {
   int64_t version = rhizome_bar_version(bar);
-  char id_hex[RHIZOME_BAR_PREFIX_BYTES + 2];
+  char id_hex[RHIZOME_BAR_PREFIX_BYTES *2 + 2];
   tohex(id_hex, RHIZOME_BAR_PREFIX_BYTES * 2, &bar[RHIZOME_BAR_PREFIX_OFFSET]);
   strcat(id_hex, "%");
   return is_interesting(id_hex, version);

From 22258b60c037414a4cf76e023a65619fd399a9ac Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Thu, 21 Nov 2013 16:35:02 +1030
Subject: [PATCH 09/56] fixes and improvements to handling of git repo with no
 tags on branch.

---
 version_string.sh | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/version_string.sh b/version_string.sh
index 998f9eed..2f9cac3f 100755
--- a/version_string.sh
+++ b/version_string.sh
@@ -115,14 +115,11 @@ if [ -n "$dirty" ] && ! $allow_modified; then
 fi
 
 # Use the "git describe" command to form the version string and append $dirty.
-extragitflags=
-set +e
-error=`git describe --match="$version_tag_glob" 2>&1 1>/dev/null`
-desc=`git describe --match="$version_tag_glob" 2>/dev/null`
-set -e
+# This ugly construction is required for use on machines with bash version 3.
+error="$(git describe --match="$version_tag_glob" 2>&1 1>/dev/null)" || true
 
-if [ "x$error" == "x" ]; then
-   echo $desc$dirty
+if [ -z "$error" ]; then
+   echo "$(git describe --match="$version_tag_glob")"$dirty
    exit 0
 fi
 

From 003603b97c5fefadf345e71133556632d67d5131 Mon Sep 17 00:00:00 2001
From: gardners <paul@servalproject.org>
Date: Thu, 21 Nov 2013 16:36:59 +1030
Subject: [PATCH 10/56] minor tweaks to bash compatibility fixes.

---
 version_string.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/version_string.sh b/version_string.sh
index 2f9cac3f..2035ee09 100755
--- a/version_string.sh
+++ b/version_string.sh
@@ -115,11 +115,11 @@ if [ -n "$dirty" ] && ! $allow_modified; then
 fi
 
 # Use the "git describe" command to form the version string and append $dirty.
-# This ugly construction is required for use on machines with bash version 3.
+# This ugly construction is required for use on machines with bash version < 4.
 error="$(git describe --match="$version_tag_glob" 2>&1 1>/dev/null)" || true
 
 if [ -z "$error" ]; then
-   echo "$(git describe --match="$version_tag_glob")"$dirty
+   echo "$(git describe --match="$version_tag_glob")$dirty"
    exit 0
 fi
 

From 8c06881fa3a5c591dabefa21ddba5631cc7da209 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 21 Nov 2013 16:49:44 +1030
Subject: [PATCH 11/56] Fix compiler warning

---
 commandline.c | 2 +-
 context1.c    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/commandline.c b/commandline.c
index 28e563d1..62372c48 100644
--- a/commandline.c
+++ b/commandline.c
@@ -2484,7 +2484,7 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont
   return 1;
 }
 
-int context_switch_test(int);
+void context_switch_test(int);
 int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
 {
   int mem_size;
diff --git a/context1.c b/context1.c
index da819447..93402c36 100644
--- a/context1.c
+++ b/context1.c
@@ -38,7 +38,7 @@ void report()
 	stop_timing=1;
 }
 
-int context_switch_test(int duration)
+void context_switch_test(int duration)
 {
 	unsigned long	check;
 	int	p1[2], p2[2];

From 702d22de99f1bd83ccf965899f3b418ef0dc7a2c Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 21 Nov 2013 17:09:25 +1030
Subject: [PATCH 12/56] Fix version_string.sh: check for .git directory was too
 early

---
 version_string.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/version_string.sh b/version_string.sh
index 2035ee09..a5b46462 100755
--- a/version_string.sh
+++ b/version_string.sh
@@ -17,11 +17,6 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
-if [ ! -e .git ]; then
-  echo "UNKNOWN-VERSION"
-  exit
-fi
-
 usage() {
    echo "Usage: ${0##*/} [options]"'
 
@@ -86,6 +81,11 @@ done
 
 cd "$repo_path" >/dev/null
 
+if [ ! -d .git ]; then
+  echo "UNKNOWN-VERSION"
+  exit 0
+fi
+
 get_author_label() {
    # See git-commit-tree(1) for the semantics of working out the author's email
    # address when committing.

From 67a724b29d253ad5f56cf050d748e5b4daa0d276 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 21 Nov 2013 17:32:58 +1030
Subject: [PATCH 13/56] Fix Android compiler warnings ("%zd" and ssize_t)

Also squash some unconditional DEBUG statements
---
 commandline.c   |  6 ++----
 meshms.c        | 16 +++++++++++-----
 rhizome_store.c |  5 +----
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/commandline.c b/commandline.c
index e40f1bbd..74bc155c 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1117,7 +1117,7 @@ int app_trace(const struct cli_parsed *parsed, struct cli_context *context)
   int ret=overlay_mdp_send(mdp_sockfd, &mdp, MDP_AWAITREPLY, 5000);
   ob_free(b);
   if (ret)
-    DEBUGF("overlay_mdp_send returned %d", ret);
+    WHYF("overlay_mdp_send returned %d", ret);
   else{
     int offset=0;
     {
@@ -2319,7 +2319,6 @@ int app_id_list(const struct cli_parsed *parsed, struct cli_context *context)
     struct mdp_header rev_header;
     unsigned char response_payload[1600];
     ssize_t len = mdp_poll_recv(mdp_sock, timeout, &rev_header, response_payload, sizeof(response_payload));
-    DEBUGF("mdp_poll_recv = %zd", len);
     if (len==-1)
       break;
     if (len==-2){
@@ -2838,11 +2837,10 @@ int app_network_scan(const struct cli_parsed *parsed, struct cli_context *contex
     return -1;
   
   if (address){
-    DEBUGF("Parsing arg %s", address);
     if (!inet_aton(address, &scan->addr))
       return WHY("Unable to parse the address");
   }else
-    DEBUGF("Scanning local networks");
+    INFO("Scanning local networks");
   
   if ((mdp_sockfd = overlay_mdp_client_socket()) < 0)
     return WHY("Cannot create MDP socket");
diff --git a/meshms.c b/meshms.c
index b60c9711..c88a984b 100644
--- a/meshms.c
+++ b/meshms.c
@@ -274,8 +274,12 @@ static int ply_read_next(struct ply_read *ply)
     return 1;
   }
   ply->read.offset -= sizeof footer;
-  if (rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer) != sizeof footer)
-    return -1;
+  ssize_t read;
+  read = rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer);
+  if (read == -1)
+    return WHYF("rhizome_read_buffered() failed");
+  if ((size_t) read != sizeof footer)
+    return WHYF("Expected %zu bytes read, got %zu", (size_t) sizeof footer, (size_t) read);
   // (rhizome_read automatically advances the offset by the number of bytes read)
   ply->record_length=read_uint16(footer);
   ply->type = ply->record_length & 0xF;
@@ -302,9 +306,11 @@ static int ply_read_next(struct ply_read *ply)
     ply->buffer = b;
   }
   
-  ssize_t read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length);
-  if (read != ply->record_length)
-    return WHYF("Expected %u bytes read, got %zd", ply->record_length, read);
+  read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length);
+  if (read == -1)
+    return WHYF("rhizome_read_buffered() failed");
+  if ((size_t) read != ply->record_length)
+    return WHYF("Expected %u bytes read, got %zu", ply->record_length, (size_t) read);
   
   ply->read.offset = record_start;
   return 0;
diff --git a/rhizome_store.c b/rhizome_store.c
index 53a6907e..336c5943 100644
--- a/rhizome_store.c
+++ b/rhizome_store.c
@@ -838,7 +838,7 @@ ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buf
   size_t bytes_copied=0;
   
   while (len>0){
-    DEBUGF("len=%zu read->length=%"PRIu64" read->offset=%"PRIu64" buffer->offset=%"PRIu64"", len, read->length, read->offset, buffer->offset);
+    //DEBUGF("len=%zu read->length=%"PRIu64" read->offset=%"PRIu64" buffer->offset=%"PRIu64"", len, read->length, read->offset, buffer->offset);
     // make sure we only attempt to read data that actually exists
     if (read->length != RHIZOME_SIZE_UNSET && read->offset + len > read->length)
       len = read->length - read->offset;
@@ -858,7 +858,6 @@ ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buf
 	  len-=size;
 	  read->offset+=size;
 	  bytes_copied+=size;
-	  DEBUGF("read->offset=%"PRIu64, read->offset);
 	  continue;
 	}
       }
@@ -876,7 +875,6 @@ ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buf
 	  bcopy(buffer->data + ofs - size, data + len - size, size);
 	  len-=size;
 	  bytes_copied+=size;
-	  DEBUGF("read->offset=%"PRIu64, read->offset);
 	  continue;
 	}
       }
@@ -892,7 +890,6 @@ ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buf
     if (r == -1)
       return -1;
     buffer->len = (size_t) r;
-    DEBUGF("read->offset=%"PRIu64, read->offset);
   }
   return bytes_copied;
 }

From a13189e554355bf4a5f8c6693480d62c11f61ad1 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 21 Nov 2013 17:27:22 +1030
Subject: [PATCH 14/56] Fix printf formats

---
 commandline.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/commandline.c b/commandline.c
index 74bc155c..6514f401 100644
--- a/commandline.c
+++ b/commandline.c
@@ -2774,8 +2774,8 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont
 void context_switch_test(int);
 int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
 {
-  int mem_size;
-  int addr;
+  size_t mem_size;
+  size_t addr;
   uint64_t count;
 
 
@@ -2785,7 +2785,7 @@ int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
   for(mem_size=1024;mem_size<=(128*1024*1024);mem_size*=2) {
     uint8_t *mem=malloc(mem_size);
     if (!mem) {
-      fprintf(stderr,"Could not allocate %dKB memory -- stopping test.\n",mem_size/1024);
+      fprintf(stderr,"Could not allocate %zdKB memory -- stopping test.\n",mem_size/1024);
       return -1;
     }
 
@@ -2793,15 +2793,16 @@ int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
     // delays when doing the reads
     for(addr=0;addr<mem_size;addr++) mem[addr]=random()&0xff;
     
-    uint64_t end_time=gettime_ms()+100;
+    time_ms_t end_time=gettime_ms()+100;
     uint64_t total=0;
-    uint32_t mem_mask=mem_size-1;
+    size_t mem_mask=mem_size-1;
 
     for(count=0;gettime_ms()<end_time;count++) {
       addr=random()&mem_mask;
       total+=mem[addr];
     }
-    printf("Memory size = %8dKB : %lld random  reads per second (irrelevant sum is %016llx)\n",mem_size/1024,count*10,
+    printf("Memory size = %8zdKB : %"PRId64" random  reads per second (irrelevant sum is %016"PRIx64")\n",
+	   mem_size/1024,count*10,
 	   /* use total so that compiler doesn't optimise away our memory accesses */
 	   total);
 
@@ -2810,7 +2811,8 @@ int app_mem_test(const struct cli_parsed *parsed, struct cli_context *context)
       addr=random()&mem_mask;
       mem[addr]=3;
     }
-    printf("Memory size = %8dKB : %lld random writes per second (irrelevant sum is %016llx)\n",mem_size/1024,count*10,
+    printf("Memory size = %8zdKB : %"PRId64" random writes per second (irrelevant sum is %016"PRIx64")\n",
+	   mem_size/1024,count*10,
 	   /* use total so that compiler doesn't optimise away our memory accesses */
 	   total);
 

From 0085291ae6230c2c710b254e94823842e416be52 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 21 Nov 2013 18:11:30 +1030
Subject: [PATCH 15/56] Remove stop file last during server shutdown to allow
 database cleanup

---
 server.c | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/server.c b/server.c
index 7be22dfe..87186455 100644
--- a/server.c
+++ b/server.c
@@ -230,19 +230,23 @@ int server_check_stopfile()
 
 void serverCleanUp()
 {
-  /* Try to remove shutdown and PID files and exit */
-  server_remove_stopfile();
-  char filename[1024];
-  if (FORM_SERVAL_INSTANCE_PATH(filename, PIDFILE_NAME))
-    unlink(filename);
-  
-  if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.socket")) {
-    unlink(filename);
+  if (serverMode){
+    rhizome_close_db();
+    dna_helper_shutdown();
   }
   
-  rhizome_close_db();
+  char filename[1024];
+  if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.socket"))
+    unlink(filename);
   
-  dna_helper_shutdown();
+  if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.2.socket"))
+    unlink(filename);
+    
+  if (FORM_SERVAL_INSTANCE_PATH(filename, "monitor.socket"))
+    unlink(filename);
+  
+  /* Try to remove shutdown and PID files and exit */
+  server_remove_stopfile();
 }
 
 static void signame(char *buf, size_t len, int signal)

From 9ddcfb87aaf55a552b5c8f2c3cc87c6fdaf5a2b4 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 22 Nov 2013 11:49:27 +1030
Subject: [PATCH 16/56] Fix output delimiter in "reverse lookup" command

---
 commandline.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commandline.c b/commandline.c
index 6514f401..5aed982a 100644
--- a/commandline.c
+++ b/commandline.c
@@ -2758,7 +2758,7 @@ int app_reverse_lookup(const struct cli_parsed *parsed, struct cli_context *cont
       
       /* Got a good DNA reply, copy it into place and stop polling */
       cli_field_name(context, "sid", ":");
-      cli_put_string(context, sidhex, ":");
+      cli_put_string(context, sidhex, "\n");
       cli_field_name(context, "did", ":");
       cli_put_string(context, did, "\n");
       cli_field_name(context, "name", ":");

From e92e7cb65e67db4e0f2f74bbb924062abc8188a2 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 22 Nov 2013 11:52:11 +1030
Subject: [PATCH 17/56] Improve sendto(2) error log messages

---
 mdp_client.c        | 2 +-
 overlay_interface.c | 6 +++++-
 overlay_mdp.c       | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/mdp_client.c b/mdp_client.c
index 02930bef..b8ef160e 100644
--- a/mdp_client.c
+++ b/mdp_client.c
@@ -137,7 +137,7 @@ int overlay_mdp_send(int mdp_sockfd, overlay_mdp_frame *mdp, int flags, int time
     mdp->packetTypeAndFlags=MDP_ERROR;
     mdp->error.error=1;
     snprintf(mdp->error.message,128,"Error sending frame to MDP server.");
-    return WHY_perror("sendto(f)");
+    return WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sockfd, (size_t)len, alloca_sockaddr(&addr, addrlen));
   } else {
     if (!(flags&MDP_AWAITREPLY)) {       
       return 0;
diff --git a/overlay_interface.c b/overlay_interface.c
index 172f4621..55b547de 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -982,7 +982,11 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
 		(struct sockaddr *)&destination->address, sizeof(destination->address));
       ob_free(buffer);
       if (sent!= len){
-	WHY_perror("sendto(c)");
+	WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)",
+	    interface->alarm.poll.fd,
+	    (size_t)len,
+	    alloca_sockaddr((struct sockaddr *)&destination->address, sizeof destination->address)
+	  );
 	// close the interface if we had any error while sending broadcast packets,
 	// unicast packets should not bring the interface down
 	if (destination == interface->destination)
diff --git a/overlay_mdp.c b/overlay_mdp.c
index ba13500c..03c0a794 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -163,7 +163,7 @@ int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrl
   int r=sendto(sock,(char *)mdpreply,replylen,0,
 	       (struct sockaddr *)recvaddr,recvaddrlen);
   if (r<replylen) { 
-    WHY_perror("sendto(d)"); 
+    WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", sock, (size_t)replylen, alloca_sockaddr((struct sockaddr *)recvaddr, recvaddrlen));
     return WHYF("sendto() failed when sending MDP reply, sock=%d, r=%d", sock, r); 
   } else
     if (0) DEBUGF("reply of %d bytes sent",r);

From 3805650736846c4e46cad85107dd81a70515ec11 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 22 Nov 2013 11:54:32 +1030
Subject: [PATCH 18/56] Use ssize_t for sendto(2) return value, not int

Also improve error checking on ssize_t values (use ==-1 not <0) test
and cast ssize_t to size_t for comparisons
---
 mdp_client.c        |  4 ++--
 overlay_interface.c |  8 ++++----
 overlay_mdp.c       | 21 +++++++++------------
 3 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/mdp_client.c b/mdp_client.c
index b8ef160e..c7231903 100644
--- a/mdp_client.c
+++ b/mdp_client.c
@@ -122,7 +122,7 @@ int overlay_mdp_send(int mdp_sockfd, overlay_mdp_frame *mdp, int flags, int time
     return -1;
   // Minimise frame length to save work and prevent accidental disclosure of memory contents.
   ssize_t len = overlay_mdp_relevant_bytes(mdp);
-  if (len < 0)
+  if (len == -1)
     return WHY("MDP frame invalid (could not compute length)");
   /* Construct name of socket to send to. */
   struct sockaddr_un addr;
@@ -131,7 +131,7 @@ int overlay_mdp_send(int mdp_sockfd, overlay_mdp_frame *mdp, int flags, int time
     return -1;
   // Send to that socket
   set_nonblock(mdp_sockfd);
-  int result = sendto(mdp_sockfd, mdp, len, 0, (struct sockaddr *)&addr, addrlen);
+  ssize_t result = sendto(mdp_sockfd, mdp, (size_t)len, 0, (struct sockaddr *)&addr, addrlen);
   set_block(mdp_sockfd);
   if (result == -1) {
     mdp->packetTypeAndFlags=MDP_ERROR;
diff --git a/overlay_interface.c b/overlay_interface.c
index 55b547de..f71f4208 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -976,12 +976,12 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
     case SOCK_DGRAM:
     {
       if (config.debug.overlayinterfaces) 
-	DEBUGF("Sending %d byte overlay frame on %s to %s",len,interface->name,inet_ntoa(destination->address.sin_addr));
-      int sent=sendto(interface->alarm.poll.fd, 
-		bytes, len, 0, 
+	DEBUGF("Sending %zu byte overlay frame on %s to %s", (size_t)len, interface->name, inet_ntoa(destination->address.sin_addr));
+      ssize_t sent = sendto(interface->alarm.poll.fd, 
+		bytes, (size_t)len, 0, 
 		(struct sockaddr *)&destination->address, sizeof(destination->address));
       ob_free(buffer);
-      if (sent!= len){
+      if (sent == -1 || (size_t)sent != (size_t)len) {
 	WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)",
 	    interface->alarm.poll.fd,
 	    (size_t)len,
diff --git a/overlay_mdp.c b/overlay_mdp.c
index 03c0a794..9ad5dfdd 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -154,19 +154,16 @@ int overlay_mdp_reply_error(int sock,
 int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
 			  overlay_mdp_frame *mdpreply)
 {
-  if (!recvaddr) return WHY("No reply address");
-
+  if (!recvaddr)
+    return WHY("No reply address");
   ssize_t replylen = overlay_mdp_relevant_bytes(mdpreply);
-  if (replylen<0) return WHY("Invalid MDP frame (could not compute length)");
-
-  errno=0;
-  int r=sendto(sock,(char *)mdpreply,replylen,0,
-	       (struct sockaddr *)recvaddr,recvaddrlen);
-  if (r<replylen) { 
+  if (replylen == -1)
+    return WHY("Invalid MDP frame (could not compute length)");
+  ssize_t r = sendto(sock, (char *)mdpreply, (size_t)replylen, 0, (struct sockaddr *)recvaddr, recvaddrlen);
+  if (r == -1 || (size_t)r < (size_t)replylen) { 
     WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", sock, (size_t)replylen, alloca_sockaddr((struct sockaddr *)recvaddr, recvaddrlen));
     return WHYF("sendto() failed when sending MDP reply, sock=%d, r=%d", sock, r); 
-  } else
-    if (0) DEBUGF("reply of %d bytes sent",r);
+  }
   return 0;  
 }
 
@@ -450,8 +447,8 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
       if (len < 0)
 	RETURN(WHY("unsupported MDP packet type"));
       socklen_t addrlen = sizeof addr.sun_family + mdp_bindings[match].name_len;
-      int r = sendto(mdp_sock.poll.fd,mdp,len,0,(struct sockaddr*)&addr, addrlen);
-      if (r == len)
+      ssize_t r = sendto(mdp_sock.poll.fd, mdp, (size_t)len, 0, (struct sockaddr*)&addr, addrlen);
+      if ((size_t)r == (size_t)len)
 	RETURN(0);
       if (r == -1 && errno == ENOENT) {
 	/* far-end of socket has died, so drop binding */

From 45450386dbdb9e0682bdc824cba2d1d4e3e74400 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 22 Nov 2013 13:12:29 +1030
Subject: [PATCH 19/56] Improve sendto(2) failure diagnostics

In MDP client and server MDP overlay code
---
 mdp_client.c        |  8 ++++++--
 overlay_interface.c | 15 ++++++++++-----
 overlay_mdp.c       | 27 ++++++++++++++++-----------
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/mdp_client.c b/mdp_client.c
index c7231903..f3694b9a 100644
--- a/mdp_client.c
+++ b/mdp_client.c
@@ -133,11 +133,15 @@ int overlay_mdp_send(int mdp_sockfd, overlay_mdp_frame *mdp, int flags, int time
   set_nonblock(mdp_sockfd);
   ssize_t result = sendto(mdp_sockfd, mdp, (size_t)len, 0, (struct sockaddr *)&addr, addrlen);
   set_block(mdp_sockfd);
-  if (result == -1) {
+  if ((size_t)result != (size_t)len) {
+    if (result == -1)
+      WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sockfd, (size_t)len, alloca_sockaddr(&addr, addrlen));
+    else
+      WHYF("sendto() sent %zu bytes of MDP reply (%zu) to socket=%d", (size_t)result, (size_t)len, mdp_sockfd); 
     mdp->packetTypeAndFlags=MDP_ERROR;
     mdp->error.error=1;
     snprintf(mdp->error.message,128,"Error sending frame to MDP server.");
-    return WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sockfd, (size_t)len, alloca_sockaddr(&addr, addrlen));
+    return -1;
   } else {
     if (!(flags&MDP_AWAITREPLY)) {       
       return 0;
diff --git a/overlay_interface.c b/overlay_interface.c
index f71f4208..8b74b60c 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -982,11 +982,16 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
 		(struct sockaddr *)&destination->address, sizeof(destination->address));
       ob_free(buffer);
       if (sent == -1 || (size_t)sent != (size_t)len) {
-	WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)",
-	    interface->alarm.poll.fd,
-	    (size_t)len,
-	    alloca_sockaddr((struct sockaddr *)&destination->address, sizeof destination->address)
-	  );
+	if (sent == -1)
+	  WHYF_perror("sendto(fd=%d,len=%zu,addr=%s) on interface %s",
+	      interface->alarm.poll.fd,
+	      (size_t)len,
+	      alloca_sockaddr((struct sockaddr *)&destination->address, sizeof destination->address),
+	      interface->name
+	    );
+	else
+	  WHYF("sendto() sent %zu bytes of overlay frame (%zu) to interface %s (socket=%d)",
+	      (size_t)sent, (size_t)len, interface->name, interface->alarm.poll.fd);
 	// close the interface if we had any error while sending broadcast packets,
 	// unicast packets should not bring the interface down
 	if (destination == interface->destination)
diff --git a/overlay_mdp.c b/overlay_mdp.c
index 9ad5dfdd..80eb3fb8 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -160,10 +160,12 @@ int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrl
   if (replylen == -1)
     return WHY("Invalid MDP frame (could not compute length)");
   ssize_t r = sendto(sock, (char *)mdpreply, (size_t)replylen, 0, (struct sockaddr *)recvaddr, recvaddrlen);
-  if (r == -1 || (size_t)r < (size_t)replylen) { 
+  if (r == -1) {
     WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", sock, (size_t)replylen, alloca_sockaddr((struct sockaddr *)recvaddr, recvaddrlen));
-    return WHYF("sendto() failed when sending MDP reply, sock=%d, r=%d", sock, r); 
+    return WHYF("sendto() failed when sending MDP reply"); 
   }
+  if ((size_t)r != (size_t)replylen)
+    return WHYF("sendto() sent %zu bytes of MDP reply (%zu) to socket=%d", (size_t)r, (size_t)replylen, sock); 
   return 0;  
 }
 
@@ -444,19 +446,22 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
       addr.sun_family = AF_UNIX;
       bcopy(mdp_bindings[match].socket_name, addr.sun_path, mdp_bindings[match].name_len);
       ssize_t len = overlay_mdp_relevant_bytes(mdp);
-      if (len < 0)
+      if (len == -1)
 	RETURN(WHY("unsupported MDP packet type"));
       socklen_t addrlen = sizeof addr.sun_family + mdp_bindings[match].name_len;
       ssize_t r = sendto(mdp_sock.poll.fd, mdp, (size_t)len, 0, (struct sockaddr*)&addr, addrlen);
-      if ((size_t)r == (size_t)len)
-	RETURN(0);
-      if (r == -1 && errno == ENOENT) {
-	/* far-end of socket has died, so drop binding */
-	INFOF("Closing dead MDP client '%s'",mdp_bindings[match].socket_name);
-	overlay_mdp_releasebindings(&addr,mdp_bindings[match].name_len);
+      if ((size_t)r != (size_t)len) {
+	if (r == -1) {
+	  WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sock.poll.fd, (size_t)len, alloca_sockaddr(&addr, addrlen));
+	  if (errno == ENOENT) {
+	    /* far-end of socket has died, so drop binding */
+	    INFOF("Closing dead MDP client '%s'",mdp_bindings[match].socket_name);
+	    overlay_mdp_releasebindings(&addr,mdp_bindings[match].name_len);
+	  }
+	} else
+	  WHYF("sendto() sent %zu bytes of MDP frame (%zu) to client, socket=%d", (size_t)r, (size_t)len, mdp_sock.poll.fd); 
+	RETURN(WHY("Failed to pass received MDP frame to client"));
       }
-      WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sock.poll.fd, (size_t)len, alloca_sockaddr(&addr, addrlen));
-      RETURN(WHY("Failed to pass received MDP frame to client"));
     } else {
       /* No socket is bound, ignore the packet ... except for magic sockets */
       RETURN(overlay_mdp_try_interal_services(frame, mdp));

From 9cdf053320a0bb0e118eeaddcee61a30e645defb Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 25 Nov 2013 12:51:13 +1030
Subject: [PATCH 20/56] Fix RETURN macros

Remove trailing semicolon

Fix RETURNNULL(X) -- was not executing X at all
---
 fdqueue.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fdqueue.h b/fdqueue.h
index cb718331..af409543 100644
--- a/fdqueue.h
+++ b/fdqueue.h
@@ -86,8 +86,8 @@ void dump_stack(int log_level);
     fd_func_enter(__HERE__, &_this_call);
 
 #define OUT() fd_func_exit(__HERE__, &_this_call)
-#define RETURN(X) do { OUT(); return (X); } while (0);
-#define RETURNNULL do { OUT(); return (NULL); } while (0);
-#define RETURNVOID do { OUT(); return; } while (0);
+#define RETURN(X) do { OUT(); return (X); } while (0)
+#define RETURNNULL(X) do { X; OUT(); return (NULL); } while (0)
+#define RETURNVOID do { OUT(); return; } while (0)
 
 #endif // __SERVALDNA__FDQUEUE_H

From a9ccd38adc2965f67a90781253f4bfc9c9b887ed Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 25 Nov 2013 16:43:32 +1030
Subject: [PATCH 21/56] Improve overlay buffer ob_xxx() primitives

All ob_append_xxx(b,...) functions return void

ob_makespace() returns 1 if successful, 0 if not

Add ob_overrun(b) predicate to check for overrun after any number of
appends
---
 commandline.c           |  27 ++--
 conf_schema.h           |   1 +
 mavlink.c               |   2 +-
 overlay_address.c       |  73 +++++----
 overlay_address.h       |   4 +-
 overlay_buffer.c        | 347 +++++++++++++++++++++-------------------
 overlay_buffer.h        |  56 +++----
 overlay_link.c          |  52 +++---
 overlay_mdp.c           |  42 +++--
 overlay_mdp_services.c  |   5 +-
 overlay_olsr.c          |   3 +-
 overlay_packetformats.c |  18 +--
 overlay_payload.c       |  66 ++++----
 overlay_queue.c         |  71 ++++----
 rhizome_packetformats.c |  38 +++--
 rhizome_sync.c          |  35 ++--
 route_link.c            | 109 ++++++-------
 serval.h                |   2 +-
 18 files changed, 458 insertions(+), 493 deletions(-)

diff --git a/commandline.c b/commandline.c
index 5aed982a..cf5c146e 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1101,24 +1101,25 @@ int app_trace(const struct cli_parsed *parsed, struct cli_context *context)
   mdp.out.dst.port=MDP_PORT_TRACE;
   mdp.packetTypeAndFlags=MDP_TX;
   struct overlay_buffer *b = ob_static(mdp.out.payload, sizeof(mdp.out.payload));
-  
   ob_append_byte(b, SID_SIZE);
   ob_append_bytes(b, srcsid.binary, SID_SIZE);
-  
   ob_append_byte(b, SID_SIZE);
   ob_append_bytes(b, dstsid.binary, SID_SIZE);
-  
-  mdp.out.payload_length = ob_position(b);
-  cli_printf(context, "Tracing the network path from %s to %s", 
-	 alloca_tohex_sid_t(srcsid), alloca_tohex_sid_t(dstsid));
-  cli_delim(context, "\n");
-  cli_flush(context);
-
-  int ret=overlay_mdp_send(mdp_sockfd, &mdp, MDP_AWAITREPLY, 5000);
+  int ret;
+  if (ob_overrun(b))
+    ret = WHY("overlay buffer overrun");
+  else {
+    mdp.out.payload_length = ob_position(b);
+    cli_printf(context, "Tracing the network path from %s to %s", 
+	  alloca_tohex_sid_t(srcsid), alloca_tohex_sid_t(dstsid));
+    cli_delim(context, "\n");
+    cli_flush(context);
+    ret = overlay_mdp_send(mdp_sockfd, &mdp, MDP_AWAITREPLY, 5000);
+    if (ret)
+      WHYF("overlay_mdp_send returned %d", ret);
+  }
   ob_free(b);
-  if (ret)
-    WHYF("overlay_mdp_send returned %d", ret);
-  else{
+  if (ret == 0) {
     int offset=0;
     {
       // skip the first two sid's
diff --git a/conf_schema.h b/conf_schema.h
index ac41acc3..82e005b3 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -246,6 +246,7 @@ ATOM(bool_t, mavlink,                   0, boolean,, "")
 ATOM(bool_t, mavlink_payloads,          0, boolean,, "")
 ATOM(bool_t, mavlinkfsm,                0, boolean,, "")
 ATOM(bool_t, peers,                     0, boolean,, "")
+ATOM(bool_t, overlaybuffer,             0, boolean,, "")
 ATOM(bool_t, overlayframes,             0, boolean,, "")
 ATOM(bool_t, overlayabbreviations,      0, boolean,, "")
 ATOM(bool_t, overlayrouting,            0, boolean,, "")
diff --git a/mavlink.c b/mavlink.c
index 3150789e..0fb88230 100644
--- a/mavlink.c
+++ b/mavlink.c
@@ -141,7 +141,7 @@ int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad);
 int mavlink_encode_packet(struct overlay_interface *interface)
 {
   int count = ob_remaining(interface->tx_packet);
-  int startP = !ob_position(interface->tx_packet);
+  int startP = (ob_position(interface->tx_packet) == 0);
   int endP = 1;
   if (count+6+32 > 255){
     count = 255-6-32;
diff --git a/overlay_address.c b/overlay_address.c
index 2d371b8b..982900a1 100644
--- a/overlay_address.c
+++ b/overlay_address.c
@@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   since for things like number resolution we are happy to send repeat requests.
  */
 
+#include <assert.h>
 #include "serval.h"
 #include "conf.h"
 #include "str.h"
@@ -234,35 +235,28 @@ int overlay_broadcast_drop_check(struct broadcast *addr)
   }
 }
 
-int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast)
+void overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast)
 {
-  return ob_append_bytes(b, broadcast->id, BROADCAST_LEN);
+  ob_append_bytes(b, broadcast->id, BROADCAST_LEN);
 }
 
 // append an appropriate abbreviation into the address
-int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber)
+void overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber)
 {
-  if (!subscriber)
-    return WHY("No address supplied");
-
-  if(context
-      && subscriber == context->point_to_point_device){
-    if (ob_append_byte(b, OA_CODE_P2P_YOU))
-      return -1;
-  }else if(context
+  assert(subscriber != NULL);
+  if (context && subscriber == context->point_to_point_device)
+    ob_append_byte(b, OA_CODE_P2P_YOU);
+  else if(context
       && !subscriber->send_full
       && subscriber == my_subscriber
       && context->point_to_point_device
-      && (context->encoding_header==0 || !context->interface->local_echo)){
-    if (ob_append_byte(b, OA_CODE_P2P_ME))
-      return -1;
-  }else if (context && subscriber==context->sender){
-    if (ob_append_byte(b, OA_CODE_SELF))
-      return -1;
-  }else if(context && subscriber==context->previous){
-    if (ob_append_byte(b, OA_CODE_PREVIOUS))
-      return -1;
-  }else{
+      && (context->encoding_header==0 || !context->interface->local_echo))
+    ob_append_byte(b, OA_CODE_P2P_ME);
+  else if (context && subscriber==context->sender)
+    ob_append_byte(b, OA_CODE_SELF);
+  else if (context && subscriber==context->previous)
+    ob_append_byte(b, OA_CODE_PREVIOUS);
+  else {
     int len=SID_SIZE;
     if (subscriber->send_full){
       subscriber->send_full=0;
@@ -273,17 +267,15 @@ int overlay_address_append(struct decode_context *context, struct overlay_buffer
       if (len>SID_SIZE)
 	len=SID_SIZE;
     }
-    if (ob_append_byte(b, len))
-      return -1;
-    if (ob_append_bytes(b, subscriber->sid.binary, len))
-      return -1;
+    ob_append_byte(b, len);
+    ob_append_bytes(b, subscriber->sid.binary, len);
   }
   if (context)
     context->previous = subscriber;
-  return 0;
 }
 
-static int add_explain_response(struct subscriber *subscriber, void *context){
+static int add_explain_response(struct subscriber *subscriber, void *context)
+{
   struct decode_context *response = context;
   // only explain a SID once every half second.
   time_ms_t now = gettime_ms();
@@ -292,8 +284,13 @@ static int add_explain_response(struct subscriber *subscriber, void *context){
   subscriber->last_explained = now;
 
   if (!response->please_explain){
-    response->please_explain = calloc(sizeof(struct overlay_frame),1);
-    response->please_explain->payload=ob_new();
+    if ((response->please_explain = emalloc_zero(sizeof(struct overlay_frame))) == NULL)
+      return 1; // stop walking
+    if ((response->please_explain->payload = ob_new()) == NULL) {
+      free(response->please_explain);
+      response->please_explain = NULL;
+      return 1; // stop walking
+    }
     ob_limitsize(response->please_explain->payload, 1024);
   }
   
@@ -309,11 +306,10 @@ static int add_explain_response(struct subscriber *subscriber, void *context){
   
   // add the whole subscriber id to the payload, stop if we run out of space
   DEBUGF("Adding full sid by way of explanation %s", alloca_tohex_sid_t(subscriber->sid));
-  if (ob_append_byte(response->please_explain->payload, SID_SIZE))
+  ob_append_byte(response->please_explain->payload, SID_SIZE);
+  ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE);
+  if (ob_overrun(response->please_explain->payload))
     return 1;
-  if (ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE))
-    return 1;
-
   // let the routing engine know that we had to explain this sid, we probably need to re-send routing info
   link_explained(subscriber);
   return 0;
@@ -345,7 +341,8 @@ static int find_subscr_buffer(struct decode_context *context, struct overlay_buf
     // add the abbreviation you told me about
     if (!context->please_explain){
       context->please_explain = calloc(sizeof(struct overlay_frame),1);
-      context->please_explain->payload=ob_new();
+      if ((context->please_explain->payload = ob_new()) == NULL)
+	return -1;
       ob_limitsize(context->please_explain->payload, MDP_MTU);
     }
     
@@ -396,7 +393,8 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer
 	// add the abbreviation you told me about
 	if (!context->please_explain){
 	  context->please_explain = calloc(sizeof(struct overlay_frame),1);
-	  context->please_explain->payload=ob_new();
+	  if ((context->please_explain->payload = ob_new()) == NULL)
+	    return -1;
 	  ob_limitsize(context->please_explain->payload, MDP_MTU);
 	}
 	
@@ -433,8 +431,9 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer
 int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination){
   IN();
   struct overlay_frame *frame=context->please_explain;
-  if (!frame)
+  if (frame == NULL)
     RETURN(0);
+  assert(frame->payload != NULL);
   frame->type = OF_TYPE_PLEASEEXPLAIN;
   
   if (source)
@@ -466,7 +465,7 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc
   }
   
   frame->queue=OQ_MESH_MANAGEMENT;
-  if (!overlay_payload_enqueue(frame))
+  if (overlay_payload_enqueue(frame) != -1)
     RETURN(0);
   op_free(frame);
   RETURN(-1);
diff --git a/overlay_address.h b/overlay_address.h
index 5a529a7b..7796e469 100644
--- a/overlay_address.h
+++ b/overlay_address.h
@@ -119,8 +119,8 @@ int process_explain(struct overlay_frame *frame);
 int overlay_broadcast_drop_check(struct broadcast *addr);
 int overlay_broadcast_generate_address(struct broadcast *addr);
 
-int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast);
-int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber);
+void overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast);
+void overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber);
 
 int overlay_broadcast_parse(struct overlay_buffer *b, struct broadcast *broadcast);
 int overlay_address_parse(struct decode_context *context, struct overlay_buffer *b, struct subscriber **subscriber);
diff --git a/overlay_buffer.c b/overlay_buffer.c
index f066838f..d6dc957f 100644
--- a/overlay_buffer.c
+++ b/overlay_buffer.c
@@ -17,6 +17,7 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
+#include <assert.h>
 #include "serval.h"
 #include "conf.h"
 #include "mem.h"
@@ -30,57 +31,56 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  In either case, functions that don't take an offset use and advance the position.
  */
 
-
-
 struct overlay_buffer *ob_new(void)
 {
-  struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
-  if (!ret) return NULL;
-  
+  struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (ret == NULL)
+    return NULL;
   ob_unlimitsize(ret);
-
   return ret;
 }
 
 // index an existing static buffer.
 // and allow other callers to use the ob_ convenience methods for reading and writing up to size bytes.
-struct overlay_buffer *ob_static(unsigned char *bytes, int size){
-  struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
-  if (!ret) return NULL;
+struct overlay_buffer *ob_static(unsigned char *bytes, int size)
+{
+  struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (ret == NULL)
+    return NULL;
   ret->bytes = bytes;
   ret->allocSize = size;
   ret->allocated = NULL;
   ob_unlimitsize(ret);
-  
   return ret;
 }
 
 // create a new overlay buffer from an existing piece of another buffer.
 // Both buffers will point to the same memory region.
 // It is up to the caller to ensure this buffer is not used after the parent buffer is freed.
-struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length){
+struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length)
+{
   if (offset+length > b->allocSize) {
     WHY("Buffer isn't long enough to slice");
-	return NULL;
+    return NULL;
   }
-      
-  struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
-  if (!ret)
+  struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (ret == NULL)
       return NULL;
   ret->bytes = b->bytes+offset;
   ret->allocSize = length;
   ret->allocated = NULL;
   ob_unlimitsize(ret);
-  
   return ret;
 }
 
-struct overlay_buffer *ob_dup(struct overlay_buffer *b){
-  struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
+struct overlay_buffer *ob_dup(struct overlay_buffer *b)
+{
+  struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (ret == NULL)
+    return NULL;
   ret->sizeLimit = b->sizeLimit;
   ret->position = b->position;
   ret->checkpointLength = b->checkpointLength;
-  
   if (b->bytes && b->allocSize){
     // duplicate any bytes that might be relevant
     int byteCount = b->sizeLimit;
@@ -88,97 +88,88 @@ struct overlay_buffer *ob_dup(struct overlay_buffer *b){
       byteCount = b->position;
     if (byteCount > b->allocSize)
       byteCount = b->allocSize;
-    
-    ob_append_bytes(ret, b->bytes, byteCount);
+    if (byteCount)
+      ob_append_bytes(ret, b->bytes, byteCount);
   }
   return ret;
 }
 
-int ob_free(struct overlay_buffer *b)
+void ob_free(struct overlay_buffer *b)
 {
-  if (!b) return WHY("Asked to free NULL");
-  if (b->bytes && b->allocated) free(b->allocated);
-  // we're about to free this anyway, why are we clearing it?
-  b->bytes=NULL;
-  b->allocated=NULL;
-  b->allocSize=0;
-  b->sizeLimit=0;
+  assert(b != NULL);
+  if (b->allocated)
+    free(b->allocated);
   free(b);
-  return 0;
 }
 
 int ob_checkpoint(struct overlay_buffer *b)
 {
-  if (!b) return WHY("Asked to checkpoint NULL");
-  b->checkpointLength=b->position;
+  assert(b != NULL);
+  b->checkpointLength = b->position;
   return 0;
 }
 
 int ob_rewind(struct overlay_buffer *b)
 {
-  if (!b) return WHY("Asked to rewind NULL");
-  b->position=b->checkpointLength;
+  assert(b != NULL);
+  b->position = b->checkpointLength;
   return 0;
 }
 
-int ob_limitsize(struct overlay_buffer *b,int bytes)
+void ob_limitsize(struct overlay_buffer *b, int bytes)
 {
-  if (!b) return WHY("Asked to limit size of NULL");
-  if (b->position>bytes) return WHY("Length of data in buffer already exceeds size limit");
-  if (b->checkpointLength>bytes) return WHY("Checkpointed length of data in buffer already exceeds size limit");
-  if (b->bytes && (!b->allocated) && bytes > b->allocSize) return WHY("Size limit exceeds buffer size");
-  if (bytes<0) return WHY("Can't limit buffer to a negative size");
+  assert(b != NULL);
+  assert(bytes >= 0);
+  assert(b->position >= 0);
+  assert(b->position <= bytes);
+  assert(b->checkpointLength <= bytes);
+  if (b->bytes && b->allocated == NULL)
+    assert(bytes <= b->allocSize);
   b->sizeLimit=bytes;
-  return 0;
 }
 
-int ob_unlimitsize(struct overlay_buffer *b)
+void ob_unlimitsize(struct overlay_buffer *b)
 {
-  if (!b) return WHY("b is NULL");
-  b->sizeLimit=-1;
-  return 0;
+  assert(b != NULL);
+  b->sizeLimit = -1;
 }
 
-int ob_flip(struct overlay_buffer *b)
+void ob_flip(struct overlay_buffer *b)
 {
-  b->checkpointLength=0;
-  if (ob_limitsize(b, b->position))
-    return -1;
-  b->position=0;
-  return 0;
+  b->checkpointLength = 0;
+  ob_limitsize(b, b->position);
+  b->position = 0;
 }
 
-int _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b,int bytes)
+/* Return 1 if space is available, 0 if not.
+ */
+ssize_t ob_makespace(struct overlay_buffer *b, size_t bytes)
 {
-  if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit) {
-    if (config.debug.packetformats)
-      DEBUGF("asked for space to %u, beyond size limit of %u", b->position + bytes, b->sizeLimit);
-    return -1;
-  }
-  
-  // already enough space?
-  if (b->position + bytes <= b->allocSize)
+  assert(b != NULL);
+  assert(b->position >= 0);
+  if (b->sizeLimit != -1)
+    assert(b->sizeLimit >= 0);
+  assert(b->allocSize >= 0);
+  if (b->position)
+    assert(b->bytes != NULL);
+  if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit)
+    return 0;
+  if (b->position + bytes <= b->allocSize)
+    return 1;
+  // Don't realloc a static buffer.
+  if (b->bytes && b->allocated == NULL)
     return 0;
-  
-  if (b->bytes && !b->allocated)
-    return WHY("Can't resize a static buffer");
-  
-  if (0)
-    DEBUGF("ob_makespace(%p,%d)\n  b->bytes=%p,b->position=%d,b->allocSize=%d\n",
-	   b,bytes,b->bytes,b->position,b->allocSize);
-
   int newSize=b->position+bytes;
   if (newSize<64) newSize=64;
   if (newSize&63) newSize+=64-(newSize&63);
-  if (newSize>1024) {
-    if (newSize&1023) newSize+=1024-(newSize&1023);
-  }
-  if (newSize>65536) {
-    if (newSize&65535) newSize+=65536-(newSize&65535);
-  }
-  if (0) DEBUGF("realloc(b->bytes=%p,newSize=%d)", b->bytes,newSize);
+  if (newSize>1024 && (newSize&1023))
+    newSize+=1024-(newSize&1023);
+  if (newSize>65536 && (newSize&65535))
+    newSize+=65536-(newSize&65535);
+  if (config.debug.overlaybuffer)
+    DEBUGF("realloc(b->bytes=%p,newSize=%d)", b->bytes,newSize);
   /* XXX OSX realloc() seems to be able to corrupt things if the heap is not happy when calling realloc(), making debugging memory corruption much harder.
-     So will do a three-stage malloc,bcopy,free to see if we can tease bugs out that way. */
+    So will do a three-stage malloc,bcopy,free to see if we can tease bugs out that way. */
   /*
     unsigned char *r=realloc(b->bytes,newSize);
     if (!r) return WHY("realloc() failed");
@@ -196,94 +187,99 @@ int _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b,int byte
       sleep_ms(36000000);
     }
   }
-  unsigned char *new=malloc(newSize+4096);
-  if (!new) return WHY("realloc() failed");
+  unsigned char *new = emalloc(newSize+4096);
   {
     int i;
     for(i=0;i<4096;i++) new[newSize+i]=0xbd;
   }
 #else
-  unsigned char *new=malloc(newSize);
+  unsigned char *new = emalloc(newSize);
 #endif
+  if (!new)
+    return 0;
   bcopy(b->bytes,new,b->position);
-  if (b->allocated) free(b->allocated);
+  if (b->allocated) {
+    assert(b->allocated == b->bytes);
+    free(b->allocated);
+  }
   b->bytes=new;
   b->allocated=new;
   b->allocSize=newSize;
-  return 0;
+  return 1;
 }
 
-
-
 /*
  Functions that append data and increase the size of the buffer if possible / required
  */
 
-int _ob_append_byte(struct __sourceloc __whence, struct overlay_buffer *b,unsigned char byte)
+void ob_append_byte(struct overlay_buffer *b,unsigned char byte)
 {
-  if (_ob_makespace(__whence, b,1)) return WHY("ob_makespace() failed");
-  b->bytes[b->position++] = byte;
-  return 0;
+  const int bytes = 1;
+  if (ob_makespace(b, bytes))
+    b->bytes[b->position] = byte;
+  b->position += bytes;
 }
 
-unsigned char *_ob_append_space(struct __sourceloc __whence, struct overlay_buffer *b,int count)
+unsigned char *ob_append_space(struct overlay_buffer *b, int count)
 {
-  if (_ob_makespace(__whence, b,count))  {
-    WHY("ob_makespace() failed");
-    return NULL;
-  }
-  
-  unsigned char *r=&b->bytes[b->position];
-  b->position+=count;
+  assert(count > 0);
+  unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL;
+  b->position += count;
   return r;
 }
 
-int _ob_append_bytes(struct __sourceloc __whence, struct overlay_buffer *b, const unsigned char *bytes, int count)
+void ob_append_bytes(struct overlay_buffer *b, const unsigned char *bytes, int count)
 {
-  if (_ob_makespace(__whence, b,count)) return WHY("ob_makespace() failed");
-  
-  bcopy(bytes,&b->bytes[b->position],count);
-  b->position+=count;
-  return 0;
+  assert(count > 0);
+  unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL;
+  if (r)
+    bcopy(bytes, r, count);
+  b->position += count;
+  if (r)
+    bcopy(bytes, r, count);
 }
 
-int _ob_append_buffer(struct __sourceloc __whence, struct overlay_buffer *b, struct overlay_buffer *s){
-  return _ob_append_bytes(__whence, b, s->bytes, s->position);
+void append_buffer(struct overlay_buffer *b, struct overlay_buffer *s)
+{
+  ob_append_bytes(b, s->bytes, s->position);
 }
 
-int _ob_append_ui16(struct __sourceloc __whence, struct overlay_buffer *b, uint16_t v)
+void ob_append_ui16(struct overlay_buffer *b, uint16_t v)
 {
-  if (_ob_makespace(__whence, b, 2)) return WHY("ob_makespace() failed");
-  b->bytes[b->position] = (v >> 8) & 0xFF;
-  b->bytes[b->position+1] = v & 0xFF;
-  b->position+=2;
-  return 0;
+  const int bytes = 2;
+  if (ob_makespace(b, bytes)) {
+    b->bytes[b->position] = (v >> 8) & 0xFF;
+    b->bytes[b->position+1] = v & 0xFF;
+  }
+  b->position += bytes;
 }
 
-int _ob_append_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
+void ob_append_ui32(struct overlay_buffer *b, uint32_t v)
 {
-  if (_ob_makespace(__whence, b, 4)) return WHY("ob_makespace() failed");
-  b->bytes[b->position] = (v >> 24) & 0xFF;
-  b->bytes[b->position+1] = (v >> 16) & 0xFF;
-  b->bytes[b->position+2] = (v >> 8) & 0xFF;
-  b->bytes[b->position+3] = v & 0xFF;
-  b->position+=4;
-  return 0;
+  const int bytes = 4;
+  if (ob_makespace(b, bytes)) {
+    b->bytes[b->position] = (v >> 24) & 0xFF;
+    b->bytes[b->position+1] = (v >> 16) & 0xFF;
+    b->bytes[b->position+2] = (v >> 8) & 0xFF;
+    b->bytes[b->position+3] = v & 0xFF;
+  }
+  b->position += bytes;
 }
 
-int _ob_append_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
+void ob_append_ui64(struct overlay_buffer *b, uint64_t v)
 {
-  if (_ob_makespace(__whence, b, 8)) return WHY("ob_makespace() failed");
-  b->bytes[b->position] = (v >> 56) & 0xFF;
-  b->bytes[b->position+1] = (v >> 48) & 0xFF;
-  b->bytes[b->position+2] = (v >> 40) & 0xFF;
-  b->bytes[b->position+3] = (v >> 32) & 0xFF;
-  b->bytes[b->position+4] = (v >> 24) & 0xFF;
-  b->bytes[b->position+5] = (v >> 16) & 0xFF;
-  b->bytes[b->position+6] = (v >> 8) & 0xFF;
-  b->bytes[b->position+7] = v & 0xFF;
-  b->position+=8;
-  return 0;
+  const int bytes = 8;
+  if (ob_makespace(b, bytes)) {
+    b->bytes[b->position] = (v >> 56) & 0xFF;
+    b->bytes[b->position+1] = (v >> 48) & 0xFF;
+    b->bytes[b->position+2] = (v >> 40) & 0xFF;
+    b->bytes[b->position+3] = (v >> 32) & 0xFF;
+    b->bytes[b->position+4] = (v >> 24) & 0xFF;
+    b->bytes[b->position+5] = (v >> 16) & 0xFF;
+    b->bytes[b->position+6] = (v >> 8) & 0xFF;
+    b->bytes[b->position+7] = v & 0xFF;
+  }
+  b->position += bytes;
 }
 
 int measure_packed_uint(uint64_t v){
@@ -320,36 +316,28 @@ int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v){
   return i;
 }
 
-int _ob_append_packed_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
+void ob_append_packed_ui32(struct overlay_buffer *b, uint32_t v)
 {
-  do{
-    
-    if (_ob_append_byte(__whence, b, (v&0x7f) | (v>0x7f?0x80:0)))
-      return -1;
-    v = v>>7;
-    
-  }while(v!=0);
-  return 0;
+  do {
+    ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0));
+    v = v >> 7;
+  } while (v != 0);
 }
 
-int _ob_append_packed_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
+void ob_append_packed_ui64(struct overlay_buffer *b, uint64_t v)
 {
-  do{
-    
-    if (ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0)))
-      return -1;
-    v = v>>7;
-    
-  }while(v!=0);
-  return 0;
+  do {
+    ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0));
+    v = v >> 7;
+  } while (v != 0);
 }
 
-int _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l)
+void ob_append_rfs(struct overlay_buffer *b, int l)
 {
-  if (l<0||l>0xffff) return -1;
-  
-  b->var_length_offset=b->position;
-  return _ob_append_ui16(__whence, b,l);
+  assert(l >= 0);
+  assert(l <= 0xffff);
+  b->var_length_offset = b->position;
+  ob_append_ui16(b, l);
 }
 
 
@@ -359,7 +347,8 @@ int _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l)
 
 
 // make sure a range of bytes is valid for reading
-int test_offset(struct overlay_buffer *b,int start,int length){
+int test_offset(struct overlay_buffer *b,int start,int length)
+{
   if (!b) return -1;
   if (start<0) return -1;
   if (b->sizeLimit>=0 && start+length>b->sizeLimit) return -1;
@@ -471,39 +460,59 @@ int ob_get(struct overlay_buffer *b){
   return b->bytes[b->position++];
 }
 
-int ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v)
+void ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v)
 {
-  if (test_offset(b, offset, 2))
-    return -1;
-  
+  assert(b != NULL);
+  assert(offset >= 0);
+  if (b->sizeLimit != -1)
+    assert(offset + 2 <= b->sizeLimit);
+  assert(offset + 2 <= b->allocSize);
   b->bytes[offset] = (v >> 8) & 0xFF;
   b->bytes[offset+1] = v & 0xFF;
-  return 0;
 }
 
-int ob_set(struct overlay_buffer *b, int ofs, unsigned char byte)
+void ob_set(struct overlay_buffer *b, int offset, unsigned char byte)
 {
-  if (test_offset(b, ofs, 1))
-    return -1;
-  b->bytes[ofs] = byte;
-  return 0;
+  assert(b != NULL);
+  assert(offset >= 0);
+  if (b->sizeLimit != -1)
+    assert(offset + 1 <= b->sizeLimit);
+  assert(offset + 1 <= b->allocSize);
+  b->bytes[offset] = byte;
 }
 
-int ob_patch_rfs(struct overlay_buffer *b){
-  return ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2));
+void ob_patch_rfs(struct overlay_buffer *b)
+{
+  ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2));
 }
 
 
-int ob_position(struct overlay_buffer *b){
+int ob_position(struct overlay_buffer *b)
+{
   return b->position;
 }
-int ob_limit(struct overlay_buffer *b){
+
+int ob_limit(struct overlay_buffer *b)
+{
   return b->sizeLimit;
 }
-int ob_remaining(struct overlay_buffer *b){
+
+int ob_remaining(struct overlay_buffer *b)
+{
+  assert(b->sizeLimit != -1);
   return b->sizeLimit - b->position;
 }
-unsigned char *ob_ptr(struct overlay_buffer *b){
+
+int ob_overrun(struct overlay_buffer *b)
+{
+  assert(b->allocSize >= 0);
+  if (b->sizeLimit != -1)
+    assert(b->sizeLimit >= 0);
+  return b->position > (b->sizeLimit != -1 && b->sizeLimit < b->allocSize ? b->sizeLimit : b->allocSize);
+}
+
+unsigned char *ob_ptr(struct overlay_buffer *b)
+{
   return b->bytes;
 }
 
diff --git a/overlay_buffer.h b/overlay_buffer.h
index 49cc8ca8..5f2b2897 100644
--- a/overlay_buffer.h
+++ b/overlay_buffer.h
@@ -17,8 +17,8 @@
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-#ifndef _SERVALD_OVERLAY_BUFFER_H
-#define _SERVALD_OVERLAY_BUFFER_H
+#ifndef __SERVALD__OVERLAY_BUFFER_H
+#define __SERVALD__OVERLAY_BUFFER_H
 
 struct overlay_buffer {
   unsigned char *bytes;
@@ -46,39 +46,28 @@ struct overlay_buffer *ob_new(void);
 struct overlay_buffer *ob_static(unsigned char *bytes, int size);
 struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length);
 struct overlay_buffer *ob_dup(struct overlay_buffer *b);
-int ob_free(struct overlay_buffer *b);
+void ob_free(struct overlay_buffer *b);
 int ob_checkpoint(struct overlay_buffer *b);
 int ob_rewind(struct overlay_buffer *b);
-int ob_limitsize(struct overlay_buffer *b,int bytes);
-int ob_flip(struct overlay_buffer *b);
-int ob_unlimitsize(struct overlay_buffer *b);
-int _ob_makespace(struct __sourceloc whence, struct overlay_buffer *b,int bytes);
-int ob_set(struct overlay_buffer *b, int ofs, unsigned char byte);
+void ob_limitsize(struct overlay_buffer *b,int bytes);
+void ob_flip(struct overlay_buffer *b);
+void ob_unlimitsize(struct overlay_buffer *b);
+ssize_t ob_makespace(struct overlay_buffer *b, size_t bytes);
+void ob_set(struct overlay_buffer *b, int ofs, unsigned char byte);
+void ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v);
+void ob_patch_rfs(struct overlay_buffer *b);
 
-int _ob_append_byte(struct __sourceloc whence, struct overlay_buffer *b,unsigned char byte);
-int _ob_append_bytes(struct __sourceloc whence, struct overlay_buffer *b,const unsigned char *bytes,int count);
-int _ob_append_buffer(struct __sourceloc whence, struct overlay_buffer *b,struct overlay_buffer *s);
-unsigned char *_ob_append_space(struct __sourceloc whence, struct overlay_buffer *b,int count);
-int _ob_append_ui16(struct __sourceloc whence, struct overlay_buffer *b, uint16_t v);
-int _ob_append_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
-int _ob_append_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
-int _ob_append_packed_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
-int _ob_append_packed_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
-int _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l);
+void ob_append_byte(struct overlay_buffer *b,unsigned char byte);
+void ob_append_bytes(struct overlay_buffer *b,const unsigned char *bytes,int count);
+void ob_append_buffer(struct overlay_buffer *b,struct overlay_buffer *s);
+unsigned char *ob_append_space(struct overlay_buffer *b,int count);
+void ob_append_ui16(struct overlay_buffer *b, uint16_t v);
+void ob_append_ui32(struct overlay_buffer *b, uint32_t v);
+void ob_append_ui64(struct overlay_buffer *b, uint64_t v);
+void ob_append_packed_ui32(struct overlay_buffer *b, uint32_t v);
+void ob_append_packed_ui64(struct overlay_buffer *b, uint64_t v);
+void ob_append_rfs(struct overlay_buffer *b,int l);
 
-#define ob_makespace(b, bytes) _ob_makespace(__WHENCE__, b, bytes)
-#define ob_append_byte(b, byte) _ob_append_byte(__WHENCE__, b, byte)
-#define ob_append_bytes(b, bytes, count) _ob_append_bytes(__WHENCE__, b, bytes, count)
-#define ob_append_buffer(b, s) _ob_append_buffer(__WHENCE__, b, s)
-#define ob_append_space(b, count) _ob_append_space(__WHENCE__, b, count)
-#define ob_append_ui16(b, v) _ob_append_ui16(__WHENCE__, b, v)
-#define ob_append_ui32(b, v) _ob_append_ui32(__WHENCE__, b, v)
-#define ob_append_ui64(b, v) _ob_append_ui64(__WHENCE__, b, v)
-#define ob_append_packed_ui32(b, v) _ob_append_packed_ui32(__WHENCE__, b, v)
-#define ob_append_packed_ui64(b, v) _ob_append_packed_ui64(__WHENCE__, b, v)
-#define ob_append_rfs(b, l) _ob_append_rfs(__WHENCE__, b, l)
-
-int ob_patch_rfs(struct overlay_buffer *b);
 // get one byte, -ve number indicates failure
 int ob_getbyte(struct overlay_buffer *b,int ofs);
 // get one byte from the current position, -ve number indicates failure
@@ -89,7 +78,6 @@ uint64_t ob_get_ui64(struct overlay_buffer *b);
 uint32_t ob_get_ui32(struct overlay_buffer *b);
 uint16_t ob_get_ui16(struct overlay_buffer *b);
 int ob_dump(struct overlay_buffer *b,char *desc);
-int ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v);
 
 uint32_t ob_get_packed_ui32(struct overlay_buffer *b);
 uint64_t ob_get_packed_ui64(struct overlay_buffer *b);
@@ -98,5 +86,7 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b);
 int ob_position(struct overlay_buffer *b);
 int ob_limit(struct overlay_buffer *b);
 int ob_remaining(struct overlay_buffer *b);
+int ob_overrun(struct overlay_buffer *b);
 unsigned char* ob_ptr(struct overlay_buffer *b);
-#endif
+
+#endif //__SERVALD__OVERLAY_BUFFER_H
diff --git a/overlay_link.c b/overlay_link.c
index e66e992c..220d605a 100644
--- a/overlay_link.c
+++ b/overlay_link.c
@@ -1,3 +1,4 @@
+#include <assert.h>
 #include "serval.h"
 #include "conf.h"
 #include "str.h"
@@ -222,14 +223,14 @@ int overlay_send_probe(struct subscriber *peer, struct network_destination *dest
   frame->ttl=1;
   frame->queue=queue;
   frame->destinations[frame->destination_count++].destination=add_destination_ref(destination);
-  frame->payload = ob_new();
-  frame->source_full = 1;
-  // TODO call mdp payload encryption / signing without calling overlay_mdp_dispatch...
-  
-  if (overlay_mdp_encode_ports(frame->payload, MDP_PORT_ECHO, MDP_PORT_PROBE)){
+  if ((frame->payload = ob_new()) == NULL) {
     op_free(frame);
     return -1;
   }
+  frame->source_full = 1;
+  // TODO call mdp payload encryption / signing without calling overlay_mdp_dispatch...
+  
+  overlay_mdp_encode_ports(frame->payload, MDP_PORT_ECHO, MDP_PORT_PROBE);
   // not worried about byte order here as we are the only node that should be parsing the contents.
   unsigned char *dst=ob_append_space(frame->payload, sizeof(struct probe_contents));
   if (!dst){
@@ -254,25 +255,21 @@ int overlay_send_probe(struct subscriber *peer, struct network_destination *dest
 }
 
 // append the address of a unicast link into a packet buffer
-static int overlay_append_unicast_address(struct subscriber *subscriber, struct overlay_buffer *buff)
+static void overlay_append_unicast_address(struct subscriber *subscriber, struct overlay_buffer *buff)
 {
-  if (subscriber->destination 
-    && subscriber->destination->unicast
-    && subscriber->destination->address.sin_family==AF_INET){
-    if (overlay_address_append(NULL, buff, subscriber))
-      return -1;
-    if (ob_append_ui32(buff, subscriber->destination->address.sin_addr.s_addr))
-      return -1;
-    if (ob_append_ui16(buff, subscriber->destination->address.sin_port))
-      return -1;
-    ob_checkpoint(buff);
+  if (   subscriber->destination 
+      && subscriber->destination->unicast
+      && subscriber->destination->address.sin_family==AF_INET
+  ) {
+    overlay_address_append(NULL, buff, subscriber);
+    ob_append_ui32(buff, subscriber->destination->address.sin_addr.s_addr);
+    ob_append_ui16(buff, subscriber->destination->address.sin_port);
     if (config.debug.overlayrouting)
       DEBUGF("Added STUN info for %s", alloca_tohex_sid_t(subscriber->sid));
   }else{
     if (config.debug.overlayrouting)
       DEBUGF("Unable to give address of %s, %d", alloca_tohex_sid_t(subscriber->sid),subscriber->reachable);
   }
-  return 0;
 }
 
 int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp)
@@ -296,22 +293,20 @@ int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp)
   struct overlay_buffer *replypayload = ob_static(reply.out.payload, sizeof(reply.out.payload));
   
   ob_checkpoint(replypayload);
-  while(ob_remaining(payload)>0){
+  while (ob_remaining(payload) > 0) {
     struct subscriber *subscriber=NULL;
-    
     if (overlay_address_parse(NULL, payload, &subscriber))
       break;
-    
     if (!subscriber){
       if (config.debug.overlayrouting)
 	DEBUGF("Unknown subscriber");
       continue;
     }
-    
-    if (overlay_append_unicast_address(subscriber, replypayload))
+    overlay_append_unicast_address(subscriber, replypayload);
+    if (ob_overrun(payload))
       break;
+    ob_checkpoint(replypayload);
   }
-  
   ob_rewind(replypayload);
   reply.out.payload_length=ob_position(replypayload);
   
@@ -388,11 +383,12 @@ int overlay_send_stun_request(struct subscriber *server, struct subscriber *requ
   
   struct overlay_buffer *payload = ob_static(mdp.out.payload, sizeof(mdp.out.payload));
   overlay_address_append(NULL, payload, request);
-  mdp.out.payload_length=ob_position(payload);
-  if (config.debug.overlayrouting)
-    DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid));
-  overlay_mdp_dispatch(&mdp,0 /* system generated */,
-		       NULL,0);
+  if (!ob_overrun(payload)) {
+    mdp.out.payload_length=ob_position(payload);
+    if (config.debug.overlayrouting)
+      DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid));
+    overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL,0);
+  }
   ob_free(payload);
   return 0;
 }
diff --git a/overlay_mdp.c b/overlay_mdp.c
index 80eb3fb8..67c58cce 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -267,9 +267,7 @@ static int overlay_mdp_decode_header(struct overlay_buffer *buff, overlay_mdp_fr
   if (!same)
     port = ob_get_packed_ui32(buff);
   mdp->in.src.port = port;
-  
-  int len=ob_remaining(buff);
-  
+  int len = ob_remaining(buff);
   if (len<0)
     return WHY("MDP payload is too short");
   mdp->in.payload_length=len;
@@ -316,10 +314,8 @@ int overlay_mdp_decrypt(struct overlay_frame *f, overlay_mdp_frame *mdp)
       if (!k) 
 	RETURN(WHY("I don't have the private key required to decrypt that"));
       
-      if (0){
-	dump("frame",&f->payload->bytes[f->payload->position],
-	     ob_remaining(f->payload));
-      }
+      if (0)
+	dump("frame",&f->payload->bytes[f->payload->position], ob_remaining(f->payload));
       
       unsigned char *nonce=ob_get_bytes_ptr(f->payload, nb);
       if (!nonce)
@@ -529,18 +525,14 @@ int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port, in
       );
 }
 
-int overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port)
+void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port)
 {
   mdp_port_t port = dst_port << 1;
   if (dst_port == src_port)
     port |= 1;
-  if (ob_append_packed_ui32(plaintext, port))
-    return -1;
-  if (dst_port != src_port){
-    if (ob_append_packed_ui32(plaintext, src_port))
-      return -1;
-  }
-  return 0;
+  ob_append_packed_ui32(plaintext, port);
+  if (dst_port != src_port)
+    ob_append_packed_ui32(plaintext, src_port);
 }
 
 static struct overlay_buffer * encrypt_payload(
@@ -564,13 +556,15 @@ static struct overlay_buffer * encrypt_payload(
   cipher_len+=zb;
   
   struct overlay_buffer *ret = ob_new();
+  if (ret == NULL)
+    return NULL;
   
   unsigned char *nonce = ob_append_space(ret, nb+cipher_len);
-  unsigned char *cipher_text = nonce + nb;
   if (!nonce){
     ob_free(ret);
     return NULL;
   }
+  unsigned char *cipher_text = nonce + nb;
 
   if (generate_nonce(nonce,nb)){
     ob_free(ret);
@@ -644,8 +638,9 @@ int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plain
   case OF_CRYPTO_SIGNED:
     // Lets just append some space into the existing payload buffer for the signature, without copying it.
     frame->payload = plaintext;
-    ob_makespace(frame->payload,SIGNATURE_BYTES);
-    if (crypto_sign_message(frame->source, ob_ptr(frame->payload), frame->payload->allocSize, &frame->payload->position)){
+    if (   ob_makespace(frame->payload, SIGNATURE_BYTES) != SIGNATURE_BYTES
+        || crypto_sign_message(frame->source, ob_ptr(frame->payload), frame->payload->allocSize, &frame->payload->position) == -1
+    ) {
       op_free(frame);
       return -1;
     }
@@ -773,14 +768,13 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
   
   // copy the plain text message into a new buffer, with the wire encoded port numbers
   struct overlay_buffer *plaintext=ob_new();
-  if (!plaintext)
+  if (plaintext == NULL)
     RETURN(-1);
   
-  if (overlay_mdp_encode_ports(plaintext, mdp->out.dst.port, mdp->out.src.port)){
-    ob_free(plaintext);
-    RETURN (-1);
-  }
-  if (ob_append_bytes(plaintext, mdp->out.payload, mdp->out.payload_length)){
+  overlay_mdp_encode_ports(plaintext, mdp->out.dst.port, mdp->out.src.port);
+  if (mdp->out.payload_length)
+    ob_append_bytes(plaintext, mdp->out.payload, mdp->out.payload_length);
+  if (ob_overrun(plaintext)) {
     ob_free(plaintext);
     RETURN(-1);
   }
diff --git a/overlay_mdp_services.c b/overlay_mdp_services.c
index 38138692..8298a135 100644
--- a/overlay_mdp_services.c
+++ b/overlay_mdp_services.c
@@ -334,15 +334,14 @@ static int overlay_mdp_service_trace(overlay_mdp_frame *mdp){
   ob_unlimitsize(b);
   // always write a full sid into the payload
   my_subscriber->send_full=1;
-  if (overlay_address_append(&context, b, my_subscriber)){
+  overlay_address_append(&context, b, my_subscriber);
+  if (ob_overrun(b)) {
     ret = WHYF("Unable to append my address to the trace");
     goto end;
   }
-  
   mdp->out.payload_length = ob_position(b);
   mdp->out.src.sid = my_subscriber->sid;
   mdp->out.dst.sid = next->sid;
-  
   ret = overlay_mdp_dispatch(mdp, 0, NULL, 0);
 end:
   ob_free(b);
diff --git a/overlay_olsr.c b/overlay_olsr.c
index 6eec98cd..9eb2d0d1 100644
--- a/overlay_olsr.c
+++ b/overlay_olsr.c
@@ -253,6 +253,8 @@ int olsr_send(struct overlay_frame *frame){
   struct decode_context context;
   bzero(&context, sizeof context);
   struct overlay_buffer *b=ob_new();
+  if (b == NULL)
+    return 0;
   
   // build olsr specific frame header
   ob_append_byte(b, PACKET_FORMAT_NUMBER);
@@ -260,7 +262,6 @@ int olsr_send(struct overlay_frame *frame){
   
   // address the packet as transmitted by me
   overlay_address_append(&context, b, my_subscriber);
-  
   overlay_address_append(&context, b, frame->source);
   overlay_broadcast_append(b, &frame->broadcast_id);
   ob_append_byte(b, frame->modifiers);
diff --git a/overlay_packetformats.c b/overlay_packetformats.c
index c61a53a8..c2c39208 100644
--- a/overlay_packetformats.c
+++ b/overlay_packetformats.c
@@ -42,20 +42,16 @@ int overlay_packet_init_header(int packet_version, int encapsulation,
   if (encapsulation !=ENCAP_OVERLAY && encapsulation !=ENCAP_SINGLE)
     return WHY("Invalid packet encapsulation");
   
-  if (ob_append_byte(buff, packet_version))
-    return -1;
-  if (ob_append_byte(buff, encapsulation))
-    return -1;
+  ob_append_byte(buff, packet_version);
+  ob_append_byte(buff, encapsulation);
   
-  if (context->interface->point_to_point 
-    && context->interface->other_device 
-    && packet_version>=1)
+  if (   context->interface->point_to_point 
+      && context->interface->other_device 
+      && packet_version>=1
+  )
     context->point_to_point_device = context->interface->other_device;
-    
   context->encoding_header=1;
-  
-  if (overlay_address_append(context, buff, my_subscriber))
-    return -1;
+  overlay_address_append(context, buff, my_subscriber);
   
   context->encoding_header=0;
   context->sender = my_subscriber;
diff --git a/overlay_payload.c b/overlay_payload.c
index ec88c117..05f43215 100644
--- a/overlay_payload.c
+++ b/overlay_payload.c
@@ -51,34 +51,28 @@ static int overlay_frame_build_header(int packet_version, struct decode_context
   if (type!=OF_TYPE_DATA)
     flags |= PAYLOAD_FLAG_LEGACY_TYPE;
   
-  if (ob_append_byte(buff, flags)) return -1;
+  ob_append_byte(buff, flags);
   
-  if (!(flags & PAYLOAD_FLAG_SENDER_SAME)){
-    if (overlay_address_append(context, buff, source)) return -1;
-  }
+  if (!(flags & PAYLOAD_FLAG_SENDER_SAME))
+    overlay_address_append(context, buff, source);
   
   if (flags & PAYLOAD_FLAG_TO_BROADCAST){
-    if (!(flags & PAYLOAD_FLAG_ONE_HOP)){
-      if (overlay_broadcast_append(buff, broadcast)) return -1;
-    }
-  }else{
-    if (overlay_address_append(context, buff, destination)) return -1;
-    if (!(flags & PAYLOAD_FLAG_ONE_HOP)){
-      if (overlay_address_append(context, buff, next_hop)) return -1;
-    }
+    if (!(flags & PAYLOAD_FLAG_ONE_HOP))
+      overlay_broadcast_append(buff, broadcast);
+  } else {
+    overlay_address_append(context, buff, destination);
+    if (!(flags & PAYLOAD_FLAG_ONE_HOP))
+      overlay_address_append(context, buff, next_hop);
   }
   
-  if (!(flags & PAYLOAD_FLAG_ONE_HOP)){
-    if (ob_append_byte(buff, ttl | ((queue&3)<<5))) return -1;
-  }
+  if (!(flags & PAYLOAD_FLAG_ONE_HOP))
+    ob_append_byte(buff, ttl | ((queue&3)<<5));
   
-  if (flags & PAYLOAD_FLAG_LEGACY_TYPE){
-    if (ob_append_byte(buff, type)) return -1;
-  }
+  if (flags & PAYLOAD_FLAG_LEGACY_TYPE)
+    ob_append_byte(buff, type);
 
   if (packet_version >= 1)
-    if (ob_append_byte(buff, sequence))
-      return -1;
+    ob_append_byte(buff, sequence);
   
   return 0;
 }
@@ -112,20 +106,17 @@ int overlay_frame_append_payload(struct decode_context *context, int encapsulati
 			     p->queue, p->type, p->modifiers, will_retransmit, 
 			     p->ttl, p->mdp_sequence&0xFF,
 			     broadcast, p->next_hop, 
-			     p->destination, p->source))
+			     p->destination, p->source) == -1)
     goto cleanup;
   
-  if (encapsulation == ENCAP_OVERLAY){
-    if (ob_append_ui16(b, ob_position(p->payload)))
-      goto cleanup;
-  }
+  if (encapsulation == ENCAP_OVERLAY)
+    ob_append_ui16(b, ob_position(p->payload));
 
-  if (ob_append_bytes(b, ob_ptr(p->payload), ob_position(p->payload))) {
-    WHYF("could not append payload of %u bytes", ob_position(p->payload));
-    goto cleanup;
-  }
-      
-  return 0;
+  if (ob_position(p->payload))
+    ob_append_bytes(b, ob_ptr(p->payload), ob_position(p->payload));
+
+  if (!ob_overrun(b))
+    return 0;
   
 cleanup:
   ob_rewind(b);
@@ -150,13 +141,18 @@ struct overlay_frame *op_dup(struct overlay_frame *in)
   if (!in) return NULL;
 
   /* clone the frame */
-  struct overlay_frame *out=malloc(sizeof(struct overlay_frame));
-  if (!out) { WHY("malloc() failed"); return NULL; }
+  struct overlay_frame *out = emalloc(sizeof(struct overlay_frame));
+  if (out == NULL)
+    return NULL;
 
   /* copy main data structure */
   bcopy(in,out,sizeof(struct overlay_frame));
 
-  if (in->payload)
-    out->payload=ob_dup(in->payload);
+  if (in->payload) {
+    if ((out->payload = ob_dup(in->payload)) == NULL) {
+      free(out);
+      return NULL;
+    }
+  }
   return out;
 }
diff --git a/overlay_queue.c b/overlay_queue.c
index 88815418..10170057 100644
--- a/overlay_queue.c
+++ b/overlay_queue.c
@@ -17,6 +17,7 @@
  */
 
 
+#include <assert.h>
 #include "serval.h"
 #include "conf.h"
 #include "overlay_buffer.h"
@@ -150,11 +151,9 @@ int overlay_payload_enqueue(struct overlay_frame *p)
    Complain if there are too many frames in the queue.
    */
   
-  if (!p) return WHY("Cannot queue NULL");
-  
-  if (p->queue>=OQ_MAX) 
-    return WHY("Invalid queue specified");
-  
+  assert(p != NULL);
+  assert(p->queue < OQ_MAX);
+  assert(p->payload != NULL);
   overlay_txqueue *queue = &overlay_tx[p->queue];
 
   if (config.debug.packettx)
@@ -162,18 +161,15 @@ int overlay_payload_enqueue(struct overlay_frame *p)
 	   p->destination?alloca_tohex_sid_t_trunc(p->destination->sid, 14): alloca_tohex(p->broadcast_id.id, BROADCAST_LEN),
 	   p->queue, queue->length);
   
-  if (p->payload && ob_remaining(p->payload)<0){
-    // HACK, maybe should be done in each caller
-    // set the size of the payload based on the position written
-    ob_limitsize(p->payload,ob_position(p->payload));
-  }
+  if (ob_overrun(p->payload))
+    return WHY("Packet content overrun -- not queueing");
   
+  if (ob_position(p->payload) >= MDP_MTU)
+    FATAL("Queued packet is too big");
+
   if (queue->length>=queue->maxLength) 
     return WHYF("Queue #%d congested (size = %d)",p->queue,queue->maxLength);
     
-  if (ob_position(p->payload)>=MDP_MTU)
-    FATAL("Queued packet is too big");
-
   // it should be safe to try sending all packets with an mdp sequence
   if (p->packet_version<=0)
     p->packet_version=1;
@@ -226,11 +222,13 @@ int overlay_payload_enqueue(struct overlay_frame *p)
   return 0;
 }
 
-static void
+static int
 overlay_init_packet(struct outgoing_packet *packet, int packet_version,
-		    struct network_destination *destination){
+		    struct network_destination *destination)
+{
   packet->context.interface = destination->interface;
-  packet->buffer=ob_new();
+  if ((packet->buffer = ob_new()) == NULL)
+    return -1;
   packet->packet_version = packet_version;
   packet->context.packet_version = packet_version;
   packet->destination = add_destination_ref(destination);
@@ -238,20 +236,24 @@ overlay_init_packet(struct outgoing_packet *packet, int packet_version,
     packet->seq=-1;
   else
     packet->seq = destination->sequence_number = (destination->sequence_number + 1) & 0xFFFF;
-  
   ob_limitsize(packet->buffer, destination->interface->mtu);
-  
-  int i=destination->interface - overlay_interfaces;
-  overlay_packet_init_header(packet_version, destination->encapsulation, 
-			     &packet->context, packet->buffer, 
-			     destination->unicast, 
-			     i, packet->seq);
+  int i = destination->interface - overlay_interfaces;
+  if (overlay_packet_init_header(packet_version, destination->encapsulation, 
+				 &packet->context, packet->buffer, 
+				 destination->unicast, 
+				 i, packet->seq) == -1
+  ) {
+    ob_free(packet->buffer);
+    packet->buffer = NULL;
+    return -1;
+  }
   packet->header_length = ob_position(packet->buffer);
   if (config.debug.overlayframes)
     DEBUGF("Creating %d packet for interface %s, seq %d, %s", 
       packet_version,
       destination->interface->name, destination->sequence_number,
       destination->unicast?"unicast":"broadcast");
+  return 0;
 }
 
 int overlay_queue_schedule_next(time_ms_t next_allowed_packet){
@@ -331,8 +333,9 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
   while(frame){
     if (frame->enqueued_at + queue->latencyTarget < now){
       if (config.debug.overlayframes)
-	DEBUGF("Dropping frame type %x for %s due to expiry timeout", 
-	       frame->type, frame->destination?alloca_tohex_sid_t(frame->destination->sid):"All");
+	DEBUGF("Dropping frame type %x (length %d) for %s due to expiry timeout", 
+	       frame->type, frame->payload->checkpointLength,
+	       frame->destination?alloca_tohex_sid_t(frame->destination->sid):"All");
       frame = overlay_queue_remove(queue, frame);
       continue;
     }
@@ -411,11 +414,11 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
 	  // send a packet to this destination
 	  if (frame->source_full)
 	    my_subscriber->send_full=1;
-
-	  overlay_init_packet(packet, frame->packet_version, dest);
-	  destination_index=i;
-	  frame->destinations[i].sent_sequence = dest->sequence_number;
-	  break;
+	  if (overlay_init_packet(packet, frame->packet_version, dest) != -1) {
+	    destination_index=i;
+	    frame->destinations[i].sent_sequence = dest->sequence_number;
+	    break;
+	  }
 	}
       }
     }
@@ -532,12 +535,12 @@ static void overlay_send_packet(struct sched_ent *alarm){
   overlay_fill_send_packet(&packet, gettime_ms());
 }
 
-int overlay_send_tick_packet(struct network_destination *destination){
+int overlay_send_tick_packet(struct network_destination *destination)
+{
   struct outgoing_packet packet;
   bzero(&packet, sizeof(struct outgoing_packet));
-  overlay_init_packet(&packet, 0, destination);
-  
-  overlay_fill_send_packet(&packet, gettime_ms());
+  if (overlay_init_packet(&packet, 0, destination) != -1)
+    overlay_fill_send_packet(&packet, gettime_ms());
   return 0;
 }
 
diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c
index 092976f4..01af7c16 100644
--- a/rhizome_packetformats.c
+++ b/rhizome_packetformats.c
@@ -168,11 +168,12 @@ static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, cons
 	DEBUG("Found a BAR that is the wrong size - ignoring");
       continue;
     }
-    if (ob_append_bytes(e, (unsigned char *)data, blob_bytes)){
+    if (ob_remaining(e) < blob_bytes) {
       // out of room
       count--;
       break;
     }
+    ob_append_bytes(e, (unsigned char *)data, blob_bytes);
     *last_rowid=rowid;
   }
   if (statement)
@@ -184,7 +185,8 @@ static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, cons
  Always advertise the most recent 3 manifests in the table, cycle through the rest of the table, adding 17 BAR's at a time
  */
 int64_t bundles_available=0;
-void overlay_rhizome_advertise(struct sched_ent *alarm){
+void overlay_rhizome_advertise(struct sched_ent *alarm)
+{
   bundles_available=0;
   static int64_t bundle_last_rowid=INT64_MAX;
   
@@ -197,7 +199,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){
   int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
 
-  // DEPRECATE REST OF THIS CODE WHICH SEEMS TO BE CAUSING TOO MUCH CHATTER
+  // TODO: DEPRECATE REST OF THIS CODE WHICH SEEMS TO BE CAUSING TOO MUCH CHATTER
   // ESPECIALLY FOR PACKET-RADIO
   goto end;
 
@@ -216,31 +218,28 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){
   frame->source = my_subscriber;
   frame->ttl = 1;
   frame->queue = OQ_OPPORTUNISTIC;
-  frame->payload = ob_new();
+  if ((frame->payload = ob_new()) == NULL) {
+    op_free(frame);
+    goto end;
+  }
   ob_limitsize(frame->payload, 800);
-  
   ob_append_byte(frame->payload, 2);
   ob_append_ui16(frame->payload, rhizome_http_server_port);
-  
   int64_t rowid=0;
   int count = append_bars(frame->payload, &retry, 
 			  "SELECT BAR,ROWID FROM MANIFESTS ORDER BY ROWID DESC LIMIT 3", 
 			  &rowid);
-  
   if (count>=3){
     if (bundle_last_rowid>rowid || bundle_last_rowid<=0)
       bundle_last_rowid=rowid;
-    
     count = append_bars(frame->payload, &retry, 
 			"SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < ? ORDER BY ROWID DESC LIMIT 17", 
 			&bundle_last_rowid);
     if (count<17)
       bundle_last_rowid=INT64_MAX;
   }
-  
-  if (overlay_payload_enqueue(frame))
+  if (overlay_payload_enqueue(frame) == -1)
     op_free(frame);
-
 end:
   sqlite_set_tracefunc(oldfunc);
   alarm->alarm = gettime_ms()+config.rhizome.advertise.interval;
@@ -262,21 +261,20 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){
   else
     frame->ttl = 1;
   frame->queue = OQ_OPPORTUNISTIC;
-  frame->payload = ob_new();
-  
+  if ((frame->payload = ob_new()) == NULL)
+    goto error;
   ob_limitsize(frame->payload, 800);
-  
-  if (ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS)) goto error;
-  if (ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0)) goto error;
-  if (ob_append_ui16(frame->payload, m->manifest_all_bytes)) goto error;
-  if (ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes)) goto error;
+  ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS);
+  ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0);
+  ob_append_ui16(frame->payload, m->manifest_all_bytes);
+  ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes);
   ob_append_byte(frame->payload, 0xFF);
-  if (overlay_payload_enqueue(frame)) goto error;
+  if (overlay_payload_enqueue(frame) == -1)
+    goto error;
   if (config.debug.rhizome_ads)
     DEBUGF("Advertising manifest %s %"PRId64" to %s", 
       alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid_t(dest->sid):"broadcast");
   return 0;
-  
 error:
   op_free(frame);
   return -1;
diff --git a/rhizome_sync.c b/rhizome_sync.c
index 415b0944..98162614 100644
--- a/rhizome_sync.c
+++ b/rhizome_sync.c
@@ -321,21 +321,16 @@ static void sync_process_bar_list(struct subscriber *subscriber, struct rhizome_
 
 }
 
-static int append_response(struct overlay_buffer *b, uint64_t token, const unsigned char *bar)
+static void append_response(struct overlay_buffer *b, uint64_t token, const unsigned char *bar)
 {
-  if (ob_append_packed_ui64(b, token))
-    return -1;
-  if (bar){
-    if (ob_append_bytes(b, bar, RHIZOME_BAR_BYTES))
-      return -1;
-  }else{
+  ob_append_packed_ui64(b, token);
+  if (bar)
+    ob_append_bytes(b, bar, RHIZOME_BAR_BYTES);
+  else{
     unsigned char *ptr = ob_append_space(b, RHIZOME_BAR_BYTES);
-    if (!ptr)
-      return -1;
-    bzero(ptr, RHIZOME_BAR_BYTES);
+    if (ptr)
+      bzero(ptr, RHIZOME_BAR_BYTES);
   }
-  ob_checkpoint(b);
-  return 0;
 }
 
 static uint64_t max_token=0;
@@ -400,24 +395,26 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
       // make sure we include the exact rowid that was requested, even if we just deleted / replaced the manifest
       if (count==0 && rowid!=token){
         if (token!=HEAD_FLAG){
-          if (append_response(b, token, NULL))
+	  ob_checkpoint(b);
+          append_response(b, token, NULL);
+	  if (ob_overrun(b))
 	    ob_rewind(b);
-	  else{
+	  else {
             count++;
             last = token;
 	  }
         }else
           token = rowid;
       }
-
-      if (append_response(b, rowid, bar))
+      ob_checkpoint(b);
+      append_response(b, rowid, bar);
+      if (ob_overrun(b))
 	ob_rewind(b);
       else {
         last = rowid;
         count++;
       }
     }
-
     if (count >= max_count && rowid <= max_token)
       break;
   }
@@ -427,7 +424,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
 
   // send a zero lower bound if we reached the end of our manifest list
   if (count && count < max_count && !forwards){
-    if (append_response(b, 0, NULL))
+    ob_checkpoint(b);
+    append_response(b, 0, NULL);
+    if (ob_overrun(b))
       ob_rewind(b);
     else {
       last = 0;
diff --git a/route_link.c b/route_link.c
index 17e1310e..e7581ec7 100644
--- a/route_link.c
+++ b/route_link.c
@@ -467,48 +467,27 @@ static int append_link_state(struct overlay_buffer *payload, char flags,
     flags|=FLAG_HAS_ACK;
   if (drop_rate!=-1)
     flags|=FLAG_HAS_DROP_RATE;
-
   int length_pos = ob_position(payload);
-  if (ob_append_byte(payload, 0))
-    return -1;
-
-  if (ob_append_byte(payload, flags))
-    return -1;
-
-  if (overlay_address_append(NULL, payload, receiver))
-    return -1;
-
-  if (ob_append_byte(payload, version))
-    return -1;
-
+  ob_append_byte(payload, 0);
+  ob_append_byte(payload, flags);
+  overlay_address_append(NULL, payload, receiver);
+  ob_append_byte(payload, version);
   if (transmitter)
-    if (overlay_address_append(NULL, payload, transmitter))
-      return -1;
-
-  if (interface!=-1)
-    if (ob_append_byte(payload, interface))
-      return -1;
-
-  if (ack_sequence!=-1){
-    if (ob_append_byte(payload, ack_sequence))
-      return -1;
-    if (ob_append_ui32(payload, ack_mask))
-      return -1;
+    overlay_address_append(NULL, payload, transmitter);
+  if (interface != -1)
+    ob_append_byte(payload, interface);
+  if (ack_sequence != -1){
+    ob_append_byte(payload, ack_sequence);
+    ob_append_ui32(payload, ack_mask);
   }
-
-  if (drop_rate!=-1)
-    if (ob_append_byte(payload, drop_rate))
-      return -1;
-
-
+  if (drop_rate != -1)
+    ob_append_byte(payload, drop_rate);
   // TODO insert future fields here
-
-
+  if (ob_overrun(payload))
+    return -1;
   // patch the record length
   int end_pos = ob_position(payload);
-  if (ob_set(payload, length_pos, end_pos - length_pos))
-    return -1;
-
+  ob_set(payload, length_pos, end_pos - length_pos);
   ob_checkpoint(payload);
   return 0;
 }
@@ -710,12 +689,15 @@ static int send_legacy_self_announce_ack(struct neighbour *neighbour, struct lin
   frame->ttl = 6;
   frame->destination = neighbour->subscriber;
   frame->source = my_subscriber;
-  frame->payload = ob_new();
+  if ((frame->payload = ob_new()) == NULL) {
+    op_free(frame);
+    return -1;
+  }
   ob_append_ui32(frame->payload, neighbour->last_update);
   ob_append_ui32(frame->payload, now);
   ob_append_byte(frame->payload, link->neighbour_interface);
   frame->queue=OQ_MESH_MANAGEMENT;
-  if (overlay_payload_enqueue(frame)){
+  if (overlay_payload_enqueue(frame) == -1) {
     op_free(frame);
     return -1;
   }
@@ -787,12 +769,16 @@ static int send_neighbour_link(struct neighbour *n)
     send_legacy_self_announce_ack(n, n->best_link, now);
     n->last_update = now;
   } else {
-    struct overlay_frame *frame=emalloc_zero(sizeof(struct overlay_frame));
+    struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame));
     frame->type=OF_TYPE_DATA;
     frame->source=my_subscriber;
     frame->ttl=1;
     frame->queue=OQ_MESH_MANAGEMENT;
-    frame->payload = ob_new();
+    if ((frame->payload = ob_new()) == NULL) {
+      op_free(frame);
+      RETURN(-1);
+    }
+
     frame->send_hook = neighbour_link_sent;
     frame->send_context = n->subscriber;
     frame->resend=-1;
@@ -825,7 +811,7 @@ static int send_neighbour_link(struct neighbour *n)
     
     append_link_state(frame->payload, flags, n->subscriber, my_subscriber, n->best_link->neighbour_interface, 1,
 	              n->best_link->ack_sequence, n->best_link->ack_mask, -1);
-    if (overlay_payload_enqueue(frame))
+    if (overlay_payload_enqueue(frame) == -1)
       op_free(frame);
 
     n->best_link->ack_counter = ACK_WINDOW;
@@ -885,27 +871,25 @@ static void link_send(struct sched_ent *alarm)
   frame->source=my_subscriber;
   frame->ttl=1;
   frame->queue=OQ_MESH_MANAGEMENT;
-  frame->payload = ob_new();
-  ob_limitsize(frame->payload, 400);
-
-  overlay_mdp_encode_ports(frame->payload, MDP_PORT_LINKSTATE, MDP_PORT_LINKSTATE);
-  ob_checkpoint(frame->payload);
-  int pos = ob_position(frame->payload);
-
-  enum_subscribers(NULL, append_link, frame->payload);
-
-  ob_rewind(frame->payload);
-
-  if (ob_position(frame->payload) == pos)
-    op_free(frame);
-  else if (overlay_payload_enqueue(frame))
-    op_free(frame);
-
-  if (neighbours){
-    alarm->deadline = alarm->alarm;
-    schedule(alarm);
-  }else
-    alarm->alarm=0;
+  if ((frame->payload = ob_new()) == NULL)
+    WHY("Cannot send link details");
+  else {
+    ob_limitsize(frame->payload, 400);
+    overlay_mdp_encode_ports(frame->payload, MDP_PORT_LINKSTATE, MDP_PORT_LINKSTATE);
+    ob_checkpoint(frame->payload);
+    int pos = ob_position(frame->payload);
+    enum_subscribers(NULL, append_link, frame->payload);
+    ob_rewind(frame->payload);
+    if (ob_position(frame->payload) == pos)
+      op_free(frame);
+    else if (overlay_payload_enqueue(frame))
+      op_free(frame);
+    if (neighbours){
+      alarm->deadline = alarm->alarm;
+      schedule(alarm);
+    }else
+      alarm->alarm=0;
+  }
 }
 
 static void update_alarm(time_ms_t limit){
@@ -1113,7 +1097,6 @@ static struct link_out *create_out_link(struct neighbour *neighbour, overlay_int
       ret->destination = create_unicast_destination(*addr, interface);
     else
       ret->destination = add_destination_ref(interface->destination);
-    
     if (config.debug.linkstate)
       DEBUGF("LINK STATE; Create possible %s link_out for neighbour %s on interface %s", 
 	unicast?"unicast":"broadcast",
diff --git a/serval.h b/serval.h
index 1c0a6373..62b2bcf6 100644
--- a/serval.h
+++ b/serval.h
@@ -576,7 +576,7 @@ int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrl
 			  overlay_mdp_frame *mdpreply);
 int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
 		     struct sockaddr_un *recvaddr, socklen_t recvaddrlen);
-int overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port);
+void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port);
 int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name);
 
 struct vomp_call_state;

From e3ceedddf86ca30ab628b4b602a40be4d69d62c8 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 25 Nov 2013 16:48:48 +1030
Subject: [PATCH 22/56] Fix link state routing timing logic

Avoid calling schedule() with zero alarm time
---
 fdqueue.c    |  2 +-
 route_link.c | 30 ++++++++++++++++++------------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/fdqueue.c b/fdqueue.c
index 4393aa22..297094e5 100644
--- a/fdqueue.c
+++ b/fdqueue.c
@@ -108,7 +108,7 @@ int _schedule(struct __sourceloc __whence, struct sched_ent *alarm)
   
   if (now - alarm->deadline > 1000){
     // 1000ms ago? thats silly, if you keep doing it noone else will get a turn.
-    WHYF("Alarm %s tried to schedule a deadline %"PRId64"ms ago",
+    FATALF("Alarm %s tried to schedule a deadline %"PRId64"ms ago",
 	   alloca_alarm_name(alarm),
            (now - alarm->deadline)
 	);
diff --git a/route_link.c b/route_link.c
index e7581ec7..c836c7c3 100644
--- a/route_link.c
+++ b/route_link.c
@@ -255,6 +255,7 @@ static struct neighbour *get_neighbour(struct subscriber *subscriber, char creat
     n->mdp_ack_sequence = -1;
     // TODO measure min/max rtt
     n->rtt = 120;
+    n->next_neighbour_update = gettime_ms() + 10;
     neighbours = n;
     if (config.debug.linkstate)
       DEBUGF("LINK STATE; new neighbour %s", alloca_tohex_sid_t(n->subscriber->sid));
@@ -892,7 +893,10 @@ static void link_send(struct sched_ent *alarm)
   }
 }
 
-static void update_alarm(time_ms_t limit){
+static void update_alarm(struct __sourceloc __whence, time_ms_t limit)
+{
+  if (limit == 0)
+    FATALF("limit == 0");
   if (link_send_alarm.alarm>limit || link_send_alarm.alarm==0){
     unschedule(&link_send_alarm);
     link_send_alarm.alarm = limit;
@@ -912,7 +916,7 @@ int link_stop_routing(struct subscriber *subscriber)
   if (subscriber->link_state){
     struct link_state *state = get_link_state(subscriber);
     state->next_update = gettime_ms();
-    update_alarm(state->next_update);
+    update_alarm(__WHENCE__, state->next_update);
   }
   return 0;
 }
@@ -1031,7 +1035,8 @@ int link_state_should_forward_broadcast(struct subscriber *transmitter)
 }
 
 // when we receive a packet from a neighbour with ourselves as the next hop, make sure we send an ack soon(ish)
-int link_state_ack_soon(struct subscriber *subscriber){
+int link_state_ack_soon(struct subscriber *subscriber)
+{
   IN();
   struct neighbour *neighbour = get_neighbour(subscriber, 0);
   if (!neighbour)
@@ -1046,8 +1051,8 @@ int link_state_ack_soon(struct subscriber *subscriber){
       if (config.debug.ack)
 	DEBUGF("Asking for next ACK Real Soon Now");
     }
+    update_alarm(__WHENCE__, neighbour->next_neighbour_update);
   }
-  update_alarm(neighbour->next_neighbour_update);
   OUT();
   return 0;
 }
@@ -1088,7 +1093,8 @@ int link_unicast_ack(struct subscriber *subscriber, struct overlay_interface *in
   return 0;
 }
 
-static struct link_out *create_out_link(struct neighbour *neighbour, overlay_interface *interface, struct sockaddr_in *addr, char unicast){
+static struct link_out *create_out_link(struct neighbour *neighbour, overlay_interface *interface, struct sockaddr_in *addr, char unicast)
+{
   struct link_out *ret=emalloc_zero(sizeof(struct link_out));
   if (ret){
     ret->_next=neighbour->out_links;
@@ -1102,9 +1108,9 @@ static struct link_out *create_out_link(struct neighbour *neighbour, overlay_int
 	unicast?"unicast":"broadcast",
 	alloca_tohex_sid_t(neighbour->subscriber->sid),
 	interface->name);
-
-    ret->timeout = gettime_ms()+ret->destination->tick_ms*3;
-    update_alarm(gettime_ms()+5);
+    time_ms_t now = gettime_ms();
+    ret->timeout = now + ret->destination->tick_ms * 3;
+    update_alarm(__WHENCE__, now + 5);
   }
   return ret;
 }
@@ -1192,7 +1198,7 @@ int link_received_packet(struct decode_context *context, int sender_seq, char un
     send_neighbour_link(neighbour);
   }
 
-  update_alarm(neighbour->next_neighbour_update);
+  update_alarm(__WHENCE__, neighbour->next_neighbour_update);
   return 0;
 }
 
@@ -1371,7 +1377,7 @@ int link_receive(struct overlay_frame *frame, overlay_mdp_frame *mdp)
 	    if (config.debug.ack)
 	      DEBUGF("LINK STATE; neighbour %s missed ack %d, queue another", alloca_tohex_sid_t(sender->sid), neighbour->last_update_seq);
 	    neighbour->next_neighbour_update=now+5;
-	    update_alarm(neighbour->next_neighbour_update);
+	    update_alarm(__WHENCE__, neighbour->next_neighbour_update);
 	  }
         }
       }
@@ -1412,8 +1418,8 @@ void link_explained(struct subscriber *subscriber)
 {
   time_ms_t now = gettime_ms();
   struct link_state *state = get_link_state(subscriber);
-  state->next_update = now+5;
-  update_alarm(now+5);
+  state->next_update = now + 5;
+  update_alarm(__WHENCE__, now + 5);
 }
 
 void link_interface_down(struct overlay_interface *interface)

From 1bb60fd8bc3fdeb5bfd53ecf778413476d99f904 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Sat, 23 Nov 2013 00:59:11 +1030
Subject: [PATCH 23/56] Add 'debug.overlaybuffer' logging

Instrument all ob_xxx() primitives with __whence
---
 overlay_buffer.c | 147 ++++++++++++++++++++++++++++++++++-------------
 overlay_buffer.h |  78 +++++++++++++++++--------
 2 files changed, 161 insertions(+), 64 deletions(-)

diff --git a/overlay_buffer.c b/overlay_buffer.c
index d6dc957f..06686e3d 100644
--- a/overlay_buffer.c
+++ b/overlay_buffer.c
@@ -31,9 +31,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  In either case, functions that don't take an offset use and advance the position.
  */
 
-struct overlay_buffer *ob_new(void)
+struct overlay_buffer *_ob_new(struct __sourceloc __whence)
 {
   struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_new() return %p", ret);
   if (ret == NULL)
     return NULL;
   ob_unlimitsize(ret);
@@ -42,9 +44,11 @@ struct overlay_buffer *ob_new(void)
 
 // index an existing static buffer.
 // and allow other callers to use the ob_ convenience methods for reading and writing up to size bytes.
-struct overlay_buffer *ob_static(unsigned char *bytes, int size)
+struct overlay_buffer *_ob_static(struct __sourceloc __whence, unsigned char *bytes, int size)
 {
   struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_static(bytes=%p, size=%d) return %p", bytes, size, ret);
   if (ret == NULL)
     return NULL;
   ret->bytes = bytes;
@@ -57,13 +61,15 @@ struct overlay_buffer *ob_static(unsigned char *bytes, int size)
 // create a new overlay buffer from an existing piece of another buffer.
 // Both buffers will point to the same memory region.
 // It is up to the caller to ensure this buffer is not used after the parent buffer is freed.
-struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length)
+struct overlay_buffer *_ob_slice(struct __sourceloc __whence, struct overlay_buffer *b, int offset, int length)
 {
   if (offset+length > b->allocSize) {
     WHY("Buffer isn't long enough to slice");
     return NULL;
   }
   struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_slice(b=%p, offset=%d, length=%d) return %p", b, offset, length, ret);
   if (ret == NULL)
       return NULL;
   ret->bytes = b->bytes+offset;
@@ -73,9 +79,11 @@ struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length
   return ret;
 }
 
-struct overlay_buffer *ob_dup(struct overlay_buffer *b)
+struct overlay_buffer *_ob_dup(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_dup(b=%p) return %p", b, ret);
   if (ret == NULL)
     return NULL;
   ret->sizeLimit = b->sizeLimit;
@@ -94,29 +102,35 @@ struct overlay_buffer *ob_dup(struct overlay_buffer *b)
   return ret;
 }
 
-void ob_free(struct overlay_buffer *b)
+void _ob_free(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   assert(b != NULL);
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_free(b=%p)", b);
   if (b->allocated)
     free(b->allocated);
   free(b);
 }
 
-int ob_checkpoint(struct overlay_buffer *b)
+int _ob_checkpoint(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   assert(b != NULL);
   b->checkpointLength = b->position;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_checkpoint(b=%p) checkpointLength=%d", b, b->checkpointLength);
   return 0;
 }
 
-int ob_rewind(struct overlay_buffer *b)
+int _ob_rewind(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   assert(b != NULL);
   b->position = b->checkpointLength;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_rewind(b=%p) position=%d", b, b->position);
   return 0;
 }
 
-void ob_limitsize(struct overlay_buffer *b, int bytes)
+void _ob_limitsize(struct __sourceloc __whence, struct overlay_buffer *b, int bytes)
 {
   assert(b != NULL);
   assert(bytes >= 0);
@@ -125,17 +139,23 @@ void ob_limitsize(struct overlay_buffer *b, int bytes)
   assert(b->checkpointLength <= bytes);
   if (b->bytes && b->allocated == NULL)
     assert(bytes <= b->allocSize);
-  b->sizeLimit=bytes;
+  b->sizeLimit = bytes;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_limitsize(b=%p, bytes=%d) sizeLimit=%d", b, bytes, b->sizeLimit);
 }
 
-void ob_unlimitsize(struct overlay_buffer *b)
+void _ob_unlimitsize(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   assert(b != NULL);
   b->sizeLimit = -1;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_unlimitsize(b=%p) sizeLimit=%d", b, b->sizeLimit);
 }
 
-void ob_flip(struct overlay_buffer *b)
+void _ob_flip(struct __sourceloc __whence, struct overlay_buffer *b)
 {
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_flip(b=%p) checkpointLength=0 position=0", b);
   b->checkpointLength = 0;
   ob_limitsize(b, b->position);
   b->position = 0;
@@ -143,22 +163,31 @@ void ob_flip(struct overlay_buffer *b)
 
 /* Return 1 if space is available, 0 if not.
  */
-ssize_t ob_makespace(struct overlay_buffer *b, size_t bytes)
+ssize_t _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b, size_t bytes)
 {
   assert(b != NULL);
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_makespace(b=%p, bytes=%d) b->bytes=%p b->position=%d b->allocSize=%d",
+	   b, bytes, b->bytes, b->position, b->allocSize);
   assert(b->position >= 0);
   if (b->sizeLimit != -1)
     assert(b->sizeLimit >= 0);
   assert(b->allocSize >= 0);
   if (b->position)
     assert(b->bytes != NULL);
-  if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit)
+  if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit) {
+    if (config.debug.packetformats)
+      DEBUGF("ob_makespace(): asked for space to %u, beyond size limit of %u", b->position + bytes, b->sizeLimit);
     return 0;
+  }
   if (b->position + bytes <= b->allocSize)
     return 1;
   // Don't realloc a static buffer.
-  if (b->bytes && b->allocated == NULL)
+  if (b->bytes && b->allocated == NULL) {
+    if (config.debug.packetformats)
+      DEBUGF("ob_makespace(): asked for space to %u, beyond static buffer size of %u", b->position + bytes, b->allocSize);
     return 0;
+  }
   int newSize=b->position+bytes;
   if (newSize<64) newSize=64;
   if (newSize&63) newSize+=64-(newSize&63);
@@ -212,49 +241,68 @@ ssize_t ob_makespace(struct overlay_buffer *b, size_t bytes)
  Functions that append data and increase the size of the buffer if possible / required
  */
 
-void ob_append_byte(struct overlay_buffer *b,unsigned char byte)
+void _ob_append_byte(struct __sourceloc __whence, struct overlay_buffer *b, unsigned char byte)
 {
   const int bytes = 1;
-  if (ob_makespace(b, bytes))
+  if (ob_makespace(b, bytes)) {
     b->bytes[b->position] = byte;
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_byte(b=%p, byte=0x%02x) %p[%d]=%02x position=%d", b, byte, b->bytes, b->position, byte, b->position + bytes);
+  } else {
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_byte(b=%p, byte=0x%02x) OVERRUN position=%d", b, byte, b->position + bytes);
+  }
   b->position += bytes;
 }
 
-unsigned char *ob_append_space(struct overlay_buffer *b, int count)
+unsigned char *_ob_append_space(struct __sourceloc __whence, struct overlay_buffer *b, int count)
 {
   assert(count > 0);
   unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL;
   b->position += count;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_append_space(b=%p, count=%d) position=%d return %p", b, count, b->position, r);
   return r;
 }
 
-void ob_append_bytes(struct overlay_buffer *b, const unsigned char *bytes, int count)
+void _ob_append_bytes(struct __sourceloc __whence, struct overlay_buffer *b, const unsigned char *bytes, int count)
 {
   assert(count > 0);
   unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL;
-  if (r)
+  if (r) {
     bcopy(bytes, r, count);
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_bytes(b=%p, bytes=%p, count=%d) position=%d return %p", b, bytes, count, b->position + count, r);
+  } else {
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_bytes(b=%p, bytes=%p, count=%d) OVERRUN position=%d return NULL", b, bytes, count, b->position + count);
+  }
+  if (config.debug.overlaybuffer)
+    dump("ob_append_bytes", bytes, count);
   b->position += count;
-  if (r)
-    bcopy(bytes, r, count);
 }
 
-void append_buffer(struct overlay_buffer *b, struct overlay_buffer *s)
+void _ob_append_buffer(struct __sourceloc __whence, struct overlay_buffer *b, struct overlay_buffer *s)
 {
   ob_append_bytes(b, s->bytes, s->position);
 }
 
-void ob_append_ui16(struct overlay_buffer *b, uint16_t v)
+void _ob_append_ui16(struct __sourceloc __whence, struct overlay_buffer *b, uint16_t v)
 {
   const int bytes = 2;
   if (ob_makespace(b, bytes)) {
     b->bytes[b->position] = (v >> 8) & 0xFF;
     b->bytes[b->position+1] = v & 0xFF;
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_ui16(b=%p, v=%u) %p[%d]=%s position=%d", b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes);
+  } else {
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_ui16(b=%p, v=%u) OVERRUN position=%d", b, v, b->position + bytes);
   }
   b->position += bytes;
 }
 
-void ob_append_ui32(struct overlay_buffer *b, uint32_t v)
+void _ob_append_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
 {
   const int bytes = 4;
   if (ob_makespace(b, bytes)) {
@@ -262,11 +310,17 @@ void ob_append_ui32(struct overlay_buffer *b, uint32_t v)
     b->bytes[b->position+1] = (v >> 16) & 0xFF;
     b->bytes[b->position+2] = (v >> 8) & 0xFF;
     b->bytes[b->position+3] = v & 0xFF;
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_ui32(b=%p, v=%"PRIu32") %p[%d]=%s position=%d",
+	  b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes);
+  } else {
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_ui32(b=%p, v=%"PRIu32") OVERRUN position=%d", b, v, b->position + bytes);
   }
   b->position += bytes;
 }
 
-void ob_append_ui64(struct overlay_buffer *b, uint64_t v)
+void _ob_append_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
 {
   const int bytes = 8;
   if (ob_makespace(b, bytes)) {
@@ -278,6 +332,12 @@ void ob_append_ui64(struct overlay_buffer *b, uint64_t v)
     b->bytes[b->position+5] = (v >> 16) & 0xFF;
     b->bytes[b->position+6] = (v >> 8) & 0xFF;
     b->bytes[b->position+7] = v & 0xFF;
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_ui64(b=%p, v=%"PRIu64") %p[%d]=%s position=%d",
+	  b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes);
+  } else {
+    if (config.debug.overlaybuffer)
+      DEBUGF("ob_append_ui64(b=%p, v=%"PRIu64") OVERRUN position=%d", b, v, b->position + bytes);
   }
   b->position += bytes;
 }
@@ -316,7 +376,7 @@ int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v){
   return i;
 }
 
-void ob_append_packed_ui32(struct overlay_buffer *b, uint32_t v)
+void _ob_append_packed_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
 {
   do {
     ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0));
@@ -324,7 +384,7 @@ void ob_append_packed_ui32(struct overlay_buffer *b, uint32_t v)
   } while (v != 0);
 }
 
-void ob_append_packed_ui64(struct overlay_buffer *b, uint64_t v)
+void _ob_append_packed_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
 {
   do {
     ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0));
@@ -332,7 +392,7 @@ void ob_append_packed_ui64(struct overlay_buffer *b, uint64_t v)
   } while (v != 0);
 }
 
-void ob_append_rfs(struct overlay_buffer *b, int l)
+void _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l)
 {
   assert(l >= 0);
   assert(l <= 0xffff);
@@ -453,35 +513,41 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b)
   return ret;
 }
 
-int ob_get(struct overlay_buffer *b){
+int ob_get(struct overlay_buffer *b)
+{
   if (test_offset(b, b->position, 1))
     return -1;
-  
   return b->bytes[b->position++];
 }
 
-void ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v)
+void _ob_set_ui16(struct __sourceloc __whence, struct overlay_buffer *b, int offset, uint16_t v)
 {
+  const int bytes = 2;
   assert(b != NULL);
   assert(offset >= 0);
   if (b->sizeLimit != -1)
-    assert(offset + 2 <= b->sizeLimit);
-  assert(offset + 2 <= b->allocSize);
+    assert(offset + bytes <= b->sizeLimit);
+  assert(offset + bytes <= b->allocSize);
   b->bytes[offset] = (v >> 8) & 0xFF;
   b->bytes[offset+1] = v & 0xFF;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_set_ui16(b=%p, offset=%d, v=%u) %p[%d]=%s", b, offset, v, b->bytes, offset, alloca_tohex(&b->bytes[offset], bytes));
 }
 
-void ob_set(struct overlay_buffer *b, int offset, unsigned char byte)
+void _ob_set(struct __sourceloc __whence, struct overlay_buffer *b, int offset, unsigned char byte)
 {
+  const int bytes = 1;
   assert(b != NULL);
   assert(offset >= 0);
   if (b->sizeLimit != -1)
-    assert(offset + 1 <= b->sizeLimit);
-  assert(offset + 1 <= b->allocSize);
+    assert(offset + bytes <= b->sizeLimit);
+  assert(offset + bytes <= b->allocSize);
   b->bytes[offset] = byte;
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_set(b=%p, offset=%d, byte=0x%02x) %p[%d]=%s", b, offset, byte, b->bytes, offset, alloca_tohex(&b->bytes[offset], bytes));
 }
 
-void ob_patch_rfs(struct overlay_buffer *b)
+void _ob_patch_rfs(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2));
 }
@@ -503,12 +569,15 @@ int ob_remaining(struct overlay_buffer *b)
   return b->sizeLimit - b->position;
 }
 
-int ob_overrun(struct overlay_buffer *b)
+int _ob_overrun(struct __sourceloc __whence, struct overlay_buffer *b)
 {
   assert(b->allocSize >= 0);
   if (b->sizeLimit != -1)
     assert(b->sizeLimit >= 0);
-  return b->position > (b->sizeLimit != -1 && b->sizeLimit < b->allocSize ? b->sizeLimit : b->allocSize);
+  int ret = b->position > (b->sizeLimit != -1 && b->sizeLimit < b->allocSize ? b->sizeLimit : b->allocSize);
+  if (config.debug.overlaybuffer)
+    DEBUGF("ob_overrun(b=%p) return %d", b, ret);
+  return ret;
 }
 
 unsigned char *ob_ptr(struct overlay_buffer *b)
diff --git a/overlay_buffer.h b/overlay_buffer.h
index 5f2b2897..dd7ba4a0 100644
--- a/overlay_buffer.h
+++ b/overlay_buffer.h
@@ -42,31 +42,57 @@ struct overlay_buffer {
   int var_length_offset;
 };
 
-struct overlay_buffer *ob_new(void);
-struct overlay_buffer *ob_static(unsigned char *bytes, int size);
-struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length);
-struct overlay_buffer *ob_dup(struct overlay_buffer *b);
-void ob_free(struct overlay_buffer *b);
-int ob_checkpoint(struct overlay_buffer *b);
-int ob_rewind(struct overlay_buffer *b);
-void ob_limitsize(struct overlay_buffer *b,int bytes);
-void ob_flip(struct overlay_buffer *b);
-void ob_unlimitsize(struct overlay_buffer *b);
-ssize_t ob_makespace(struct overlay_buffer *b, size_t bytes);
-void ob_set(struct overlay_buffer *b, int ofs, unsigned char byte);
-void ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v);
-void ob_patch_rfs(struct overlay_buffer *b);
+struct overlay_buffer *_ob_new(struct __sourceloc __whence);
+struct overlay_buffer *_ob_static(struct __sourceloc __whence, unsigned char *bytes, int size);
+struct overlay_buffer *_ob_slice(struct __sourceloc __whence, struct overlay_buffer *b, int offset, int length);
+struct overlay_buffer *_ob_dup(struct __sourceloc __whence, struct overlay_buffer *b);
+void _ob_free(struct __sourceloc __whence, struct overlay_buffer *b);
+int _ob_checkpoint(struct __sourceloc __whence, struct overlay_buffer *b);
+int _ob_rewind(struct __sourceloc __whence, struct overlay_buffer *b);
+void _ob_limitsize(struct __sourceloc __whence, struct overlay_buffer *b,int bytes);
+void _ob_flip(struct __sourceloc __whence, struct overlay_buffer *b);
+void _ob_unlimitsize(struct __sourceloc __whence, struct overlay_buffer *b);
+ssize_t _ob_makespace(struct __sourceloc whence, struct overlay_buffer *b, size_t bytes);
+void _ob_set(struct __sourceloc __whence, struct overlay_buffer *b, int ofs, unsigned char byte);
+void _ob_set_ui16(struct __sourceloc __whence, struct overlay_buffer *b, int offset, uint16_t v);
+void _ob_patch_rfs(struct __sourceloc __whence, struct overlay_buffer *b);
 
-void ob_append_byte(struct overlay_buffer *b,unsigned char byte);
-void ob_append_bytes(struct overlay_buffer *b,const unsigned char *bytes,int count);
-void ob_append_buffer(struct overlay_buffer *b,struct overlay_buffer *s);
-unsigned char *ob_append_space(struct overlay_buffer *b,int count);
-void ob_append_ui16(struct overlay_buffer *b, uint16_t v);
-void ob_append_ui32(struct overlay_buffer *b, uint32_t v);
-void ob_append_ui64(struct overlay_buffer *b, uint64_t v);
-void ob_append_packed_ui32(struct overlay_buffer *b, uint32_t v);
-void ob_append_packed_ui64(struct overlay_buffer *b, uint64_t v);
-void ob_append_rfs(struct overlay_buffer *b,int l);
+void _ob_append_byte(struct __sourceloc whence, struct overlay_buffer *b,unsigned char byte);
+void _ob_append_bytes(struct __sourceloc whence, struct overlay_buffer *b,const unsigned char *bytes,int count);
+void _ob_append_buffer(struct __sourceloc whence, struct overlay_buffer *b,struct overlay_buffer *s);
+unsigned char *_ob_append_space(struct __sourceloc whence, struct overlay_buffer *b,int count);
+void _ob_append_ui16(struct __sourceloc whence, struct overlay_buffer *b, uint16_t v);
+void _ob_append_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
+void _ob_append_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
+void _ob_append_packed_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
+void _ob_append_packed_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
+void _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l);
+
+#define ob_new() _ob_new(__WHENCE__)
+#define ob_static(bytes, size) _ob_static(__WHENCE__, bytes, size)
+#define ob_slice(b, off, len) _ob_slice(__WHENCE__, b, off, len)
+#define ob_dup(b) _ob_dup(__WHENCE__, b)
+#define ob_free(b) _ob_free(__WHENCE__, b)
+#define ob_checkpoint(b) _ob_checkpoint(__WHENCE__, b)
+#define ob_rewind(b) _ob_rewind(__WHENCE__, b)
+#define ob_limitsize(b, size) _ob_limitsize(__WHENCE__, b, size)
+#define ob_flip(b) _ob_flip(__WHENCE__, b)
+#define ob_unlimitsize(b) _ob_unlimitsize(__WHENCE__, b)
+#define ob_makespace(b, bytes) _ob_makespace(__WHENCE__, b, bytes)
+#define ob_set(b, off, byte) _ob_set(__WHENCE__, b, off, byte)
+#define ob_set_ui16(b, off, v) _ob_set_ui16(__WHENCE__, b, off, v)
+#define ob_patch_rfs(b) _ob_patch_rfs(__WHENCE__, b)
+
+#define ob_append_byte(b, byte) _ob_append_byte(__WHENCE__, b, byte)
+#define ob_append_bytes(b, bytes, count) _ob_append_bytes(__WHENCE__, b, bytes, count)
+#define ob_append_buffer(b, s) _ob_append_buffer(__WHENCE__, b, s)
+#define ob_append_space(b, count) _ob_append_space(__WHENCE__, b, count)
+#define ob_append_ui16(b, v) _ob_append_ui16(__WHENCE__, b, v)
+#define ob_append_ui32(b, v) _ob_append_ui32(__WHENCE__, b, v)
+#define ob_append_ui64(b, v) _ob_append_ui64(__WHENCE__, b, v)
+#define ob_append_packed_ui32(b, v) _ob_append_packed_ui32(__WHENCE__, b, v)
+#define ob_append_packed_ui64(b, v) _ob_append_packed_ui64(__WHENCE__, b, v)
+#define ob_append_rfs(b, l) _ob_append_rfs(__WHENCE__, b, l)
 
 // get one byte, -ve number indicates failure
 int ob_getbyte(struct overlay_buffer *b,int ofs);
@@ -86,7 +112,9 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b);
 int ob_position(struct overlay_buffer *b);
 int ob_limit(struct overlay_buffer *b);
 int ob_remaining(struct overlay_buffer *b);
-int ob_overrun(struct overlay_buffer *b);
+int _ob_overrun(struct __sourceloc, struct overlay_buffer *b);
 unsigned char* ob_ptr(struct overlay_buffer *b);
 
+#define ob_overrun(b) _ob_overrun(__WHENCE__, b)
+
 #endif //__SERVALD__OVERLAY_BUFFER_H

From 97ce07b76cc4bd47f52c97f97b73dab7cd63ca10 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 25 Nov 2013 12:52:43 +1030
Subject: [PATCH 24/56] Improve MDP debug logging

---
 fdqueue.h            |  1 +
 overlay_mdp.c        | 76 ++++++++++++++++++++++++++++++++++----------
 overlay_queue.c      |  2 +-
 performance_timing.c |  9 ++++++
 4 files changed, 71 insertions(+), 17 deletions(-)

diff --git a/fdqueue.h b/fdqueue.h
index af409543..39d1b37d 100644
--- a/fdqueue.h
+++ b/fdqueue.h
@@ -80,6 +80,7 @@ int fd_checkalarms();
 int fd_func_enter(struct __sourceloc, struct call_stats *this_call);
 int fd_func_exit(struct __sourceloc, struct call_stats *this_call);
 void dump_stack(int log_level);
+unsigned fd_depth();
 
 #define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \
     struct call_stats _this_call={.totals=&_aggregate_stats}; \
diff --git a/overlay_mdp.c b/overlay_mdp.c
index 67c58cce..ebda591a 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -445,6 +445,9 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
       if (len == -1)
 	RETURN(WHY("unsupported MDP packet type"));
       socklen_t addrlen = sizeof addr.sun_family + mdp_bindings[match].name_len;
+      if (config.debug.mdprequests) 
+	DEBUGF("Resolved bound socket on port %"PRImdp_port_t", addr=%s",
+	    mdp_bindings[match].port, alloca_sockaddr(&addr, addrlen));
       ssize_t r = sendto(mdp_sock.poll.fd, mdp, (size_t)len, 0, (struct sockaddr*)&addr, addrlen);
       if ((size_t)r != (size_t)len) {
 	if (r == -1) {
@@ -459,6 +462,7 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
 	RETURN(WHY("Failed to pass received MDP frame to client"));
       }
     } else {
+	DEBUGF("No socket bound to port %"PRImdp_port_t", try internal service", mdp->out.dst.port);
       /* No socket is bound, ignore the packet ... except for magic sockets */
       RETURN(overlay_mdp_try_interal_services(frame, mdp));
     }
@@ -474,6 +478,13 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
 
 int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name)
 {
+  if (config.debug.mdprequests)
+    DEBUGF("MDP_PORT_DNALOOKUP resolved_sid=%s uri=%s did=%s name=%s",
+	  alloca_tohex_sid_t(*resolved_sidp),
+	  alloca_str_toprint(uri),
+	  alloca_str_toprint(did),
+	  alloca_str_toprint(name)
+	);
   overlay_mdp_frame mdpreply;
   bzero(&mdpreply, sizeof mdpreply);
   mdpreply.packetTypeAndFlags = MDP_TX; // outgoing MDP message
@@ -538,9 +549,9 @@ void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_p
 static struct overlay_buffer * encrypt_payload(
   struct subscriber *source, 
   struct subscriber *dest, 
-  const unsigned char *buffer, int cipher_len){
-    
-  int nm=crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES;
+  const unsigned char *buffer,
+  int cipher_len)
+{
   int zb=crypto_box_curve25519xsalsa20poly1305_ZEROBYTES;
   int nb=crypto_box_curve25519xsalsa20poly1305_NONCEBYTES;
   int cz=crypto_box_curve25519xsalsa20poly1305_BOXZEROBYTES;
@@ -585,30 +596,31 @@ static struct overlay_buffer * encrypt_payload(
   }
   
   /* Actually authcrypt the payload */
-  if (crypto_box_curve25519xsalsa20poly1305_afternm
-      (cipher_text,plain,cipher_len,nonce,k)){
+  if (crypto_box_curve25519xsalsa20poly1305_afternm(cipher_text, plain,cipher_len, nonce, k)) {
     ob_free(ret);
     WHY("crypto_box_afternm() failed");
     return NULL;
   }
   
-  if (0) {
+#if 0
+  if (config.debug.crypto) {
     DEBUG("authcrypted mdp frame");
-    dump("nm",k,nm);
+    dump("nm",k,crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES);
     dump("plain text",plain,sizeof(plain));
     dump("nonce",nonce,nb);
     dump("cipher text",cipher_text,cipher_len);
   }
+#endif
   
-  /* now shuffle down to get rid of the temporary space that crypto_box
-   uses. 
+  /* now shuffle down to get rid of the temporary space that crypto_box uses. 
    TODO extend overlay buffer so we don't need this.
    */
   bcopy(&cipher_text[cz],&cipher_text[0],cipher_len-cz);
   ret->position-=cz;
-  if (0){
+#if 0
+  if (config.debug.crypto)
     dump("frame", &ret->bytes[0], ret->position);
-  }
+#endif
   
   return ret;
 }
@@ -633,17 +645,25 @@ int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plain
       op_free(frame);
       return -1;
     }
+#if 0
+    if (config.debug.crypto)
+      dump("Frame signed ciphertext", ob_ptr(frame->payload), ob_position(frame->payload));
+#endif
     break;
       
   case OF_CRYPTO_SIGNED:
     // Lets just append some space into the existing payload buffer for the signature, without copying it.
     frame->payload = plaintext;
-    if (   ob_makespace(frame->payload, SIGNATURE_BYTES) != SIGNATURE_BYTES
+    if (   !ob_makespace(frame->payload, SIGNATURE_BYTES)
         || crypto_sign_message(frame->source, ob_ptr(frame->payload), frame->payload->allocSize, &frame->payload->position) == -1
     ) {
       op_free(frame);
       return -1;
     }
+#if 0
+    if (config.debug.crypto)
+      dump("Frame signed plaintext", ob_ptr(frame->payload), ob_position(frame->payload));
+#endif
     break;
       
   case 0:
@@ -673,6 +693,17 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
 			 struct sockaddr_un *recvaddr, socklen_t recvaddrlen)
 {
   IN();
+  unsigned __d = 0;
+  if (config.debug.mdprequests) {
+    __d = fd_depth();
+    DEBUGF("[%u] src=%s*:%"PRImdp_port_t", dst=%s*:%"PRImdp_port_t", userGen=%d, recv=%s",
+	   __d,
+	   alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14), mdp->out.src.port,
+	   alloca_tohex_sid_t_trunc(mdp->out.dst.sid, 14), mdp->out.dst.port,
+	   userGeneratedFrameP,
+	   recvaddr ? alloca_sockaddr(recvaddr, recvaddrlen) : "NULL"
+	);
+  }
 
   if (mdp->out.payload_length > sizeof(mdp->out.payload))
     FATAL("Payload length is past the end of the buffer");
@@ -710,6 +741,8 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
   
   /* Work out if destination is broadcast or not */
   if (is_sid_t_broadcast(mdp->out.dst.sid)){
+    if (config.debug.mdprequests) 
+      DEBUGF("[%u] Broadcast packet", __d);
     /* broadcast packets cannot be encrypted, so complain if MDP_NOCRYPT
      flag is not set. Also, MDP_NOSIGN must also be applied, until
      NaCl cryptobox keys can be used for signing. */	
@@ -731,12 +764,15 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
   if (mdp->out.queue == 0)
     mdp->out.queue = OQ_ORDINARY;
     
-  if (!destination || destination->reachable == REACHABLE_SELF){
+  if (config.debug.mdprequests) 
+    DEBUGF("[%u] destination->sid=%s", __d, destination ? alloca_tohex_sid_t(destination->sid) : "NULL");
+  if (!destination || destination->reachable == REACHABLE_SELF) {
     /* Packet is addressed to us / broadcast, we should process it first. */
     overlay_saw_mdp_frame(NULL,mdp,gettime_ms());
     if (destination) {
-      /* Is local, and is not broadcast, so shouldn't get sent out
-	 on the wire. */
+      /* Is local, and is not broadcast, so shouldn't get sent out on the wire. */
+      if (config.debug.mdprequests) 
+	DEBUGF("[%u] Local packet, not transmitting", __d);
       RETURN(0);
     }
   }
@@ -775,9 +811,16 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
   if (mdp->out.payload_length)
     ob_append_bytes(plaintext, mdp->out.payload, mdp->out.payload_length);
   if (ob_overrun(plaintext)) {
+    if (config.debug.mdprequests) 
+      DEBUGF("[%u] Frame overrun: position=%d allocSize=%d sizeLimit=%d", __d,
+	  plaintext->position, plaintext->allocSize, plaintext->sizeLimit);
     ob_free(plaintext);
     RETURN(-1);
   }
+  if (config.debug.mdprequests) {
+    DEBUGF("[%u] Send frame %zu bytes", __d, ob_position(plaintext));
+    dump("Frame plaintext", ob_ptr(plaintext), ob_position(plaintext));
+  }
   
   /* Prepare the overlay frame for dispatch */
   struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame));
@@ -823,7 +866,8 @@ static int search_subscribers(struct subscriber *subscriber, void *context){
   return 0;
 }
 
-int overlay_mdp_address_list(overlay_mdp_addrlist *request, overlay_mdp_addrlist *response){
+int overlay_mdp_address_list(overlay_mdp_addrlist *request, overlay_mdp_addrlist *response)
+{
   if (config.debug.mdprequests)
     DEBUGF("MDP_GETADDRS first_sid=%u mode=%d", request->first_sid, request->mode);
   
diff --git a/overlay_queue.c b/overlay_queue.c
index 10170057..4ba12c9f 100644
--- a/overlay_queue.c
+++ b/overlay_queue.c
@@ -157,7 +157,7 @@ int overlay_payload_enqueue(struct overlay_frame *p)
   overlay_txqueue *queue = &overlay_tx[p->queue];
 
   if (config.debug.packettx)
-    DEBUGF("Enqueuing packet for %s* (q[%d]length = %d)",
+    DEBUGF("Enqueuing packet for %s* (q[%d].length = %d)",
 	   p->destination?alloca_tohex_sid_t_trunc(p->destination->sid, 14): alloca_tohex(p->broadcast_id.id, BROADCAST_LEN),
 	   p->queue, queue->length);
   
diff --git a/performance_timing.c b/performance_timing.c
index 5db8f80a..2ac8b0b8 100644
--- a/performance_timing.c
+++ b/performance_timing.c
@@ -208,6 +208,15 @@ void dump_stack(int log_level)
   }
 }
 
+unsigned fd_depth()
+{
+  unsigned depth = 0;
+  struct call_stats *call;
+  for (call = current_call; call; call = call->prev)
+    ++depth;
+  return depth;
+}
+
 int fd_func_enter(struct __sourceloc __whence, struct call_stats *this_call)
 {
   if (config.debug.profiling)

From 59a6ded10c7cb9f09c901d945e24012182eb24fa Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 25 Nov 2013 12:57:09 +1030
Subject: [PATCH 25/56] Update 'dnaprotocol' debug flags

---
 tests/dnaprotocol | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/dnaprotocol b/tests/dnaprotocol
index c4a1aa0d..d53ccce6 100755
--- a/tests/dnaprotocol
+++ b/tests/dnaprotocol
@@ -47,8 +47,10 @@ set_server_vars() {
       set log.console.show_time on \
       set mdp.iftype.wifi.tick_ms 100 \
       set rhizome.enable No \
-      set debug.overlayinterfaces Yes \
+      set debug.overlayinterfaces No \
+      set debug.overlaybuffer No \
       set debug.packetformats No \
+      set debug.overlayframes No \
       set debug.overlayrouting No \
       set debug.packettx No \
       set debug.packetrx No \

From 851dbb4d64dfc4d2d6cbd27a0ca20d4332cefe6a Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 25 Nov 2013 22:40:14 +1030
Subject: [PATCH 26/56] Add 'debug.subscriber' config option

Add debug logging to find_subscriber() function
---
 conf_schema.h     |   1 +
 overlay_address.c | 102 ++++++++++++++++++++++++++++------------------
 overlay_address.h |   4 +-
 tests/routing     |   1 +
 4 files changed, 68 insertions(+), 40 deletions(-)

diff --git a/conf_schema.h b/conf_schema.h
index 82e005b3..2e10fefb 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -270,6 +270,7 @@ ATOM(bool_t, rhizome_rx,                0, boolean,, "")
 ATOM(bool_t, rhizome_ads,               0, boolean,, "")
 ATOM(bool_t, rhizome_nohttptx,          0, boolean,, "")
 ATOM(bool_t, rhizome_mdp_rx,            0, boolean,, "")
+ATOM(bool_t, subscriber,                0, boolean,, "")
 ATOM(bool_t, throttling,                0, boolean,, "")
 ATOM(bool_t, meshms,                    0, boolean,, "")
 ATOM(bool_t, manifests,                 0, boolean,, "")
diff --git a/overlay_address.c b/overlay_address.c
index 982900a1..399e147f 100644
--- a/overlay_address.c
+++ b/overlay_address.c
@@ -104,57 +104,69 @@ void free_subscribers()
 }
 
 // find a subscriber struct from a whole or abbreviated subscriber id
-struct subscriber *find_subscriber(const unsigned char *sidp, int len, int create)
+struct subscriber *_find_subscriber(struct __sourceloc __whence, const unsigned char *sidp, int len, int create)
 {
+  IN();
+  if (config.debug.subscriber)
+    DEBUGF("find_subscriber(sid=%s, create=%d)", alloca_tohex(sidp, len), create);
   struct tree_node *ptr = &root;
   int pos=0;
   if (len!=SID_SIZE)
     create =0;
-  
-  do{
+  struct subscriber *ret = NULL;
+  do {
     unsigned char nibble = get_nibble(sidp, pos++);
-    
     if (ptr->is_tree & (1<<nibble)){
       ptr = ptr->tree_nodes[nibble];
-      
     }else if(!ptr->subscribers[nibble]){
       // subscriber is not yet known
-      
-      if (create){
-	struct subscriber *ret=(struct subscriber *)malloc(sizeof(struct subscriber));
-	memset(ret,0,sizeof(struct subscriber));
-	ptr->subscribers[nibble]=ret;
+      if (create && (ret = (struct subscriber *) emalloc_zero(sizeof(struct subscriber)))) {
+	ptr->subscribers[nibble] = ret;
 	ret->sid = *(const sid_t *)sidp;
-	ret->abbreviate_len=pos;
+	ret->abbreviate_len = pos;
+	if (config.debug.subscriber)
+	  DEBUGF("set node[%.*s].subscribers[%c]=%p (sid=%s, abbrev_len=%d)",
+		pos - 1, alloca_tohex(sidp, len), hexdigit_upper[nibble],
+		ret, alloca_tohex_sid_t(ret->sid), ret->abbreviate_len
+	      );
       }
-      return ptr->subscribers[nibble];
-      
+      goto done;
     }else{
       // there's a subscriber in this slot, does it match the rest of the sid we've been given?
-      struct subscriber *ret = ptr->subscribers[nibble];
+      ret = ptr->subscribers[nibble];
       if (memcmp(ret->sid.binary, sidp, len) == 0)
-	return ret;
-      
+	goto done;
       // if we need to insert this subscriber, we have to make a new tree node first
-      if (!create)
-	return NULL;
-      
+      if (!create) {
+	ret = NULL;
+	goto done;
+      }
       // create a new tree node and move the existing subscriber into it
-      struct tree_node *new=(struct tree_node *)malloc(sizeof(struct tree_node));
-      memset(new,0,sizeof(struct tree_node));
-      ptr->tree_nodes[nibble]=new;
+      struct tree_node *new = (struct tree_node *) emalloc_zero(sizeof(struct tree_node));
+      if (new == NULL) {
+	ret = NULL;
+	goto done;
+      }
+      if (config.debug.subscriber)
+	DEBUGF("create node[%.*s]", pos, alloca_tohex(sidp, len));
+      ptr->tree_nodes[nibble] = new;
       ptr->is_tree |= (1<<nibble);
-      
-      ptr=new;
-      nibble=get_nibble(ret->sid.binary, pos);
-      ptr->subscribers[nibble]=ret;
-      ret->abbreviate_len=pos+1;
+      ptr = new;
+      nibble = get_nibble(ret->sid.binary, pos);
+      ptr->subscribers[nibble] = ret;
+      ret->abbreviate_len = pos + 1;
+      if (config.debug.subscriber)
+	DEBUGF("set node[%.*s].subscribers[%c]=%p(sid=%s, abbrev_len=%d)",
+	    pos, alloca_tohex(sidp, len), hexdigit_upper[nibble],
+	    ret, alloca_tohex_sid_t(ret->sid), ret->abbreviate_len
+	  );
       // then go around the loop again to compare the next nibble against the sid until we find an empty slot.
     }
-  }while(pos < len*2);
-  
-  // abbreviation is not unique
-  return NULL;
+  } while(pos < len*2);
+done:
+  if (config.debug.subscriber)
+    DEBUGF("find_subscriber() return %p", ret);
+  RETURN(ret);
 }
 
 /* 
@@ -298,6 +310,8 @@ static int add_explain_response(struct subscriber *subscriber, void *context)
   // the header of this packet must include our full sid.
   if (subscriber->reachable==REACHABLE_SELF){
     if (subscriber==my_subscriber){
+      if (config.debug.subscriber)
+	DEBUGF("Explaining SELF sid=%s", alloca_tohex_sid_t(subscriber->sid));
       response->please_explain->source_full=1;
       return 0;
     }
@@ -305,17 +319,22 @@ static int add_explain_response(struct subscriber *subscriber, void *context)
   }
   
   // add the whole subscriber id to the payload, stop if we run out of space
-  DEBUGF("Adding full sid by way of explanation %s", alloca_tohex_sid_t(subscriber->sid));
+  if (config.debug.subscriber)
+    DEBUGF("Explaining sid=%s", alloca_tohex_sid_t(subscriber->sid));
+  ob_checkpoint(response->please_explain->payload);
   ob_append_byte(response->please_explain->payload, SID_SIZE);
   ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE);
-  if (ob_overrun(response->please_explain->payload))
+  if (ob_overrun(response->please_explain->payload)) {
+    ob_rewind(response->please_explain->payload);
     return 1;
+  }
   // let the routing engine know that we had to explain this sid, we probably need to re-send routing info
   link_explained(subscriber);
   return 0;
 }
 
-static int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int len, struct subscriber **subscriber){
+static int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int len, struct subscriber **subscriber)
+{
   if (len<=0 || len>SID_SIZE){
     return WHYF("Invalid abbreviation length %d", len);
   }
@@ -428,7 +447,8 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer
 }
 
 // once we've finished parsing a packet, complete and send a please explain if required.
-int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination){
+int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination)
+{
   IN();
   struct overlay_frame *frame=context->please_explain;
   if (frame == NULL)
@@ -473,7 +493,8 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc
 }
 
 // process an incoming request for explanation of subscriber abbreviations
-int process_explain(struct overlay_frame *frame){
+int process_explain(struct overlay_frame *frame)
+{
   struct overlay_buffer *b=frame->payload;
   
   struct decode_context context;
@@ -498,14 +519,17 @@ int process_explain(struct overlay_frame *frame){
     if (len==SID_SIZE){
       // This message is also used to inform people of previously unknown subscribers
       // make sure we know this one
+      INFOF("Storing explain response for %s", alloca_tohex(sid, len));
       find_subscriber(sid,len,1);
     }else{
       // reply to the sender with all subscribers that match this abbreviation
-      INFOF("Sending responses for %s", alloca_tohex(sid, len));
+      INFOF("Sending explain responses for %s", alloca_tohex(sid, len));
       walk_tree(&root, 0, sid, len, sid, len, add_explain_response, &context);
     }
   }
-  
-  send_please_explain(&context, frame->destination, frame->source);
+  if (context.please_explain)
+    send_please_explain(&context, frame->destination, frame->source);
+  else if (config.debug.subscriber)
+    DEBUG("No explain responses");
   return 0;
 }
diff --git a/overlay_address.h b/overlay_address.h
index 7796e469..1de7801b 100644
--- a/overlay_address.h
+++ b/overlay_address.h
@@ -110,7 +110,9 @@ struct decode_context{
 extern struct subscriber *my_subscriber;
 extern struct subscriber *directory_service;
 
-struct subscriber *find_subscriber(const unsigned char *sid, int len, int create);
+struct subscriber *_find_subscriber(struct __sourceloc, const unsigned char *sid, int len, int create);
+#define find_subscriber(sid, len, create) _find_subscriber(__WHENCE__, sid, len, create)
+
 void enum_subscribers(struct subscriber *start, int(*callback)(struct subscriber *, void *), void *context);
 int set_reachable(struct subscriber *subscriber, struct network_destination *destination, struct subscriber *next_hop);
 int load_subscriber_address(struct subscriber *subscriber);
diff --git a/tests/routing b/tests/routing
index c2966c14..5a14c026 100755
--- a/tests/routing
+++ b/tests/routing
@@ -81,6 +81,7 @@ start_routing_instance() {
       set debug.mdprequests yes \
       set debug.linkstate yes \
       set debug.verbose yes \
+      set debug.subscriber yes \
       set debug.overlayrouting yes \
       set log.console.level debug \
       set log.console.show_pid on \

From 39d6c4fc3a7b77e9002ed89bbefa63bf4f8f5883 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 26 Nov 2013 13:41:46 +1030
Subject: [PATCH 27/56] strbuf_append_sockaddr_in()

---
 strbuf_helpers.c | 25 +++++++++++++++++--------
 strbuf_helpers.h |  7 +++++++
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 093c5cc8..1e2c459e 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -316,7 +316,7 @@ strbuf strbuf_append_socket_type(strbuf sb, int type)
 
 strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr)
 {
-  strbuf_sprintf(sb, " %u.%u.%u.%u",
+  strbuf_sprintf(sb, "%u.%u.%u.%u",
       ((unsigned char *) &addr->s_addr)[0],
       ((unsigned char *) &addr->s_addr)[1],
       ((unsigned char *) &addr->s_addr)[2],
@@ -324,13 +324,21 @@ strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr)
   return sb;
 }
 
+strbuf strbuf_append_sockaddr_in(strbuf sb, const struct sockaddr_in *addr)
+{
+  assert(addr->sin_family == AF_INET);
+  strbuf_puts(sb, "AF_INET:");
+  strbuf_append_in_addr(sb, &addr->sin_addr);
+  strbuf_sprintf(sb, ":%u", ntohs(addr->sin_port));
+  return sb;
+}
+
 strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen)
 {
-  strbuf_append_socket_domain(sb, addr->sa_family);
   switch (addr->sa_family) {
   case AF_UNIX: {
+      strbuf_puts(sb, "AF_UNIX:");
       size_t len = addrlen > sizeof addr->sa_family ? addrlen - sizeof addr->sa_family : 0;
-      strbuf_putc(sb, ' ');
       if (addr->sa_data[0]) {
 	strbuf_toprint_quoted_len(sb, "\"\"", addr->sa_data, len);
 	if (len < 2)
@@ -347,18 +355,19 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t
     break;
   case AF_INET: {
       const struct sockaddr_in *addr_in = (const struct sockaddr_in *) addr;
-      strbuf_putc(sb, ' ');
-      strbuf_append_in_addr(sb, &addr_in->sin_addr);
-      strbuf_sprintf(sb, ":%u", ntohs(addr_in->sin_port));
+      strbuf_append_sockaddr_in(sb, addr_in);
       if (addrlen != sizeof(struct sockaddr_in))
 	strbuf_sprintf(sb, " (addrlen=%d should be %zd)", (int)addrlen, sizeof(struct sockaddr_in));
     }
     break;
   default: {
+      strbuf_append_socket_domain(sb, addr->sa_family);
       size_t len = addrlen > sizeof addr->sa_family ? addrlen - sizeof addr->sa_family : 0;
       int i;
-      for (i = 0; i < len; ++i)
-	strbuf_sprintf(sb, " %02x", addr->sa_data[i]);
+      for (i = 0; i < len; ++i) {
+	strbuf_putc(sb, i ? ',' : ':');
+	strbuf_sprintf(sb, "%02x", addr->sa_data[i]);
+      }
     }
     break;
   }
diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index 5dca8a90..e10a346a 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -128,6 +128,13 @@ strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr);
 /* Append a textual description of a struct sockaddr_in.
  * @author Andrew Bettison <andrew@servalproject.com>
  */
+struct sockaddr_in;
+strbuf strbuf_append_sockaddr_in(strbuf sb, const struct sockaddr_in *addr);
+#define alloca_sockaddr_in(addr)    strbuf_str(strbuf_append_sockaddr_in(strbuf_alloca(45), (const struct sockaddr_in *)(addr)))
+
+/* Append a textual description of a struct sockaddr.
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
 struct sockaddr;
 strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen);
 #define alloca_sockaddr(addr, addrlen)    strbuf_str(strbuf_append_sockaddr(strbuf_alloca(200), (const struct sockaddr *)(addr), (addrlen)))

From fb39dd84ef7ad50a991440cf58a7eca920bb2ee8 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 26 Nov 2013 13:52:29 +1030
Subject: [PATCH 28/56] Improve dummy file Read/Write debug logging

---
 overlay_interface.c | 34 +++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/overlay_interface.c b/overlay_interface.c
index 8b74b60c..842685d7 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -657,9 +657,6 @@ static void interface_read_file(struct overlay_interface *interface)
       return;
     }
     
-    if (config.debug.overlayinterfaces)
-      DEBUGF("Read interface %s (size=%"PRId64") at offset=%d",interface->name, (int64_t)length, interface->recv_offset);
-    
     ssize_t nread = read(interface->alarm.poll.fd, &packet, sizeof packet);
     if (nread == -1){
       WHY_perror("read");
@@ -668,11 +665,22 @@ static void interface_read_file(struct overlay_interface *interface)
     }
     
     if (nread == sizeof packet) {
+      if (config.debug.overlayinterfaces)
+	DEBUGF("Read from interface %s (filesize=%"PRId64") at offset=%d: src_addr=%s dst_addr=%s pid=%d length=%d",
+	      interface->name, (int64_t)length, interface->recv_offset,
+	      alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr),
+	      alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr),
+	      packet.pid,
+	      packet.payload_length
+	    );
       interface->recv_offset += nread;
       if (should_drop(interface, packet.dst_addr) || (packet.pid == getpid() && !interface->local_echo)){
 	if (config.debug.packetrx)
-	  DEBUGF("Ignoring packet from %d, addressed to %s:%d", packet.pid,
-	      inet_ntoa(packet.dst_addr.sin_addr), ntohs(packet.dst_addr.sin_port));
+	  DEBUGF("Ignoring packet from pid=%d src_addr=%s dst_addr=%s",
+		packet.pid,
+		alloca_sockaddr_in(&packet.src_addr),
+		alloca_sockaddr_in(&packet.dst_addr)
+	      );
       }else{
 	packetOkOverlay(interface, packet.payload, packet.payload_length, -1, 
 			    (struct sockaddr*)&packet.src_addr, (socklen_t) sizeof(packet.src_addr));
@@ -961,9 +969,21 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
 	    not support seeking. */
 	  if (errno != ESPIPE)
 	    return WHY_perror("lseek");
-	  DEBUGF("Write to interface %s at unknown offset", interface->name);
+	  DEBUGF("Write to interface %s at offset unknown: src_addr=%s dst_addr=%s pid=%d length=%d",
+		interface->name,
+		alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr),
+		alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr),
+		packet.pid,
+		packet.payload_length
+	      );
 	} else
-	  DEBUGF("Write to interface %s at offset=%"PRId64, interface->name, (int64_t)fsize);
+	  DEBUGF("Write to interface %s at offset=%"PRId64": src_addr=%s dst_addr=%s pid=%d length=%d",
+		interface->name, (int64_t)fsize,
+		alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr),
+		alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr),
+		packet.pid,
+		packet.payload_length
+	      );
       }
       ssize_t nwrite = write(interface->alarm.poll.fd, &packet, sizeof(packet));
       if (nwrite == -1)

From e97e3db444c07c987362dbbd0f1e9987573b6b75 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 26 Nov 2013 17:44:25 +1030
Subject: [PATCH 29/56] Silence unconditional DEBUG log message

---
 overlay_mdp.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/overlay_mdp.c b/overlay_mdp.c
index ebda591a..d749ba51 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -445,7 +445,7 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
       if (len == -1)
 	RETURN(WHY("unsupported MDP packet type"));
       socklen_t addrlen = sizeof addr.sun_family + mdp_bindings[match].name_len;
-      if (config.debug.mdprequests) 
+      if (config.debug.mdprequests)
 	DEBUGF("Resolved bound socket on port %"PRImdp_port_t", addr=%s",
 	    mdp_bindings[match].port, alloca_sockaddr(&addr, addrlen));
       ssize_t r = sendto(mdp_sock.poll.fd, mdp, (size_t)len, 0, (struct sockaddr*)&addr, addrlen);
@@ -462,6 +462,7 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
 	RETURN(WHY("Failed to pass received MDP frame to client"));
       }
     } else {
+      if (config.debug.mdprequests)
 	DEBUGF("No socket bound to port %"PRImdp_port_t", try internal service", mdp->out.dst.port);
       /* No socket is bound, ignore the packet ... except for magic sockets */
       RETURN(overlay_mdp_try_interal_services(frame, mdp));

From 46363db257a3be450c908f4f4ef174e6137f91f9 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 26 Nov 2013 17:44:49 +1030
Subject: [PATCH 30/56] Format MDP port numbers as 0xXXXXXXXX

---
 serval.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/serval.h b/serval.h
index 62b2bcf6..eb775a10 100644
--- a/serval.h
+++ b/serval.h
@@ -513,7 +513,7 @@ int overlay_mdp_reply_error(int sock,
 			    int error_number,char *message);
 
 typedef uint32_t mdp_port_t;
-#define PRImdp_port_t "08" PRIx32
+#define PRImdp_port_t "#08" PRIx32
 
 typedef struct sockaddr_mdp {
   sid_t sid;

From 6ec6a6c966c6a8264c16e3c6f11205566e167bea Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Wed, 27 Nov 2013 12:59:52 +1030
Subject: [PATCH 31/56] Fix printf formats

---
 overlay_buffer.c | 6 +++---
 overlay_mdp.c    | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/overlay_buffer.c b/overlay_buffer.c
index 06686e3d..8eb3bdd1 100644
--- a/overlay_buffer.c
+++ b/overlay_buffer.c
@@ -167,7 +167,7 @@ ssize_t _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b, siz
 {
   assert(b != NULL);
   if (config.debug.overlaybuffer)
-    DEBUGF("ob_makespace(b=%p, bytes=%d) b->bytes=%p b->position=%d b->allocSize=%d",
+    DEBUGF("ob_makespace(b=%p, bytes=%zd) b->bytes=%p b->position=%d b->allocSize=%d",
 	   b, bytes, b->bytes, b->position, b->allocSize);
   assert(b->position >= 0);
   if (b->sizeLimit != -1)
@@ -177,7 +177,7 @@ ssize_t _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b, siz
     assert(b->bytes != NULL);
   if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit) {
     if (config.debug.packetformats)
-      DEBUGF("ob_makespace(): asked for space to %u, beyond size limit of %u", b->position + bytes, b->sizeLimit);
+      DEBUGF("ob_makespace(): asked for space to %zu, beyond size limit of %u", b->position + bytes, b->sizeLimit);
     return 0;
   }
   if (b->position + bytes <= b->allocSize)
@@ -185,7 +185,7 @@ ssize_t _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b, siz
   // Don't realloc a static buffer.
   if (b->bytes && b->allocated == NULL) {
     if (config.debug.packetformats)
-      DEBUGF("ob_makespace(): asked for space to %u, beyond static buffer size of %u", b->position + bytes, b->allocSize);
+      DEBUGF("ob_makespace(): asked for space to %zu, beyond static buffer size of %u", b->position + bytes, b->allocSize);
     return 0;
   }
   int newSize=b->position+bytes;
diff --git a/overlay_mdp.c b/overlay_mdp.c
index d749ba51..a1391220 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -819,7 +819,7 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
     RETURN(-1);
   }
   if (config.debug.mdprequests) {
-    DEBUGF("[%u] Send frame %zu bytes", __d, ob_position(plaintext));
+    DEBUGF("[%u] Send frame %u bytes", __d, ob_position(plaintext));
     dump("Frame plaintext", ob_ptr(plaintext), ob_position(plaintext));
   }
   

From bea9d36411d0f6ba4e636b368580de515ca38c2a Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 4 Nov 2013 15:14:53 +1030
Subject: [PATCH 32/56] Drop radio packets based on the probability of
 receiving the header

---
 fakeradio.c | 35 ++++++++++++++++++++++++++---------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/fakeradio.c b/fakeradio.c
index b63eff9f..8095635e 100644
--- a/fakeradio.c
+++ b/fakeradio.c
@@ -339,17 +339,33 @@ int transfer_bytes(struct radio_state *radios)
     log_time();
     fprintf(stderr, "Transferring %d byte packet from %s to %s\n", bytes, t->name, r->name);
   }
+  
   int i, j;
-  for (i=0;i<bytes && r->rxb_len<sizeof(r->rxbuffer);i++){
-    char byte = t->txbuffer[i];
-    // introduce bit errors
-    for(j=0;j<8;j++) {
-      if (random()<ber) {
-	byte^=(1<<j);
-	fprintf(stderr,"Flipped a bit\n");
+  int dropped=0;
+  
+// preamble length in bits that must arrive intact
+#define PREAMBLE_LENGTH (20+8)
+
+  // simulate the probability of a bit error in the packet pre-amble and drop the whole packet
+  for (i=0;i<PREAMBLE_LENGTH;i++){
+    if (random()<ber)
+      dropped=1;
+  }
+  
+  if (dropped){
+    fprintf(stderr,"Dropped the whole radio packet due to bit flip in the pre-amble\n");
+  }else{
+    for (i=0;i<bytes && r->rxb_len<sizeof(r->rxbuffer);i++){
+      char byte = t->txbuffer[i];
+      // introduce bit errors
+      for(j=0;j<8;j++) {
+	if (random()<ber) {
+	  byte^=(1<<j);
+	  fprintf(stderr,"Flipped a bit\n");
+	}
       }
+      r->rxbuffer[r->rxb_len++]=byte;
     }
-    r->rxbuffer[r->rxb_len++]=byte;
   }
   
   if (bytes>0 && bytes < t->txb_len)
@@ -362,7 +378,8 @@ int transfer_bytes(struct radio_state *radios)
     r->tx_count=6;
   }
   // set the wait time for the next transmission
-  next_transmit_time = gettime_ms() + (bytes+10)/chars_per_ms;
+  // minimum 20 bit preamble, 1 byte sync, 1 byte length, 2 byte tdm + turn around air-time
+  next_transmit_time = gettime_ms() + (bytes+8+10)/chars_per_ms;
   return bytes;
 }
 

From 5ea19ab53819789b8714276ea4fa5eec4bd9b6f7 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 21 Nov 2013 15:58:50 +1030
Subject: [PATCH 33/56] Better approximation of TDMA timing

---
 fakeradio.c           | 16 +++++++++-------
 tests/rhizomeprotocol |  4 ++--
 tests/routing         |  1 -
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/fakeradio.c b/fakeradio.c
index 8095635e..5aef5fdf 100644
--- a/fakeradio.c
+++ b/fakeradio.c
@@ -372,14 +372,16 @@ int transfer_bytes(struct radio_state *radios)
     bcopy(&t->txbuffer[bytes], t->txbuffer, t->txb_len - bytes);
   t->txb_len-=bytes;
   
-  if (bytes==0 || --t->tx_count<=0){
-    // swap who's turn it is to transmit
-    transmitter = receiver;
-    r->tx_count=6;
-  }
   // set the wait time for the next transmission
-  // minimum 20 bit preamble, 1 byte sync, 1 byte length, 2 byte tdm + turn around air-time
-  next_transmit_time = gettime_ms() + (bytes+8+10)/chars_per_ms;
+  next_transmit_time = gettime_ms() + 5 + bytes/chars_per_ms;
+  
+  if (bytes==0 || --t->tx_count<=0){
+    // swap who's turn it is to transmit after sending 3 packets or running out of data.
+    transmitter = receiver;
+    r->tx_count=3;
+    // add Tx->Rx change time (it's about 40ms between receiving empty packets)
+    next_transmit_time+=15;
+  }
   return bytes;
 }
 
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index ff9a58f3..750d91a5 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -720,7 +720,7 @@ start_radio_instance() {
 doc_SimulatedRadio="MDP Transfer over simulated radio link (~90% packet arrival)"
 setup_SimulatedRadio() {
    setup_common
-   $servald_build_root/fakeradio 6 0.9 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   $servald_build_root/fakeradio 4 0.9 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
    FAKERADIO_PID=$!
    sleep 5
    local END1=`head "$SERVALD_VAR/radioout" -n 1`
@@ -748,7 +748,7 @@ teardown_SimulatedRadio() {
 doc_SimulatedRadio2="MDP Transfer over simulated radio link (~50% packet arrival)"
 setup_SimulatedRadio2() {
    setup_common
-   $servald_build_root/fakeradio 6 0.5 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   $servald_build_root/fakeradio 4 0.5 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
    FAKERADIO_PID=$!
    sleep 1
    local END1=`head "$SERVALD_VAR/radioout" -n 1`
diff --git a/tests/routing b/tests/routing
index 5a14c026..1871a262 100755
--- a/tests/routing
+++ b/tests/routing
@@ -285,7 +285,6 @@ setup_simulate_extender() {
          set debug.mavlink on \
          set interfaces.1.type CATEAR \
          set interfaces.1.mdp.tick_ms 5000 \
-         set interfaces.1.mdp.packet_interval 5000 \
          set interfaces.1.socket_type STREAM \
          set interfaces.1.encapsulation SINGLE \
          set interfaces.1.point_to_point on

From 0d5d87852190f4d494935a1162d9188b841ba441 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 25 Nov 2013 12:15:20 +1030
Subject: [PATCH 34/56] Refactor socket name and address length handling

---
 commandline.c           | 168 +++++----
 directory_client.c      |   2 +-
 headerfiles.mk          |   1 +
 keyring.c               |  10 +-
 mdp_client.c            | 152 ++++----
 mdp_client.h            |  17 +-
 monitor-client.c        |  13 +-
 monitor.c               |  10 +-
 overlay_link.c          |   4 +-
 overlay_mdp.c           | 789 ++++++++++++++++++++++++++--------------
 overlay_mdp_services.c  |  17 +-
 rhizome_fetch.c         |   3 +-
 rhizome_packetformats.c |   2 +-
 rhizome_sync.c          |   6 +-
 serval.h                |  33 +-
 socket.c                | 118 +++---
 socket.h                |  56 +++
 strbuf_helpers.c        |   6 +
 strbuf_helpers.h        |   4 +
 vomp.c                  |   4 +-
 20 files changed, 874 insertions(+), 541 deletions(-)
 create mode 100644 socket.h

diff --git a/commandline.c b/commandline.c
index cf5c146e..8a9c7d62 100644
--- a/commandline.c
+++ b/commandline.c
@@ -931,9 +931,6 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context)
   int64_t interval_ms = 1000;
   str_to_uint64_interval_ms(opt_interval, &interval_ms, NULL);
 
-  overlay_mdp_frame mdp;
-  bzero(&mdp, sizeof(overlay_mdp_frame));
-
   /* First sequence number in the echo frames */
   unsigned int firstSeq=random();
   unsigned int sequence_number=firstSeq;
@@ -941,22 +938,23 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context)
   int broadcast = is_sid_t_broadcast(ping_sid);
 
   /* Bind to MDP socket and await confirmation */
-  if ((mdp_sockfd = overlay_mdp_client_socket()) < 0)
+  if ((mdp_sockfd = mdp_socket()) < 0)
     return WHY("Cannot create MDP socket");
 
-  sid_t srcsid;
-  mdp_port_t port=32768+(random()&32767);
-  if (overlay_mdp_getmyaddr(mdp_sockfd, 0, &srcsid)) {
-    overlay_mdp_client_close(mdp_sockfd);
-    return WHY("Could not get local address");
-  }
-  if (overlay_mdp_bind(mdp_sockfd, &srcsid, port)) {
-    overlay_mdp_client_close(mdp_sockfd);
-    return WHY("Could not bind to MDP socket");
-  }
+  struct mdp_header mdp_header;
+  bzero(&mdp_header, sizeof(mdp_header));
+
+  mdp_header.local.sid = BIND_PRIMARY;
+  mdp_header.remote.sid = ping_sid;
+  mdp_header.remote.port = MDP_PORT_ECHO;
+  mdp_header.qos = OQ_MESH_MANAGEMENT;
+  mdp_header.ttl = PAYLOAD_TTL_DEFAULT;
+  mdp_header.flags = MDP_FLAG_BIND;
+  if (broadcast)
+    mdp_header.flags |= MDP_FLAG_NO_CRYPT;
   
   /* TODO Eventually we should try to resolve SID to phone number and vice versa */
-  cli_printf(context, "MDP PING %s (%s): 12 data bytes", alloca_tohex_sid_t(ping_sid), alloca_tohex_sid_t(ping_sid));
+  cli_printf(context, "MDP PING %s: 12 data bytes", alloca_tohex_sid_t(ping_sid));
   cli_delim(context, "\n");
   cli_flush(context);
 
@@ -968,81 +966,91 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context)
 
   if (broadcast)
     WARN("broadcast ping packets will not be encrypted");
+  
   for (; icount==0 || tx_count<icount; ++sequence_number) {
-    /* Now send the ping packets */
-    mdp.packetTypeAndFlags=MDP_TX;
-    if (broadcast) mdp.packetTypeAndFlags|=MDP_NOCRYPT;
-    mdp.out.src.port=port;
-    mdp.out.src.sid = srcsid;
-    mdp.out.dst.sid = ping_sid;
-    mdp.out.queue=OQ_MESH_MANAGEMENT;
-    /* Set port to well known echo port */
-    mdp.out.dst.port=MDP_PORT_ECHO;
-    mdp.out.payload_length=4+8;
-    int *seq=(int *)&mdp.out.payload;
-    *seq=sequence_number;
-    write_uint64(&mdp.out.payload[4], gettime_ms());
     
-    int res=overlay_mdp_send(mdp_sockfd, &mdp, 0, 0);
-    if (res) {
-      WHYF("could not dispatch PING frame #%d (error %d)%s%s",
-	  sequence_number - firstSeq,
-	  res,
-	  mdp.packetTypeAndFlags == MDP_ERROR ? ": " : "",
-	  mdp.packetTypeAndFlags == MDP_ERROR ? mdp.error.message : ""
-	);
-    } else
-      tx_count++;
-
+    // send a ping packet
+    {
+      uint8_t payload[12];
+      int *seq=(int *)payload;
+      *seq=sequence_number;
+      write_uint64(&payload[4], gettime_ms());
+      
+      int r = mdp_send(mdp_sockfd, &mdp_header, payload, sizeof(payload));
+      if (r<0)
+	WHY_perror("mdp_send");
+      else
+	tx_count++;
+    }
+    
     /* Now look for replies until one second has passed, and print any replies
        with appropriate information as required */
     time_ms_t now = gettime_ms();
     time_ms_t finish = now + (tx_count < icount?interval_ms:timeout_ms);
     for (; !servalShutdown && now < finish; now = gettime_ms()) {
       time_ms_t poll_timeout_ms = finish - gettime_ms();
-      int result = overlay_mdp_client_poll(mdp_sockfd, poll_timeout_ms);
-
-      if (result>0) {
-	int ttl=-1;
-	if (overlay_mdp_recv(mdp_sockfd, &mdp, port, &ttl)==0) {
-	  switch(mdp.packetTypeAndFlags&MDP_TYPE_MASK) {
-	  case MDP_ERROR:
-	    WHYF("mdpping: overlay_mdp_recv: %s (code %d)", mdp.error.message, mdp.error.error);
-	    break;
-	  case MDP_TX:
-	    {
-	      int *rxseq=(int *)&mdp.in.payload;
-	      time_ms_t txtime = read_uint64(&mdp.in.payload[4]);
-	      int hop_count = 64 - mdp.in.ttl;
-	      time_ms_t delay = gettime_ms() - txtime;
-	      cli_printf(context, "%s: seq=%d time=%"PRId64"ms hops=%d %s%s",
-		     alloca_tohex_sid_t(mdp.in.src.sid),
-		     (*rxseq)-firstSeq+1,
-		     (int64_t)delay,
-		     hop_count,
-		     mdp.packetTypeAndFlags&MDP_NOCRYPT?"":" ENCRYPTED",
-		     mdp.packetTypeAndFlags&MDP_NOSIGN?"":" SIGNED");
-	      cli_delim(context, "\n");
-	      cli_flush(context);
-	      // TODO Put duplicate pong detection here so that stats work properly.
-	      rx_count++;
-	      ret=0;
-	      rx_ms+=delay;
-	      if (rx_mintime>delay||rx_mintime==-1) rx_mintime=delay;
-	      if (delay>rx_maxtime) rx_maxtime=delay;
-	      rx_times[rx_count%1024]=delay;
-	    }
-	    break;
-	  default:
-	    WHYF("mdpping: overlay_mdp_recv: Unexpected MDP frame type 0x%x", mdp.packetTypeAndFlags);
-	    break;
-	  }
-	}
+      
+      if (mdp_poll(mdp_sockfd, poll_timeout_ms)<=0)
+	continue;
+	
+      struct mdp_header mdp_recv_header;
+      uint8_t recv_payload[12];
+      ssize_t len = mdp_recv(mdp_sockfd, &mdp_recv_header, recv_payload, sizeof(recv_payload));
+      
+      if (len<0){
+	WHY_perror("mdp_recv");
+	break;
       }
+      
+      if (mdp_recv_header.flags & MDP_FLAG_ERROR){
+	WHY("Serval daemon reported an error, please check the log for more information");
+	break;
+      }
+      
+      if (mdp_recv_header.flags & MDP_FLAG_BIND){
+	// received port binding confirmation
+	mdp_header.local = mdp_recv_header.local;
+	mdp_header.flags &= ~MDP_FLAG_BIND;
+	DEBUGF("Bound to %s:%d", alloca_tohex_sid_t(mdp_header.local.sid), mdp_header.local.port);
+	continue;
+      }
+      
+      if (len<sizeof(recv_payload)){
+	DEBUGF("Ignoring ping response as it is too short");
+	continue;
+      }
+      
+      int *rxseq=(int *)&recv_payload;
+      time_ms_t txtime = read_uint64(&recv_payload[4]);
+      int hop_count = 64 - mdp_recv_header.ttl;
+      time_ms_t delay = gettime_ms() - txtime;
+      
+      cli_put_hexvalue(context, mdp_recv_header.remote.sid.binary, SID_SIZE, ": seq=");
+      cli_put_long(context, (*rxseq)-firstSeq+1, " time=");
+      cli_put_long(context, delay, "ms hops=");
+      cli_put_long(context, hop_count, "");
+      
+      if (mdp_recv_header.flags&MDP_FLAG_NO_CRYPT)
+	cli_put_string(context, "", "");
+      else
+	cli_put_string(context, " ENCRYPTED", "");
+	
+      if (mdp_recv_header.flags&MDP_FLAG_NO_SIGN)
+	cli_put_string(context, "", "\n");
+      else
+	cli_put_string(context, " SIGNED", "\n");
+      cli_flush(context);
+      // TODO Put duplicate pong detection here so that stats work properly.
+      rx_count++;
+      ret=0;
+      rx_ms+=delay;
+      if (rx_mintime>delay||rx_mintime==-1) rx_mintime=delay;
+      if (delay>rx_maxtime) rx_maxtime=delay;
+      rx_times[rx_count%1024]=delay;
     }
   }
 
-  overlay_mdp_client_close(mdp_sockfd);
+  mdp_close(mdp_sockfd);
   
   {
     float rx_stddev=0;
@@ -2264,7 +2272,7 @@ static int handle_pins(const struct cli_parsed *parsed, struct cli_context *cont
       WHYF("Timeout while waiting for response");
       break;
     }
-    if (rev_header.flags & MDP_FLAG_OK){
+    if (rev_header.flags & MDP_FLAG_CLOSE){
       ret=0;
       break;
     }
@@ -2334,7 +2342,7 @@ int app_id_list(const struct cli_parsed *parsed, struct cli_context *context)
       // TODO receive and decode other details about this identity
     }
     
-    if (rev_header.flags & MDP_FLAG_OK){
+    if (rev_header.flags & MDP_FLAG_CLOSE){
       ret=0;
       break;
     }
diff --git a/directory_client.c b/directory_client.c
index 5f16e565..1f542464 100644
--- a/directory_client.c
+++ b/directory_client.c
@@ -52,7 +52,7 @@ static void directory_send(struct subscriber *directory_service, const sid_t *si
   // Used by tests
   INFOF("Sending directory registration for %s*, %s, %s to %s*", 
 	alloca_tohex_sid_t_trunc(*sidp, 14), did, name, alloca_tohex_sid_t_trunc(directory_service->sid, 14));
-  overlay_mdp_dispatch(&request, 0, NULL, 0);
+  overlay_mdp_dispatch(&request, NULL);
 }
 
 // send a registration packet for each unlocked identity
diff --git a/headerfiles.mk b/headerfiles.mk
index b9169ec2..422874d3 100644
--- a/headerfiles.mk
+++ b/headerfiles.mk
@@ -6,6 +6,7 @@ HDRS=	fifo.h \
 	rhizome.h \
 	serval.h \
 	keyring.h \
+	socket.h \
 	cli.h \
 	str.h \
 	rotbuf.h \
diff --git a/keyring.c b/keyring.c
index ca558a95..9d0df5c6 100644
--- a/keyring.c
+++ b/keyring.c
@@ -1764,7 +1764,7 @@ static int keyring_respond_sas(keyring_file *k, overlay_mdp_frame *req)
 	  alloca_tohex_sid_t(req->out.src.sid), req->out.src.port,
 	  alloca_tohex_sid_t(req->out.dst.sid), req->out.dst.port
 	);
-  return overlay_mdp_dispatch(req,0,NULL,0);
+  return overlay_mdp_dispatch(req, NULL);
 }
 
 // someone else is claiming to be me on this network
@@ -1790,7 +1790,7 @@ int keyring_send_unlock(struct subscriber *subscriber)
   if (crypto_sign_message(subscriber, mdp.out.payload, sizeof(mdp.out.payload), &len))
     return -1;
   mdp.out.payload_length=len;
-  return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0);
+  return overlay_mdp_dispatch(&mdp, NULL);
 }
 
 static int keyring_send_challenge(struct subscriber *source, struct subscriber *dest)
@@ -1815,7 +1815,7 @@ static int keyring_send_challenge(struct subscriber *source, struct subscriber *
   bcopy(source->identity->challenge, &mdp.out.payload[1], sizeof(source->identity->challenge));
   mdp.out.payload_length+=sizeof(source->identity->challenge);
   
-  return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0);
+  return overlay_mdp_dispatch(&mdp, NULL);
 }
 
 static int keyring_respond_challenge(struct subscriber *subscriber, overlay_mdp_frame *req)
@@ -1839,7 +1839,7 @@ static int keyring_respond_challenge(struct subscriber *subscriber, overlay_mdp_
   if (crypto_sign_message(subscriber, mdp.out.payload, sizeof(mdp.out.payload), &len))
     return -1;
   mdp.out.payload_length=len;
-  return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0);
+  return overlay_mdp_dispatch(&mdp, NULL);
 }
 
 static int keyring_process_challenge(keyring_file *k, struct subscriber *subscriber, overlay_mdp_frame *req)
@@ -1925,7 +1925,7 @@ int keyring_send_sas_request(struct subscriber *subscriber){
   mdp.out.payload_length=1;
   mdp.out.payload[0]=KEYTYPE_CRYPTOSIGN;
   
-  if (overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0))
+  if (overlay_mdp_dispatch(&mdp, NULL))
     return WHY("Failed to send SAS resolution request");
   if (config.debug.keyring)
     DEBUGF("Dispatched SAS resolution request");
diff --git a/mdp_client.c b/mdp_client.c
index f3694b9a..3db4ca9f 100644
--- a/mdp_client.c
+++ b/mdp_client.c
@@ -27,6 +27,7 @@
 #include "overlay_address.h"
 #include "overlay_packet.h"
 #include "mdp_client.h"
+#include "socket.h"
 
 int mdp_socket(void)
 {
@@ -34,49 +35,70 @@ int mdp_socket(void)
   return overlay_mdp_client_socket();
 }
 
-int mdp_close(int socket)
+static void mdp_unlink(int mdp_sock)
 {
-  // use the same process for closing sockets, though this will need to change once bind is implemented
-  return overlay_mdp_client_close(socket);
+  // get the socket name and unlink it from the filesystem if not abstract
+  struct socket_address addr;
+  addr.addrlen = sizeof addr.store;
+  if (getsockname(mdp_sock, &addr.addr, &addr.addrlen))
+    WHYF_perror("getsockname(%d)", mdp_sock);
+  else if (addr.addr.sa_family==AF_UNIX 
+    && addr.addrlen > sizeof addr.addr_un.sun_family 
+    && addr.addrlen <= sizeof addr.addr_un && addr.addr_un.sun_path[0] != '\0') {
+    if (unlink(addr.addr_un.sun_path) == -1)
+      WARNF_perror("unlink(%s)", alloca_str_toprint(addr.addr_un.sun_path));
+  }
+  close(mdp_sock);
 }
 
-int mdp_send(int socket, const struct mdp_header *header, const unsigned char *payload, ssize_t len)
+int mdp_close(int socket)
 {
-  struct sockaddr_un addr;
-  socklen_t addrlen;
-  if (make_local_sockaddr(&addr, &addrlen, "mdp.2.socket") == -1)
+  // tell the daemon to drop all bindings
+  struct mdp_header header={
+    .flags = MDP_FLAG_CLOSE,
+    .local.port = 0,
+  };
+  
+  mdp_send(socket, &header, NULL, 0);
+  
+  // remove socket
+  mdp_unlink(socket);
+  return 0;
+}
+
+int mdp_send(int socket, const struct mdp_header *header, const uint8_t *payload, size_t len)
+{
+  struct socket_address addr;
+  if (make_local_sockaddr(&addr, "mdp.2.socket") == -1)
     return -1;
   
-  struct iovec iov[]={
-    {
-      .iov_base = (void *)header,
-      .iov_len = sizeof(struct mdp_header)
-    },
-    {
-      .iov_base = (void *)payload,
-      .iov_len = len
+  struct fragmented_data data={
+    .fragment_count=2,
+    .iov={
+      {
+	.iov_base = (void*)header,
+	.iov_len = sizeof(struct mdp_header)
+      },
+      {
+	.iov_base = (void*)payload,
+	.iov_len = len
+      }
     }
   };
   
-  struct msghdr hdr={
-    .msg_name=&addr,
-    .msg_namelen=addrlen,
-    .msg_iov=iov,
-    .msg_iovlen=2,
-  };
   
-  return sendmsg(socket, &hdr, 0);
+  return send_message(socket, &addr, &data);
 }
 
-ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, ssize_t max_len)
+ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_t max_len)
 {
   /* Construct name of socket to receive from. */
-  struct sockaddr_un mdp_addr;
-  socklen_t mdp_addrlen;
-  if (make_local_sockaddr(&mdp_addr, &mdp_addrlen, "mdp.2.socket") == -1)
-    return -1;
+  errno=0;
+  struct socket_address mdp_addr;
+  if (make_local_sockaddr(&mdp_addr, "mdp.2.socket") == -1)
+    return WHY("Failed to build socket address");
   
-  struct sockaddr_un addr;
+  struct socket_address addr;
   struct iovec iov[]={
     {
       .iov_base = (void *)header,
@@ -89,25 +111,26 @@ ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload,
   };
   
   struct msghdr hdr={
-    .msg_name=&addr,
-    .msg_namelen=sizeof(struct sockaddr_un),
+    .msg_name=&addr.addr,
+    .msg_namelen=sizeof(addr.store),
     .msg_iov=iov,
     .msg_iovlen=2,
   };
   
   ssize_t len = recvmsg(socket, &hdr, 0);
   if (len<sizeof(struct mdp_header))
-    return -1;
-  
+    return WHYF("Received message is too short (%d)", (int)len);
+  addr.addrlen=hdr.msg_namelen;
   // double check that the incoming address matches the servald daemon
-  if (cmp_sockaddr((struct sockaddr *)&addr, hdr.msg_namelen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
-      && (   addr.sun_family != AF_UNIX
-	  || real_sockaddr(&addr, hdr.msg_namelen, &addr, &hdr.msg_namelen) <= 0
-	  || cmp_sockaddr((struct sockaddr *)&addr, hdr.msg_namelen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
+  if (cmp_sockaddr(&addr, &mdp_addr) != 0
+      && (   addr.addr_un.sun_family != AF_UNIX
+	  || real_sockaddr(&addr, &addr) <= 0
+	  || cmp_sockaddr(&addr, &mdp_addr) != 0
 	 )
   )
-    return -1;
-  
+    return WHYF("Received message came from %s instead of %s?",
+      alloca_socket_address(&addr),
+      alloca_socket_address(&mdp_addr));
   return len - sizeof(struct mdp_header);
 }
 
@@ -125,19 +148,18 @@ int overlay_mdp_send(int mdp_sockfd, overlay_mdp_frame *mdp, int flags, int time
   if (len == -1)
     return WHY("MDP frame invalid (could not compute length)");
   /* Construct name of socket to send to. */
-  struct sockaddr_un addr;
-  socklen_t addrlen;
-  if (make_local_sockaddr(&addr, &addrlen, "mdp.socket") == -1)
+  struct socket_address addr;
+  if (make_local_sockaddr(&addr, "mdp.socket") == -1)
     return -1;
   // Send to that socket
   set_nonblock(mdp_sockfd);
-  ssize_t result = sendto(mdp_sockfd, mdp, (size_t)len, 0, (struct sockaddr *)&addr, addrlen);
+  ssize_t result = sendto(mdp_sockfd, mdp, (size_t)len, 0, &addr.addr, addr.addrlen);
   set_block(mdp_sockfd);
   if ((size_t)result != (size_t)len) {
     if (result == -1)
-      WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sockfd, (size_t)len, alloca_sockaddr(&addr, addrlen));
+      WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sockfd, (size_t)len, alloca_socket_address(&addr));
     else
-      WHYF("sendto() sent %zu bytes of MDP reply (%zu) to socket=%d", (size_t)result, (size_t)len, mdp_sockfd); 
+      WHYF("sendto() sent %zu bytes of MDP reply (%zu) to %s", (size_t)result, (size_t)len, alloca_socket_address(&addr)); 
     mdp->packetTypeAndFlags=MDP_ERROR;
     mdp->error.error=1;
     snprintf(mdp->error.message,128,"Error sending frame to MDP server.");
@@ -181,16 +203,15 @@ int overlay_mdp_client_socket(void)
 {
   /* Create local per-client socket to MDP server (connection is always local) */
   int mdp_sockfd;
-  struct sockaddr_un addr;
-  socklen_t addrlen;
+  struct socket_address addr;
   uint32_t random_value;
   if (urandombytes((unsigned char *)&random_value, sizeof random_value) == -1)
     return WHY("urandombytes() failed");
-  if (make_local_sockaddr(&addr, &addrlen, "mdp.client.%u.%08lx.socket", getpid(), (unsigned long)random_value) == -1)
+  if (make_local_sockaddr(&addr, "mdp.client.%u.%08lx.socket", getpid(), (unsigned long)random_value) == -1)
     return -1;
   if ((mdp_sockfd = esocket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
     return -1;
-  if (socket_bind(mdp_sockfd, (struct sockaddr *)&addr, addrlen) == -1) {
+  if (socket_bind(mdp_sockfd, &addr.addr, addr.addrlen) == -1) {
     close(mdp_sockfd);
     return -1;
   }
@@ -204,16 +225,8 @@ int overlay_mdp_client_close(int mdp_sockfd)
   overlay_mdp_frame mdp;
   mdp.packetTypeAndFlags = MDP_GOODBYE;
   overlay_mdp_send(mdp_sockfd, &mdp, 0, 0);
-  // get the socket name and unlink it from the filesystem if not abstract
-  struct sockaddr_un addr;
-  socklen_t addrlen = sizeof addr;
-  if (getsockname(mdp_sockfd, (struct sockaddr *)&addr, &addrlen))
-    WHYF_perror("getsockname(%d)", mdp_sockfd);
-  else if (addrlen > sizeof addr.sun_family && addrlen <= sizeof addr && addr.sun_path[0] != '\0') {
-    if (unlink(addr.sun_path) == -1)
-      WARNF_perror("unlink(%s)", alloca_str_toprint(addr.sun_path));
-  }
-  close(mdp_sockfd);
+  
+  mdp_unlink(mdp_sockfd);
   return 0;
 }
 
@@ -236,36 +249,35 @@ int overlay_mdp_client_poll(int mdp_sockfd, time_ms_t timeout_ms)
 int overlay_mdp_recv(int mdp_sockfd, overlay_mdp_frame *mdp, mdp_port_t port, int *ttl)
 {
   /* Construct name of socket to receive from. */
-  struct sockaddr_un mdp_addr;
-  socklen_t mdp_addrlen;
-  if (make_local_sockaddr(&mdp_addr, &mdp_addrlen, "mdp.socket") == -1)
+  struct socket_address mdp_addr;
+  if (make_local_sockaddr(&mdp_addr, "mdp.socket") == -1)
     return -1;
   
   /* Check if reply available */
-  struct sockaddr_un recvaddr;
-  socklen_t recvaddrlen = sizeof recvaddr;
+  struct socket_address recvaddr;
+  recvaddr.addrlen = sizeof recvaddr.store;
   ssize_t len;
   mdp->packetTypeAndFlags = 0;
   set_nonblock(mdp_sockfd);
-  len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, (struct sockaddr *)&recvaddr, &recvaddrlen);
+  len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, &recvaddr.addr, &recvaddr.addrlen);
   set_block(mdp_sockfd);
   if (len <= 0)
     return -1; // no packet received
 
   // If the received address overflowed the buffer, then it cannot have come from the server, whose
   // address must always fit within a struct sockaddr_un.
-  if (recvaddrlen > sizeof recvaddr)
+  if (recvaddr.addrlen > sizeof recvaddr.store)
     return WHY("reply did not come from server: address overrun");
 
   // Compare the address of the sender with the address of our server, to ensure they are the same.
   // If the comparison fails, then try using realpath(3) on the sender address and compare again.
-  if (	cmp_sockaddr((struct sockaddr *)&recvaddr, recvaddrlen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
-      && (   recvaddr.sun_family != AF_UNIX
-	  || real_sockaddr(&recvaddr, recvaddrlen, &recvaddr, &recvaddrlen) <= 0
-	  || cmp_sockaddr((struct sockaddr *)&recvaddr, recvaddrlen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
+  if (	cmp_sockaddr(&recvaddr, &mdp_addr) != 0
+      && (   recvaddr.addr_un.sun_family != AF_UNIX
+	  || real_sockaddr(&recvaddr, &recvaddr) <= 0
+	  || cmp_sockaddr(&recvaddr, &mdp_addr) != 0
 	 )
   )
-    return WHYF("reply did not come from server: %s", alloca_sockaddr(&recvaddr, recvaddrlen));
+    return WHYF("reply did not come from server: %s", alloca_socket_address(&recvaddr));
   
   // silently drop incoming packets for the wrong port number
   if (port>0 && port != mdp->in.dst.port){
diff --git a/mdp_client.h b/mdp_client.h
index 608e8c3a..47fbd33c 100644
--- a/mdp_client.h
+++ b/mdp_client.h
@@ -31,8 +31,9 @@ struct mdp_sockaddr {
 
 #define MDP_FLAG_NO_CRYPT (1<<0)
 #define MDP_FLAG_NO_SIGN (1<<1)
-#define MDP_FLAG_BIND_ALL (1<<2)
-#define MDP_FLAG_OK (1<<3)
+#define MDP_FLAG_BIND (1<<2)
+
+#define MDP_FLAG_CLOSE (1<<3)
 #define MDP_FLAG_ERROR (1<<4)
 
 struct mdp_header {
@@ -43,6 +44,9 @@ struct mdp_header {
   uint8_t ttl;
 };
 
+#define BIND_PRIMARY SID_ANY
+#define BIND_ALL SID_BROADCAST
+
 #define TYPE_SID 1
 #define TYPE_PIN 2
 #define ACTION_LOCK 1
@@ -50,6 +54,7 @@ struct mdp_header {
 
 /* Port numbers for commands sent to the local daemon*/
 
+#define MDP_LISTEN 0
 /* lock and unlock identities from the local keyring
  * Requests start with an mdp_identity_request structure followed by a list of pins or SIDs
 */ 
@@ -82,13 +87,15 @@ struct overlay_mdp_scan{
   struct in_addr addr;
 };
 
-/* V2 interface */
+/* low level V2 mdp interface */
 int mdp_socket(void);
 int mdp_close(int socket);
-int mdp_send(int socket, const struct mdp_header *header, const unsigned char *payload, ssize_t len);
-ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, ssize_t max_len);
+int mdp_send(int socket, const struct mdp_header *header, const uint8_t *payload, size_t len);
+ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_t max_len);
 int mdp_poll(int socket, time_ms_t timeout_ms);
 
+
+
 /* Client-side MDP function */
 int overlay_mdp_client_socket(void);
 int overlay_mdp_client_close(int mdp_sockfd);
diff --git a/monitor-client.c b/monitor-client.c
index a404a2f1..35ef276e 100644
--- a/monitor-client.c
+++ b/monitor-client.c
@@ -36,15 +36,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #endif
 #include <sys/un.h>
 #include <fcntl.h>
+#include <ctype.h>
 
 #include "constants.h"
 #include "conf.h"
 #include "log.h"
 #include "str.h"
 #include "strbuf_helpers.h"
-
+#include "socket.h"
 #include "monitor-client.h"
-#include <ctype.h>
 
 #define STATE_INIT 0
 #define STATE_DATA 1
@@ -72,12 +72,11 @@ int monitor_client_open(struct monitor_state **res)
   int fd;
   if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1)
     return WHYF_perror("socket(AF_UNIX, SOCK_STREAM, 0)");
-  struct sockaddr_un addr;
-  socklen_t addrlen;
-  if (make_local_sockaddr(&addr, &addrlen, "monitor.socket") == -1)
+  struct socket_address addr;
+  if (make_local_sockaddr(&addr, "monitor.socket") == -1)
     return -1;
-  INFOF("Attempting to connect to %s", alloca_sockaddr(&addr, addrlen));
-  if (socket_connect(fd, (struct sockaddr*)&addr, addrlen) == -1) {
+  INFOF("Attempting to connect to %s", alloca_socket_address(&addr));
+  if (socket_connect(fd, &addr.addr, addr.addrlen) == -1) {
     close(fd);
     return -1;
   }
diff --git a/monitor.c b/monitor.c
index a1d9611c..20a710e8 100644
--- a/monitor.c
+++ b/monitor.c
@@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "strbuf_helpers.h"
 #include "overlay_address.h"
 #include "monitor-client.h"
+#include "socket.h"
 
 #ifdef HAVE_UCRED_H
 #include <ucred.h>
@@ -80,11 +81,10 @@ int monitor_setup_sockets()
   int sock = -1;
   if ((sock = esocket(AF_UNIX, SOCK_STREAM, 0)) == -1)
     goto error;
-  struct sockaddr_un addr;
-  socklen_t addrlen;
-  if (make_local_sockaddr(&addr, &addrlen, "monitor.socket") == -1)
+  struct socket_address addr;
+  if (make_local_sockaddr(&addr, "monitor.socket") == -1)
     goto error;
-  if (socket_bind(sock, (struct sockaddr*)&addr, addrlen) == -1)
+  if (socket_bind(sock, &addr.addr, addr.addrlen) == -1)
     goto error;
   if (socket_listen(sock, MAX_MONITOR_SOCKETS) == -1)
     goto error;
@@ -97,7 +97,7 @@ int monitor_setup_sockets()
   named_socket.poll.fd=sock;
   named_socket.poll.events=POLLIN;
   watch(&named_socket);
-  INFOF("Monitor socket: fd=%d %s", sock, alloca_sockaddr(&addr, addrlen));
+  INFOF("Monitor socket: fd=%d %s", sock, alloca_socket_address(&addr));
   return 0;
   
 error:
diff --git a/overlay_link.c b/overlay_link.c
index 220d605a..781b456c 100644
--- a/overlay_link.c
+++ b/overlay_link.c
@@ -313,7 +313,7 @@ int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp)
   if (reply.out.payload_length){
     if (config.debug.overlayrouting)
       DEBUGF("Sending reply");
-    overlay_mdp_dispatch(&reply,0 /* system generated */, NULL,0);
+    overlay_mdp_dispatch(&reply, NULL);
   }
   ob_free(replypayload);
   ob_free(payload);
@@ -387,7 +387,7 @@ int overlay_send_stun_request(struct subscriber *server, struct subscriber *requ
     mdp.out.payload_length=ob_position(payload);
     if (config.debug.overlayrouting)
       DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid));
-    overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL,0);
+    overlay_mdp_dispatch(&mdp, NULL);
   }
   ob_free(payload);
   return 0;
diff --git a/overlay_mdp.c b/overlay_mdp.c
index a1391220..03008593 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -30,9 +30,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "mdp_client.h"
 #include "crypto.h"
 #include "keyring.h"
+#include "socket.h"
 
 static void overlay_mdp_poll(struct sched_ent *alarm);
 static void mdp_poll2(struct sched_ent *alarm);
+static int overlay_mdp_releasebindings(struct socket_address *client);
 
 static struct profile_total mdp_stats = { .name="overlay_mdp_poll" };
 static struct sched_ent mdp_sock = {
@@ -49,6 +51,8 @@ static struct sched_ent mdp_sock2 = {
 };
 
 static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame *mdp, time_ms_t now);
+static int mdp_send2(struct socket_address *client, struct mdp_header *header, 
+  const uint8_t *payload, size_t payload_len);
 
 /* Delete all UNIX socket files in instance directory. */
 static void overlay_mdp_clean_socket_files()
@@ -77,23 +81,22 @@ static void overlay_mdp_clean_socket_files()
 
 static int mdp_bind_socket(const char *name)
 {
-  struct sockaddr_un addr;
-  socklen_t addrlen;
+  struct socket_address addr;
   int sock;
   
-  if (make_local_sockaddr(&addr, &addrlen, "%s", name) == -1)
+  if (make_local_sockaddr(&addr, "%s", name) == -1)
     return -1;
-  if ((sock = esocket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
+  if ((sock = esocket(addr.addr.sa_family, SOCK_DGRAM, 0)) == -1)
     return -1;
   if (socket_set_reuseaddr(sock, 1) == -1)
     WARN("Could not set socket to reuse addresses");
-  if (socket_bind(sock, (struct sockaddr *)&addr, addrlen) == -1) {
+  if (socket_bind(sock, &addr.addr, addr.addrlen) == -1) {
     close(sock);
     return -1;
   }
   socket_set_rcvbufsize(sock, 64 * 1024);
   
-  INFOF("Socket %s: fd=%d %s", name, sock, alloca_sockaddr(&addr, addrlen));
+  INFOF("Socket %s: fd=%d %s", name, sock, alloca_socket_address(&addr));
   return sock;
 }
 
@@ -122,17 +125,48 @@ int overlay_mdp_setup_sockets()
 struct mdp_binding{
   struct subscriber *subscriber;
   mdp_port_t port;
-  char socket_name[MDP_MAX_SOCKET_NAME_LEN];
-  int name_len;
+  int version;
+  struct socket_address client;
   time_ms_t binding_time;
 };
 
 struct mdp_binding mdp_bindings[MDP_MAX_BINDINGS];
 int mdp_bindings_initialised=0;
+mdp_port_t next_port_binding=256;
 
-int overlay_mdp_reply_error(int sock,
-			    struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
-			    int error_number,char *message)
+static int compare_client(struct socket_address *one, struct socket_address *two)
+{
+  if (one->addrlen==two->addrlen
+    && memcmp(&one->addr, &two->addr, two->addrlen)==0)
+    return 1;
+  return 0;
+}
+
+static int overlay_mdp_reply(int sock, struct socket_address *client,
+			  overlay_mdp_frame *mdpreply)
+{
+  if (!client) return WHY("No reply address");
+
+  ssize_t replylen = overlay_mdp_relevant_bytes(mdpreply);
+  if (replylen<0) return WHY("Invalid MDP frame (could not compute length)");
+
+  ssize_t r=sendto(sock,(char *)mdpreply,replylen,0, &client->addr, client->addrlen);
+  if (r == -1){
+    WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", sock, (size_t)replylen, alloca_socket_address(client));
+    if (errno == ENOENT){
+      /* far-end of socket has died, so drop binding */
+      INFOF("Closing dead MDP client '%s'", alloca_socket_address(client));
+      overlay_mdp_releasebindings(client);
+    }
+    return -1;
+  }
+  if (r != replylen)
+    return WHYF("sendto() sent %zu bytes of MDP reply (%zu) to %s", (size_t)r, (size_t)replylen, alloca_socket_address(client));
+  return 0;  
+}
+
+static int overlay_mdp_reply_error(int sock, struct socket_address *client,
+			    int error_number, char *message)
 {
   overlay_mdp_frame mdpreply;
 
@@ -148,49 +182,29 @@ int overlay_mdp_reply_error(int sock,
   }
   mdpreply.error.message[127]=0;
 
-  return overlay_mdp_reply(sock,recvaddr,recvaddrlen,&mdpreply);
+  return overlay_mdp_reply(sock, client, &mdpreply);
 }
 
-int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
-			  overlay_mdp_frame *mdpreply)
-{
-  if (!recvaddr)
-    return WHY("No reply address");
-  ssize_t replylen = overlay_mdp_relevant_bytes(mdpreply);
-  if (replylen == -1)
-    return WHY("Invalid MDP frame (could not compute length)");
-  ssize_t r = sendto(sock, (char *)mdpreply, (size_t)replylen, 0, (struct sockaddr *)recvaddr, recvaddrlen);
-  if (r == -1) {
-    WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", sock, (size_t)replylen, alloca_sockaddr((struct sockaddr *)recvaddr, recvaddrlen));
-    return WHYF("sendto() failed when sending MDP reply"); 
-  }
-  if ((size_t)r != (size_t)replylen)
-    return WHYF("sendto() sent %zu bytes of MDP reply (%zu) to socket=%d", (size_t)r, (size_t)replylen, sock); 
-  return 0;  
-}
-
-int overlay_mdp_reply_ok(int sock,
-			 struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
+static int overlay_mdp_reply_ok(int sock, struct socket_address *client, 
 			 char *message)
 {
-  return overlay_mdp_reply_error(sock,recvaddr,recvaddrlen,0,message);
+  return overlay_mdp_reply_error(sock, client, 0, message);
 }
 
-int overlay_mdp_releasebindings(struct sockaddr_un *recvaddr, socklen_t recvaddrlen)
+static int overlay_mdp_releasebindings(struct socket_address *client)
 {
   /* Free up any MDP bindings held by this client. */
   int i;
   for(i=0;i<MDP_MAX_BINDINGS;i++)
-    if (mdp_bindings[i].name_len==recvaddrlen)
-      if (!memcmp(mdp_bindings[i].socket_name,recvaddr->sun_path,recvaddrlen))
-	mdp_bindings[i].port=0;
+    if (compare_client(&mdp_bindings[i].client, client))
+      mdp_bindings[i].port=0;
 
   return 0;
 
 }
 
-int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, mdp_port_t port,
-				     int flags, struct sockaddr_un *recvaddr,  socklen_t recvaddrlen)
+static int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, mdp_port_t port,
+				     int flags, struct socket_address *client)
 {
   if (config.debug.mdprequests) 
     DEBUGF("Bind request %s:%"PRImdp_port_t, subscriber ? alloca_tohex_sid_t(subscriber->sid) : "NULL", port);
@@ -213,8 +227,7 @@ int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, md
     for(i=0;i<MDP_MAX_BINDINGS;i++) {
       /* Look for duplicate bindings */
       if (mdp_bindings[i].port == port && mdp_bindings[i].subscriber == subscriber) {
-	if (mdp_bindings[i].name_len==recvaddrlen &&
-	    !memcmp(mdp_bindings[i].socket_name,recvaddr->sun_path,recvaddrlen)) {
+	if (compare_client(&mdp_bindings[i].client, client)) {
 	  // this client already owns this port binding?
 	  INFO("Identical binding exists");
 	  return 0;
@@ -250,9 +263,9 @@ int overlay_mdp_process_bind_request(int sock, struct subscriber *subscriber, md
   /* Okay, record binding and report success */
   mdp_bindings[free].port=port;
   mdp_bindings[free].subscriber=subscriber;
-  
-  mdp_bindings[free].name_len = recvaddrlen - sizeof recvaddr->sun_family;
-  memcpy(mdp_bindings[free].socket_name, recvaddr->sun_path, mdp_bindings[free].name_len);
+  mdp_bindings[free].version=0;
+  mdp_bindings[free].client.addrlen = client->addrlen;
+  memcpy(&mdp_bindings[free].client.addr, &client->addr, client->addrlen);
   mdp_bindings[free].binding_time=gettime_ms();
   return 0;
 }
@@ -280,7 +293,6 @@ int overlay_mdp_decrypt(struct overlay_frame *f, overlay_mdp_frame *mdp)
 
   /* Indicate MDP message type */
   mdp->packetTypeAndFlags=MDP_TX;
-  
   switch(f->modifiers&(OF_CRYPTO_CIPHERED|OF_CRYPTO_SIGNED))  {
   case 0: 
     /* nothing to do, b already points to the plain text */
@@ -401,76 +413,82 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
   int i;
   int match=-1;
 
-  switch(mdp->packetTypeAndFlags&MDP_TYPE_MASK) {
-  case MDP_TX: 
-    /* Regular MDP frame addressed to us.  Look for matching port binding,
-       and if available, push to client.  Else do nothing, or if we feel nice
-       send back a connection refused type message? Silence is probably the
-       more prudent path.
-    */
+  /* Regular MDP frame addressed to us.  Look for matching port binding,
+     and if available, push to client.  Else do nothing, or if we feel nice
+     send back a connection refused type message? Silence is probably the
+     more prudent path.
+  */
 
-    if (config.debug.mdprequests) 
-      DEBUGF("Received packet with listener (MDP ports: src=%s*:%"PRImdp_port_t", dst=%"PRImdp_port_t")",
-	   alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14),
-	   mdp->out.src.port, mdp->out.dst.port);
+  if (config.debug.mdprequests) 
+    DEBUGF("Received packet with listener (MDP ports: src=%s*:%"PRImdp_port_t", dst=%"PRImdp_port_t")",
+	 alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14),
+	 mdp->out.src.port, mdp->out.dst.port);
 
-    // TODO pass in dest subscriber as an argument, we should know it by now
-    struct subscriber *destination = NULL;
-    if (frame)
-      destination = frame->destination;
-    else if (!is_sid_t_broadcast(mdp->out.dst.sid)){
-      destination = find_subscriber(mdp->out.dst.sid.binary, SID_SIZE, 1);
-    }
-    
-    for(i=0;i<MDP_MAX_BINDINGS;i++)
-      {
-	if (mdp_bindings[i].port!=mdp->out.dst.port)
-	  continue;
-	
-	if ((!destination) || mdp_bindings[i].subscriber == destination){
-	  /* exact match, so stop searching */
-	  match=i;
-	  break;
-	}else if (!mdp_bindings[i].subscriber){
-	  /* If we find an "ANY" binding, remember it. But we will prefer an exact match if we find one */
-	  match=i;
-	}
+  // TODO pass in dest subscriber as an argument, we should know it by now
+  struct subscriber *destination = NULL;
+  if (frame)
+    destination = frame->destination;
+  else if (!is_sid_t_broadcast(mdp->out.dst.sid)){
+    destination = find_subscriber(mdp->out.dst.sid.binary, SID_SIZE, 1);
+  }
+  
+  for(i=0;i<MDP_MAX_BINDINGS;i++)
+    {
+      if (mdp_bindings[i].port!=mdp->out.dst.port)
+	continue;
+      
+      if ((!destination) || mdp_bindings[i].subscriber == destination){
+	/* exact match, so stop searching */
+	match=i;
+	break;
+      }else if (!mdp_bindings[i].subscriber){
+	/* If we find an "ANY" binding, remember it. But we will prefer an exact match if we find one */
+	match=i;
       }
-    
-    if (match>-1) {
-      struct sockaddr_un addr;
-      addr.sun_family = AF_UNIX;
-      bcopy(mdp_bindings[match].socket_name, addr.sun_path, mdp_bindings[match].name_len);
-      ssize_t len = overlay_mdp_relevant_bytes(mdp);
-      if (len == -1)
-	RETURN(WHY("unsupported MDP packet type"));
-      socklen_t addrlen = sizeof addr.sun_family + mdp_bindings[match].name_len;
-      if (config.debug.mdprequests)
-	DEBUGF("Resolved bound socket on port %"PRImdp_port_t", addr=%s",
-	    mdp_bindings[match].port, alloca_sockaddr(&addr, addrlen));
-      ssize_t r = sendto(mdp_sock.poll.fd, mdp, (size_t)len, 0, (struct sockaddr*)&addr, addrlen);
-      if ((size_t)r != (size_t)len) {
-	if (r == -1) {
-	  WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sock.poll.fd, (size_t)len, alloca_sockaddr(&addr, addrlen));
-	  if (errno == ENOENT) {
-	    /* far-end of socket has died, so drop binding */
-	    INFOF("Closing dead MDP client '%s'",mdp_bindings[match].socket_name);
-	    overlay_mdp_releasebindings(&addr,mdp_bindings[match].name_len);
+    }
+  
+  if (match>-1) {
+    switch(mdp_bindings[match].version){
+      case 0:
+	{
+	  ssize_t len = overlay_mdp_relevant_bytes(mdp);
+	  if (len < 0)
+	    RETURN(WHY("unsupported MDP packet type"));
+	  struct socket_address *client = &mdp_bindings[match].client;
+	  ssize_t r = sendto(mdp_sock.poll.fd,mdp,len,0, &client->addr, client->addrlen);
+	  if (r == -1){
+	    WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sock.poll.fd, (size_t)len, alloca_socket_address(client));
+	    if (errno == ENOENT){
+	      /* far-end of socket has died, so drop binding */
+	      INFOF("Closing dead MDP client '%s'", alloca_socket_address(client));
+	      overlay_mdp_releasebindings(client);
+	    }
+	    RETURN(-1);
 	  }
-	} else
-	  WHYF("sendto() sent %zu bytes of MDP frame (%zu) to client, socket=%d", (size_t)r, (size_t)len, mdp_sock.poll.fd); 
-	RETURN(WHY("Failed to pass received MDP frame to client"));
-      }
-    } else {
-      if (config.debug.mdprequests)
-	DEBUGF("No socket bound to port %"PRImdp_port_t", try internal service", mdp->out.dst.port);
-      /* No socket is bound, ignore the packet ... except for magic sockets */
-      RETURN(overlay_mdp_try_interal_services(frame, mdp));
+	  if (r != len)
+	    RETURN(WHYF("sendto() sent %zu bytes of MDP reply (%zu) to %s", (size_t)r, (size_t)len, alloca_socket_address(client)));
+	  RETURN(0);
+	}
+      case 1:
+	{
+	  struct mdp_header header;
+	  header.local.sid=mdp->out.dst.sid;
+	  header.local.port=mdp->out.dst.port;
+	  header.remote.sid=mdp->out.src.sid;
+	  header.remote.port=mdp->out.src.port;
+	  header.qos=mdp->out.queue;
+	  header.ttl=mdp->out.ttl;
+	  header.flags=0;
+	  if (mdp->packetTypeAndFlags & MDP_NOCRYPT)
+	    header.flags|=MDP_FLAG_NO_CRYPT;
+	  if (mdp->packetTypeAndFlags & MDP_NOSIGN)
+	    header.flags|=MDP_FLAG_NO_SIGN;
+	  RETURN(mdp_send2(&mdp_bindings[match].client, &header, mdp->out.payload, mdp->out.payload_length));
+	}
     }
-    break;
-  default:
-    RETURN(WHYF("We should only see MDP_TX frames here (MDP message type = 0x%x)",
-		mdp->packetTypeAndFlags));
+  } else {
+    /* No socket is bound, ignore the packet ... except for magic sockets */
+    RETURN(overlay_mdp_try_internal_services(frame, mdp));
   }
 
   RETURN(0);
@@ -501,14 +519,14 @@ int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolv
     return WHY("MDP payload overrun");
   mdpreply.out.payload_length = strbuf_len(b) + 1;
   /* deliver reply */
-  return overlay_mdp_dispatch(&mdpreply, 0 /* system generated */, NULL, 0);
+  return overlay_mdp_dispatch(&mdpreply, NULL);
 }
 
-int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port, int userGeneratedFrameP,
-			      struct sockaddr_un *recvaddr,  socklen_t recvaddrlen)
+static int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port,
+			      struct socket_address *client)
 {
   /* System generated frames can send anything they want */
-  if (!userGeneratedFrameP)
+  if (!client)
     return 0;
 
   /* Check if the address is in the list of bound addresses,
@@ -520,18 +538,14 @@ int overlay_mdp_check_binding(struct subscriber *subscriber, mdp_port_t port, in
       continue;
     if ((!mdp_bindings[i].subscriber) || mdp_bindings[i].subscriber == subscriber) {
       /* Binding matches, now make sure the sockets match */
-      if (  mdp_bindings[i].name_len == recvaddrlen - sizeof(sa_family_t)
-	&&  memcmp(mdp_bindings[i].socket_name, recvaddr->sun_path, recvaddrlen - sizeof(sa_family_t)) == 0
-      ) {
+      if (compare_client(&mdp_bindings[i].client, client)) {
 	/* Everything matches, so this unix socket and MDP address combination is valid */
 	return 0;
       }
     }
   }
 
-  return WHYF("No such binding: recvaddr=%p %s addr=%s port=%"PRImdp_port_t" -- possible spoofing attack",
-	recvaddr,
-	recvaddr ? alloca_toprint(-1, recvaddr->sun_path, recvaddrlen - sizeof(sa_family_t)) : "",
+  return WHYF("No matching binding: addr=%s port=%"PRImdp_port_t" -- possible spoofing attack",
 	alloca_tohex_sid_t(subscriber->sid),
 	port
       );
@@ -627,18 +641,75 @@ static struct overlay_buffer * encrypt_payload(
 }
 
 // encrypt or sign the plaintext, then queue the frame for transmission.
-int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plaintext){
+static int overlay_send_frame(
+  struct subscriber *source, mdp_port_t src_port,
+  struct subscriber *destination, mdp_port_t dst_port,
+  const uint8_t *payload, size_t payload_len,
+  uint8_t ttl, uint8_t qos, uint8_t modifiers)
+{
+  if (destination && destination->reachable == REACHABLE_SELF)
+    return 0;
   
-  if (!frame->source)
-    frame->source = my_subscriber;
+  if (ttl == 0) 
+    ttl = PAYLOAD_TTL_DEFAULT;
+  else if (ttl > PAYLOAD_TTL_MAX)
+    return WHYF("Invalid TTL");
+  
+  if (qos == 0)
+    qos = OQ_ORDINARY;
     
+  if (!source)
+    return WHYF("No source specified");
+  
+  /* Prepare the overlay frame for dispatch */
+  struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame));
+  if (!frame)
+    return -1;
+  
+  frame->source = source;
+  frame->destination = destination;
+  frame->ttl = ttl;
+  frame->queue = qos;
+  frame->type = OF_TYPE_DATA;
+  frame->modifiers = modifiers;
+  
+  // copy the plain text message into a new buffer, with the wire encoded port numbers
+  struct overlay_buffer *plaintext=ob_new();
+  if (!plaintext){
+    op_free(frame);
+    return -1;
+  }
+  
+  overlay_mdp_encode_ports(plaintext, dst_port, src_port);
+  if (payload && payload_len){
+    ob_append_bytes(plaintext, payload, payload_len);
+  }
+  
+  if (ob_overrun(plaintext)) {
+    if (config.debug.mdprequests) 
+      DEBUGF("Frame overrun: position=%d allocSize=%d sizeLimit=%d",
+	  plaintext->position, plaintext->allocSize, plaintext->sizeLimit);
+    op_free(frame);
+    ob_free(plaintext);
+    return -1;
+  }
+  if (config.debug.mdprequests) {
+    DEBUGF("Send frame %u bytes", ob_position(plaintext));
+    dump("Frame plaintext", ob_ptr(plaintext), ob_position(plaintext));
+  }
+  
   /* Work out the disposition of the frame->  For now we are only worried
      about the crypto matters, and not compression that may be applied
      before encryption (since applying it after is useless as ciphered
      text should have maximum entropy). */
-  switch(frame->modifiers) {
-  default:
+  switch(modifiers) {
   case OF_CRYPTO_SIGNED|OF_CRYPTO_CIPHERED:
+    if (!frame->destination){
+      ob_free(plaintext);
+      op_free(frame);
+      return WHY("Cannot encrypt to broadcast destinations");
+    }
+  
     /* crypted and signed (using CryptoBox authcryption primitive) */
     frame->payload = encrypt_payload(frame->source, frame->destination, ob_ptr(plaintext), ob_position(plaintext));
     if (!frame->payload){
@@ -671,6 +742,11 @@ int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plain
     /* clear text and no signature */
     frame->payload = plaintext;
     break;
+    
+  default:
+    ob_free(plaintext);
+    op_free(frame);
+    return WHY("Invalid encrypt / sign combination");
   }
   
   if (!frame->destination && frame->ttl>1)
@@ -690,19 +766,17 @@ int overlay_send_frame(struct overlay_frame *frame, struct overlay_buffer *plain
    This is for use by the SERVER. 
    Clients should use overlay_mdp_send()
  */
-int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
-			 struct sockaddr_un *recvaddr, socklen_t recvaddrlen)
+int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *client)
 {
   IN();
   unsigned __d = 0;
   if (config.debug.mdprequests) {
     __d = fd_depth();
-    DEBUGF("[%u] src=%s*:%"PRImdp_port_t", dst=%s*:%"PRImdp_port_t", userGen=%d, recv=%s",
+    DEBUGF("[%u] src=%s*:%"PRImdp_port_t", dst=%s*:%"PRImdp_port_t", recv=%s",
 	   __d,
 	   alloca_tohex_sid_t_trunc(mdp->out.src.sid, 14), mdp->out.src.port,
 	   alloca_tohex_sid_t_trunc(mdp->out.dst.sid, 14), mdp->out.dst.port,
-	   userGeneratedFrameP,
-	   recvaddr ? alloca_sockaddr(recvaddr, recvaddrlen) : "NULL"
+	   client ? alloca_socket_address(client) : "NULL"
 	);
   }
 
@@ -731,11 +805,10 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
     }
   }
   
-  if (overlay_mdp_check_binding(source, mdp->out.src.port, userGeneratedFrameP, recvaddr, recvaddrlen)){
+  if (overlay_mdp_check_binding(source, mdp->out.src.port, client)){
     RETURN(overlay_mdp_reply_error
 	   (mdp_sock.poll.fd,
-	    (struct sockaddr_un *)recvaddr,
-	    recvaddrlen,8,
+	    client,8,
 	    "Source address is invalid (you must bind to a source address before"
 	    " you can send packets"));
   }
@@ -749,25 +822,20 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
      NaCl cryptobox keys can be used for signing. */	
     if (!(mdp->packetTypeAndFlags&MDP_NOCRYPT))
       RETURN(overlay_mdp_reply_error(mdp_sock.poll.fd,
-				     recvaddr,recvaddrlen,5,
+				     client,5,
 				     "Broadcast packets cannot be encrypted "));
   }else{
     destination = find_subscriber(mdp->out.dst.sid.binary, SID_SIZE, 1);
     // should we reply with an error if the destination is not currently routable?
   }
   
-  if (mdp->out.ttl == 0) 
-    mdp->out.ttl = PAYLOAD_TTL_DEFAULT;
-  else if (mdp->out.ttl > PAYLOAD_TTL_MAX) {
-    RETURN(overlay_mdp_reply_error(mdp_sock.poll.fd, recvaddr,recvaddrlen,9, "TTL out of range"));
+  if (mdp->out.ttl > PAYLOAD_TTL_MAX) {
+    RETURN(overlay_mdp_reply_error(mdp_sock.poll.fd, client, 9, "TTL out of range"));
   }
   
-  if (mdp->out.queue == 0)
-    mdp->out.queue = OQ_ORDINARY;
-    
   if (config.debug.mdprequests) 
     DEBUGF("[%u] destination->sid=%s", __d, destination ? alloca_tohex_sid_t(destination->sid) : "NULL");
-  if (!destination || destination->reachable == REACHABLE_SELF) {
+  if (!destination || destination->reachable == REACHABLE_SELF){
     /* Packet is addressed to us / broadcast, we should process it first. */
     overlay_saw_mdp_frame(NULL,mdp,gettime_ms());
     if (destination) {
@@ -777,7 +845,7 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
       RETURN(0);
     }
   }
-  
+
   int modifiers=0;
   
   switch(mdp->packetTypeAndFlags&(MDP_NOCRYPT|MDP_NOSIGN)) {
@@ -803,43 +871,12 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
     RETURN(WHY("Not implemented"));
   };
   
-  // copy the plain text message into a new buffer, with the wire encoded port numbers
-  struct overlay_buffer *plaintext=ob_new();
-  if (plaintext == NULL)
-    RETURN(-1);
-  
-  overlay_mdp_encode_ports(plaintext, mdp->out.dst.port, mdp->out.src.port);
-  if (mdp->out.payload_length)
-    ob_append_bytes(plaintext, mdp->out.payload, mdp->out.payload_length);
-  if (ob_overrun(plaintext)) {
-    if (config.debug.mdprequests) 
-      DEBUGF("[%u] Frame overrun: position=%d allocSize=%d sizeLimit=%d", __d,
-	  plaintext->position, plaintext->allocSize, plaintext->sizeLimit);
-    ob_free(plaintext);
-    RETURN(-1);
-  }
-  if (config.debug.mdprequests) {
-    DEBUGF("[%u] Send frame %u bytes", __d, ob_position(plaintext));
-    dump("Frame plaintext", ob_ptr(plaintext), ob_position(plaintext));
-  }
-  
-  /* Prepare the overlay frame for dispatch */
-  struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame));
-  if (!frame){
-    ob_free(plaintext);
-    RETURN(-1);
-  }
-  
-  frame->source = source;
-  frame->destination = destination;
-  frame->ttl = mdp->out.ttl;
-  frame->queue = mdp->out.queue;
-  frame->type=OF_TYPE_DATA;
-  frame->prev=NULL;
-  frame->next=NULL;
-  frame->modifiers=modifiers;
-  
-  RETURN(overlay_send_frame(frame, plaintext));
+  RETURN(overlay_send_frame(
+    source, mdp->out.src.port,
+    destination, mdp->out.dst.port, 
+    mdp->out.payload, mdp->out.payload_length,
+    mdp->out.ttl, mdp->out.queue, modifiers
+  ));
   OUT();
 }
 
@@ -893,8 +930,7 @@ int overlay_mdp_address_list(overlay_mdp_addrlist *request, overlay_mdp_addrlist
 }
 
 struct routing_state{
-  struct sockaddr_un *recvaddr_un;
-  socklen_t recvaddrlen;
+  struct socket_address *client;
   int fd;
 };
 
@@ -918,7 +954,7 @@ static int routing_table(struct subscriber *subscriber, void *context)
     strcpy(r->interface_name, subscriber->destination->interface->name);
   else
     r->interface_name[0]=0;
-  overlay_mdp_reply(mdp_sock.poll.fd, state->recvaddr_un, state->recvaddrlen, &reply);
+  overlay_mdp_reply(mdp_sock.poll.fd, state->client, &reply);
   return 0;
 }
 
@@ -966,12 +1002,7 @@ static void overlay_mdp_scan(struct sched_ent *alarm)
   }
 }
 
-struct mdp_client{
-  struct sockaddr_un *addr;
-  socklen_t addrlen;
-};
-
-static int mdp_reply2(const struct mdp_client *client, const struct mdp_header *header, 
+static int mdp_reply2(const struct socket_address *client, const struct mdp_header *header, 
   int flags, const unsigned char *payload, int payload_len)
 {
   struct mdp_header response_header;
@@ -990,22 +1021,22 @@ static int mdp_reply2(const struct mdp_client *client, const struct mdp_header *
   };
   
   struct msghdr hdr={
-    .msg_name=client->addr,
+    .msg_name=(void *)&client->addr,
     .msg_namelen=client->addrlen,
     .msg_iov=iov,
     .msg_iovlen=2,
   };
   
   if (config.debug.mdprequests)
-    DEBUGF("Replying to %s with code %d", alloca_sockaddr(client->addr, client->addrlen), flags);
+    DEBUGF("Replying to %s with flags %d", alloca_socket_address(client), flags);
   return sendmsg(mdp_sock2.poll.fd, &hdr, 0);
 }
 
 #define mdp_reply_error(A,B)  mdp_reply2(A,B,MDP_FLAG_ERROR,NULL,0)
-#define mdp_reply_ok(A,B)  mdp_reply2(A,B,MDP_FLAG_OK,NULL,0)
+#define mdp_reply_ok(A,B)  mdp_reply2(A,B,MDP_FLAG_CLOSE,NULL,0)
 
-static int mdp_process_identity_request(struct mdp_client *client, struct mdp_header *header, 
-  const unsigned char *payload, size_t payload_len)
+static int mdp_process_identity_request(struct socket_address *client, struct mdp_header *header, 
+  const uint8_t *payload, size_t payload_len)
 {
   if (payload_len<sizeof(struct mdp_identity_request)){
     mdp_reply_error(client, header);
@@ -1078,8 +1109,8 @@ static int mdp_process_identity_request(struct mdp_client *client, struct mdp_he
 }
 
 // return one response per matching identity
-static int mdp_search_identities(struct mdp_client *client, struct mdp_header *header, 
-  const unsigned char *payload, size_t payload_len)
+static int mdp_search_identities(struct socket_address *client, struct mdp_header *header, 
+  const uint8_t *payload, size_t payload_len)
 {
   int cn=0, in=0, kp=0;
   const char *tag=NULL;
@@ -1110,16 +1141,15 @@ static int mdp_search_identities(struct mdp_client *client, struct mdp_header *h
       if (!keyring_next_identity(keyring, &cn, &in, &kp))
 	break;
     }
-    
     keyring_identity *id = keyring->contexts[cn]->identities[in];
     unsigned char reply_payload[1200];
-    int ofs=0;
+    size_t ofs=0;
     
     bcopy(id->subscriber->sid.binary, &reply_payload[ofs], sizeof(id->subscriber->sid));
     ofs+=sizeof(id->subscriber->sid);
-    
+     
     // TODO return other details of this identity
-    
+     
     mdp_reply2(client, header, 0, reply_payload, ofs);
     kp++;
   }
@@ -1127,54 +1157,268 @@ static int mdp_search_identities(struct mdp_client *client, struct mdp_header *h
   return 0;
 }
 
-static void mdp_poll2(struct sched_ent *alarm)
+static void mdp_process_packet(struct socket_address *client, struct mdp_header *header, 
+  const uint8_t *payload, size_t payload_len)
 {
-  if (alarm->poll.revents & POLLIN) {
-    unsigned char buffer[1600];
-    struct sockaddr_storage addr;
-    struct mdp_client client={
-      .addr = (struct sockaddr_un *)&addr,
-      .addrlen = sizeof(addr)
-    };
-    int ttl=-1;
-    
-    ssize_t len = recvwithttl(alarm->poll.fd, buffer, sizeof(buffer), &ttl, (struct sockaddr *)&addr, &client.addrlen);
-    
-    if (len<sizeof(struct mdp_header)){
-      WHYF("Expected length %d, got %d from %s", (int)sizeof(struct mdp_header), (int)len, alloca_sockaddr(client.addr, client.addrlen));
+  if ((header->flags & MDP_FLAG_CLOSE) && header->local.port==0){
+    int i;
+    for(i=0;i<MDP_MAX_BINDINGS;i++) {
+      if (mdp_bindings[i].port!=0 
+	&& compare_client(&mdp_bindings[i].client, client)){
+	if (config.debug.mdprequests)
+	  DEBUGF("Unbind MDP %s:%d from %s", 
+	    mdp_bindings[i].subscriber?alloca_tohex_sid_t(mdp_bindings[i].subscriber->sid):"All",
+	    mdp_bindings[i].port,
+	    alloca_socket_address(client));
+	mdp_bindings[i].port=0;
+      }
+    }
+    // should we expect clients to wait for this?
+    // mdp_reply_ok(client, header);
+    return;
+  }
+  
+  // find the source subscriber
+  struct subscriber *source=NULL;
+  
+  if (is_sid_t_broadcast(header->local.sid)){
+    // leave source NULL to indicate listening on all local SID's
+    // note that attempting anything else will fail
+  }else if (is_sid_t_any(header->local.sid)){
+    // leaving the sid blank indicates that we should use our main identity
+    source = my_subscriber;
+    header->local.sid = source->sid;
+  }else{
+    // find the matching sid from our keyring
+    source = find_subscriber(header->local.sid.binary, sizeof(header->local.sid), 0);
+    if (!source || source->reachable != REACHABLE_SELF){
+      mdp_reply_error(client, header);
+      WHY("Subscriber is not local");
+    }
+  }
+  
+  struct mdp_binding *binding=NULL, *free_slot=NULL;
+  
+  // assign the next available port number
+  if (header->local.port==0 && header->flags & MDP_FLAG_BIND){
+    if (next_port_binding > 32*1024)
+      next_port_binding=256;
+    else
+      next_port_binding++;
+    header->local.port=next_port_binding;
+  }
+  
+  // find matching binding
+  {
+    int i;
+    for(i=0;i<MDP_MAX_BINDINGS;i++) {
+      if ((!free_slot) && mdp_bindings[i].port==0)
+	free_slot=&mdp_bindings[i];
+      
+      if (mdp_bindings[i].port == header->local.port 
+	&& mdp_bindings[i].subscriber == source){
+	
+	binding = &mdp_bindings[i];
+	break;
+      }
+    }
+  }
+  
+  if (header->flags & MDP_FLAG_BIND){
+    if (binding){
+      mdp_reply_error(client, header);
+      WHYF("Port %d already bound", header->local.port);
       return;
     }
     
-    struct mdp_header *header = (struct mdp_header *)buffer;
-    
-    unsigned char *payload = &buffer[sizeof(struct mdp_header)];
-    size_t payload_len = len - sizeof(struct mdp_header);
-    
-    if (is_sid_t_any(header->remote.sid)){
-      // process local commands
-      switch(header->remote.port){
-	// lock and unlock identities
-	case MDP_IDENTITY:
-	  if (config.debug.mdprequests)
-	    DEBUGF("Processing MDP_IDENTITY from %s", alloca_sockaddr(client.addr, client.addrlen));
-	  mdp_process_identity_request(&client, header, payload, payload_len);
-	  break;
-	// seach unlocked identities
-	case MDP_SEARCH_IDS:
-	  if (config.debug.mdprequests)
-	    DEBUGF("Processing MDP_SEARCH_IDS from %s", alloca_sockaddr(client.addr, client.addrlen));
-	  mdp_search_identities(&client, header, payload, payload_len);
-	  break;
-	default:
-	  mdp_reply_error(&client, header);
-	  WHY("Unknown port number");
-	  break;
-      }
-    }else{
-      // TODO transmit packet
-      mdp_reply_error(&client, header);
-      WHY("Transmitting packets is not yet supported");
+    if (!free_slot){
+      mdp_reply_error(client, header);
+      WHY("Max supported bindings reached");
+      return;
     }
+    
+    if (config.debug.mdprequests)
+      DEBUGF("Bind MDP %s:%d to %s", 
+	alloca_tohex_sid_t(header->local.sid),
+	header->local.port,
+	alloca_socket_address(client));
+    
+    // claim binding
+    binding = free_slot;
+    binding->port = header->local.port;
+    binding->subscriber = source;
+    bcopy(&client->addr, &binding->client.addr, client->addrlen);
+    binding->client.addrlen = client->addrlen;
+    binding->binding_time=gettime_ms();
+    binding->version=1;
+    
+    // tell the client what we actually bound (with flags & MDP_FLAG_BIND still set)
+    mdp_reply2(client, header, MDP_FLAG_BIND, NULL, 0);
+  }
+  
+  if (is_sid_t_any(header->remote.sid)){
+    // process local commands
+    switch(header->remote.port){
+      case MDP_LISTEN:
+	// double check that this binding belongs to this connection
+	if (!binding
+	  || !compare_client(&binding->client, client))
+	  mdp_reply_error(client, header);
+	break;
+      case MDP_IDENTITY:
+	if (config.debug.mdprequests)
+	  DEBUGF("Processing MDP_IDENTITY from %s", alloca_socket_address(client));
+	mdp_process_identity_request(client, header, payload, payload_len);
+	break;
+      // seach unlocked identities
+      case MDP_SEARCH_IDS:
+	if (config.debug.mdprequests)
+	  DEBUGF("Processing MDP_SEARCH_IDS from %s", alloca_socket_address(client));
+	mdp_search_identities(client, header, payload, payload_len);
+	break;
+      default:
+	mdp_reply_error(client, header);
+	WHYF("Unknown command port %d", header->remote.port);
+	break;
+    }
+    
+  }else{
+    // double check that this binding belongs to this connection
+    if (!binding
+      || !source
+      || header->local.port == 0 
+      || !compare_client(&binding->client, client)){
+      mdp_reply_error(client, header);
+      WHY("No matching binding found");
+      return;
+    }
+    
+    struct subscriber *destination=NULL;
+    if (!is_sid_t_broadcast(header->remote.sid))
+      destination = find_subscriber(header->remote.sid.binary, SID_SIZE, 1);
+    
+    int modifiers=0;
+    if ((header->flags & MDP_FLAG_NO_CRYPT) == 0)
+      modifiers|=OF_CRYPTO_CIPHERED;
+    if ((header->flags & MDP_FLAG_NO_SIGN) == 0)
+      modifiers|=OF_CRYPTO_SIGNED;
+    
+    if (!destination || destination->reachable==REACHABLE_SELF){
+      // TODO deprecate this mdp struct, deal with local delivery in send_frame
+      overlay_mdp_frame mdp;
+      mdp.out.src.sid = header->local.sid;
+      mdp.out.src.port = header->local.port;
+      mdp.out.dst.sid = header->remote.sid;
+      mdp.out.dst.port = header->remote.port;
+      bcopy(payload, mdp.out.payload, payload_len);
+      mdp.out.payload_length = payload_len;
+      mdp.out.ttl = header->ttl;
+      mdp.out.queue = header->qos;
+      mdp.packetTypeAndFlags=MDP_TX;
+      if (header->flags&MDP_FLAG_NO_CRYPT)
+	mdp.packetTypeAndFlags |= MDP_NOCRYPT;
+      if (header->flags&MDP_FLAG_NO_SIGN)
+	mdp.packetTypeAndFlags |= MDP_NOSIGN;
+      
+      if (config.debug.mdprequests)
+	DEBUGF("Attempting to process mdp packet locally");
+      overlay_saw_mdp_frame(NULL, &mdp, gettime_ms());
+    }
+    
+    if (config.debug.mdprequests)
+      DEBUGF("Attempting to queue mdp packet");
+      
+    // construct, encrypt, sign and queue the packet
+    if (overlay_send_frame(
+      source, header->local.port,
+      destination, header->remote.port, 
+      payload, payload_len, 
+      header->ttl, header->qos, modifiers
+    )){
+      mdp_reply_error(client, header);
+      return;
+    }
+  }
+  
+  // remove binding
+  if (binding 
+    && header->flags & MDP_FLAG_CLOSE
+    && compare_client(&binding->client, client)){
+    if (config.debug.mdprequests)
+      DEBUGF("Unbind MDP %s:%d from %s", 
+	binding->subscriber?alloca_tohex_sid_t(binding->subscriber->sid):"All",
+	binding->port,
+	alloca_socket_address(client));
+    binding->port=0;
+    binding=NULL;
+  }
+}
+
+static int mdp_send2(struct socket_address *client, struct mdp_header *header, 
+  const uint8_t *payload, size_t payload_len)
+{
+  struct iovec iov[]={
+    {
+      .iov_base = (void *)header,
+      .iov_len = sizeof(struct mdp_header)
+    },
+    {
+      .iov_base = (void *)payload,
+      .iov_len = payload_len
+    }
+  };
+  
+  struct msghdr hdr={
+    .msg_name=&client->addr,
+    .msg_namelen=client->addrlen,
+    .msg_iov=iov,
+    .msg_iovlen=2,
+  };
+  
+  if (sendmsg(mdp_sock2.poll.fd, &hdr, 0)<0)
+    return WHY_perror("sendmsg");
+  return 0;
+}
+
+static void mdp_poll2(struct sched_ent *alarm)
+{
+  if (alarm->poll.revents & POLLIN) {
+    uint8_t payload[1200];
+    struct mdp_header header;
+    struct socket_address client;
+    client.addrlen=sizeof(client.addr);
+    
+    struct iovec iov[]={
+      {
+	.iov_base = (void *)&header,
+	.iov_len = sizeof(struct mdp_header)
+      },
+      {
+	.iov_base = (void *)payload,
+	.iov_len = sizeof(payload)
+      }
+    };
+    
+    struct msghdr hdr={
+      .msg_name=&client.addr,
+      .msg_namelen=sizeof(client.store),
+      .msg_iov=iov,
+      .msg_iovlen=2,
+    };
+    
+    ssize_t len = recvmsg(alarm->poll.fd, &hdr, 0);
+    if (len<0){
+      WHY_perror("recvmsg");
+      return;
+    }
+    if (len<sizeof(struct mdp_header)){
+      WHYF("Expected length %d, got %d from %s", (int)sizeof(struct mdp_header), (int)len, alloca_socket_address(&client));
+      return;
+    }
+    
+    client.addrlen = hdr.msg_namelen;
+    size_t payload_len = len - sizeof(header);
+    mdp_process_packet(&client, &header, payload, payload_len);
   }
 }
 
@@ -1183,20 +1427,16 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
   if (alarm->poll.revents & POLLIN) {
     unsigned char buffer[16384];
     int ttl;
-    unsigned char recvaddrbuffer[1024];
-    struct sockaddr *recvaddr=(struct sockaddr *)&recvaddrbuffer[0];
-    socklen_t recvaddrlen=sizeof(recvaddrbuffer);
-    struct sockaddr_un *recvaddr_un=NULL;
+    struct socket_address client;
+    client.addrlen=sizeof(client.store);
 
     ttl=-1;
-    bzero((void *)recvaddrbuffer,sizeof(recvaddrbuffer));
     
-    ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, recvaddr, &recvaddrlen);
-    recvaddr_un=(struct sockaddr_un *)recvaddr;
+    ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, (struct sockaddr *)&client.addr, &client.addrlen);
 
     if (len > 0) {
-      if (recvaddrlen <= sizeof(sa_family_t))
-	WHYF("got recvaddrlen=%d too short -- ignoring frame len=%zu", (int)recvaddrlen, (size_t)len);
+      if (client.addrlen <= sizeof(sa_family_t))
+	WHYF("got client.addrlen=%d too short -- ignoring frame len=%zu", (int)client.addrlen, (size_t)len);
       else {
 	/* Look at overlay_mdp_frame we have received */
 	overlay_mdp_frame *mdp=(overlay_mdp_frame *)&buffer[0];      
@@ -1205,17 +1445,16 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	switch (mdp_type) {
 	case MDP_GOODBYE:
 	  if (config.debug.mdprequests)
-	    DEBUGF("MDP_GOODBYE from %s", alloca_sockaddr(recvaddr, recvaddrlen));
-	  overlay_mdp_releasebindings(recvaddr_un,recvaddrlen);
+	    DEBUGF("MDP_GOODBYE from %s", alloca_socket_address(&client));
+	  overlay_mdp_releasebindings(&client);
 	  return;
 	    
 	case MDP_ROUTING_TABLE:
 	  if (config.debug.mdprequests)
-	    DEBUGF("MDP_ROUTING_TABLE from %s", alloca_sockaddr(recvaddr, recvaddrlen));
+	    DEBUGF("MDP_ROUTING_TABLE from %s", alloca_socket_address(&client));
 	  {
 	    struct routing_state state={
-	      .recvaddr_un=recvaddr_un,
-	      .recvaddrlen=recvaddrlen,
+	      .client = &client,
 	    };
 	    
 	    enum_subscribers(NULL, routing_table, &state);
@@ -1225,7 +1464,7 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	
 	case MDP_GETADDRS:
 	  if (config.debug.mdprequests)
-	    DEBUGF("MDP_GETADDRS from %s", alloca_sockaddr(recvaddr, recvaddrlen));
+	    DEBUGF("MDP_GETADDRS from %s", alloca_socket_address(&client));
 	  {
 	    overlay_mdp_frame mdpreply;
 	    bzero(&mdpreply, sizeof(overlay_mdp_frame));
@@ -1233,7 +1472,7 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	    if (!overlay_mdp_address_list(&mdp->addrlist, &mdpreply.addrlist))
 	    /* Send back to caller */
 	      overlay_mdp_reply(alarm->poll.fd,
-				(struct sockaddr_un *)recvaddr,recvaddrlen,
+				&client,
 				&mdpreply);
 	      
 	    return;
@@ -1242,18 +1481,18 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	    
 	case MDP_TX: /* Send payload (and don't treat it as system privileged) */
 	  if (config.debug.mdprequests)
-	    DEBUGF("MDP_TX from %s", alloca_sockaddr(recvaddr, recvaddrlen));
+	    DEBUGF("MDP_TX from %s", alloca_socket_address(&client));
 	    
 	  // Dont allow mdp clients to send very high priority payloads
 	  if (mdp->out.queue<=OQ_MESH_MANAGEMENT)
 	    mdp->out.queue=OQ_ORDINARY;
-	  overlay_mdp_dispatch(mdp,1,(struct sockaddr_un*)recvaddr,recvaddrlen);
+	  overlay_mdp_dispatch(mdp, &client);
 	  return;
 	  break;
 	    
 	case MDP_BIND: /* Bind to port */
 	  if (config.debug.mdprequests)
-	    DEBUGF("MDP_BIND from %s", alloca_sockaddr(recvaddr, recvaddrlen));
+	    DEBUGF("MDP_BIND from %s", alloca_socket_address(&client));
 	  {
 	    struct subscriber *subscriber=NULL;
 	    /* Make sure source address is either all zeros (listen on all), or a valid
@@ -1264,24 +1503,24 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	      if ((!subscriber) || subscriber->reachable != REACHABLE_SELF){
 		WHYF("Invalid bind request for sid=%s", alloca_tohex_sid_t(mdp->bind.sid));
 		/* Source address is invalid */
-		overlay_mdp_reply_error(alarm->poll.fd, recvaddr_un, recvaddrlen, 7,
+		overlay_mdp_reply_error(alarm->poll.fd, &client, 7,
 					      "Bind address is not valid (must be a local MDP address, or all zeroes).");
 		return;
 	      }
 	      
 	    }
 	    if (overlay_mdp_process_bind_request(alarm->poll.fd, subscriber, mdp->bind.port,
-						mdp->packetTypeAndFlags, recvaddr_un, recvaddrlen))
-	      overlay_mdp_reply_error(alarm->poll.fd,recvaddr_un,recvaddrlen,3, "Port already in use");
+						mdp->packetTypeAndFlags, &client))
+	      overlay_mdp_reply_error(alarm->poll.fd, &client, 3, "Port already in use");
 	    else
-	      overlay_mdp_reply_ok(alarm->poll.fd,recvaddr_un,recvaddrlen,"Port bound");
+	      overlay_mdp_reply_ok(alarm->poll.fd, &client, "Port bound");
 	    return;
 	  }
 	  break;
 	    
 	case MDP_SCAN:
 	  if (config.debug.mdprequests)
-	    DEBUGF("MDP_SCAN from %s", alloca_sockaddr(recvaddr, recvaddrlen));
+	    DEBUGF("MDP_SCAN from %s", alloca_socket_address(&client));
 	  {
 	    struct overlay_mdp_scan *scan = (struct overlay_mdp_scan *)&mdp->raw;
 	    time_ms_t start=gettime_ms();
@@ -1312,7 +1551,7 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	    }else{
 	      struct overlay_interface *interface = overlay_interface_find(scan->addr, 1);
 	      if (!interface){
-		overlay_mdp_reply_error(alarm->poll.fd,recvaddr_un,recvaddrlen, 1, "Unable to find matching interface");
+		overlay_mdp_reply_error(alarm->poll.fd, &client, 1, "Unable to find matching interface");
 		return;
 	      }
 	      int i = interface - overlay_interfaces;
@@ -1327,22 +1566,14 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
 	      }
 	    }
 	    
-	    overlay_mdp_reply_ok(alarm->poll.fd,recvaddr_un,recvaddrlen,"Scan initiated");
+	    overlay_mdp_reply_ok(alarm->poll.fd, &client, "Scan initiated");
 	  }
 	  break;
 	  
 	default:
 	  /* Client is not allowed to send any other frame type */
-	  WARNF("Unsupported MDP frame type [%d] from %s", mdp_type, alloca_sockaddr(recvaddr, recvaddrlen));
-	  mdp->packetTypeAndFlags=MDP_ERROR;
-	  mdp->error.error=2;
-	  snprintf(mdp->error.message,128,"Illegal request type.  Clients may use only MDP_TX or MDP_BIND.");
-	  int len=4+4+strlen(mdp->error.message)+1;
-	  errno=0;
-	  /* We ignore the result of the following, because it is just sending an
-	    error message back to the client.  If this fails, where would we report
-	    the error to? My point exactly. */
-	  sendto(alarm->poll.fd,mdp,len,0,(struct sockaddr *)recvaddr,recvaddrlen);
+	  WARNF("Unsupported MDP frame type [%d] from %s", mdp_type, alloca_socket_address(&client));
+	  overlay_mdp_reply_error(alarm->poll.fd, &client, 2, "Illegal request type.  Clients may use only MDP_TX or MDP_BIND.");
 	}
       }
     }
diff --git a/overlay_mdp_services.c b/overlay_mdp_services.c
index 8298a135..3c1fbc57 100644
--- a/overlay_mdp_services.c
+++ b/overlay_mdp_services.c
@@ -99,7 +99,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, ui
       reply.out.payload[0]='T';
     
     // send packet
-    if (overlay_mdp_dispatch(&reply,0 /* system generated */, NULL,0))
+    if (overlay_mdp_dispatch(&reply, NULL))
       break;
   }
 
@@ -252,8 +252,7 @@ int overlay_mdp_service_echo(overlay_mdp_frame *mdp)
   mdp->packetTypeAndFlags&=~(MDP_NOCRYPT|MDP_NOSIGN);
   
   /* queue frame for delivery */
-  overlay_mdp_dispatch(mdp,0 /* system generated */,
-		       NULL,0);
+  overlay_mdp_dispatch(mdp, NULL);
   mdp->packetTypeAndFlags=preserved;
   
   /* and switch addresses back around in case the caller was planning on
@@ -342,20 +341,20 @@ static int overlay_mdp_service_trace(overlay_mdp_frame *mdp){
   mdp->out.payload_length = ob_position(b);
   mdp->out.src.sid = my_subscriber->sid;
   mdp->out.dst.sid = next->sid;
-  ret = overlay_mdp_dispatch(mdp, 0, NULL, 0);
+  ret = overlay_mdp_dispatch(mdp, NULL);
 end:
   ob_free(b);
   RETURN(ret);
 }
 
-static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp)
+static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, const uint8_t *payload, size_t len)
 {
   int offset=0;
-  while (offset<mdp->out.payload_length) {
+  while (offset<len) {
     rhizome_manifest *m = rhizome_new_manifest();
     if (!m)
       return WHY("Unable to allocate manifest");
-    unsigned char *bar = &mdp->out.payload[offset];
+    const unsigned char *bar = &payload[offset];
     if (!rhizome_retrieve_manifest_by_prefix(&bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES, m)){
       rhizome_advertise_manifest(frame->source, m);
       // pre-emptively send the payload if it will fit in a single packet
@@ -368,7 +367,7 @@ static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, ov
   return 0;
 }
 
-int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp)
+int overlay_mdp_try_internal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp)
 {
   IN();
   switch(mdp->out.dst.port) {
@@ -383,7 +382,7 @@ int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_fr
   case MDP_PORT_STUN:             RETURN(overlay_mdp_service_stun(mdp));
   case MDP_PORT_RHIZOME_REQUEST:  RETURN(overlay_mdp_service_rhizomerequest(frame, mdp));
   case MDP_PORT_RHIZOME_RESPONSE: RETURN(overlay_mdp_service_rhizomeresponse(mdp));    
-  case MDP_PORT_RHIZOME_MANIFEST_REQUEST: RETURN(overlay_mdp_service_manifest_requests(frame, mdp));
+  case MDP_PORT_RHIZOME_MANIFEST_REQUEST: RETURN(overlay_mdp_service_manifest_requests(frame, mdp->out.payload, mdp->out.payload_length));
   case MDP_PORT_RHIZOME_SYNC: RETURN(overlay_mdp_service_rhizome_sync(frame, mdp));
   }
    
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 7f1d5d74..56071a02 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "str.h"
 #include "strbuf_helpers.h"
 #include "overlay_address.h"
+#include "socket.h"
 
 /* Represents a queued fetch of a bundle payload, for which the manifest is already known.
  */
@@ -1122,7 +1123,7 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot)
 	   slot->write_state.file_offset,
 	   slot->bidVersion);
 
-  overlay_mdp_dispatch(&mdp,0 /* system generated */,NULL,0);
+  overlay_mdp_dispatch(&mdp, NULL);
   
   // remember when we sent the request so that we can adjust the inter-request
   // interval based on how fast the packets arrive.
diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c
index 01af7c16..f6d845e5 100644
--- a/rhizome_packetformats.c
+++ b/rhizome_packetformats.c
@@ -482,7 +482,7 @@ next:
     lookup_time = (end_time - start_time);
 
   if (mdp.out.payload_length>0)
-    overlay_mdp_dispatch(&mdp,0 /* system generated */,NULL,0);
+    overlay_mdp_dispatch(&mdp, NULL);
 
 end:
   sqlite_set_tracefunc(oldfunc);
diff --git a/rhizome_sync.c b/rhizome_sync.c
index 98162614..be5c7fcb 100644
--- a/rhizome_sync.c
+++ b/rhizome_sync.c
@@ -93,7 +93,7 @@ static void rhizome_sync_request(struct subscriber *subscriber, uint64_t token,
   mdp.out.payload_length = ob_position(b);
   if (config.debug.rhizome)
     DEBUGF("Sending request to %s for BARs from %"PRIu64" %s", alloca_tohex_sid_t(subscriber->sid), token, forwards?"forwards":"backwards");
-  overlay_mdp_dispatch(&mdp,0,NULL,0);
+  overlay_mdp_dispatch(&mdp, NULL);
   ob_free(b);
 }
 
@@ -147,7 +147,7 @@ static void rhizome_sync_send_requests(struct subscriber *subscriber, struct rhi
       break;
   }
   if (mdp.out.payload_length!=0)
-    overlay_mdp_dispatch(&mdp,0,NULL,0);
+    overlay_mdp_dispatch(&mdp, NULL);
 
   // send request for more bars if we have room to cache them
   if (state->bar_count >= CACHE_BARS)
@@ -440,7 +440,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
     mdp.out.payload_length = ob_position(b);
     if (config.debug.rhizome_ads)
       DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last);
-    overlay_mdp_dispatch(&mdp,0,NULL,0);
+    overlay_mdp_dispatch(&mdp, NULL);
   }
   ob_free(b);
   OUT();
diff --git a/serval.h b/serval.h
index eb775a10..b02199a5 100644
--- a/serval.h
+++ b/serval.h
@@ -195,28 +195,6 @@ int formf_serval_instance_path(struct __sourceloc, char *buf, size_t bufsiz, con
 int vformf_serval_instance_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list);
 void serval_setinstancepath(const char *instancepath);
 
-/* Basic socket operations.
- */
-int _make_local_sockaddr(struct __sourceloc, struct sockaddr_un *sockname, socklen_t *addrlen, const char *fmt, ...)
-    __attribute__((format(printf, 4, 5)));
-int _esocket(struct __sourceloc, int domain, int type, int protocol);
-int _socket_bind(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
-int _socket_connect(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
-int _socket_listen(struct __sourceloc, int sock, int backlog);
-int _socket_set_reuseaddr(struct __sourceloc, int sock, int reuseP);
-int _socket_set_rcvbufsize(struct __sourceloc, int sock, unsigned buffer_size);
-
-#define make_local_sockaddr(sockname, addrlenp, fmt,...) _make_local_sockaddr(__WHENCE__, (sockname), (addrlenp), (fmt), ##__VA_ARGS__)
-#define esocket(domain, type, protocol)             _esocket(__WHENCE__, (domain), (type), (protocol))
-#define socket_bind(sock, addr, addrlen)            _socket_bind(__WHENCE__, (sock), (addr), (addrlen))
-#define socket_connect(sock, addr, addrlen)         _socket_connect(__WHENCE__, (sock), (addr), (addrlen))
-#define socket_listen(sock, backlog)                _socket_listen(__WHENCE__, (sock), (backlog))
-#define socket_set_reuseaddr(sock, reuseP)          _socket_set_reuseaddr(__WHENCE__, (sock), (reuseP))
-#define socket_set_rcvbufsize(sock, buffer_size)    _socket_set_rcvbufsize(__WHENCE__, (sock), (buffer_size))
-
-int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, struct sockaddr_un *dst_addr, socklen_t *dst_addrlen);
-int cmp_sockaddr(const struct sockaddr *, socklen_t, const struct sockaddr *, socklen_t);
-
 #define SERVER_CONFIG_RELOAD_INTERVAL_MS	1000
 
 struct cli_parsed;
@@ -508,9 +486,6 @@ int rhizome_opendb();
 int parseCommandLine(struct cli_context *context, const char *argv0, int argc, const char *const *argv);
 
 int overlay_mdp_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
-int overlay_mdp_reply_error(int sock,
-			    struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
-			    int error_number,char *message);
 
 typedef uint32_t mdp_port_t;
 #define PRImdp_port_t "#08" PRIx32
@@ -572,10 +547,8 @@ typedef struct overlay_mdp_frame {
 
 /* Server-side MDP functions */
 int overlay_mdp_swap_src_dst(overlay_mdp_frame *mdp);
-int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
-			  overlay_mdp_frame *mdpreply);
-int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
-		     struct sockaddr_un *recvaddr, socklen_t recvaddrlen);
+struct socket_address;
+int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *client);
 void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port);
 int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name);
 
@@ -688,7 +661,7 @@ int overlay_packetradio_tx_packet(struct overlay_frame *frame);
 void overlay_dummy_poll(struct sched_ent *alarm);
 void server_config_reload(struct sched_ent *alarm);
 void server_shutdown_check(struct sched_ent *alarm);
-int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp);
+int overlay_mdp_try_internal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp);
 int overlay_send_probe(struct subscriber *peer, struct network_destination *destination, int queue);
 int overlay_send_stun_request(struct subscriber *server, struct subscriber *request);
 void fd_periodicstats(struct sched_ent *alarm);
diff --git a/socket.c b/socket.c
index e9839d01..1839980e 100644
--- a/socket.c
+++ b/socket.c
@@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "conf.h"
 #include "log.h"
 #include "strbuf_helpers.h"
+#include "socket.h"
 
 /* Form the name of an AF_UNIX (local) socket in the instance directory as an absolute path.
  * Under Linux, this will create a socket name in the abstract namespace.  This permits us to use
@@ -40,25 +41,28 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  * @author Andrew Bettison <andrew@servalproject.com>
  * @author Daniel O'Connor <daniel@servalproject.com>
  */
-int _make_local_sockaddr(struct __sourceloc __whence, struct sockaddr_un *addr, socklen_t *addrlen, const char *fmt, ...)
+int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *addr, const char *fmt, ...)
 {
   bzero(addr, sizeof(*addr));
+  addr->addr_un.sun_family = AF_UNIX;
   va_list ap;
   va_start(ap, fmt);
-  int r = vformf_serval_instance_path(__WHENCE__, addr->sun_path, sizeof addr->sun_path, fmt, ap);
+  int r = vformf_serval_instance_path(__WHENCE__, addr->addr_un.sun_path, sizeof addr->addr_un.sun_path, fmt, ap);
   va_end(ap);
   if (!r)
     return WHY("socket name overflow");
-  if (real_sockaddr(addr, sizeof addr->sun_family + strlen(addr->sun_path) + 1, addr, addrlen) == -1)
-    return -1;
+  addr->addrlen=sizeof addr->addr_un.sun_family + strlen(addr->addr_un.sun_path) + 1;
+// TODO perform real path transformation in making the serval instance path
+//  if (real_sockaddr(addr, addr) == -1)
+//    return -1;
+
 #ifdef USE_ABSTRACT_NAMESPACE
   // For the abstract name we use the absolute path name with the initial '/' replaced by the
   // leading nul.  This ensures that different instances of the Serval daemon have different socket
   // names.
-  addr->sun_path[0] = '\0'; // mark as Linux abstract socket
-  --*addrlen; // do not count trailing nul in abstract socket name
+  addr->addr_un.sun_path[0] = '\0'; // mark as Linux abstract socket
+  --addr->addrlen; // do not count trailing nul in abstract socket name
 #endif // USE_ABSTRACT_NAMESPACE
-  addr->sun_family = AF_UNIX;
   return 0;
 }
 
@@ -74,31 +78,33 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct sockaddr_un *addr,
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, struct sockaddr_un *dst_addr, socklen_t *dst_addrlen)
+int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr)
 {
-  int src_path_len = src_addrlen - sizeof src_addr->sun_family;
-  if (	 src_addrlen >= sizeof src_addr->sun_family + 1
-      && src_addr->sun_family == AF_UNIX
-      && src_addr->sun_path[0] != '\0'
-      && src_addr->sun_path[src_path_len - 1] == '\0'
+  int src_path_len = src_addr->addrlen - sizeof src_addr->addr_un.sun_family;
+  if (	 src_addr->addrlen >= sizeof src_addr->addr_un.sun_family + 1
+      && src_addr->addr_un.sun_family == AF_UNIX
+      && src_addr->addr_un.sun_path[0] != '\0'
+      && src_addr->addr_un.sun_path[src_path_len - 1] == '\0'
   ) {
     char real_path[PATH_MAX];
     size_t real_path_len;
-    if (realpath(src_addr->sun_path, real_path) == NULL)
-      return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->sun_path));
-    else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->sun_path)
-      return WHYF("sockaddr overrun: realpath(%s) returned %s", alloca_str_toprint(src_addr->sun_path), alloca_str_toprint(real_path));
+    if (realpath(src_addr->addr_un.sun_path, real_path) == NULL)
+      return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->addr_un.sun_path));
+    else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->addr_un.sun_path)
+      return WHYF("sockaddr overrun: realpath(%s) returned %s", 
+	  alloca_str_toprint(src_addr->addr_un.sun_path), alloca_str_toprint(real_path));
     else if (   real_path_len != src_path_len
-	     || memcmp(real_path, src_addr->sun_path, src_path_len) != 0
+	     || memcmp(real_path, src_addr->addr_un.sun_path, src_path_len) != 0
     ) {
-      memcpy(dst_addr->sun_path, real_path, real_path_len);
-      *dst_addrlen = real_path_len + sizeof dst_addr->sun_family;
+      memcpy(dst_addr->addr_un.sun_path, real_path, real_path_len);
+      dst_addr->addrlen = real_path_len + sizeof dst_addr->addr_un.sun_family;
       return 1;
     }
   }
-  if (dst_addr != src_addr)
-    memcpy(dst_addr, src_addr, src_addrlen);
-  *dst_addrlen = src_addrlen;
+  if (dst_addr != src_addr){
+    memcpy(&dst_addr->addr, &src_addr->addr, src_addr->addrlen);
+    dst_addr->addrlen = src_addr->addrlen;
+  }
   return 0;
 }
 
@@ -107,43 +113,43 @@ int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, str
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-int cmp_sockaddr(const struct sockaddr *addrA, socklen_t addrlenA, const struct sockaddr *addrB, socklen_t addrlenB)
+int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address *addrB)
 {
   // Two zero-length sockaddrs are equal.
-  if (addrlenA == 0 && addrlenB == 0)
+  if (addrA->addrlen == 0 && addrB->addrlen == 0)
     return 0;
   // If either sockaddr is truncated, then we compare the bytes we have.
-  if (addrlenA < sizeof addrA->sa_family || addrlenB < sizeof addrB->sa_family) {
-    int c = memcmp(addrA, addrB, addrlenA < addrlenB ? addrlenA : addrlenB);
+  if (addrA->addrlen < sizeof addrA->addr.sa_family || addrB->addrlen < sizeof addrB->addr.sa_family) {
+    int c = memcmp(addrA, addrB, addrA->addrlen < addrB->addrlen ? addrA->addrlen : addrB->addrlen);
     if (c == 0)
-      c = addrlenA < addrlenB ? -1 : addrlenA > addrlenB ? 1 : 0;
+      c = addrA->addrlen < addrB->addrlen ? -1 : addrA->addrlen > addrB->addrlen ? 1 : 0;
     return c;
   }
   // Order first by address family.
-  if (addrA->sa_family < addrB->sa_family)
+  if (addrA->addr.sa_family < addrB->addr.sa_family)
     return -1;
-  if (addrA->sa_family > addrB->sa_family)
+  if (addrA->addr.sa_family > addrB->addr.sa_family)
     return 1;
   // Both addresses are in the same family...
-  switch (addrA->sa_family) {
+  switch (addrA->addr.sa_family) {
   case AF_UNIX: {
-      unsigned pathlenA = addrlenA - sizeof ((const struct sockaddr_un *)addrA)->sun_family;
-      unsigned pathlenB = addrlenB - sizeof ((const struct sockaddr_un *)addrB)->sun_family;
+      unsigned pathlenA = addrA->addrlen - sizeof (addrA->addr_un.sun_family);
+      unsigned pathlenB = addrB->addrlen - sizeof (addrB->addr_un.sun_family);
       int c;
       if (   pathlenA > 1 && pathlenB > 1
-	  && ((const struct sockaddr_un *)addrA)->sun_path[0] == '\0'
-	  && ((const struct sockaddr_un *)addrB)->sun_path[0] == '\0'
+	  && addrA->addr_un.sun_path[0] == '\0'
+	  && addrB->addr_un.sun_path[0] == '\0'
       ) {
 	// Both abstract sockets - just compare names, nul bytes are not terminators.
-	c = memcmp(&((const struct sockaddr_un *)addrA)->sun_path[1],
-		   &((const struct sockaddr_un *)addrB)->sun_path[1],
+	c = memcmp(&addrA->addr_un.sun_path[1],
+		   &addrB->addr_un.sun_path[1],
 		   (pathlenA < pathlenB ? pathlenA : pathlenB) - 1);
       } else {
 	// Either or both are named local file sockets.  If the file names are identical up to the
 	// first nul, then the addresses are equal.  This collates abstract socket names, whose first
 	// character is a nul, ahead of all non-empty file socket names.
-	c = strncmp(((const struct sockaddr_un *)addrA)->sun_path,
-		    ((const struct sockaddr_un *)addrB)->sun_path,
+	c = strncmp(addrA->addr_un.sun_path,
+		    addrB->addr_un.sun_path,
 		    (pathlenA < pathlenB ? pathlenA : pathlenB));
       }
       if (c == 0)
@@ -153,9 +159,10 @@ int cmp_sockaddr(const struct sockaddr *addrA, socklen_t addrlenA, const struct
     break;
   }
   // Fall back to comparing raw data bytes.
-  int c = memcmp(addrA->sa_data, addrB->sa_data, (addrlenA < addrlenB ? addrlenA : addrlenB) - sizeof addrA->sa_family);
+  int c = memcmp(addrA->addr.sa_data, addrB->addr.sa_data, 
+      (addrA->addrlen < addrB->addrlen ? addrA->addrlen : addrB->addrlen) - sizeof addrA->addr.sa_family);
   if (c == 0)
-    c = addrlenA < addrlenB ? -1 : addrlenA > addrlenB ? 1 : 0;
+    c = addrA->addrlen < addrB->addrlen ? -1 : addrA->addrlen > addrB->addrlen ? 1 : 0;
   return c;
 }
 
@@ -225,3 +232,32 @@ int _socket_set_rcvbufsize(struct __sourceloc __whence, int sock, unsigned buffe
     DEBUGF("setsockopt(%d, SOL_SOCKET, SO_RCVBUF, &%u, %u)", sock, buffer_size, (unsigned)sizeof buffer_size);
   return 0;
 }
+
+ssize_t _send_message(struct __sourceloc __whence, int fd, const struct socket_address *address, const struct fragmented_data *data)
+{
+  struct msghdr hdr={
+    .msg_name=(void *)&address->addr,
+    .msg_namelen=address->addrlen,
+    .msg_iov=(struct iovec*)data->iov,
+    .msg_iovlen=data->fragment_count,
+  };
+  
+  ssize_t ret = sendmsg(fd, &hdr, 0);
+  if (ret==-1)
+    WHYF_perror("sendmsg(%d,%s,%lu)", fd, alloca_socket_address(address), (unsigned long)address->addrlen);
+  return ret;
+}
+
+ssize_t _recv_message(struct __sourceloc __whence, int fd, struct socket_address *address, struct fragmented_data *data)
+{
+  struct msghdr hdr={
+    .msg_name=(void *)&address->addr,
+    .msg_namelen=address->addrlen,
+    .msg_iov=data->iov,
+    .msg_iovlen=data->fragment_count,
+  };
+  ssize_t ret = recvmsg(fd, &hdr, 0);
+  if (ret==-1)
+    WHYF_perror("recvmsg(%d,%s,%lu)", fd, alloca_socket_address(address), (unsigned long)address->addrlen);
+  return ret;
+}
diff --git a/socket.h b/socket.h
new file mode 100644
index 00000000..88a05549
--- /dev/null
+++ b/socket.h
@@ -0,0 +1,56 @@
+#ifndef __SERVALD_SOCKET_H
+#define __SERVALD_SOCKET_H
+
+#ifndef WIN32
+#include <sys/un.h>
+#endif
+
+struct socket_address{
+  socklen_t addrlen;
+  union{
+    struct sockaddr addr;
+    struct sockaddr_un addr_un;
+    struct sockaddr_storage store;
+  };
+};
+
+/* Basic socket operations.
+ */
+int _make_local_sockaddr(struct __sourceloc, struct socket_address *addr, const char *fmt, ...)
+    __attribute__((format(printf, 3, 4)));
+int _esocket(struct __sourceloc, int domain, int type, int protocol);
+int _socket_bind(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
+int _socket_connect(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
+int _socket_listen(struct __sourceloc, int sock, int backlog);
+int _socket_set_reuseaddr(struct __sourceloc, int sock, int reuseP);
+int _socket_set_rcvbufsize(struct __sourceloc, int sock, unsigned buffer_size);
+
+#define make_local_sockaddr(sockname, fmt,...) _make_local_sockaddr(__WHENCE__, (sockname), (fmt), ##__VA_ARGS__)
+#define esocket(domain, type, protocol)             _esocket(__WHENCE__, (domain), (type), (protocol))
+#define socket_bind(sock, addr, addrlen)            _socket_bind(__WHENCE__, (sock), (addr), (addrlen))
+#define socket_connect(sock, addr, addrlen)         _socket_connect(__WHENCE__, (sock), (addr), (addrlen))
+#define socket_listen(sock, backlog)                _socket_listen(__WHENCE__, (sock), (backlog))
+#define socket_set_reuseaddr(sock, reuseP)          _socket_set_reuseaddr(__WHENCE__, (sock), (reuseP))
+#define socket_set_rcvbufsize(sock, buffer_size)    _socket_set_rcvbufsize(__WHENCE__, (sock), (buffer_size))
+
+int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr);
+int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address *addrB);
+
+// helper functions for manipulating fragmented packet data
+#define MAX_FRAGMENTS 8
+struct fragmented_data{
+  int fragment_count;
+  struct iovec iov[MAX_FRAGMENTS];
+};
+
+int prepend_fragment(struct fragmented_data *data, const uint8_t *payload, size_t len);
+int append_fragment(struct fragmented_data *data, const uint8_t *payload, size_t len);
+size_t copy_fragment(struct fragmented_data *src, uint8_t *dest, size_t length);
+
+ssize_t _send_message(struct __sourceloc, int fd, const struct socket_address *address, const struct fragmented_data *data);
+ssize_t _recv_message(struct __sourceloc, int fd, struct socket_address *address, struct fragmented_data *data);
+
+#define send_message(fd, address, data)    _send_message(__WHENCE__, (fd), (address), (data))
+#define recv_message(fd, address, data)    _recv_message(__WHENCE__, (fd), (address), (data))
+
+#endif
\ No newline at end of file
diff --git a/strbuf_helpers.c b/strbuf_helpers.c
index 1e2c459e..363172f5 100644
--- a/strbuf_helpers.c
+++ b/strbuf_helpers.c
@@ -37,6 +37,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "http_server.h"
 #include "strbuf_helpers.h"
 #include "str.h"
+#include "socket.h"
 
 static inline strbuf _toprint(strbuf sb, char c)
 {
@@ -374,6 +375,11 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t
   return sb;
 }
 
+strbuf strbuf_append_socket_address(strbuf sb, const struct socket_address *addr)
+{
+  return strbuf_append_sockaddr(sb, &addr->addr, addr->addrlen);
+}
+
 strbuf strbuf_append_strftime(strbuf sb, const char *format, const struct tm *tm)
 {
   // First, try calling strftime(3) directly on the buffer in the strbuf, if there is one and it
diff --git a/strbuf_helpers.h b/strbuf_helpers.h
index e10a346a..4fee86cb 100644
--- a/strbuf_helpers.h
+++ b/strbuf_helpers.h
@@ -139,6 +139,10 @@ struct sockaddr;
 strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen);
 #define alloca_sockaddr(addr, addrlen)    strbuf_str(strbuf_append_sockaddr(strbuf_alloca(200), (const struct sockaddr *)(addr), (addrlen)))
 
+struct socket_address;
+strbuf strbuf_append_socket_address(strbuf sb, const struct socket_address *addr);
+#define alloca_socket_address(addr)    strbuf_str(strbuf_append_socket_address(strbuf_alloca(200), (addr)))
+
 /* Append a strftime(3) string.
  * @author Andrew Bettison <andrew@servalproject.com>
  */
diff --git a/vomp.c b/vomp.c
index 72d03a36..f87fd688 100644
--- a/vomp.c
+++ b/vomp.c
@@ -523,7 +523,7 @@ static int vomp_send_status_remote(struct vomp_call_state *call)
 
   call->local.sequence++;
   
-  overlay_mdp_dispatch(&mdp,0,NULL,0);
+  overlay_mdp_dispatch(&mdp, NULL);
   
   return 0;
 }
@@ -561,7 +561,7 @@ int vomp_received_audio(struct vomp_call_state *call, int audio_codec, int time,
     
   mdp.out.queue=OQ_ISOCHRONOUS_VOICE;
   
-  overlay_mdp_dispatch(&mdp,0,NULL,0);
+  overlay_mdp_dispatch(&mdp, NULL);
   
   return 0;
 }

From a77642fc9e9c41adcf4594a927edcfad35ae4656 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 18 Nov 2013 13:12:52 +1030
Subject: [PATCH 35/56] Support for binding internal handlers to mdp ports

---
 overlay_mdp.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
 serval.h      |  7 +++++++
 2 files changed, 58 insertions(+)

diff --git a/overlay_mdp.c b/overlay_mdp.c
index 03008593..72652a8e 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -126,6 +126,7 @@ struct mdp_binding{
   struct subscriber *subscriber;
   mdp_port_t port;
   int version;
+  int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len);
   struct socket_address client;
   time_ms_t binding_time;
 };
@@ -270,6 +271,49 @@ static int overlay_mdp_process_bind_request(int sock, struct subscriber *subscri
   return 0;
 }
 
+int mdp_bind_internal(struct subscriber *subscriber, mdp_port_t port,
+  int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len))
+{
+  
+  int i;
+  struct mdp_binding *free_slot=NULL;
+  
+  for(i=0;i<MDP_MAX_BINDINGS;i++) {
+    if ((!free_slot) && mdp_bindings[i].port==0)
+      free_slot=&mdp_bindings[i];
+    
+    if (mdp_bindings[i].port == port 
+      && mdp_bindings[i].subscriber == subscriber)
+      return -1;
+  }
+  
+  if (!free_slot)
+    return -1;
+    
+  free_slot->subscriber=subscriber;
+  free_slot->port=port;
+  free_slot->version=1;
+  free_slot->internal=internal;
+  free_slot->binding_time=gettime_ms();
+  return 0;
+}
+
+int mdp_unbind_internal(struct subscriber *subscriber, mdp_port_t port,
+  int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len))
+{
+  int i;
+  for(i=0;i<MDP_MAX_BINDINGS;i++) {
+    if (mdp_bindings[i].port == port
+      && mdp_bindings[i].subscriber == subscriber
+      && mdp_bindings[i].internal == internal){
+      mdp_bindings[i].port=0;
+      mdp_bindings[i].subscriber=NULL;
+      mdp_bindings[i].internal=NULL;
+    }
+  }
+  return 0;
+}
+
 static int overlay_mdp_decode_header(struct overlay_buffer *buff, overlay_mdp_frame *mdp)
 {
   /* extract MDP port numbers */
@@ -483,6 +527,10 @@ static int overlay_saw_mdp_frame(struct overlay_frame *frame, overlay_mdp_frame
 	    header.flags|=MDP_FLAG_NO_CRYPT;
 	  if (mdp->packetTypeAndFlags & MDP_NOSIGN)
 	    header.flags|=MDP_FLAG_NO_SIGN;
+	    
+	  if (mdp_bindings[match].internal)
+	    RETURN(mdp_bindings[match].internal(&header, mdp->out.payload, mdp->out.payload_length));
+	    
 	  RETURN(mdp_send2(&mdp_bindings[match].client, &header, mdp->out.payload, mdp->out.payload_length));
 	}
     }
@@ -1262,6 +1310,7 @@ static void mdp_process_packet(struct socket_address *client, struct mdp_header
       case MDP_LISTEN:
 	// double check that this binding belongs to this connection
 	if (!binding
+	  || binding->internal
 	  || !compare_client(&binding->client, client))
 	  mdp_reply_error(client, header);
 	break;
@@ -1285,6 +1334,7 @@ static void mdp_process_packet(struct socket_address *client, struct mdp_header
   }else{
     // double check that this binding belongs to this connection
     if (!binding
+      || binding->internal
       || !source
       || header->local.port == 0 
       || !compare_client(&binding->client, client)){
@@ -1342,6 +1392,7 @@ static void mdp_process_packet(struct socket_address *client, struct mdp_header
   
   // remove binding
   if (binding 
+    && !binding->internal
     && header->flags & MDP_FLAG_CLOSE
     && compare_client(&binding->client, client)){
     if (config.debug.mdprequests)
diff --git a/serval.h b/serval.h
index b02199a5..32b3d3fa 100644
--- a/serval.h
+++ b/serval.h
@@ -552,6 +552,13 @@ int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *client);
 void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port);
 int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name);
 
+struct mdp_header;
+int mdp_bind_internal(struct subscriber *subscriber, mdp_port_t port,
+  int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len));
+int mdp_unbind_internal(struct subscriber *subscriber, mdp_port_t port,
+  int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len));
+
+
 struct vomp_call_state;
 
 void set_codec_flag(int codec, unsigned char *flags);

From ad33264834ea31f24adb5331e588f8809395cea8 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Wed, 30 Oct 2013 17:09:37 +1030
Subject: [PATCH 36/56] Rename mavlink to radio_link

- split link state from slip structures and encapsulate it completely
- allocate interface specific buffers on demand
---
 conf_schema.h             |   4 +-
 headerfiles.mk            |   1 +
 overlay_interface.c       |  29 +++++---
 mavlink.c => radio_link.c | 146 +++++++++++++++++++-------------------
 radio_link.h              |  10 +++
 serval.h                  |  15 +---
 sourcefiles.mk            |   2 +-
 tests/rhizomeprotocol     |   2 +-
 tests/routing             |   2 +-
 9 files changed, 111 insertions(+), 100 deletions(-)
 rename mavlink.c => radio_link.c (73%)
 create mode 100644 radio_link.h

diff --git a/conf_schema.h b/conf_schema.h
index 2e10fefb..db680364 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -242,9 +242,7 @@ ATOM(bool_t, gateway,                   0, boolean,, "")
 ATOM(bool_t, keyring,                   0, boolean,, "")
 ATOM(bool_t, security,                  0, boolean,, "")
 ATOM(bool_t, mdprequests,               0, boolean,, "")
-ATOM(bool_t, mavlink,                   0, boolean,, "")
-ATOM(bool_t, mavlink_payloads,          0, boolean,, "")
-ATOM(bool_t, mavlinkfsm,                0, boolean,, "")
+ATOM(bool_t, radio_link,                0, boolean,, "")
 ATOM(bool_t, peers,                     0, boolean,, "")
 ATOM(bool_t, overlaybuffer,             0, boolean,, "")
 ATOM(bool_t, overlayframes,             0, boolean,, "")
diff --git a/headerfiles.mk b/headerfiles.mk
index 422874d3..08a0b2e4 100644
--- a/headerfiles.mk
+++ b/headerfiles.mk
@@ -27,4 +27,5 @@ HDRS=	fifo.h \
 	constants.h \
 	monitor-client.h \
 	mdp_client.h \
+	radio_link.h \
 	sqlite-amalgamation-3070900/sqlite3.h
diff --git a/overlay_interface.c b/overlay_interface.c
index 842685d7..4eedac0f 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "overlay_buffer.h"
 #include "overlay_packet.h"
 #include "str.h"
+#include "radio_link.h"
 
 #ifdef HAVE_IFADDRS_H
 #include <ifaddrs.h>
@@ -58,6 +59,12 @@ overlay_interface_close(overlay_interface *interface){
   unschedule(&interface->alarm);
   unwatch(&interface->alarm);
   close(interface->alarm.poll.fd);
+  if (interface->txbuffer){
+    free(interface->txbuffer);
+    interface->txbuffer=NULL;
+  }
+  if (interface->radio_link_state)
+    radio_link_free(interface);
   interface->alarm.poll.fd=-1;
   interface->state=INTERFACE_STATE_DOWN;
 }
@@ -531,10 +538,7 @@ overlay_interface_init(const char *name, struct in_addr src_addr, struct in_addr
     
     switch (ifconfig->socket_type) {
     case SOCK_STREAM:
-      interface->slip_decode_state.dst_offset=0;
-      /* The encapsulation type should be configurable, but for now default to the one that should
-         be safe on the RFD900 radios, and that also allows us to receive RSSI reports inline */
-      interface->slip_decode_state.encapsulator=SLIP_FORMAT_MAVLINK;
+      radio_link_init(interface);
       interface->alarm.poll.events=POLLIN|POLLOUT;
       watch(&interface->alarm);
 
@@ -717,11 +721,13 @@ static void interface_read_stream(struct overlay_interface *interface){
     OUT();
     return;
   }
-  struct slip_decode_state *state=&interface->slip_decode_state;
+  
+  if (config.debug.packetradio)
+    dump("read bytes", buffer, nread);
   
   int i;
   for (i=0;i<nread;i++)
-    mavlink_decode(interface, state, buffer[i]);
+    radio_link_decode(interface, buffer[i]);
     
   OUT();
 }
@@ -738,14 +744,21 @@ static void write_stream_buffer(overlay_interface *interface){
     
     if (interface->tx_bytes_pending==0){
       if (interface->next_heartbeat <= now){
+	
+	if (!interface->txbuffer){
+	  interface->txbuffer=emalloc(OVERLAY_INTERFACE_RX_BUFFER_SIZE);
+	  if (!interface->txbuffer)
+	    break;
+	}
+	
 	// Queue a hearbeat now
-	mavlink_heartbeat(interface->txbuffer,&interface->tx_bytes_pending);
+	radio_link_heartbeat(interface->txbuffer,&interface->tx_bytes_pending);
 	if (config.debug.packetradio)
 	  DEBUGF("Sending heartbeat");
 	interface->next_heartbeat = now+1000;
       }else if(interface->tx_packet && interface->remaining_space >= 256 + 8+9){
 	// prepare a new link layer packet in txbuffer
-	if (mavlink_encode_packet(interface))
+	if (radio_link_encode_packet(interface))
 	  break;
 	if (interface->remaining_space - interface->tx_bytes_pending < 256 + 8+9)
 	  interface->next_heartbeat = now;
diff --git a/mavlink.c b/radio_link.c
similarity index 73%
rename from mavlink.c
rename to radio_link.c
index 0fb88230..50a97fe7 100644
--- a/mavlink.c
+++ b/radio_link.c
@@ -51,38 +51,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #define MAVLINK_MSG_ID_RADIO 166
 #define MAVLINK_MSG_ID_DATASTREAM 67
-int MAVLINK_MESSAGE_CRCS[]={72, 39, 190, 92, 191, 217, 104, 119, 0, 219, 60, 186, 10, 0, 0, 0, 0, 0, 0, 0, 89, 159, 162, 121, 0, 149, 222, 110, 179, 136, 66, 126, 185, 147, 112, 252, 162, 215, 229, 128, 9, 106, 101, 213, 4, 229, 21, 214, 215, 14, 206, 50, 157, 126, 108, 213, 95, 5, 127, 0, 0, 0, 57, 126, 130, 119, 193, 191, 236, 158, 143, 0, 0, 104, 123, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 29, 208, 188, 118, 242, 19, 97, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 224, 60, 106, 7};
 
 // use '3D' for 3DRadio
 #define RADIO_SOURCE_SYSTEM '3'
 #define RADIO_SOURCE_COMPONENT 'D'
 
-uint16_t mavlink_crc(unsigned char *buf,int length)
-{
-  uint16_t sum = 0xFFFF;
-  uint8_t i, stoplen;
-  
-  stoplen = length + 6;
-  
-  // MAVLink 1.0 has an extra CRC seed
-  buf[length+6] = MAVLINK_MESSAGE_CRCS[buf[5]];
-  stoplen++;
-  
-  i = 1;
-  while (i<stoplen) {
-    uint8_t tmp;
-    tmp = buf[i] ^ (uint8_t)(sum&0xff);
-    tmp ^= (tmp<<4);
-    sum = (sum>>8) ^ (tmp<<8) ^ (tmp<<3) ^ (tmp>>4);
-    i++;
-  }
-  buf[length+6]=sum&0xff;
-  buf[length+7]=sum>>8;
-  
-  return sum;
-}
-
-
 /*
   we use a hand-crafted MAVLink packet based on the following
   message definition
@@ -117,6 +90,18 @@ struct mavlink_RADIO_v10 {
   uint8_t remnoise;
 };
 
+struct radio_link_state{
+  int tx_seq;
+  int payload_length;
+  int seq;
+  int payload_start;
+  int payload_offset;
+  uint8_t payload[1024];
+  int packet_length;
+  uint8_t dst[OVERLAY_INTERFACE_RX_BUFFER_SIZE];
+  int dst_offset;
+};
+
 /*
   Each mavlink frame consists of 0xfe followed by a standard 6 byte header.
   Normally the payload plus a 2-byte CRC follows.
@@ -138,7 +123,20 @@ struct mavlink_RADIO_v10 {
 void encode_rs_8(data_t *data, data_t *parity,int pad);
 int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad);
 
-int mavlink_encode_packet(struct overlay_interface *interface)
+int radio_link_free(struct overlay_interface *interface)
+{
+  free(interface->radio_link_state);
+  interface->radio_link_state=NULL;
+  return 0;
+}
+
+int radio_link_init(struct overlay_interface *interface)
+{
+  interface->radio_link_state = emalloc_zero(sizeof(struct radio_link_state));
+  return 0;
+}
+
+int radio_link_encode_packet(struct overlay_interface *interface)
 {
   int count = ob_remaining(interface->tx_packet);
   int startP = (ob_position(interface->tx_packet) == 0);
@@ -162,7 +160,7 @@ int mavlink_encode_packet(struct overlay_interface *interface)
   interface->txbuffer[3]=0;
   golay_encode(&interface->txbuffer[1]);
   
-  interface->txbuffer[4]=(interface->mavlink_seq++) & 0x3f;
+  interface->txbuffer[4]=(interface->radio_link_state->tx_seq++) & 0x3f;
   if (startP) interface->txbuffer[4]|=0x40;
   if (endP) interface->txbuffer[4]|=0x80;
   interface->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM;
@@ -179,7 +177,7 @@ int mavlink_encode_packet(struct overlay_interface *interface)
   return 0;
 }
 
-int mavlink_heartbeat(unsigned char *frame,int *outlen)
+int radio_link_heartbeat(unsigned char *frame,int *outlen)
 {
   int count=9;
   bzero(frame, count+8);
@@ -233,7 +231,7 @@ static int parse_heartbeat(struct overlay_interface *interface, const unsigned c
   return 0;
 }
 
-static int mavlink_parse(struct overlay_interface *interface, struct slip_decode_state *state, 
+static int radio_link_parse(struct overlay_interface *interface, struct radio_link_state *state, 
   int packet_length, unsigned char *payload, int *backtrack)
 {
   *backtrack=0;
@@ -256,7 +254,7 @@ static int mavlink_parse(struct overlay_interface *interface, struct slip_decode
   int pad=223 - (data_bytes + 2);
   int errors=decode_rs_8(&payload[4], NULL, 0, pad);
   if (errors==-1){
-    if (config.debug.mavlink)
+    if (config.debug.radio_link)
       DEBUGF("Reed-Solomon error correction failed");
     return 0;
   }
@@ -264,7 +262,7 @@ static int mavlink_parse(struct overlay_interface *interface, struct slip_decode
   
   int seq=payload[4]&0x3f;
   
-  if (config.debug.mavlink){
+  if (config.debug.radio_link){
     DEBUGF("Received RS protected message, len: %d, errors: %d, seq: %d, flags:%s%s", 
       data_bytes,
       errors,
@@ -273,10 +271,10 @@ static int mavlink_parse(struct overlay_interface *interface, struct slip_decode
       payload[4]&0x80?" end":"");
   }
   
-  if (seq != ((state->mavlink_seq+1)&0x3f)){
+  if (seq != ((state->seq+1)&0x3f)){
     // reject partial packet if we missed a sequence number
-    if (config.debug.mavlink) 
-      DEBUGF("Rejecting packet, sequence jumped from %d to %d", state->mavlink_seq, seq);
+    if (config.debug.radio_link) 
+      DEBUGF("Rejecting packet, sequence jumped from %d to %d", state->seq, seq);
     state->packet_length=sizeof(state->dst)+1;
   }
   
@@ -285,9 +283,9 @@ static int mavlink_parse(struct overlay_interface *interface, struct slip_decode
     state->packet_length=0;
   }
   
-  state->mavlink_seq=payload[4]&0x3f;
+  state->seq=payload[4]&0x3f;
   if (state->packet_length + data_bytes > sizeof(state->dst)){
-    if (config.debug.mavlink)
+    if (config.debug.radio_link)
       DEBUG("Fragmented packet is too long or a previous piece was missed - discarding");
     state->packet_length=sizeof(state->dst)+1;
     return 1;
@@ -297,7 +295,7 @@ static int mavlink_parse(struct overlay_interface *interface, struct slip_decode
   state->packet_length+=data_bytes;
     
   if (payload[4]&0x80) {
-    if (config.debug.mavlink) 
+    if (config.debug.radio_link) 
       DEBUGF("PDU Complete (length=%d)",state->packet_length);
     state->dst_offset=0;
     
@@ -307,7 +305,7 @@ static int mavlink_parse(struct overlay_interface *interface, struct slip_decode
   return 1;
 }
 
-static int decode_length(struct slip_decode_state *state, unsigned char *p)
+static int decode_length(struct radio_link_state *state, unsigned char *p)
 {
   // look for a valid golay encoded length
   int errs=0;
@@ -318,90 +316,92 @@ static int decode_length(struct slip_decode_state *state, unsigned char *p)
   if (length!=9 && (length<31 || length+8>255))
     return -1;
   
-  if (config.debug.mavlink && (errs || state->mavlink_payload_length!=*p))
+  if (config.debug.radio_link && (errs || state->payload_length!=*p))
     DEBUGF("Decoded length %d to %d with %d errs", *p, length, errs);
   
-  state->mavlink_payload_length=length;
+  state->payload_length=length;
   return 0;
 }
 
-int mavlink_decode(struct overlay_interface *interface, struct slip_decode_state *state, uint8_t c)
+int radio_link_decode(struct overlay_interface *interface, uint8_t c)
 {
-  if (state->mavlink_payload_start + state->mavlink_payload_offset >= sizeof(state->mavlink_payload)){
+  struct radio_link_state *state=interface->radio_link_state;
+  
+  if (state->payload_start + state->payload_offset >= sizeof(state->payload)){
     // drop one byte if we run out of space
-    if (config.debug.mavlink)
-      DEBUGF("Dropped %02x, buffer full", state->mavlink_payload[0]);
-    bcopy(state->mavlink_payload+1, state->mavlink_payload, sizeof(state->mavlink_payload) -1);
-    state->mavlink_payload_start--;
+    if (config.debug.radio_link)
+      DEBUGF("Dropped %02x, buffer full", state->payload[0]);
+    bcopy(state->payload+1, state->payload, sizeof(state->payload) -1);
+    state->payload_start--;
   }
   
-  unsigned char *p = &state->mavlink_payload[state->mavlink_payload_start];
-  p[state->mavlink_payload_offset++]=c;
+  unsigned char *p = &state->payload[state->payload_start];
+  p[state->payload_offset++]=c;
   
   while(1){
     // look for packet length headers
-    p = &state->mavlink_payload[state->mavlink_payload_start];
-    while(state->mavlink_payload_length==0 && state->mavlink_payload_offset>=6){
+    p = &state->payload[state->payload_start];
+    while(state->payload_length==0 && state->payload_offset>=6){
       if (p[0]==0xFE 
 	&& p[1]==9
 	&& p[3]==RADIO_SOURCE_SYSTEM
 	&& p[4]==RADIO_SOURCE_COMPONENT
 	&& p[5]==MAVLINK_MSG_ID_RADIO){
 	//looks like a valid heartbeat response header, read the rest and process it
-	state->mavlink_payload_length=9;
+	state->payload_length=9;
 	break;
       }
       
       if (decode_length(state, &p[1])==0)
 	break;
       
-      state->mavlink_payload_start++;
-      state->mavlink_payload_offset--;
+      state->payload_start++;
+      state->payload_offset--;
       p++;
     }
     
     // wait for a whole packet
-    if (!state->mavlink_payload_length || state->mavlink_payload_offset < state->mavlink_payload_length+8)
+    if (!state->payload_length || state->payload_offset < state->payload_length+8)
       return 0;
     
     if (parse_heartbeat(interface, p)){
       // cut the bytes of the heartbeat out of the buffer
-      state->mavlink_payload_offset -= state->mavlink_payload_length+8;
-      if (state->mavlink_payload_offset){
+      state->payload_offset -= state->payload_length+8;
+      if (state->payload_offset){
 	// shuffle bytes backwards
-	bcopy(&p[state->mavlink_payload_length+8], p, state->mavlink_payload_offset);
+	bcopy(&p[state->payload_length+8], p, state->payload_offset);
       }
       // restart parsing for a valid header from the beginning of out buffer
-      state->mavlink_payload_offset+=state->mavlink_payload_start;
-      state->mavlink_payload_start=0;
-      state->mavlink_payload_length=0;
+      state->payload_offset+=state->payload_start;
+      state->payload_start=0;
+      state->payload_length=0;
       continue;
     }
     
     // is this a well formed packet?
     int backtrack=0;
-    if (mavlink_parse(interface, state, state->mavlink_payload_length, p, &backtrack)==1){
+    if (radio_link_parse(interface, state, state->payload_length, p, &backtrack)==1){
       // Since we know we've synced with the remote party, 
       // and there's nothing we can do about any earlier data
       // throw away everything before the end of this packet
-      if (state->mavlink_payload_start && config.debug.mavlink)
-	dump("Skipped", state->mavlink_payload, state->mavlink_payload_start);
+      if (state->payload_start && config.debug.radio_link)
+	dump("Skipped", state->payload, state->payload_start);
       
       // If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet, 
       // but we may need to examine the last few bytes to find the start of the next packet.
-      state->mavlink_payload_offset -= state->mavlink_payload_length+8-backtrack;
-      if (state->mavlink_payload_offset){
+      state->payload_offset -= state->payload_length+8-backtrack;
+      if (state->payload_offset){
 	// shuffle all remaining bytes back to the start of the buffer
-	bcopy(&state->mavlink_payload[state->mavlink_payload_start + state->mavlink_payload_length+8-backtrack], 
-	  state->mavlink_payload, state->mavlink_payload_offset);
+	bcopy(&state->payload[state->payload_start + state->payload_length+8-backtrack], 
+	  state->payload, state->payload_offset);
       }
-      state->mavlink_payload_start=0;
+      state->payload_start=0;
     }else{
       // ignore the first byte for now and start looking for another packet header
       // we may find a heartbeat in the middle that we need to cut out first
-      state->mavlink_payload_start++;
-      state->mavlink_payload_offset--;
+      state->payload_start++;
+      state->payload_offset--;
     }
-    state->mavlink_payload_length=0;
+    state->payload_length=0;
   };
 }
diff --git a/radio_link.h b/radio_link.h
new file mode 100644
index 00000000..9a3060b3
--- /dev/null
+++ b/radio_link.h
@@ -0,0 +1,10 @@
+#ifndef __SERVALD_RADIO_LINK_H
+#define __SERVALD_RADIO_LINK_H
+
+int radio_link_free(struct overlay_interface *interface);
+int radio_link_init(struct overlay_interface *interface);
+int radio_link_decode(struct overlay_interface *interface, uint8_t c);
+int radio_link_heartbeat(unsigned char *frame, int *outlen);
+int radio_link_encode_packet(struct overlay_interface *interface);
+
+#endif
\ No newline at end of file
diff --git a/serval.h b/serval.h
index 32b3d3fa..5abdcb31 100644
--- a/serval.h
+++ b/serval.h
@@ -277,12 +277,6 @@ struct slip_decode_state{
   uint32_t crc;
   int src_offset;
   int dst_offset;
-
-  int mavlink_payload_length;
-  int mavlink_seq;
-  int mavlink_payload_start;
-  int mavlink_payload_offset;
-  uint8_t mavlink_payload[1024];
 };
 
 struct overlay_interface;
@@ -357,7 +351,7 @@ typedef struct overlay_interface {
   
   // stream socket tx state;
   struct overlay_buffer *tx_packet;
-  unsigned char txbuffer[OVERLAY_INTERFACE_RX_BUFFER_SIZE];
+  uint8_t *txbuffer;
   int tx_bytes_pending;
   // Throttle TX rate if required (stream interfaces only for now)
   uint32_t throttle_bytes_per_second;
@@ -365,11 +359,10 @@ typedef struct overlay_interface {
   uint64_t next_tx_allowed;
   int32_t remaining_space;
   time_ms_t next_heartbeat;
-  int mavlink_seq;
   int radio_rssi;
   int remote_rssi;
   
-  struct slip_decode_state slip_decode_state;
+  struct radio_link_state *radio_link_state;
 
   // copy of ifconfig flags
   uint16_t drop_packets;
@@ -733,8 +726,4 @@ int link_stop_routing(struct subscriber *subscriber);
 
 int generate_nonce(unsigned char *nonce,int bytes);
 
-int mavlink_decode(struct overlay_interface *interface, struct slip_decode_state *state,uint8_t c);
-int mavlink_heartbeat(unsigned char *frame,int *outlen);
-int mavlink_encode_packet(struct overlay_interface *interface);
-
 #endif // __SERVALD_SERVALD_H
diff --git a/sourcefiles.mk b/sourcefiles.mk
index ddd229f8..86dc3dcd 100644
--- a/sourcefiles.mk
+++ b/sourcefiles.mk
@@ -19,7 +19,7 @@ SERVAL_SOURCES = \
 	$(SERVAL_BASE)log.c \
 	$(SERVAL_BASE)lsif.c \
 	$(SERVAL_BASE)main.c \
-	$(SERVAL_BASE)mavlink.c \
+	$(SERVAL_BASE)radio_link.c \
 	$(SERVAL_BASE)meshms.c \
 	$(SERVAL_BASE)mdp_client.c \
 	$(SERVAL_BASE)os.c \
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 750d91a5..c9462338 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -702,7 +702,7 @@ start_radio_instance() {
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \
       set debug.throttling on \
-      set debug.mavlink on \
+      set debug.radio_link on \
       set rhizome.advertise.interval 5000 \
       set rhizome.rhizome_mdp_block_size 350 \
       set log.console.level debug \
diff --git a/tests/routing b/tests/routing
index 1871a262..595b054b 100755
--- a/tests/routing
+++ b/tests/routing
@@ -282,7 +282,7 @@ setup_simulate_extender() {
       executeOk_servald config \
          set debug.throttling on \
          set debug.packetradio on \
-         set debug.mavlink on \
+         set debug.radio_link on \
          set interfaces.1.type CATEAR \
          set interfaces.1.mdp.tick_ms 5000 \
          set interfaces.1.socket_type STREAM \

From 1b34c2dc132a882a64d27891bb77701f85656374 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 25 Nov 2013 12:54:29 +1030
Subject: [PATCH 37/56] Improve test reliability

---
 fakeradio.c           | 45 ++++++++++++++++++++++++++++---------------
 tests/rhizomeprotocol | 29 ++++++++++++++--------------
 tests/routing         | 17 ++++++++++------
 3 files changed, 55 insertions(+), 36 deletions(-)

diff --git a/fakeradio.c b/fakeradio.c
index 5aef5fdf..2ed5441f 100644
--- a/fakeradio.c
+++ b/fakeradio.c
@@ -399,21 +399,33 @@ int calc_ber(double target_packet_fraction)
   // so no point starting smaller than that.
   // Only ~30,000,000 reduces packet delivery rate to
   // ~1%, so the search range is fairly narrow.
-  ber=9000000;
-  if (target_packet_fraction<=0.9) ber=13000000;
-  if (target_packet_fraction<=0.5) ber=18000000;
-  if (target_packet_fraction<=0.25) ber=21000000;
-  if (target_packet_fraction<=0.1) ber=24000000;
-  if (target_packet_fraction<=0.05) ber=26000000;
+  ber=0;
+  if (target_packet_fraction<=0.9) ber=6900000;
+  if (target_packet_fraction<=0.5) ber=16900000;
+  if (target_packet_fraction<=0.25) ber=20600000;
+  if (target_packet_fraction<=0.1) ber=23400000;
+  if (target_packet_fraction<=0.05) ber=28600000;
+
   for(;ber<0x70ffffff;ber+=100000)
     {
       int packet_errors=0;
       for(p=0;p<1000;p++) {
 	int byte_errors=0;
-	for(byte=0;byte<byte_count;byte++) {
-	  for(bit=0;bit<8;bit++) if (random()<ber) { byte_errors++; break; }
-	  if (byte_errors>max_error_bytes) { packet_errors++; break; }
+	int dropped = 0;
+	for (byte=0;byte<PREAMBLE_LENGTH;byte++){
+	  if (random()<ber){
+	    dropped = 1;
+	    break;
+	  }
 	}
+	if (!dropped){
+	  for(byte=0;byte<byte_count;byte++) {
+	    for(bit=0;bit<8;bit++) if (random()<ber) { byte_errors++; break; }
+	    if (byte_errors>max_error_bytes) { dropped=1; break; }
+	  }
+	}
+	if (dropped)
+	  packet_errors++;
       }
       if (packet_errors>=((1.0-target_packet_fraction)*1000)) break;
     }
@@ -423,13 +435,11 @@ int calc_ber(double target_packet_fraction)
 
 int main(int argc,char **argv)
 {
-  if (argv[1]) {
+  if (argc>=1) {
     chars_per_ms=atol(argv[1]);
-    if (argv[2]) 
+    if (argc>=2) 
       ber=calc_ber(atof(argv[2]));
   }
-  fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms);
-  fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF);
 
   struct pollfd fds[2];
   struct radio_state radios[2];
@@ -437,18 +447,21 @@ int main(int argc,char **argv)
   bzero(&radios,sizeof radios);
   
   int i;
+  radios[0].name="left";
+  radios[1].name="right";
   for (i=0;i<2;i++){
     radios[i].fd=posix_openpt(O_RDWR|O_NOCTTY);
     grantpt(radios[i].fd);
     unlockpt(radios[i].fd);
     fcntl(radios[i].fd,F_SETFL,fcntl(radios[i].fd, F_GETFL, NULL)|O_NONBLOCK);
-    fprintf(stdout,"%s\n",ptsname(radios[i].fd));
+    fprintf(stdout,"%s:%s\n", radios[i].name, ptsname(radios[i].fd));
     fds[i].fd = radios[i].fd;
   }
-  radios[0].name="left";
-  radios[1].name="right";
   fflush(stdout);
 
+  fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms);
+  fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF);
+  
   while(1) {
     // what events do we need to poll for? how long can we block?
     int64_t now = gettime_ms();
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index c9462338..761f1952 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -717,17 +717,23 @@ start_radio_instance() {
    wait_until interface_up
 }
 
+start_fakeradio() {
+   $servald_build_root/fakeradio $1 $2 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   FAKERADIO_PID=$!
+   wait_until $GREP "^right:" "$SERVALD_VAR/radioout"
+   local _line=`head "$SERVALD_VAR/radioout" -n 1`
+   END1="${_line#*:}"
+   _line=`tail "$SERVALD_VAR/radioout" -n 1`
+   END2="${_line#*:}"
+   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+}
+
 doc_SimulatedRadio="MDP Transfer over simulated radio link (~90% packet arrival)"
 setup_SimulatedRadio() {
    setup_common
-   $servald_build_root/fakeradio 4 0.9 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
-   FAKERADIO_PID=$!
-   sleep 5
-   local END1=`head "$SERVALD_VAR/radioout" -n 1`
-   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
-   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+   start_fakeradio 4 0.9
    set_instance +A
-   rhizome_add_file file1 10000
+   rhizome_add_file file1 5000
    executeOk_servald config \
       set interfaces.1.file "$END1"
    set_instance +B
@@ -748,14 +754,9 @@ teardown_SimulatedRadio() {
 doc_SimulatedRadio2="MDP Transfer over simulated radio link (~50% packet arrival)"
 setup_SimulatedRadio2() {
    setup_common
-   $servald_build_root/fakeradio 4 0.5 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
-   FAKERADIO_PID=$!
-   sleep 1
-   local END1=`head "$SERVALD_VAR/radioout" -n 1`
-   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
-   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+   start_fakeradio 4 0.5
    set_instance +A
-   rhizome_add_file file1 10000
+   rhizome_add_file file1 5000
    executeOk_servald config \
       set interfaces.1.file "$END1"
    set_instance +B
diff --git a/tests/routing b/tests/routing
index 595b054b..2d2381e6 100755
--- a/tests/routing
+++ b/tests/routing
@@ -260,18 +260,23 @@ setup_slip_encoding() {
 test_slip_encoding() {
    executeOk_servald test slip --seed=1 --iterations=2000
 }
+start_fakeradio() {
+   $servald_build_root/fakeradio 4 1 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
+   FAKERADIO_PID=$!
+   wait_until $GREP "^right:" "$SERVALD_VAR/radioout"
+   local _line=`head "$SERVALD_VAR/radioout" -n 1`
+   END1="${_line#*:}"
+   _line=`tail "$SERVALD_VAR/radioout" -n 1`
+   END2="${_line#*:}"
+   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+}
 
 doc_simulate_extender="Simulate a mesh extender radio link"
 setup_simulate_extender() {
    setup_servald
    assert_no_servald_processes
    foreach_instance +A +B create_single_identity
-   $servald_build_root/fakeradio 1 0.8 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
-   FAKERADIO_PID=$!
-   sleep 1
-   local END1=`head "$SERVALD_VAR/radioout" -n 1`
-   local END2=`tail "$SERVALD_VAR/radioout" -n 1`
-   tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
+   start_fakeradio
    set_instance +A
    executeOk_servald config \
       set interfaces.1.file "$END1"

From 844afcff0ec4a337781b82291d81a4823f50a9fc Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 25 Nov 2013 17:48:38 +1030
Subject: [PATCH 38/56] Calculate golay parity regardless of CPU endian

---
 golay.c | 45 +++++++++++++++++++++++----------------------
 golay.h | 10 ++++++++--
 2 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/golay.c b/golay.c
index 1f973814..119cdae5 100644
--- a/golay.c
+++ b/golay.c
@@ -1,13 +1,14 @@
 
 #define POLY  0xAE3  /* or use the other polynomial, 0xC75 */
+#include <inttypes.h>
 
-static unsigned long golay(unsigned long cw) 
+static uint32_t golay(uint32_t cw) 
 /* This function calculates [23,12] Golay codewords. 
    The format of the returned longint is 
    [checkbits(11),data(12)]. */ 
 { 
   int i; 
-  unsigned long c; 
+  uint32_t c; 
   cw&=0xfffl; 
   c=cw; /* save original codeword */ 
   for (i=1; i<=12; i++)  /* examine each data bit */ 
@@ -19,16 +20,16 @@ static unsigned long golay(unsigned long cw)
   return((cw<<12)|c);    /* assemble codeword */ 
 }
 
-static int parity(unsigned long cw) 
+static int parity(uint32_t cw) 
 /* This function checks the overall parity of codeword cw.
    If parity is even, 0 is returned, else 1. */ 
 { 
-  unsigned char p; 
-
+  uint8_t p; 
+  
   /* XOR the bytes of the codeword */ 
-  p=*(unsigned char*)&cw; 
-  p^=*((unsigned char*)&cw+1); 
-  p^=*((unsigned char*)&cw+2); 
+  p=cw & 0xFF; 
+  p^=(cw>>8) & 0xFF; 
+  p^=(cw>>16) & 0xFF; 
 
   /* XOR the halves of the intermediate result */ 
   p=p ^ (p>>4); 
@@ -39,9 +40,9 @@ static int parity(unsigned long cw)
   return(p & 1); 
 }
 
-int golay_encode(unsigned char *data)
+int golay_encode(uint8_t *data)
 {
-  unsigned long cw = data[0] | (data[1]<<8) | (data[2]<<16);
+  uint32_t cw = data[0] | (data[1]<<8) | (data[2]<<16);
   cw = golay(cw);
   if (parity(cw))
     cw|=0x800000l;
@@ -51,7 +52,7 @@ int golay_encode(unsigned char *data)
   return 0;
 }
 
-static unsigned long syndrome(unsigned long cw) 
+static uint32_t syndrome(uint32_t cw) 
 /* This function calculates and returns the syndrome 
    of a [23,12] Golay codeword. */ 
 { 
@@ -66,7 +67,7 @@ static unsigned long syndrome(unsigned long cw)
   return(cw<<12);        /* value pairs with upper bits of cw */
 }
 
-static int weight(unsigned long cw) 
+static int weight(uint32_t cw) 
 /* This function calculates the weight of 
    23 bit codeword cw. */ 
 { 
@@ -88,7 +89,7 @@ static int weight(unsigned long cw)
   return(bits); 
 } 
 
-static unsigned long rotate_left(unsigned long cw, int n) 
+static uint32_t rotate_left(uint32_t cw, int n) 
 /* This function rotates 23 bit codeword cw left by n bits. */ 
 { 
   int i; 
@@ -107,7 +108,7 @@ static unsigned long rotate_left(unsigned long cw, int n)
   return(cw & 0x7fffffl); 
 } 
 
-static unsigned long rotate_right(unsigned long cw, int n) 
+static uint32_t rotate_right(uint32_t cw, int n) 
 /* This function rotates 23 bit codeword cw right by n bits. */ 
 { 
   int i; 
@@ -126,20 +127,20 @@ static unsigned long rotate_right(unsigned long cw, int n)
   return(cw & 0x7fffffl); 
 }
 
-static unsigned long correct(unsigned long cw, int *errs) 
+static uint32_t correct(uint32_t cw, int *errs) 
 /* This function corrects Golay [23,12] codeword cw, returning the 
    corrected codeword. This function will produce the corrected codeword 
    for three or fewer errors. It will produce some other valid Golay 
    codeword for four or more errors, possibly not the intended 
    one. *errs is set to the number of bit errors corrected. */ 
 { 
-  unsigned char 
+  uint8_t 
     w;                /* current syndrome limit weight, 2 or 3 */ 
-  unsigned long 
+  uint32_t 
     mask;             /* mask for bit flipping */ 
   int 
     i,j;              /* index */ 
-  unsigned long 
+  uint32_t 
     s,                /* calculated syndrome */ 
     cwsaver;          /* saves initial value of cw */ 
 
@@ -187,17 +188,17 @@ static unsigned long correct(unsigned long cw, int *errs)
   return(cwsaver); /* return original if no corrections */ 
 } /* correct */ 
 
-int golay_decode(int *errs, const unsigned char *data)
+int golay_decode(int *errs, const uint8_t *data)
 /* This function decodes codeword *cw , error correction is attempted, 
    with *errs set to the number of bits corrected, and returning 0 if 
    no errors exist, or 1 if parity errors exist. */ 
 { 
-  unsigned long cw = data[0] | (data[1]<<8) | (data[2]<<16);
-  unsigned long parity_bit=cw & 0x800000l;
+  uint32_t cw = data[0] | (data[1]<<8) | (data[2]<<16);
+  uint32_t parity_bit=cw & 0x800000l;
   cw&=~0x800000l;            /* remove parity bit for correction */
   cw=correct(cw, errs);     /* correct up to three bits */ 
   cw|=parity_bit;
   if (parity(cw))
-    return -1;
+    *errs++;
   return cw&0xFFF;
 } /* decode */ 
diff --git a/golay.h b/golay.h
index 44ecfd88..506eeacc 100644
--- a/golay.h
+++ b/golay.h
@@ -1,2 +1,8 @@
-int golay_encode(unsigned char *data);
-int golay_decode(int *errs, unsigned char *data);
+
+#ifndef __SERVALD_GOLAY_H
+#define __SERVALD_GOLAY_H
+
+int golay_encode(uint8_t *data);
+int golay_decode(int *errs, uint8_t *data);
+
+#endif
\ No newline at end of file

From 62cd8c75554b5081245a052b6592adbdb0cb20c1 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Wed, 27 Nov 2013 12:27:15 +1030
Subject: [PATCH 39/56] Log any alarm we call more than 10,000 times a second
 (probably indicates a bug)

---
 performance_timing.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/performance_timing.c b/performance_timing.c
index 2ac8b0b8..e45c3f6a 100644
--- a/performance_timing.c
+++ b/performance_timing.c
@@ -168,8 +168,8 @@ int fd_showstats()
       while(stats!=NULL){
 	/* If a function spends more than 1 second in any 
 	   notionally 3 second period, then dob on it */
-	if (stats->total_time>1000
-	    &&strcmp(stats->name,"Idle (in poll)"))
+	if ((stats->total_time>1000 || stats->calls > 10000)
+	    && strcmp(stats->name,"Idle (in poll)"))
 	  fd_showstat(&total,stats);
 	stats = stats->_next;
       }

From 1681abd620379023ec60f14c3ba98eb7ffcd1f88 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 4 Nov 2013 12:12:58 +1030
Subject: [PATCH 40/56] Increase default mdp timeout & make it configurable

---
 conf_schema.h         | 1 +
 rhizome_fetch.c       | 2 +-
 tests/rhizomeprotocol | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/conf_schema.h b/conf_schema.h
index db680364..fad38f39 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -430,6 +430,7 @@ ATOM(bool_t,                external_blobs, 0, boolean,, "Store rhizome bundles
 
 ATOM(uint64_t,              rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")
 ATOM(uint64_t,              idle_timeout,           RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.")
+ATOM(uint64_t,              mdp_stall_timeout,      1000, uint64_scaled,, "Timeout to request more data via mdp.")
 ATOM(uint32_t,              fetch_delay_ms,         50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
 SUB_STRUCT(rhizome_direct,  direct,)
 SUB_STRUCT(rhizome_api,     api,)
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 56071a02..08452e4f 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -1065,7 +1065,7 @@ static int rhizome_fetch_mdp_touch_timeout(struct rhizome_fetch_slot *slot)
   // For now, we will just make the timeout 1 second from the time of the last
   // received block.
   unschedule(&slot->alarm);
-  slot->alarm.alarm=gettime_ms()+1000; 
+  slot->alarm.alarm=gettime_ms()+config.rhizome.mdp_stall_timeout; 
   slot->alarm.deadline=slot->alarm.alarm+500;
   schedule(&slot->alarm);
   return 0;
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 761f1952..265dc926 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -704,7 +704,7 @@ start_radio_instance() {
       set debug.throttling on \
       set debug.radio_link on \
       set rhizome.advertise.interval 5000 \
-      set rhizome.rhizome_mdp_block_size 350 \
+      set rhizome.rhizome_mdp_block_size 375 \
       set log.console.level debug \
       set log.console.show_pid on \
       set log.console.show_time on \

From 28fc1da2d27ffd0026d432f6ed884500fa279999 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 4 Nov 2013 14:15:08 +1030
Subject: [PATCH 41/56] Encapsulate rhizome fetch logging

---
 performance_timing.c | 13 ++-----------
 rhizome.h            |  1 -
 rhizome_fetch.c      | 32 +++++++++++++++++++++-----------
 serval.h             |  3 +--
 4 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/performance_timing.c b/performance_timing.c
index e45c3f6a..2d91fd3a 100644
--- a/performance_timing.c
+++ b/performance_timing.c
@@ -149,17 +149,8 @@ int fd_showstats()
     stats = stats->_next;
   }
 
-  // Show periodic rhizome transfer information, but only
-  // if there are some active rhizome transfers.
-  if (rhizome_active_fetch_count()!=0)
-    INFOF("Rhizome transfer progress: %"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64" (remaining %"PRIu64")",
-	  rhizome_active_fetch_bytes_received(0),
-	  rhizome_active_fetch_bytes_received(1),
-	  rhizome_active_fetch_bytes_received(2),
-	  rhizome_active_fetch_bytes_received(3),
-	  rhizome_active_fetch_bytes_received(4),
-	  rhizome_active_fetch_bytes_received(5),
-          rhizome_fetch_queue_bytes());
+  // Show periodic rhizome transfer information
+  rhizome_fetch_log_short_status();
 
   // Report any functions that take too much time
   if (!config.debug.timing)
diff --git a/rhizome.h b/rhizome.h
index 7b171a15..a45cd823 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -898,7 +898,6 @@ enum rhizome_start_fetch_result {
 enum rhizome_start_fetch_result rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip, const sid_t *sidp, const unsigned char *prefix, size_t prefix_length);
 int rhizome_any_fetch_active();
 int rhizome_any_fetch_queued();
-uint64_t rhizome_fetch_queue_bytes();
 int rhizome_fetch_status_html(struct strbuf *b);
 int rhizome_fetch_has_queue_space(unsigned char log2_size);
 
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 08452e4f..8c70bf93 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -150,23 +150,14 @@ static const char * fetch_state(int state)
   }
 }
 
-int rhizome_active_fetch_count()
-{
-  int i,active=0;
-  for(i=0;i<NQUEUES;i++)
-    if (rhizome_fetch_queues[i].active.state!=RHIZOME_FETCH_FREE)
-      active++;
-  return active;
-}
-
-uint64_t rhizome_active_fetch_bytes_received(int q)
+static uint64_t rhizome_active_fetch_bytes_received(int q)
 {
   if (q<0 || q>=NQUEUES) return -1;
   if (rhizome_fetch_queues[q].active.state==RHIZOME_FETCH_FREE) return -1;
   return rhizome_fetch_queues[q].active.write_state.file_offset;
 }
 
-uint64_t rhizome_fetch_queue_bytes()
+static uint64_t rhizome_fetch_queue_bytes()
 {
   uint64_t bytes = 0;
   unsigned i;
@@ -186,6 +177,25 @@ uint64_t rhizome_fetch_queue_bytes()
   return bytes;
 }
 
+void rhizome_fetch_log_short_status()
+{
+  int i,active=0;
+  for(i=0;i<NQUEUES;i++)
+    if (rhizome_fetch_queues[i].active.state!=RHIZOME_FETCH_FREE)
+      active++;
+  if (!active)
+    return;
+  
+  INFOF("Rhizome transfer progress: %"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64" (remaining %"PRIu64")",
+	rhizome_active_fetch_bytes_received(0),
+	rhizome_active_fetch_bytes_received(1),
+	rhizome_active_fetch_bytes_received(2),
+	rhizome_active_fetch_bytes_received(3),
+	rhizome_active_fetch_bytes_received(4),
+	rhizome_active_fetch_bytes_received(5),
+	rhizome_fetch_queue_bytes());
+}
+
 int rhizome_fetch_status_html(strbuf b)
 {
   unsigned i;
diff --git a/serval.h b/serval.h
index 5abdcb31..983e0daf 100644
--- a/serval.h
+++ b/serval.h
@@ -703,8 +703,7 @@ int slip_decode(struct slip_decode_state *state);
 int upper7_decode(struct slip_decode_state *state,unsigned char byte);
 uint32_t Crc32_ComputeBuf( uint32_t inCrc32, const void *buf,
 			  size_t bufLen );
-int rhizome_active_fetch_count();
-uint64_t rhizome_active_fetch_bytes_received(int q);
+void rhizome_fetch_log_short_status();
 extern int64_t bundles_available;
 extern char crash_handler_clue[1024];
 

From 0e6a3cd7cc7c0509ea77ce7df571af1c24fb4c94 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Thu, 7 Nov 2013 15:14:36 +1030
Subject: [PATCH 42/56] Use interval parameter correctly even for unlimited
 pings

---
 commandline.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/commandline.c b/commandline.c
index 8a9c7d62..ab474b41 100644
--- a/commandline.c
+++ b/commandline.c
@@ -986,7 +986,7 @@ int app_mdp_ping(const struct cli_parsed *parsed, struct cli_context *context)
     /* Now look for replies until one second has passed, and print any replies
        with appropriate information as required */
     time_ms_t now = gettime_ms();
-    time_ms_t finish = now + (tx_count < icount?interval_ms:timeout_ms);
+    time_ms_t finish = now + ((tx_count < icount || icount==0)?interval_ms:timeout_ms);
     for (; !servalShutdown && now < finish; now = gettime_ms()) {
       time_ms_t poll_timeout_ms = finish - gettime_ms();
       

From 6dbc62364387a3b4f4fb7c525a8d24d94de2ebb6 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 11 Nov 2013 13:39:52 +1030
Subject: [PATCH 43/56] Increase mdp timeout based on file size

---
 rhizome_fetch.c | 39 +++++++++++++++++++++------------------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 8c70bf93..6546fd4e 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -241,10 +241,9 @@ static struct profile_total fetch_stats = { .name="rhizome_fetch_poll" };
  *
  * @author Andrew Bettison <andrew@servalproject.com>
  */
-static struct rhizome_fetch_queue *rhizome_find_queue(uint64_t size)
+static struct rhizome_fetch_queue *rhizome_find_queue(unsigned char log_size)
 {
   int i;
-  unsigned char log_size = log2ll(size);
   for (i = 0; i < NQUEUES; ++i) {
     struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
     if (log_size < q->log_size_threshold)
@@ -836,17 +835,14 @@ static void rhizome_start_next_queued_fetches(struct sched_ent *alarm)
 
 /* Do we have space to add a fetch candidate of this size? */
 int rhizome_fetch_has_queue_space(unsigned char log2_size){
-  int i;
-  for (i = 0; i < NQUEUES; ++i) {
-    struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
-    if (log2_size < q->log_size_threshold){
-      // is there an empty candidate?
-      unsigned j;
-      for (j=0;j < q->candidate_queue_size;j++)
-	if (!q->candidate_queue[j].manifest)
-	  return 1;
-      return 0;
-    }
+  struct rhizome_fetch_queue *q = rhizome_find_queue(log2_size);
+  if (q){
+    // is there an empty candidate?
+    unsigned j=0;
+    for (j=0;j < q->candidate_queue_size;j++)
+      if (!q->candidate_queue[j].manifest)
+	return 1;
+    return 0;
   }
   return 0;
 }
@@ -911,7 +907,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
   }
 
   // Find the proper queue for the payload.  If there is none suitable, it is an error.
-  struct rhizome_fetch_queue *qi = rhizome_find_queue(m->filesize);
+  struct rhizome_fetch_queue *qi = rhizome_find_queue(log2ll(m->filesize));
   if (!qi) {
     WHYF("No suitable fetch queue for bundle size=%"PRIu64, m->filesize);
     rhizome_manifest_free(m);
@@ -1048,8 +1044,8 @@ static void rhizome_fetch_mdp_slot_callback(struct sched_ent *alarm)
   struct rhizome_fetch_slot *slot=(struct rhizome_fetch_slot*)alarm;
 
   time_ms_t now = gettime_ms();
-  if (now-slot->last_write_time>slot->mdpIdleTimeout) {
-    DEBUGF("MDP connection timed out: last RX %"PRId64"ms ago (read %"PRIu64" of %"PRIu64" bytes)",
+  if (now - slot->last_write_time > slot->mdpIdleTimeout) {
+    DEBUGF("MDP connection timed out: last RX %"PRId64"ms ago (read %"PRId64" of %"PRId64" bytes)",
 	   now-slot->last_write_time,
 	   slot->write_state.file_offset,
 	   slot->write_state.file_length);
@@ -1233,8 +1229,15 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
        down too much.  Much careful thought is required to optimise this
        transport.
     */
-  slot->mdpIdleTimeout=config.rhizome.idle_timeout; // give up if nothing received for 5 seconds
-  slot->mdpRXBlockLength=config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size
+  slot->mdpIdleTimeout = config.rhizome.idle_timeout; // give up if nothing received for 5 seconds
+  
+  unsigned char log_size=log2ll(slot->manifest->filesize);
+  struct rhizome_fetch_queue *q=rhizome_find_queue(log_size);
+  // increase the timeout based on the queue number
+  if (q)
+    slot->mdpIdleTimeout *= 1+(q - rhizome_fetch_queues);
+  
+  slot->mdpRXBlockLength = config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size
   rhizome_fetch_mdp_requestblocks(slot);
 
   RETURN(0);

From 99d5d9fa1f8a1f5b87828ec8fa6f9e19bbd0c951 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 25 Nov 2013 13:09:54 +1030
Subject: [PATCH 44/56] Refactor dataformats into separate header

---
 cli.c                  |  1 +
 commandline.c          |  1 +
 conf_schema.c          |  1 +
 dataformats.c          |  9 +++++----
 dataformats.h          | 26 ++++++++++++++++++++++++++
 dna_helper.c           |  1 +
 headerfiles.mk         |  1 +
 keyring.c              |  1 +
 meshms.c               |  1 +
 monitor.c              |  1 +
 overlay_mdp_services.c |  1 +
 rhizome.c              |  1 +
 rhizome.h              | 10 ----------
 rhizome_bundle.c       |  1 +
 rhizome_crypto.c       |  2 +-
 rhizome_fetch.c        |  1 +
 serval.h               | 12 ------------
 slip.c                 |  1 +
 18 files changed, 45 insertions(+), 27 deletions(-)
 create mode 100644 dataformats.h

diff --git a/cli.c b/cli.c
index 9d1ea003..2013f262 100644
--- a/cli.c
+++ b/cli.c
@@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "rhizome.h"
 #include "strbuf_helpers.h"
+#include "dataformats.h"
 
 int cli_usage(const struct cli_schema *commands, XPRINTF xpf)
 {
diff --git a/commandline.c b/commandline.c
index ab474b41..e579d982 100644
--- a/commandline.c
+++ b/commandline.c
@@ -46,6 +46,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "overlay_address.h"
 #include "overlay_buffer.h"
 #include "keyring.h"
+#include "dataformats.h"
 
 extern struct cli_schema command_line_options[];
 
diff --git a/conf_schema.c b/conf_schema.c
index 02348b8a..c53c74f5 100644
--- a/conf_schema.c
+++ b/conf_schema.c
@@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "strbuf.h"
 #include "strbuf_helpers.h"
 #include "conf.h"
+#include "dataformats.h"
 
 int cf_opt_boolean(bool_t *booleanp, const char *text)
 {
diff --git a/dataformats.c b/dataformats.c
index 5180775d..ed841ca2 100644
--- a/dataformats.c
+++ b/dataformats.c
@@ -17,10 +17,11 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 */
 
+#include <ctype.h>
 #include "serval.h"
 #include "rhizome.h"
 #include "str.h"
-#include <ctype.h>
+#include "dataformats.h"
 
 int cmp_sid_t(const sid_t *a, const sid_t *b)
 {
@@ -214,7 +215,7 @@ void write_uint16(unsigned char *o,uint16_t v)
   { *(o++)=v&0xff; v=v>>8; }
 }
 
-uint64_t read_uint64(unsigned char *o)
+uint64_t read_uint64(const unsigned char *o)
 {
   int i;
   uint64_t v=0;
@@ -222,7 +223,7 @@ uint64_t read_uint64(unsigned char *o)
   return v;
 }
 
-uint32_t read_uint32(unsigned char *o)
+uint32_t read_uint32(const unsigned char *o)
 {
   int i;
   uint32_t v=0;
@@ -230,7 +231,7 @@ uint32_t read_uint32(unsigned char *o)
   return v;
 }
 
-uint16_t read_uint16(unsigned char *o)
+uint16_t read_uint16(const unsigned char *o)
 {
   int i;
   uint16_t v=0;
diff --git a/dataformats.h b/dataformats.h
new file mode 100644
index 00000000..67afb746
--- /dev/null
+++ b/dataformats.h
@@ -0,0 +1,26 @@
+#ifndef __SERVALD_DATA_FORMATS_H
+#define __SERVALD_DATA_FORMATS_H
+
+int str_is_subscriber_id(const char *sid);
+int strn_is_subscriber_id(const char *sid, size_t *lenp);
+int str_is_did(const char *did);
+int strn_is_did(const char *did, size_t *lenp);
+
+int rhizome_strn_is_manifest_id(const char *text);
+int rhizome_str_is_manifest_id(const char *text);
+int rhizome_strn_is_bundle_key(const char *text);
+int rhizome_str_is_bundle_key(const char *text);
+int rhizome_strn_is_bundle_crypt_key(const char *text);
+int rhizome_str_is_bundle_crypt_key(const char *text);
+int rhizome_strn_is_file_hash(const char *text);
+int rhizome_str_is_file_hash(const char *text);
+int rhizome_str_is_manifest_service(const char *text);
+
+void write_uint64(unsigned char *o,uint64_t v);
+void write_uint16(unsigned char *o,uint16_t v);
+void write_uint32(unsigned char *o,uint32_t v);
+uint64_t read_uint64(const unsigned char *o);
+uint32_t read_uint32(const unsigned char *o);
+uint16_t read_uint16(const unsigned char *o);
+
+#endif
diff --git a/dna_helper.c b/dna_helper.c
index d1d18b42..66bceb96 100644
--- a/dna_helper.c
+++ b/dna_helper.c
@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "strbuf.h"
 #include "strbuf_helpers.h"
 #include "overlay_address.h"
+#include "dataformats.h"
 
 /*
   The challenge with making an interface for calling an external program to 
diff --git a/headerfiles.mk b/headerfiles.mk
index 08a0b2e4..fc9ae543 100644
--- a/headerfiles.mk
+++ b/headerfiles.mk
@@ -19,6 +19,7 @@ HDRS=	fifo.h \
 	conf.h \
 	conf_schema.h \
 	crypto.h \
+	dataformats.h \
 	log.h \
 	net.h \
 	fdqueue.h \
diff --git a/keyring.c b/keyring.c
index 9d0df5c6..3182e6bb 100644
--- a/keyring.c
+++ b/keyring.c
@@ -30,6 +30,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "crypto.h"
 #include "overlay_packet.h"
 #include "keyring.h"
+#include "dataformats.h"
 
 static void keyring_free_keypair(keypair *kp);
 static void keyring_free_context(keyring_context *c);
diff --git a/meshms.c b/meshms.c
index c88a984b..3c3d18ca 100644
--- a/meshms.c
+++ b/meshms.c
@@ -6,6 +6,7 @@
 #include "crypto.h"
 #include "strlcpy.h"
 #include "keyring.h"
+#include "dataformats.h"
 
 #define MESHMS_BLOCK_TYPE_ACK 0x01
 #define MESHMS_BLOCK_TYPE_MESSAGE 0x02
diff --git a/monitor.c b/monitor.c
index 20a710e8..e28016d9 100644
--- a/monitor.c
+++ b/monitor.c
@@ -33,6 +33,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "overlay_address.h"
 #include "monitor-client.h"
 #include "socket.h"
+#include "dataformats.h"
 
 #ifdef HAVE_UCRED_H
 #include <ucred.h>
diff --git a/overlay_mdp_services.c b/overlay_mdp_services.c
index 3c1fbc57..2abc30d4 100644
--- a/overlay_mdp_services.c
+++ b/overlay_mdp_services.c
@@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "crypto.h"
 #include "log.h"
 #include "keyring.h"
+#include "dataformats.h"
 
 int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength)
 {
diff --git a/rhizome.c b/rhizome.c
index 3db8cfc5..68495b3b 100644
--- a/rhizome.c
+++ b/rhizome.c
@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "conf.h"
 #include "str.h"
 #include "rhizome.h"
+#include "dataformats.h"
 
 int is_rhizome_enabled()
 {
diff --git a/rhizome.h b/rhizome.h
index a45cd823..d99f900a 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -423,16 +423,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report);
 int rhizome_manifest_createid(rhizome_manifest *m);
 int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
 
-int rhizome_strn_is_manifest_id(const char *text);
-int rhizome_str_is_manifest_id(const char *text);
-int rhizome_strn_is_bundle_key(const char *text);
-int rhizome_str_is_bundle_key(const char *text);
-int rhizome_strn_is_bundle_crypt_key(const char *text);
-int rhizome_str_is_bundle_crypt_key(const char *text);
-int rhizome_strn_is_file_hash(const char *text);
-int rhizome_str_is_file_hash(const char *text);
-int rhizome_str_is_manifest_service(const char *text);
-
 int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
 
 typedef struct sqlite_retry_state {
diff --git a/rhizome_bundle.c b/rhizome_bundle.c
index 8ecf5b51..a8949032 100644
--- a/rhizome_bundle.c
+++ b/rhizome_bundle.c
@@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "str.h"
 #include "mem.h"
 #include "keyring.h"
+#include "dataformats.h"
 
 static const char *rhizome_manifest_get(const rhizome_manifest *m, const char *var)
 {
diff --git a/rhizome_crypto.c b/rhizome_crypto.c
index 3055204f..fc2868f5 100644
--- a/rhizome_crypto.c
+++ b/rhizome_crypto.c
@@ -23,13 +23,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include "crypto_sign_edwards25519sha512batch.h"
 #include "nacl/src/crypto_sign_edwards25519sha512batch_ref/ge.h"
-
 #include "serval.h"
 #include "conf.h"
 #include "str.h"
 #include "rhizome.h"
 #include "crypto.h"
 #include "keyring.h"
+#include "dataformats.h"
 
 /* Work out the encrypt/decrypt key for the supplied manifest.
    If the manifest is not encrypted, then return NULL.
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 6546fd4e..d2614b10 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "strbuf_helpers.h"
 #include "overlay_address.h"
 #include "socket.h"
+#include "dataformats.h"
 
 /* Represents a queued fetch of a bundle payload, for which the manifest is already known.
  */
diff --git a/serval.h b/serval.h
index 983e0daf..a84d5e81 100644
--- a/serval.h
+++ b/serval.h
@@ -163,11 +163,6 @@ int cmp_sid_t(const sid_t *a, const sid_t *b);
 int str_to_sid_t(sid_t *sid, const char *hex);
 int strn_to_sid_t(sid_t *sid, const char *hex, const char **endp);
 
-int str_is_subscriber_id(const char *sid);
-int strn_is_subscriber_id(const char *sid, size_t *lenp);
-int str_is_did(const char *did);
-int strn_is_did(const char *did, size_t *lenp);
-
 #define alloca_tohex_sas(sas)           alloca_tohex((sas), SAS_SIZE)
 
 /*
@@ -686,13 +681,6 @@ int limit_init(struct limit_state *state, int rate_micro_seconds);
 int olsr_init_socket(void);
 int olsr_send(struct overlay_frame *frame);
 
-void write_uint64(unsigned char *o,uint64_t v);
-void write_uint16(unsigned char *o,uint16_t v);
-void write_uint32(unsigned char *o,uint32_t v);
-uint64_t read_uint64(unsigned char *o);
-uint32_t read_uint32(unsigned char *o);
-uint16_t read_uint16(unsigned char *o);
-
 int pack_uint(unsigned char *buffer, uint64_t v);
 int measure_packed_uint(uint64_t v);
 int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v);
diff --git a/slip.c b/slip.c
index daa37052..134d3e39 100644
--- a/slip.c
+++ b/slip.c
@@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "conf.h"
 #include "log.h"
+#include "dataformats.h"
 
 #define DEBUG_packet_visualise(M,P,N) logServalPacket(LOG_LEVEL_DEBUG, __WHENCE__, (M), (P), (N))
 

From cd639ba3b679ab9b3db0405acdb63113aaca2cf9 Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 25 Nov 2013 15:12:21 +1030
Subject: [PATCH 45/56] Use constants instead of magic numbers

---
 overlay_interface.c |  23 ++++----
 radio_link.c        | 127 +++++++++++++++++++++++---------------------
 radio_link.h        |   3 ++
 3 files changed, 83 insertions(+), 70 deletions(-)

diff --git a/overlay_interface.c b/overlay_interface.c
index 4eedac0f..0dbdff19 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -743,26 +743,31 @@ static void write_stream_buffer(overlay_interface *interface){
   while (interface->tx_bytes_pending>0 || interface->tx_packet || interface->next_heartbeat <= now) {
     
     if (interface->tx_bytes_pending==0){
+      // allocate tx buffer on first use
+      if (!interface->txbuffer){
+	interface->txbuffer=emalloc(OVERLAY_INTERFACE_RX_BUFFER_SIZE);
+	if (!interface->txbuffer)
+	  break;
+      }
+      
       if (interface->next_heartbeat <= now){
-	
-	if (!interface->txbuffer){
-	  interface->txbuffer=emalloc(OVERLAY_INTERFACE_RX_BUFFER_SIZE);
-	  if (!interface->txbuffer)
-	    break;
-	}
-	
 	// Queue a hearbeat now
 	radio_link_heartbeat(interface->txbuffer,&interface->tx_bytes_pending);
 	if (config.debug.packetradio)
 	  DEBUGF("Sending heartbeat");
 	interface->next_heartbeat = now+1000;
-      }else if(interface->tx_packet && interface->remaining_space >= 256 + 8+9){
+	
+      }else if(interface->remaining_space >= LINK_MTU + HEARTBEAT_SIZE){
 	// prepare a new link layer packet in txbuffer
 	if (radio_link_encode_packet(interface))
 	  break;
-	if (interface->remaining_space - interface->tx_bytes_pending < 256 + 8+9)
+	if (interface->remaining_space - interface->tx_bytes_pending < LINK_MTU + HEARTBEAT_SIZE)
 	  interface->next_heartbeat = now;
       }
+      
+      // nothing interesting to send, just break
+      if (interface->tx_bytes_pending==0)
+	break;
     }
     
     if (interface->next_tx_allowed > now)
diff --git a/radio_link.c b/radio_link.c
index 50a97fe7..90734b15 100644
--- a/radio_link.c
+++ b/radio_link.c
@@ -48,6 +48,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "conf.h"
 #include "overlay_buffer.h"
 #include "golay.h"
+#include "radio_link.h"
 
 #define MAVLINK_MSG_ID_RADIO 166
 #define MAVLINK_MSG_ID_DATASTREAM 67
@@ -60,46 +61,46 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   we use a hand-crafted MAVLink packet based on the following
   message definition
   
-  <message name="RADIO" id="166">
-  <description>Status generated by radio</description>
-  <field type="uint8_t" name="rssi">local signal strength</field>
-  <field type="uint8_t" name="remrssi">remote signal strength</field>
-  <field type="uint8_t" name="txbuf">percentage free space in transmit buffer</field>
-  <field type="uint8_t" name="noise">background noise level</field>
-  <field type="uint8_t" name="remnoise">remote background noise level</field>
-  <field type="uint16_t" name="rxerrors">receive errors</field>
-  <field type="uint16_t" name="fixed">count of error corrected packets</field>
-  </message>
-*/
-struct mavlink_RADIO_v09 {
-  uint8_t rssi;
-  uint8_t remrssi;
-  uint8_t txbuf;
-  uint8_t noise;
-  uint8_t remnoise;
-  uint16_t rxerrors;
-  uint16_t fixed;
-};
 struct mavlink_RADIO_v10 {
-  uint16_t rxerrors;
-  uint16_t fixed;
-  uint8_t rssi;
-  uint8_t remrssi;
-  uint8_t txbuf;
-  uint8_t noise;
-  uint8_t remnoise;
+  uint16_t rxerrors;  // receive errors
+  uint16_t fixed;     // count of error corrected packets
+  uint8_t rssi;       // local signal strength
+  uint8_t remrssi;    // remote signal strength
+  uint8_t txbuf;      // percentage free space in transmit buffer
+  uint8_t noise;      // background noise level
+  uint8_t remnoise;   // remote background noise level
 };
 
+*/
+
+#define FEC_LENGTH 32
+#define FEC_BYTES 223
+#define RADIO_HEADER_LENGTH 6
+#define RADIO_CRC_LENGTH 2
+
+#define LINK_PAYLOAD_MTU (LINK_MTU - FEC_LENGTH - RADIO_HEADER_LENGTH - RADIO_CRC_LENGTH)
+
 struct radio_link_state{
+  // next seq for transmission
   int tx_seq;
+  // decoded length of next link layer packet
   int payload_length;
+  // last rx seq for reassembly
   int seq;
+  // offset within payload that we have found a valid looking header
   int payload_start;
+  // offset after payload_start for incoming bytes
   int payload_offset;
-  uint8_t payload[1024];
-  int packet_length;
+  
+  // small buffer for parsing incoming bytes from the serial interface, 
+  // looking for recoverable link layer packets
+  uint8_t payload[LINK_MTU*3];
+  
+  // small buffer for assembling mdp payloads.
+  // should be large enough to hold MDP_MTU
   uint8_t dst[OVERLAY_INTERFACE_RX_BUFFER_SIZE];
-  int dst_offset;
+  // length of recovered packet
+  int packet_length;
 };
 
 /*
@@ -136,30 +137,34 @@ int radio_link_init(struct overlay_interface *interface)
   return 0;
 }
 
+// write a new link layer packet to interface->txbuffer
+// consuming more bytes from the next interface->tx_packet if required
 int radio_link_encode_packet(struct overlay_interface *interface)
 {
+  // if we have nothing interesting left to send, don't create a packet at all
+  if (!interface->tx_packet)
+    return 0;
+    
   int count = ob_remaining(interface->tx_packet);
   int startP = (ob_position(interface->tx_packet) == 0);
   int endP = 1;
-  if (count+6+32 > 255){
-    count = 255-6-32;
+  if (count > LINK_PAYLOAD_MTU){
+    count = LINK_PAYLOAD_MTU;
     endP = 0;
   }
-  interface->txbuffer[0]=0xfe; // mavlink v1.0 frame
-  /* payload len, excluding 6 byte header and 2 byte CRC.
-     But we use a 4-byte CRC, so need to add two to count to make packet lengths
-     be as expected.
-     Note that this construction will result in CRC errors by non-servald
-     programmes, which is probably more helpful than otherwise.
-  */
-  // we need 32 bytes for the parity, but this field assumes
-  // that there is a 2 byte CRC, so we can save two bytes
-  int len = count+32 - 2;
-  interface->txbuffer[1]=len;
+  
+  interface->txbuffer[0]=0xfe; // mavlink v1.0 magic header
+  
+  // we need to add FEC_LENGTH for FEC, but the length field doesn't include the expected headers or CRC
+  int len = count + FEC_LENGTH - RADIO_CRC_LENGTH;
+  interface->txbuffer[1]=len; // mavlink payload length
   interface->txbuffer[2]=(len & 0xF);
   interface->txbuffer[3]=0;
+  
+  // add golay encoding so that decoding the actual length is more reliable
   golay_encode(&interface->txbuffer[1]);
   
+  
   interface->txbuffer[4]=(interface->radio_link_state->tx_seq++) & 0x3f;
   if (startP) interface->txbuffer[4]|=0x40;
   if (endP) interface->txbuffer[4]|=0x80;
@@ -167,8 +172,8 @@ int radio_link_encode_packet(struct overlay_interface *interface)
   
   ob_get_bytes(interface->tx_packet, &interface->txbuffer[6], count);
   
-  encode_rs_8(&interface->txbuffer[4], &interface->txbuffer[6+count], 223 - (count+2));
-  interface->tx_bytes_pending=len + 8;
+  encode_rs_8(&interface->txbuffer[4], &interface->txbuffer[6+count], FEC_BYTES - (count+2));
+  interface->tx_bytes_pending=len + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
   if (endP){
     ob_free(interface->tx_packet);
     interface->tx_packet=NULL;
@@ -177,27 +182,28 @@ int radio_link_encode_packet(struct overlay_interface *interface)
   return 0;
 }
 
-int radio_link_heartbeat(unsigned char *frame,int *outlen)
+int radio_link_heartbeat(unsigned char *frame, int *outlen)
 {
   int count=9;
-  bzero(frame, count+8);
+  bzero(frame, count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH);
   
   frame[0]=0xfe; // mavlink v1.0 frame
   // Must be 9 to indicate heartbeat
   frame[1]=count; // payload len, excluding 6 byte header and 2 byte CRC
   frame[2]=(count & 0xF); // packet sequence
   frame[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC)
+  // we're golay encoding the length to improve the probability of skipping it correctly
   golay_encode(&frame[1]);
   frame[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE)
   // Must be zero to indicate heartbeat
   frame[5]=0; // message ID type of this frame: DATA_STREAM
 
-  // extra magic number to detect remote heartbeat requests
+  // extra magic number to help correctly detect remote heartbeat requests
   frame[14]=0x55;
   frame[15]=0x05;
   golay_encode(&frame[14]);
   
-  *outlen=count+8;
+  *outlen=count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
   
   return 0;
 }
@@ -245,13 +251,13 @@ static int radio_link_parse(struct overlay_interface *interface, struct radio_li
     return 0;
   }
   
-  int data_bytes = packet_length - (32 - 2);
+  int data_bytes = packet_length - (FEC_LENGTH - RADIO_CRC_LENGTH);
   // preserve the last 16 bytes of data
-  unsigned char old_footer[32];
-  unsigned char *payload_footer=&payload[packet_length+8-sizeof(old_footer)];
+  unsigned char old_footer[FEC_LENGTH];
+  unsigned char *payload_footer=&payload[packet_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH - sizeof(old_footer)];
   bcopy(payload_footer, old_footer, sizeof(old_footer));
   
-  int pad=223 - (data_bytes + 2);
+  int pad=FEC_BYTES - (data_bytes + RADIO_CRC_LENGTH);
   int errors=decode_rs_8(&payload[4], NULL, 0, pad);
   if (errors==-1){
     if (config.debug.radio_link)
@@ -291,13 +297,12 @@ static int radio_link_parse(struct overlay_interface *interface, struct radio_li
     return 1;
   }
   
-  bcopy(&payload[6], &state->dst[state->packet_length], data_bytes);
+  bcopy(&payload[RADIO_HEADER_LENGTH], &state->dst[state->packet_length], data_bytes);
   state->packet_length+=data_bytes;
     
   if (payload[4]&0x80) {
     if (config.debug.radio_link) 
       DEBUGF("PDU Complete (length=%d)",state->packet_length);
-    state->dst_offset=0;
     
     packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL, 0);
     state->packet_length=sizeof(state->dst)+1;
@@ -313,7 +318,7 @@ static int decode_length(struct radio_link_state *state, unsigned char *p)
   if (length<0 || ((length >>8) & 0xF) != (length&0xF))
     return -1;
   length=length&0xFF;
-  if (length!=9 && (length<31 || length+8>255))
+  if (length!=9 && (length <= FEC_LENGTH - RADIO_CRC_LENGTH || length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH > LINK_MTU))
     return -1;
   
   if (config.debug.radio_link && (errs || state->payload_length!=*p))
@@ -361,15 +366,15 @@ int radio_link_decode(struct overlay_interface *interface, uint8_t c)
     }
     
     // wait for a whole packet
-    if (!state->payload_length || state->payload_offset < state->payload_length+8)
+    if (!state->payload_length || state->payload_offset < state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH)
       return 0;
     
     if (parse_heartbeat(interface, p)){
       // cut the bytes of the heartbeat out of the buffer
-      state->payload_offset -= state->payload_length+8;
+      state->payload_offset -= state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH;
       if (state->payload_offset){
 	// shuffle bytes backwards
-	bcopy(&p[state->payload_length+8], p, state->payload_offset);
+	bcopy(&p[state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH], p, state->payload_offset);
       }
       // restart parsing for a valid header from the beginning of out buffer
       state->payload_offset+=state->payload_start;
@@ -389,10 +394,10 @@ int radio_link_decode(struct overlay_interface *interface, uint8_t c)
       
       // If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet, 
       // but we may need to examine the last few bytes to find the start of the next packet.
-      state->payload_offset -= state->payload_length+8-backtrack;
+      state->payload_offset -= state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH - backtrack;
       if (state->payload_offset){
 	// shuffle all remaining bytes back to the start of the buffer
-	bcopy(&state->payload[state->payload_start + state->payload_length+8-backtrack], 
+	bcopy(&state->payload[state->payload_start + state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH - backtrack], 
 	  state->payload, state->payload_offset);
       }
       state->payload_start=0;
diff --git a/radio_link.h b/radio_link.h
index 9a3060b3..6651623f 100644
--- a/radio_link.h
+++ b/radio_link.h
@@ -1,6 +1,9 @@
 #ifndef __SERVALD_RADIO_LINK_H
 #define __SERVALD_RADIO_LINK_H
 
+#define HEARTBEAT_SIZE (8+9)
+#define LINK_MTU 255
+
 int radio_link_free(struct overlay_interface *interface);
 int radio_link_init(struct overlay_interface *interface);
 int radio_link_decode(struct overlay_interface *interface, uint8_t c);

From d45470ce81b170d53496932eb7570eacf4b457fc Mon Sep 17 00:00:00 2001
From: Jeremy Lakeman <Jeremy.Lakeman@gmail.com>
Date: Mon, 25 Nov 2013 15:05:33 +1030
Subject: [PATCH 46/56] Refactor radio link for better encapsulation

---
 conf_schema.h       |   2 -
 overlay_interface.c | 156 ++----------------------
 overlay_queue.c     |   6 +-
 radio_link.c        | 288 ++++++++++++++++++++++++++++++++------------
 radio_link.h        |   6 +-
 serval.h            |  13 --
 6 files changed, 228 insertions(+), 243 deletions(-)

diff --git a/conf_schema.h b/conf_schema.h
index fad38f39..bab3c5fd 100644
--- a/conf_schema.h
+++ b/conf_schema.h
@@ -476,8 +476,6 @@ ATOM(bool_t,                debug,           0, boolean,, "If true, log details
 ATOM(bool_t,                point_to_point,  0, boolean,, "If true, assume there will only be two devices on this interface")
 ATOM(bool_t,                ctsrts,          0, boolean,, "If true, enable CTS/RTS hardware handshaking")
 ATOM(int32_t,               uartbps,         57600, int32_rs232baudrate,, "Speed of serial UART link speed (which may be different to serial device link speed)")
-ATOM(int32_t,               throttle,        0, int32_nonneg,, "Limit transmit speed of serial interface (bytes per second)") 
-ATOM(int32_t,               burst_size,      0, int32_nonneg,, "Write no more than this many bytes at a time to a serial interface") 
 END_STRUCT
 
 ARRAY(interface_list, NO_DUPLICATES)
diff --git a/overlay_interface.c b/overlay_interface.c
index 0dbdff19..48e96c91 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -49,7 +49,6 @@ struct profile_total sock_any_stats;
 
 static void overlay_interface_poll(struct sched_ent *alarm);
 static int re_init_socket(int interface_index);
-static void write_stream_buffer(overlay_interface *interface);
 
 static void
 overlay_interface_close(overlay_interface *interface){
@@ -59,10 +58,6 @@ overlay_interface_close(overlay_interface *interface){
   unschedule(&interface->alarm);
   unwatch(&interface->alarm);
   close(interface->alarm.poll.fd);
-  if (interface->txbuffer){
-    free(interface->txbuffer);
-    interface->txbuffer=NULL;
-  }
   if (interface->radio_link_state)
     radio_link_free(interface);
   interface->alarm.poll.fd=-1;
@@ -82,8 +77,7 @@ void interface_state_html(struct strbuf *b, struct overlay_interface *interface)
   switch(interface->type){
     case OVERLAY_INTERFACE_PACKETRADIO:
       strbuf_puts(b, "Type: Packet Radio<br>");
-      strbuf_sprintf(b, "RSSI: %ddB<br>",interface->radio_rssi);
-      strbuf_sprintf(b, "Remote RSSI: %ddB<br>",interface->remote_rssi);
+      radio_link_state_html(b, interface);
       break;
     case OVERLAY_INTERFACE_ETHERNET:
       strbuf_puts(b, "Type: Ethernet<br>");
@@ -417,8 +411,6 @@ overlay_interface_init(const char *name, struct in_addr src_addr, struct in_addr
   set_destination_ref(&interface->destination, NULL);
   interface->destination = new_destination(interface, ifconfig->encapsulation);
   
-  interface->throttle_bytes_per_second = ifconfig->throttle;
-  interface->throttle_burst_write_size = ifconfig->burst_size;
   /* Pick a reasonable default MTU.
      This will ultimately get tuned by the bandwidth and other properties of the interface */
   interface->mtu = 1200;
@@ -722,8 +714,6 @@ static void interface_read_stream(struct overlay_interface *interface){
     return;
   }
   
-  if (config.debug.packetradio)
-    dump("read bytes", buffer, nread);
   
   int i;
   for (i=0;i<nread;i++)
@@ -732,104 +722,6 @@ static void interface_read_stream(struct overlay_interface *interface){
   OUT();
 }
 
-static void write_stream_buffer(overlay_interface *interface){
-  time_ms_t now = gettime_ms();
-  
-  // Throttle output to a prescribed bit-rate
-  // first, reduce the number of bytes based on the configured burst size
-  int bytes_allowed=interface->throttle_burst_write_size;
-    
-  int total_written=0;
-  while (interface->tx_bytes_pending>0 || interface->tx_packet || interface->next_heartbeat <= now) {
-    
-    if (interface->tx_bytes_pending==0){
-      // allocate tx buffer on first use
-      if (!interface->txbuffer){
-	interface->txbuffer=emalloc(OVERLAY_INTERFACE_RX_BUFFER_SIZE);
-	if (!interface->txbuffer)
-	  break;
-      }
-      
-      if (interface->next_heartbeat <= now){
-	// Queue a hearbeat now
-	radio_link_heartbeat(interface->txbuffer,&interface->tx_bytes_pending);
-	if (config.debug.packetradio)
-	  DEBUGF("Sending heartbeat");
-	interface->next_heartbeat = now+1000;
-	
-      }else if(interface->remaining_space >= LINK_MTU + HEARTBEAT_SIZE){
-	// prepare a new link layer packet in txbuffer
-	if (radio_link_encode_packet(interface))
-	  break;
-	if (interface->remaining_space - interface->tx_bytes_pending < LINK_MTU + HEARTBEAT_SIZE)
-	  interface->next_heartbeat = now;
-      }
-      
-      // nothing interesting to send, just break
-      if (interface->tx_bytes_pending==0)
-	break;
-    }
-    
-    if (interface->next_tx_allowed > now)
-      break;
-    
-    int bytes = interface->tx_bytes_pending;
-    if (interface->throttle_burst_write_size && bytes>bytes_allowed)
-      bytes=bytes_allowed;
-    if (bytes<=0)
-      break;
-    
-    int written=write(interface->alarm.poll.fd, interface->txbuffer, bytes);
-    if (written<=0){
-      DEBUGF("Blocking for POLLOUT");
-      break;
-    }
-    
-    interface->remaining_space-=written;
-    interface->tx_bytes_pending-=written;
-    total_written+=written;
-    bytes_allowed-=written;
-    if (interface->tx_bytes_pending){
-      bcopy(&interface->txbuffer[written],&interface->txbuffer[0],
-	    interface->tx_bytes_pending);
-      DEBUGF("Partial write, %d left", interface->tx_bytes_pending);
-    }
-  }
-  
-  if (total_written>0){
-    // Now when are we allowed to send more?
-    int rate = interface->throttle_bytes_per_second;
-    if (interface->remaining_space<=0)
-      rate = 600;
-    if (rate){
-      int delay = total_written*1000/rate;
-      if (config.debug.throttling)
-	DEBUGF("Throttling for %dms (%d).", delay, interface->remaining_space);
-      interface->next_tx_allowed = now + delay;
-    }
-  }
-  
-  time_ms_t next_write = interface->next_tx_allowed;
-  if (interface->tx_bytes_pending<=0){
-    next_write = interface->next_heartbeat;
-  }
-  
-  if (interface->alarm.alarm==-1 || next_write < interface->alarm.alarm){
-    interface->alarm.alarm = next_write;
-    interface->alarm.deadline = interface->alarm.alarm+10;
-  }
-  
-  if (interface->tx_bytes_pending>0 && next_write <= now){
-    // more to write, so set the POLLOUT flag
-    interface->alarm.poll.events|=POLLOUT;
-  } else {
-    // Nothing to write, so clear POLLOUT flag
-    interface->alarm.poll.events&=~POLLOUT;
-  }
-  watch(&interface->alarm);
-}
-
-
 static void overlay_interface_poll(struct sched_ent *alarm)
 {
   struct overlay_interface *interface = (overlay_interface *)alarm;
@@ -841,7 +733,7 @@ static void overlay_interface_poll(struct sched_ent *alarm)
     if (interface->state==INTERFACE_STATE_UP 
       && interface->destination->tick_ms>0
       && interface->send_broadcasts
-      && !interface->tx_packet){
+      && !radio_link_is_busy(interface)){
       
       if (now >= interface->destination->last_tx+interface->destination->tick_ms)
         overlay_send_tick_packet(interface->destination);
@@ -852,8 +744,8 @@ static void overlay_interface_poll(struct sched_ent *alarm)
     
     switch(interface->socket_type){
       case SOCK_STREAM:
-	write_stream_buffer(interface);
-	break;
+	radio_link_tx(interface);
+	return;
       case SOCK_DGRAM:
 	break;
       case SOCK_FILE:
@@ -873,14 +765,8 @@ static void overlay_interface_poll(struct sched_ent *alarm)
   if (alarm->poll.revents & POLLOUT){
     switch(interface->socket_type){
       case SOCK_STREAM:
-	write_stream_buffer(interface);
-	if (alarm->alarm!=-1 && interface->state==INTERFACE_STATE_UP) {
-	  if (alarm->alarm < now)
-	    alarm->alarm = now;
-	  unschedule(alarm);
-	  schedule(alarm);
-	}
-	break;
+	radio_link_tx(interface);
+	return;
       case SOCK_DGRAM:
       case SOCK_FILE:
 	//XXX error? fatal?
@@ -896,14 +782,9 @@ static void overlay_interface_poll(struct sched_ent *alarm)
       case SOCK_STREAM:
 	interface_read_stream(interface);
 	// if we read a valid heartbeat packet, we may be able to write more bytes now.
-	if (interface->state==INTERFACE_STATE_UP && interface->remaining_space>0){
-	  write_stream_buffer(interface);
-	  if (alarm->alarm!=-1 && interface->state==INTERFACE_STATE_UP) {
-	    if (alarm->alarm < now)
-	      alarm->alarm = now;
-	    unschedule(alarm);
-	    schedule(alarm);
-	  }
+	if (interface->state==INTERFACE_STATE_UP){
+	  radio_link_tx(interface);
+	  return;
 	}
 	break;
       case SOCK_FILE:
@@ -943,24 +824,7 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
   
   switch(interface->socket_type){
     case SOCK_STREAM:
-    {
-      if (interface->tx_packet){
-	ob_free(buffer);
-	return WHYF("Cannot send two packets to a stream at the same time");
-      }
-      
-      // prepare the buffer for reading
-      ob_flip(buffer);
-      interface->tx_packet = buffer;
-      write_stream_buffer(interface);
-      
-      if (interface->alarm.alarm!=-1){
-	unschedule(&interface->alarm);
-	schedule(&interface->alarm);
-      }
-      
-      return 0;
-    }
+      return radio_link_queue_packet(interface, buffer);
       
     case SOCK_FILE:
     {
diff --git a/overlay_queue.c b/overlay_queue.c
index 4ba12c9f..b239b79a 100644
--- a/overlay_queue.c
+++ b/overlay_queue.c
@@ -22,6 +22,7 @@
 #include "conf.h"
 #include "overlay_buffer.h"
 #include "overlay_packet.h"
+#include "radio_link.h"
 #include "str.h"
 #include "strbuf.h"
 
@@ -290,7 +291,7 @@ overlay_calc_queue_time(overlay_txqueue *queue, struct overlay_frame *frame){
     int i;
     for(i=0;i<frame->destination_count;i++)
     {
-      if (frame->destinations[i].destination->interface->tx_packet)
+      if (radio_link_is_busy(frame->destinations[i].destination->interface))
 	continue;
       time_ms_t next_packet = limit_next_allowed(&frame->destinations[i].destination->transfer_limit);
       if (frame->destinations[i].transmit_time){
@@ -403,8 +404,7 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
 	  }
 	}else{
 	  // skip this interface if the stream tx buffer has data
-	  if (dest->interface->socket_type==SOCK_STREAM 
-	    && dest->interface->tx_packet)
+	  if (radio_link_is_busy(dest->interface))
 	    continue;
 	    
 	  // can we send a packet on this interface now?
diff --git a/radio_link.c b/radio_link.c
index 90734b15..628a4bf9 100644
--- a/radio_link.c
+++ b/radio_link.c
@@ -74,8 +74,9 @@ struct mavlink_RADIO_v10 {
 */
 
 #define FEC_LENGTH 32
-#define FEC_BYTES 223
+#define FEC_MAX_BYTES 223
 #define RADIO_HEADER_LENGTH 6
+#define RADIO_USED_HEADER_LENGTH 4
 #define RADIO_CRC_LENGTH 2
 
 #define LINK_PAYLOAD_MTU (LINK_MTU - FEC_LENGTH - RADIO_HEADER_LENGTH - RADIO_CRC_LENGTH)
@@ -83,7 +84,15 @@ struct mavlink_RADIO_v10 {
 struct radio_link_state{
   // next seq for transmission
   int tx_seq;
+
+  // small buffer for parsing incoming bytes from the serial interface, 
+  // looking for recoverable link layer packets
+  // should be large enough to hold at least one packet from the remote end
+  // plus one heartbeat packet from the local firmware
+  uint8_t payload[LINK_MTU*3];
+  
   // decoded length of next link layer packet
+  // including all header and footer bytes
   int payload_length;
   // last rx seq for reassembly
   int seq;
@@ -92,15 +101,31 @@ struct radio_link_state{
   // offset after payload_start for incoming bytes
   int payload_offset;
   
-  // small buffer for parsing incoming bytes from the serial interface, 
-  // looking for recoverable link layer packets
-  uint8_t payload[LINK_MTU*3];
-  
   // small buffer for assembling mdp payloads.
-  // should be large enough to hold MDP_MTU
-  uint8_t dst[OVERLAY_INTERFACE_RX_BUFFER_SIZE];
+  uint8_t dst[MDP_MTU];
   // length of recovered packet
   int packet_length;
+  
+  // next firmware heartbeat
+  time_ms_t next_heartbeat;
+  
+  time_ms_t last_packet;
+  
+  // parsed rssi
+  int radio_rssi;
+  int remote_rssi;
+  // estimated firmware buffer space
+  int32_t remaining_space;
+  
+  // next serial write
+  uint64_t next_tx_allowed;
+  // partially sent packet
+  struct overlay_buffer *tx_packet;
+  
+  // serial write buffer
+  uint8_t txbuffer[LINK_MTU];
+  int tx_bytes;
+  int tx_pos;
 };
 
 /*
@@ -126,8 +151,10 @@ int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad);
 
 int radio_link_free(struct overlay_interface *interface)
 {
-  free(interface->radio_link_state);
-  interface->radio_link_state=NULL;
+  if (interface->radio_link_state){
+    free(interface->radio_link_state);
+    interface->radio_link_state=NULL;
+  }
   return 0;
 }
 
@@ -137,78 +164,183 @@ int radio_link_init(struct overlay_interface *interface)
   return 0;
 }
 
+void radio_link_state_html(struct strbuf *b, struct overlay_interface *interface)
+{
+  struct radio_link_state *state = interface->radio_link_state;
+  strbuf_sprintf(b, "RSSI: %ddB<br>", state->radio_rssi);
+  strbuf_sprintf(b, "Remote RSSI: %ddB<br>", state->remote_rssi);
+}
+
 // write a new link layer packet to interface->txbuffer
 // consuming more bytes from the next interface->tx_packet if required
-int radio_link_encode_packet(struct overlay_interface *interface)
+static int radio_link_encode_packet(struct radio_link_state *link_state)
 {
   // if we have nothing interesting left to send, don't create a packet at all
-  if (!interface->tx_packet)
+  if (!link_state->tx_packet)
     return 0;
     
-  int count = ob_remaining(interface->tx_packet);
-  int startP = (ob_position(interface->tx_packet) == 0);
+  int count = ob_remaining(link_state->tx_packet);
+  int startP = (ob_position(link_state->tx_packet) == 0);
   int endP = 1;
   if (count > LINK_PAYLOAD_MTU){
     count = LINK_PAYLOAD_MTU;
     endP = 0;
   }
   
-  interface->txbuffer[0]=0xfe; // mavlink v1.0 magic header
+  link_state->txbuffer[0]=0xfe; // mavlink v1.0 magic header
   
   // we need to add FEC_LENGTH for FEC, but the length field doesn't include the expected headers or CRC
   int len = count + FEC_LENGTH - RADIO_CRC_LENGTH;
-  interface->txbuffer[1]=len; // mavlink payload length
-  interface->txbuffer[2]=(len & 0xF);
-  interface->txbuffer[3]=0;
+  link_state->txbuffer[1]=len; // mavlink payload length
+  link_state->txbuffer[2]=(len & 0xF);
+  link_state->txbuffer[3]=0;
   
   // add golay encoding so that decoding the actual length is more reliable
-  golay_encode(&interface->txbuffer[1]);
+  golay_encode(&link_state->txbuffer[1]);
   
   
-  interface->txbuffer[4]=(interface->radio_link_state->tx_seq++) & 0x3f;
-  if (startP) interface->txbuffer[4]|=0x40;
-  if (endP) interface->txbuffer[4]|=0x80;
-  interface->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM;
+  link_state->txbuffer[4]=(link_state->tx_seq++) & 0x3f;
+  if (startP) link_state->txbuffer[4]|=0x40;
+  if (endP) link_state->txbuffer[4]|=0x80;
+  link_state->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM;
   
-  ob_get_bytes(interface->tx_packet, &interface->txbuffer[6], count);
+  ob_get_bytes(link_state->tx_packet, &link_state->txbuffer[6], count);
   
-  encode_rs_8(&interface->txbuffer[4], &interface->txbuffer[6+count], FEC_BYTES - (count+2));
-  interface->tx_bytes_pending=len + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
+  encode_rs_8(&link_state->txbuffer[4], &link_state->txbuffer[6+count], FEC_MAX_BYTES - (count+2));
+  link_state->tx_bytes=len + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
   if (endP){
-    ob_free(interface->tx_packet);
-    interface->tx_packet=NULL;
+    ob_free(link_state->tx_packet);
+    link_state->tx_packet=NULL;
     overlay_queue_schedule_next(gettime_ms());
   }
   return 0;
 }
 
-int radio_link_heartbeat(unsigned char *frame, int *outlen)
+int radio_link_is_busy(struct overlay_interface *interface)
 {
-  int count=9;
-  bzero(frame, count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH);
-  
-  frame[0]=0xfe; // mavlink v1.0 frame
-  // Must be 9 to indicate heartbeat
-  frame[1]=count; // payload len, excluding 6 byte header and 2 byte CRC
-  frame[2]=(count & 0xF); // packet sequence
-  frame[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC)
-  // we're golay encoding the length to improve the probability of skipping it correctly
-  golay_encode(&frame[1]);
-  frame[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE)
-  // Must be zero to indicate heartbeat
-  frame[5]=0; // message ID type of this frame: DATA_STREAM
+  if (interface->radio_link_state && interface->radio_link_state->tx_packet)
+    return 1;
+  return 0;
+}
 
-  // extra magic number to help correctly detect remote heartbeat requests
-  frame[14]=0x55;
-  frame[15]=0x05;
-  golay_encode(&frame[14]);
+int radio_link_queue_packet(struct overlay_interface *interface, struct overlay_buffer *buffer)
+{
+  struct radio_link_state *link_state = interface->radio_link_state;
   
-  *outlen=count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
+  if (link_state->tx_packet){
+    ob_free(buffer);
+    return WHYF("Cannot send two packets to a stream at the same time");
+  }
+
+  // prepare the buffer for reading
+  ob_flip(buffer);
+  link_state->tx_packet = buffer;
+  radio_link_tx(interface);
   
   return 0;
 }
 
-static int parse_heartbeat(struct overlay_interface *interface, const unsigned char *payload)
+static int build_heartbeat(struct radio_link_state *link_state)
+{
+  int count=9;
+  bzero(link_state->txbuffer, count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH);
+  
+  link_state->txbuffer[0]=0xfe; // mavlink v1.0 link_state->txbuffer
+  // Must be 9 to indicate heartbeat
+  link_state->txbuffer[1]=count; // payload len, excluding 6 byte header and 2 byte CRC
+  link_state->txbuffer[2]=(count & 0xF); // packet sequence
+  link_state->txbuffer[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC)
+  // we're golay encoding the length to improve the probability of skipping it correctly
+  golay_encode(&link_state->txbuffer[1]);
+  link_state->txbuffer[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE)
+  // Must be zero to indicate heartbeat
+  link_state->txbuffer[5]=0; // message ID type of this link_state->txbuffer: DATA_STREAM
+
+  // extra magic number to help correctly detect remote heartbeat requests
+  link_state->txbuffer[14]=0x55;
+  link_state->txbuffer[15]=0x05;
+  golay_encode(&link_state->txbuffer[14]);
+  link_state->tx_bytes = count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
+  if (config.debug.radio_link)
+    DEBUGF("Produced heartbeat");
+  
+  return 0;
+}
+
+// write a new link layer packet to interface->txbuffer
+// consuming more bytes from the next interface->tx_packet if required
+int radio_link_tx(struct overlay_interface *interface)
+{
+  struct radio_link_state *link_state = interface->radio_link_state;
+  
+  unschedule(&interface->alarm);
+  interface->alarm.alarm = 0;
+  time_ms_t next_tick = interface->destination->last_tx+interface->destination->tick_ms;
+  time_ms_t now = gettime_ms();
+  
+  while(1){
+    
+    if (link_state->tx_bytes){
+      if (link_state->next_tx_allowed > now){
+	interface->alarm.alarm = link_state->next_tx_allowed;
+	break;
+      }
+      
+      int written=write(interface->alarm.poll.fd, &link_state->txbuffer[link_state->tx_pos], link_state->tx_bytes);
+      if (written<=0){
+	interface->alarm.poll.events|=POLLOUT;
+	break;
+      }
+      link_state->remaining_space-=written;
+      link_state->tx_bytes-=written;
+      if (link_state->tx_bytes)
+	link_state->tx_pos+=written;
+      else
+	link_state->tx_pos=0;
+      continue;
+    }
+  
+    interface->alarm.poll.events&=~POLLOUT;
+    
+    if (link_state->next_heartbeat<=now){
+      build_heartbeat(link_state);
+      link_state->next_heartbeat = now + 1000;
+      continue;
+    }
+    
+    // out of space? Don't bother to send anything interesting 
+    // until we hear the next heartbeat response
+    if (link_state->remaining_space < LINK_MTU + HEARTBEAT_SIZE){
+      interface->alarm.alarm = link_state->next_heartbeat;
+      break;
+    }
+    
+    if (link_state->remaining_space < LINK_MTU + HEARTBEAT_SIZE)
+      link_state->next_heartbeat = now;
+    
+    if (!link_state->tx_packet){
+      // finished current packet, wait for more.
+      interface->alarm.alarm = next_tick;
+      break;
+    }
+    
+    // encode another packet fragment
+    radio_link_encode_packet(link_state);
+    link_state->last_packet = now;
+  }
+  
+  watch(&interface->alarm);
+  if (interface->alarm.alarm<now)
+    interface->alarm.alarm=now;
+  if (interface->alarm.alarm){
+    interface->alarm.deadline = interface->alarm.alarm+100;
+    schedule(&interface->alarm);
+  }
+  
+  return 0;
+}
+
+static int parse_heartbeat(struct radio_link_state *state, const unsigned char *payload)
 {
   if (payload[0]==0xFE 
     && payload[1]==9
@@ -217,59 +349,56 @@ static int parse_heartbeat(struct overlay_interface *interface, const unsigned c
     && payload[5]==MAVLINK_MSG_ID_RADIO){
     
     // we can assume that radio status packets arrive without corruption
-    interface->radio_rssi=(1.0*payload[10]-payload[13])/1.9;
-    interface->remote_rssi=(1.0*payload[11] - payload[14])/1.9;
+    state->radio_rssi=(1.0*payload[10]-payload[13])/1.9;
+    state->remote_rssi=(1.0*payload[11] - payload[14])/1.9;
     int free_space = payload[12];
     int free_bytes = (free_space * 1280) / 100 - 30;
-    interface->remaining_space = free_bytes;
+    state->remaining_space = free_bytes;
     if (free_bytes>0)
-      interface->next_tx_allowed = gettime_ms();
+      state->next_tx_allowed = gettime_ms();
     if (free_bytes>720)
-      interface->next_heartbeat=gettime_ms()+1000;
-    if (config.debug.packetradio) {
+      state->next_heartbeat=gettime_ms()+1000;
+    if (config.debug.packetradio)
       INFOF("Link budget = %+ddB, remote link budget = %+ddB, buffer space = %d%% (approx %d)",
-	    interface->radio_rssi,
-	    interface->remote_rssi,
+	    state->radio_rssi,
+	    state->remote_rssi,
 	    free_space, free_bytes);
-    }
     return 1;
   }
   return 0;
 }
 
 static int radio_link_parse(struct overlay_interface *interface, struct radio_link_state *state, 
-  int packet_length, unsigned char *payload, int *backtrack)
+  size_t packet_length, uint8_t *payload, int *backtrack)
 {
   *backtrack=0;
-  if (packet_length==9){
-    // make sure we've heard the start and end of a remote heartbeat request
+  if (packet_length==17){
+    // if we've heard the start and end of a remote heartbeat request
+    // we can skip it without checking anything else
     int errs=0;
     int tail = golay_decode(&errs, &payload[14]);
     if (tail == 0x555){
+      if (config.debug.radio_link)
+	DEBUGF("Decoded remote heartbeat request");
       return 1;
     }
     return 0;
   }
   
-  int data_bytes = packet_length - (FEC_LENGTH - RADIO_CRC_LENGTH);
-  // preserve the last 16 bytes of data
-  unsigned char old_footer[FEC_LENGTH];
-  unsigned char *payload_footer=&payload[packet_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH - sizeof(old_footer)];
-  bcopy(payload_footer, old_footer, sizeof(old_footer));
+  size_t data_bytes = packet_length - (RADIO_USED_HEADER_LENGTH + FEC_LENGTH);
   
-  int pad=FEC_BYTES - (data_bytes + RADIO_CRC_LENGTH);
-  int errors=decode_rs_8(&payload[4], NULL, 0, pad);
+  int errors=decode_rs_8(&payload[4], NULL, 0, FEC_MAX_BYTES - data_bytes);
   if (errors==-1){
     if (config.debug.radio_link)
       DEBUGF("Reed-Solomon error correction failed");
     return 0;
   }
   *backtrack=errors;
-  
+  data_bytes -= 2;
   int seq=payload[4]&0x3f;
   
   if (config.debug.radio_link){
-    DEBUGF("Received RS protected message, len: %d, errors: %d, seq: %d, flags:%s%s", 
+    DEBUGF("Received RS protected message, len: %zd, errors: %d, seq: %d, flags:%s%s", 
       data_bytes,
       errors,
       seq,
@@ -318,7 +447,9 @@ static int decode_length(struct radio_link_state *state, unsigned char *p)
   if (length<0 || ((length >>8) & 0xF) != (length&0xF))
     return -1;
   length=length&0xFF;
-  if (length!=9 && (length <= FEC_LENGTH - RADIO_CRC_LENGTH || length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH > LINK_MTU))
+  length += RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH;
+  
+  if (length!=17 && (length <= FEC_LENGTH || length > LINK_MTU))
     return -1;
   
   if (config.debug.radio_link && (errs || state->payload_length!=*p))
@@ -328,8 +459,10 @@ static int decode_length(struct radio_link_state *state, unsigned char *p)
   return 0;
 }
 
+// add one byte at a time from the serial link, and attempt to decode packets
 int radio_link_decode(struct overlay_interface *interface, uint8_t c)
 {
+  IN();
   struct radio_link_state *state=interface->radio_link_state;
   
   if (state->payload_start + state->payload_offset >= sizeof(state->payload)){
@@ -353,7 +486,7 @@ int radio_link_decode(struct overlay_interface *interface, uint8_t c)
 	&& p[4]==RADIO_SOURCE_COMPONENT
 	&& p[5]==MAVLINK_MSG_ID_RADIO){
 	//looks like a valid heartbeat response header, read the rest and process it
-	state->payload_length=9;
+	state->payload_length=17;
 	break;
       }
       
@@ -366,15 +499,15 @@ int radio_link_decode(struct overlay_interface *interface, uint8_t c)
     }
     
     // wait for a whole packet
-    if (!state->payload_length || state->payload_offset < state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH)
-      return 0;
+    if (!state->payload_length || state->payload_offset < state->payload_length)
+      RETURN(0);
     
-    if (parse_heartbeat(interface, p)){
+    if (parse_heartbeat(state, p)){
       // cut the bytes of the heartbeat out of the buffer
-      state->payload_offset -= state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH;
+      state->payload_offset -= state->payload_length;
       if (state->payload_offset){
 	// shuffle bytes backwards
-	bcopy(&p[state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH], p, state->payload_offset);
+	bcopy(&p[state->payload_length], p, state->payload_offset);
       }
       // restart parsing for a valid header from the beginning of out buffer
       state->payload_offset+=state->payload_start;
@@ -394,10 +527,10 @@ int radio_link_decode(struct overlay_interface *interface, uint8_t c)
       
       // If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet, 
       // but we may need to examine the last few bytes to find the start of the next packet.
-      state->payload_offset -= state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH - backtrack;
+      state->payload_offset -= state->payload_length - backtrack;
       if (state->payload_offset){
 	// shuffle all remaining bytes back to the start of the buffer
-	bcopy(&state->payload[state->payload_start + state->payload_length + RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH - backtrack], 
+	bcopy(&state->payload[state->payload_start + state->payload_length - backtrack], 
 	  state->payload, state->payload_offset);
       }
       state->payload_start=0;
@@ -409,4 +542,5 @@ int radio_link_decode(struct overlay_interface *interface, uint8_t c)
     }
     state->payload_length=0;
   };
+  RETURN(0);
 }
diff --git a/radio_link.h b/radio_link.h
index 6651623f..1dd92ccf 100644
--- a/radio_link.h
+++ b/radio_link.h
@@ -7,7 +7,9 @@
 int radio_link_free(struct overlay_interface *interface);
 int radio_link_init(struct overlay_interface *interface);
 int radio_link_decode(struct overlay_interface *interface, uint8_t c);
-int radio_link_heartbeat(unsigned char *frame, int *outlen);
-int radio_link_encode_packet(struct overlay_interface *interface);
+int radio_link_tx(struct overlay_interface *interface);
+void radio_link_state_html(struct strbuf *b, struct overlay_interface *interface);
+int radio_link_is_busy(struct overlay_interface *interface);
+int radio_link_queue_packet(struct overlay_interface *interface, struct overlay_buffer *buffer);
 
 #endif
\ No newline at end of file
diff --git a/serval.h b/serval.h
index a84d5e81..d97115f2 100644
--- a/serval.h
+++ b/serval.h
@@ -344,19 +344,6 @@ typedef struct overlay_interface {
   int recv_count;
   int tx_count;
   
-  // stream socket tx state;
-  struct overlay_buffer *tx_packet;
-  uint8_t *txbuffer;
-  int tx_bytes_pending;
-  // Throttle TX rate if required (stream interfaces only for now)
-  uint32_t throttle_bytes_per_second;
-  uint32_t throttle_burst_write_size;
-  uint64_t next_tx_allowed;
-  int32_t remaining_space;
-  time_ms_t next_heartbeat;
-  int radio_rssi;
-  int remote_rssi;
-  
   struct radio_link_state *radio_link_state;
 
   // copy of ifconfig flags

From dafa1fc186f184d85f4369de2f31a4926e96db52 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Fri, 29 Nov 2013 12:56:59 +1030
Subject: [PATCH 47/56] Improve struct socket_address

Add struct sockaddr_in 'inet' union field, rename 'addr_un' union field
to 'local'

Replace recvwithttl()'s (struct sockaddr *) and socklen_t pair of args
with single (struct socket_address *) arg
---
 mdp_client.c            | 14 ++++++------
 net.c                   | 10 ++++-----
 net.h                   |  4 +++-
 overlay_interface.c     | 40 ++++++++++++++++++----------------
 overlay_mdp.c           |  4 ++--
 overlay_packetformats.c | 11 +++++-----
 radio_link.c            |  2 +-
 serval.h                |  2 +-
 socket.c                | 48 ++++++++++++++++++++---------------------
 socket.h                |  5 +++--
 10 files changed, 73 insertions(+), 67 deletions(-)

diff --git a/mdp_client.c b/mdp_client.c
index 3db4ca9f..cb0459d3 100644
--- a/mdp_client.c
+++ b/mdp_client.c
@@ -43,10 +43,10 @@ static void mdp_unlink(int mdp_sock)
   if (getsockname(mdp_sock, &addr.addr, &addr.addrlen))
     WHYF_perror("getsockname(%d)", mdp_sock);
   else if (addr.addr.sa_family==AF_UNIX 
-    && addr.addrlen > sizeof addr.addr_un.sun_family 
-    && addr.addrlen <= sizeof addr.addr_un && addr.addr_un.sun_path[0] != '\0') {
-    if (unlink(addr.addr_un.sun_path) == -1)
-      WARNF_perror("unlink(%s)", alloca_str_toprint(addr.addr_un.sun_path));
+    && addr.addrlen > sizeof addr.local.sun_family 
+    && addr.addrlen <= sizeof addr.local && addr.local.sun_path[0] != '\0') {
+    if (unlink(addr.local.sun_path) == -1)
+      WARNF_perror("unlink(%s)", alloca_str_toprint(addr.local.sun_path));
   }
   close(mdp_sock);
 }
@@ -123,7 +123,7 @@ ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_
   addr.addrlen=hdr.msg_namelen;
   // double check that the incoming address matches the servald daemon
   if (cmp_sockaddr(&addr, &mdp_addr) != 0
-      && (   addr.addr_un.sun_family != AF_UNIX
+      && (   addr.local.sun_family != AF_UNIX
 	  || real_sockaddr(&addr, &addr) <= 0
 	  || cmp_sockaddr(&addr, &mdp_addr) != 0
 	 )
@@ -259,7 +259,7 @@ int overlay_mdp_recv(int mdp_sockfd, overlay_mdp_frame *mdp, mdp_port_t port, in
   ssize_t len;
   mdp->packetTypeAndFlags = 0;
   set_nonblock(mdp_sockfd);
-  len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, &recvaddr.addr, &recvaddr.addrlen);
+  len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, &recvaddr);
   set_block(mdp_sockfd);
   if (len <= 0)
     return -1; // no packet received
@@ -272,7 +272,7 @@ int overlay_mdp_recv(int mdp_sockfd, overlay_mdp_frame *mdp, mdp_port_t port, in
   // Compare the address of the sender with the address of our server, to ensure they are the same.
   // If the comparison fails, then try using realpath(3) on the sender address and compare again.
   if (	cmp_sockaddr(&recvaddr, &mdp_addr) != 0
-      && (   recvaddr.addr_un.sun_family != AF_UNIX
+      && (   recvaddr.local.sun_family != AF_UNIX
 	  || real_sockaddr(&recvaddr, &recvaddr) <= 0
 	  || cmp_sockaddr(&recvaddr, &mdp_addr) != 0
 	 )
diff --git a/net.c b/net.c
index e5e2da00..ec622ec0 100644
--- a/net.c
+++ b/net.c
@@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "conf.h"
 #include "net.h"
+#include "socket.h"
 #include "str.h"
 #include "strbuf_helpers.h"
 
@@ -138,8 +139,7 @@ ssize_t _write_str_nonblock(int fd, const char *str, struct __sourceloc __whence
   return _write_all_nonblock(fd, str, strlen(str), __whence);
 }
 
-ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
-		    struct sockaddr *recvaddr, socklen_t *recvaddrlen)
+ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl, struct socket_address *recvaddr)
 {
   struct msghdr msg;
   struct iovec iov[1];
@@ -147,8 +147,8 @@ ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
   iov[0].iov_base=buffer;
   iov[0].iov_len=bufferlen;
   bzero(&msg,sizeof(msg));
-  msg.msg_name = recvaddr;
-  msg.msg_namelen = *recvaddrlen;
+  msg.msg_name = &recvaddr->store;
+  msg.msg_namelen = recvaddr->addrlen;
   msg.msg_iov = &iov[0];
   msg.msg_iovlen = 1;
   // setting the following makes the data end up in the wrong place
@@ -192,7 +192,7 @@ ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
       }	 
     }
   }
-  *recvaddrlen=msg.msg_namelen;
+  recvaddr->addrlen = msg.msg_namelen;
   
   return len;
 }
diff --git a/net.h b/net.h
index 108b3f56..6ef3e1c0 100644
--- a/net.h
+++ b/net.h
@@ -52,6 +52,8 @@ ssize_t _write_all_nonblock(int fd, const void *buf, size_t len, struct __source
 ssize_t _writev_all(int fd, const struct iovec *iov, int iovcnt, struct __sourceloc __whence);
 ssize_t _write_str(int fd, const char *str, struct __sourceloc __whence);
 ssize_t _write_str_nonblock(int fd, const char *str, struct __sourceloc __whence);
-ssize_t recvwithttl(int sock, unsigned char *buffer, size_t bufferlen, int *ttl, struct sockaddr *recvaddr, socklen_t *recvaddrlen);
+
+struct socket_address;
+ssize_t recvwithttl(int sock, unsigned char *buffer, size_t bufferlen, int *ttl, struct socket_address *);
 
 #endif // __SERVALD_NET_H
diff --git a/overlay_interface.c b/overlay_interface.c
index 48e96c91..9568c992 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -25,6 +25,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include <fnmatch.h>
 #include "serval.h"
 #include "conf.h"
+#include "net.h"
+#include "socket.h"
 #include "strbuf.h"
 #include "strbuf_helpers.h"
 #include "overlay_buffer.h"
@@ -248,18 +250,19 @@ int overlay_interface_compare(overlay_interface *one, overlay_interface *two)
 // OSX doesn't recieve broadcast packets on sockets bound to an interface's address
 // So we have to bind a socket to INADDR_ANY to receive these packets.
 static void
-overlay_interface_read_any(struct sched_ent *alarm){
+overlay_interface_read_any(struct sched_ent *alarm)
+{
   if (alarm->poll.revents & POLLIN) {
     int plen=0;
     int recvttl=1;
     unsigned char packet[16384];
     overlay_interface *interface=NULL;
-    struct sockaddr src_addr;
-    socklen_t addrlen = sizeof(src_addr);
+    struct socket_address recvaddr;
+    recvaddr.addrlen = sizeof recvaddr.store;
     
     /* Read only one UDP packet per call to share resources more fairly, and also
      enable stats to accurately count packets received */
-    plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &src_addr, &addrlen);
+    plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &recvaddr);
     if (plen == -1) {
       WHY_perror("recvwithttl(c)");
       unwatch(alarm);
@@ -267,18 +270,16 @@ overlay_interface_read_any(struct sched_ent *alarm){
       return;
     }
     
-    struct in_addr src = ((struct sockaddr_in *)&src_addr)->sin_addr;
-    
     /* Try to identify the real interface that the packet arrived on */
-    interface = overlay_interface_find(src, 0);
+    interface = overlay_interface_find(recvaddr.inet.sin_addr, 0);
     
     /* Drop the packet if we don't find a match */
     if (!interface){
       if (config.debug.overlayinterfaces)
-	DEBUGF("Could not find matching interface for packet received from %s", inet_ntoa(src));
+	DEBUGF("Could not find matching interface for packet received from %s", inet_ntoa(recvaddr.inet.sin_addr));
       return;
     }
-    packetOkOverlay(interface, packet, plen, recvttl, &src_addr, addrlen);
+    packetOkOverlay(interface, packet, plen, recvttl, &recvaddr);
   }
   if (alarm->poll.revents & (POLLHUP | POLLERR)) {
     INFO("Closing broadcast socket due to error");
@@ -569,25 +570,24 @@ cleanup:
   return cleanup_ret;
 }
 
-static void interface_read_dgram(struct overlay_interface *interface){
+static void interface_read_dgram(struct overlay_interface *interface)
+{
   int plen=0;
   unsigned char packet[8096];
   
-  struct sockaddr src_addr;
-  socklen_t addrlen = sizeof(src_addr);
-  
-  
+  struct socket_address recvaddr;
+  recvaddr.addrlen = sizeof recvaddr.store;
+
   /* Read only one UDP packet per call to share resources more fairly, and also
    enable stats to accurately count packets received */
   int recvttl=1;
-  plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &src_addr, &addrlen);
+  plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &recvaddr);
   if (plen == -1) {
     WHY_perror("recvwithttl(c)");
     overlay_interface_close(interface);
     return;
   }
-  
-  packetOkOverlay(interface, packet, plen, recvttl, &src_addr, addrlen);
+  packetOkOverlay(interface, packet, plen, recvttl, &recvaddr);
 }
 
 struct file_packet{
@@ -678,8 +678,10 @@ static void interface_read_file(struct overlay_interface *interface)
 		alloca_sockaddr_in(&packet.dst_addr)
 	      );
       }else{
-	packetOkOverlay(interface, packet.payload, packet.payload_length, -1, 
-			    (struct sockaddr*)&packet.src_addr, (socklen_t) sizeof(packet.src_addr));
+	struct socket_address srcaddr;
+	srcaddr.addrlen = sizeof packet.src_addr;
+	srcaddr.inet = packet.src_addr;
+	packetOkOverlay(interface, packet.payload, packet.payload_length, -1, &srcaddr);
       }
     }
   }
diff --git a/overlay_mdp.c b/overlay_mdp.c
index 72652a8e..4f570bc0 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -1479,11 +1479,11 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
     unsigned char buffer[16384];
     int ttl;
     struct socket_address client;
-    client.addrlen=sizeof(client.store);
+    client.addrlen = sizeof client.store;
 
     ttl=-1;
     
-    ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, (struct sockaddr *)&client.addr, &client.addrlen);
+    ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, &client);
 
     if (len > 0) {
       if (client.addrlen <= sizeof(sa_family_t))
diff --git a/overlay_packetformats.c b/overlay_packetformats.c
index c2c39208..b9261e05 100644
--- a/overlay_packetformats.c
+++ b/overlay_packetformats.c
@@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 #include "serval.h"
 #include "conf.h"
+#include "socket.h"
 #include "str.h"
 #include "strbuf.h"
 #include "overlay_buffer.h"
@@ -320,7 +321,7 @@ int parseEnvelopeHeader(struct decode_context *context, struct overlay_interface
 }
 
 int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, size_t len,
-		    int recvttl, struct sockaddr *recvaddr, socklen_t recvaddrlen)
+		    int recvttl, struct socket_address *recvaddr)
 {
   IN();
   /* 
@@ -385,8 +386,8 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
     }
   }
   
-  if (recvaddr&&recvaddr->sa_family!=AF_INET)
-    RETURN(WHYF("Unexpected protocol family %d",recvaddr->sa_family));
+  if (recvaddr && recvaddr->addr.sa_family != AF_INET)
+    RETURN(WHYF("Unexpected protocol family %d", recvaddr->addr.sa_family));
   
   struct overlay_frame f;
   struct decode_context context;
@@ -399,11 +400,11 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
   
   f.interface = interface;
   if (recvaddr)
-    f.recvaddr = *((struct sockaddr_in *)recvaddr); 
+    f.recvaddr = recvaddr->inet;
   else 
     bzero(&f.recvaddr, sizeof f.recvaddr);
   
-  int ret=parseEnvelopeHeader(&context, interface, (struct sockaddr_in *)recvaddr, b);
+  int ret=parseEnvelopeHeader(&context, interface, recvaddr ? &recvaddr->inet : NULL, b);
   if (ret){
     ob_free(b);
     RETURN(ret);
diff --git a/radio_link.c b/radio_link.c
index 628a4bf9..2b2f9224 100644
--- a/radio_link.c
+++ b/radio_link.c
@@ -433,7 +433,7 @@ static int radio_link_parse(struct overlay_interface *interface, struct radio_li
     if (config.debug.radio_link) 
       DEBUGF("PDU Complete (length=%d)",state->packet_length);
     
-    packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL, 0);
+    packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL);
     state->packet_length=sizeof(state->dst)+1;
   }
   return 1;
diff --git a/serval.h b/serval.h
index d97115f2..d213165e 100644
--- a/serval.h
+++ b/serval.h
@@ -404,7 +404,7 @@ void insertTransactionInCache(unsigned char *transaction_id);
 
 int overlay_forward_payload(struct overlay_frame *f);
 int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, size_t len,
-		    int recvttl, struct sockaddr *recvaddr, socklen_t recvaddrlen);
+		    int recvttl, struct socket_address *recvaddr);
 int parseMdpPacketHeader(struct decode_context *context, struct overlay_frame *frame, 
 			 struct overlay_buffer *buffer, struct subscriber **nexthop);
 int parseEnvelopeHeader(struct decode_context *context, struct overlay_interface *interface, 
diff --git a/socket.c b/socket.c
index 1839980e..e81482d3 100644
--- a/socket.c
+++ b/socket.c
@@ -44,14 +44,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *addr, const char *fmt, ...)
 {
   bzero(addr, sizeof(*addr));
-  addr->addr_un.sun_family = AF_UNIX;
+  addr->local.sun_family = AF_UNIX;
   va_list ap;
   va_start(ap, fmt);
-  int r = vformf_serval_instance_path(__WHENCE__, addr->addr_un.sun_path, sizeof addr->addr_un.sun_path, fmt, ap);
+  int r = vformf_serval_instance_path(__WHENCE__, addr->local.sun_path, sizeof addr->local.sun_path, fmt, ap);
   va_end(ap);
   if (!r)
     return WHY("socket name overflow");
-  addr->addrlen=sizeof addr->addr_un.sun_family + strlen(addr->addr_un.sun_path) + 1;
+  addr->addrlen=sizeof addr->local.sun_family + strlen(addr->local.sun_path) + 1;
 // TODO perform real path transformation in making the serval instance path
 //  if (real_sockaddr(addr, addr) == -1)
 //    return -1;
@@ -60,7 +60,7 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *add
   // For the abstract name we use the absolute path name with the initial '/' replaced by the
   // leading nul.  This ensures that different instances of the Serval daemon have different socket
   // names.
-  addr->addr_un.sun_path[0] = '\0'; // mark as Linux abstract socket
+  addr->local.sun_path[0] = '\0'; // mark as Linux abstract socket
   --addr->addrlen; // do not count trailing nul in abstract socket name
 #endif // USE_ABSTRACT_NAMESPACE
   return 0;
@@ -80,24 +80,24 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *add
  */
 int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr)
 {
-  int src_path_len = src_addr->addrlen - sizeof src_addr->addr_un.sun_family;
-  if (	 src_addr->addrlen >= sizeof src_addr->addr_un.sun_family + 1
-      && src_addr->addr_un.sun_family == AF_UNIX
-      && src_addr->addr_un.sun_path[0] != '\0'
-      && src_addr->addr_un.sun_path[src_path_len - 1] == '\0'
+  int src_path_len = src_addr->addrlen - sizeof src_addr->local.sun_family;
+  if (	 src_addr->addrlen >= sizeof src_addr->local.sun_family + 1
+      && src_addr->local.sun_family == AF_UNIX
+      && src_addr->local.sun_path[0] != '\0'
+      && src_addr->local.sun_path[src_path_len - 1] == '\0'
   ) {
     char real_path[PATH_MAX];
     size_t real_path_len;
-    if (realpath(src_addr->addr_un.sun_path, real_path) == NULL)
-      return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->addr_un.sun_path));
-    else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->addr_un.sun_path)
+    if (realpath(src_addr->local.sun_path, real_path) == NULL)
+      return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->local.sun_path));
+    else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->local.sun_path)
       return WHYF("sockaddr overrun: realpath(%s) returned %s", 
-	  alloca_str_toprint(src_addr->addr_un.sun_path), alloca_str_toprint(real_path));
+	  alloca_str_toprint(src_addr->local.sun_path), alloca_str_toprint(real_path));
     else if (   real_path_len != src_path_len
-	     || memcmp(real_path, src_addr->addr_un.sun_path, src_path_len) != 0
+	     || memcmp(real_path, src_addr->local.sun_path, src_path_len) != 0
     ) {
-      memcpy(dst_addr->addr_un.sun_path, real_path, real_path_len);
-      dst_addr->addrlen = real_path_len + sizeof dst_addr->addr_un.sun_family;
+      memcpy(dst_addr->local.sun_path, real_path, real_path_len);
+      dst_addr->addrlen = real_path_len + sizeof dst_addr->local.sun_family;
       return 1;
     }
   }
@@ -133,23 +133,23 @@ int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address
   // Both addresses are in the same family...
   switch (addrA->addr.sa_family) {
   case AF_UNIX: {
-      unsigned pathlenA = addrA->addrlen - sizeof (addrA->addr_un.sun_family);
-      unsigned pathlenB = addrB->addrlen - sizeof (addrB->addr_un.sun_family);
+      unsigned pathlenA = addrA->addrlen - sizeof (addrA->local.sun_family);
+      unsigned pathlenB = addrB->addrlen - sizeof (addrB->local.sun_family);
       int c;
       if (   pathlenA > 1 && pathlenB > 1
-	  && addrA->addr_un.sun_path[0] == '\0'
-	  && addrB->addr_un.sun_path[0] == '\0'
+	  && addrA->local.sun_path[0] == '\0'
+	  && addrB->local.sun_path[0] == '\0'
       ) {
 	// Both abstract sockets - just compare names, nul bytes are not terminators.
-	c = memcmp(&addrA->addr_un.sun_path[1],
-		   &addrB->addr_un.sun_path[1],
+	c = memcmp(&addrA->local.sun_path[1],
+		   &addrB->local.sun_path[1],
 		   (pathlenA < pathlenB ? pathlenA : pathlenB) - 1);
       } else {
 	// Either or both are named local file sockets.  If the file names are identical up to the
 	// first nul, then the addresses are equal.  This collates abstract socket names, whose first
 	// character is a nul, ahead of all non-empty file socket names.
-	c = strncmp(addrA->addr_un.sun_path,
-		    addrB->addr_un.sun_path,
+	c = strncmp(addrA->local.sun_path,
+		    addrB->local.sun_path,
 		    (pathlenA < pathlenB ? pathlenA : pathlenB));
       }
       if (c == 0)
diff --git a/socket.h b/socket.h
index 88a05549..5a167f43 100644
--- a/socket.h
+++ b/socket.h
@@ -9,7 +9,8 @@ struct socket_address{
   socklen_t addrlen;
   union{
     struct sockaddr addr;
-    struct sockaddr_un addr_un;
+    struct sockaddr_un local; // name "unix" is a predefined macro
+    struct sockaddr_in inet;
     struct sockaddr_storage store;
   };
 };
@@ -53,4 +54,4 @@ ssize_t _recv_message(struct __sourceloc, int fd, struct socket_address *address
 #define send_message(fd, address, data)    _send_message(__WHENCE__, (fd), (address), (data))
 #define recv_message(fd, address, data)    _recv_message(__WHENCE__, (fd), (address), (data))
 
-#endif
\ No newline at end of file
+#endif

From 1d14f430506ba08ae9c595cb8a7fea27975a232b Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Sun, 1 Dec 2013 05:01:23 +1030
Subject: [PATCH 48/56] Improve test framework: --timeout option in usage
 message

---
 testframework.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/testframework.sh b/testframework.sh
index 004d2e90..903a7cc1 100644
--- a/testframework.sh
+++ b/testframework.sh
@@ -81,6 +81,7 @@ Options:
    --jobs=N                Run tests in parallel, at most N at a time
    -E, --stop-on-error     Do not execute any tests after an ERROR occurs
    -F, --stop-on-failure   Do not execute any tests after a FAIL occurs
+   --timeout=N             Override default timeout, make it N seconds instead of 60
    --filter=PREFIX         Only execute tests whose names start with PREFIX
    --filter=N              Only execute test number N
    --filter=M-N            Only execute tests with numbers in range M-N inclusive

From 16995f1041d2a80265f0d2aba474f94371944eeb Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 28 Nov 2013 17:45:32 +1030
Subject: [PATCH 49/56] Minor refactor in strn_edup()

---
 mem.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/mem.c b/mem.c
index 569226a8..17ddcb9f 100644
--- a/mem.c
+++ b/mem.c
@@ -51,10 +51,8 @@ void *_emalloc_zero(struct __sourceloc __whence, size_t bytes)
 char *_strn_edup(struct __sourceloc __whence, const char *str, size_t len)
 {
   char *new = _emalloc(__whence, len + 1);
-  if (new) {
-    strncpy(new, str, len);
-    new[len] = '\0';
-  }
+  if (new)
+    strncpy(new, str, len)[len] = '\0';
   return new;
 }
 

From 72482b34276a428d29e01d69c8c9645b0ac7a5c1 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 28 Nov 2013 17:46:16 +1030
Subject: [PATCH 50/56] Better error logging for recvwithttl()

---
 mdp_client.c        |  2 ++
 net.c               | 13 ++++---------
 overlay_interface.c |  8 ++++----
 overlay_mdp.c       | 12 ++++++++----
 4 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/mdp_client.c b/mdp_client.c
index f3694b9a..6a7d4aba 100644
--- a/mdp_client.c
+++ b/mdp_client.c
@@ -248,6 +248,8 @@ int overlay_mdp_recv(int mdp_sockfd, overlay_mdp_frame *mdp, mdp_port_t port, in
   mdp->packetTypeAndFlags = 0;
   set_nonblock(mdp_sockfd);
   len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, (struct sockaddr *)&recvaddr, &recvaddrlen);
+  if (len == -1)
+    WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p,%p)", mdp_sockfd, mdp, sizeof(overlay_mdp_frame), *ttl, &recvaddr, &recvaddrlen);
   set_block(mdp_sockfd);
   if (len <= 0)
     return -1; // no packet received
diff --git a/net.c b/net.c
index e5e2da00..1871febe 100644
--- a/net.c
+++ b/net.c
@@ -143,7 +143,7 @@ ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
 {
   struct msghdr msg;
   struct iovec iov[1];
-  
+  struct cmsghdr cmsgcmsg[16];
   iov[0].iov_base=buffer;
   iov[0].iov_len=bufferlen;
   bzero(&msg,sizeof(msg));
@@ -151,18 +151,13 @@ ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
   msg.msg_namelen = *recvaddrlen;
   msg.msg_iov = &iov[0];
   msg.msg_iovlen = 1;
-  // setting the following makes the data end up in the wrong place
-  //  msg.msg_iov->iov_base=iov_buffer;
-  // msg.msg_iov->iov_len=sizeof(iov_buffer);
-  
-  struct cmsghdr cmsgcmsg[16];
-  msg.msg_control = &cmsgcmsg[0];
-  msg.msg_controllen = sizeof(struct cmsghdr)*16;
+  msg.msg_control = cmsgcmsg;
+  msg.msg_controllen = sizeof cmsgcmsg;
   msg.msg_flags = 0;
   
   ssize_t len = recvmsg(sock,&msg,0);
   if (len == -1 && errno != EAGAIN && errno != EWOULDBLOCK)
-    return WHY_perror("recvmsg");
+    return WHYF_perror("recvmsg(%d,%p,0)", sock, &msg);
   
 #if 0
   if (config.debug.packetrx) {
diff --git a/overlay_interface.c b/overlay_interface.c
index 842685d7..c3f8ab3a 100644
--- a/overlay_interface.c
+++ b/overlay_interface.c
@@ -258,9 +258,9 @@ overlay_interface_read_any(struct sched_ent *alarm){
     
     /* Read only one UDP packet per call to share resources more fairly, and also
      enable stats to accurately count packets received */
-    plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &src_addr, &addrlen);
+    plen = recvwithttl(alarm->poll.fd, packet, sizeof packet, &recvttl, &src_addr, &addrlen);
     if (plen == -1) {
-      WHY_perror("recvwithttl(c)");
+      WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p,%p)", alarm->poll.fd, packet, sizeof packet, recvttl, &src_addr, &addrlen);
       unwatch(alarm);
       close(alarm->poll.fd);
       return;
@@ -584,9 +584,9 @@ static void interface_read_dgram(struct overlay_interface *interface){
   /* Read only one UDP packet per call to share resources more fairly, and also
    enable stats to accurately count packets received */
   int recvttl=1;
-  plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &src_addr, &addrlen);
+  plen = recvwithttl(interface->alarm.poll.fd, packet, sizeof packet, &recvttl, &src_addr, &addrlen);
   if (plen == -1) {
-    WHY_perror("recvwithttl(c)");
+    WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p,%p)", interface->alarm.poll.fd, packet, sizeof packet, recvttl, &src_addr, &addrlen);
     overlay_interface_close(interface);
     return;
   }
diff --git a/overlay_mdp.c b/overlay_mdp.c
index d749ba51..78de8048 100644
--- a/overlay_mdp.c
+++ b/overlay_mdp.c
@@ -1138,7 +1138,9 @@ static void mdp_poll2(struct sched_ent *alarm)
     };
     int ttl=-1;
     
-    ssize_t len = recvwithttl(alarm->poll.fd, buffer, sizeof(buffer), &ttl, (struct sockaddr *)&addr, &client.addrlen);
+    ssize_t len = recvwithttl(alarm->poll.fd, buffer, sizeof buffer, &ttl, (struct sockaddr *)&addr, &client.addrlen);
+    if (len == -1)
+      WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p,%p)", alarm->poll.fd, buffer, sizeof buffer, ttl, &addr, &client.addrlen);
     
     if (len<sizeof(struct mdp_header)){
       WHYF("Expected length %d, got %d from %s", (int)sizeof(struct mdp_header), (int)len, alloca_sockaddr(client.addr, client.addrlen));
@@ -1184,14 +1186,16 @@ static void overlay_mdp_poll(struct sched_ent *alarm)
     unsigned char buffer[16384];
     int ttl;
     unsigned char recvaddrbuffer[1024];
-    struct sockaddr *recvaddr=(struct sockaddr *)&recvaddrbuffer[0];
-    socklen_t recvaddrlen=sizeof(recvaddrbuffer);
+    struct sockaddr *recvaddr = (struct sockaddr *)recvaddrbuffer;
+    socklen_t recvaddrlen = sizeof recvaddrbuffer;
     struct sockaddr_un *recvaddr_un=NULL;
 
     ttl=-1;
     bzero((void *)recvaddrbuffer,sizeof(recvaddrbuffer));
     
-    ssize_t len = recvwithttl(alarm->poll.fd,buffer,sizeof(buffer),&ttl, recvaddr, &recvaddrlen);
+    ssize_t len = recvwithttl(alarm->poll.fd, buffer, sizeof buffer, &ttl, recvaddr, &recvaddrlen);
+    if (len == -1)
+      WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p,%p)", alarm->poll.fd, buffer, sizeof buffer, ttl, recvaddr, &recvaddrlen);
     recvaddr_un=(struct sockaddr_un *)recvaddr;
 
     if (len > 0) {

From 5e5e097d7f97fe3ae510fb2b83cea08c966d9e93 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Tue, 26 Nov 2013 17:46:07 +1030
Subject: [PATCH 51/56] Accept any well formed Rhizome service field

Do not limit service field to only recognised supported values 'file',
'MeshMS1' and 'MeshMS2'

New test case checks for malformed service field, fails
---
 commandline.c    |  2 +-
 rhizome.c        | 36 +++++++++++++++++-------------------
 tests/rhizomeops | 22 +++++++++++++++++-----
 3 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/commandline.c b/commandline.c
index cf5c146e..b5baeb0b 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1332,7 +1332,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
   if (!m){
     keyring_free(keyring);
     return WHY("Manifest struct could not be allocated -- not added to rhizome");
-  }  
+  }
   if (manifestpath && *manifestpath && access(manifestpath, R_OK) == 0) {
     if (config.debug.rhizome)
       DEBUGF("reading manifest from %s", manifestpath);
diff --git a/rhizome.c b/rhizome.c
index 3db8cfc5..f063ee69 100644
--- a/rhizome.c
+++ b/rhizome.c
@@ -156,32 +156,30 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
 int rhizome_manifest_check_sanity(rhizome_manifest *m)
 {
   /* Ensure manifest meets basic sanity checks. */
-  if (m->filesize == RHIZOME_SIZE_UNSET)
-    return WHY("Manifest missing 'filesize' field");
-  if (m->filesize && rhizome_filehash_t_is_zero(m->filehash))
-    return WHY("Manifest 'filehash' field has not been set");
-  if (m->service == NULL || !m->service[0])
-    return WHY("Manifest missing 'service' field");
-  if (!m->has_date)
-    return WHY("Manifest missing 'date' field");
+  int ret = 0;
   if (m->version == 0)
-    return WHY("Manifest must have a version number");
-  if (strcasecmp(m->service, RHIZOME_SERVICE_FILE) == 0) {
+    ret = WHY("Manifest must have a version number");
+  if (m->filesize == RHIZOME_SIZE_UNSET)
+    ret = WHY("Manifest missing 'filesize' field");
+  else if (m->filesize && rhizome_filehash_t_is_zero(m->filehash))
+    ret = WHY("Manifest 'filehash' field has not been set");
+  if (m->service == NULL)
+    ret = WHY("Manifest missing 'service' field");
+  else if (strcasecmp(m->service, RHIZOME_SERVICE_FILE) == 0) {
     if (m->name == NULL)
-      return WHY("Manifest missing 'name' field");
+      ret = WHY("Manifest with service='" RHIZOME_SERVICE_FILE "' missing 'name' field");
   } else if (strcasecmp(m->service, RHIZOME_SERVICE_MESHMS) == 0 
 	  || strcasecmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) {
     if (!m->has_sender)
-      return WHY("MeshMS Manifest missing 'sender' field");
+      ret = WHYF("Manifest with service='%s' missing 'sender' field", m->service);
     if (!m->has_recipient)
-      return WHY("MeshMS Manifest missing 'recipient' field");
-  } else {
-    return WHYF("Invalid service=%s", m->service);
+      ret = WHYF("Manifest with service='%s' missing 'recipient' field", m->service);
   }
-  if (config.debug.rhizome)
-    DEBUGF("sender=%s", m->has_sender ? alloca_tohex_sid_t(m->sender) : "(null)");
-  /* passes all sanity checks */
-  return 0;
+  else if (!rhizome_str_is_manifest_service(m->service))
+    ret = WHYF("Manifest invalid 'service' field %s", alloca_str_toprint(m->service));
+  if (!m->has_date)
+    ret = WHY("Manifest missing 'date' field");
+  return ret;
 }
 
 /* Sets the bundle key "BK" field of a manifest.  Returns 1 if the field was set, 0 if not.
diff --git a/tests/rhizomeops b/tests/rhizomeops
index 0668db07..615357ff 100755
--- a/tests/rhizomeops
+++ b/tests/rhizomeops
@@ -714,17 +714,29 @@ test_AddUpdateAutoVersion() {
    assert_rhizome_list --fromhere=1 file1_2 file2
 }
 
-doc_AddUnsupportedService="Add with unsupported service fails"
-setup_AddUnsupportedService() {
+doc_AddServiceInvalid="Add with invalid service fails"
+setup_AddServiceInvalid() {
+   setup_servald
+   setup_rhizome
+   echo "Message1" >file1
+   echo 'service=Fubar!' >file1.manifest
+}
+test_AddServiceInvalid() {
+   execute $servald rhizome add file $SIDB1 file1 file1.manifest
+   tfw_cat --stdout --stderr
+   assertExitStatus '!=' 0
+}
+
+doc_AddServiceUnsupported="Add with unsupported service succeeds"
+setup_AddServiceUnsupported() {
    setup_servald
    setup_rhizome
    echo "Message1" >file1
    echo 'service=Fubar' >file1.manifest
 }
-test_AddUnsupportedService() {
-   execute $servald rhizome add file $SIDB1 file1 file1.manifest
+test_AddServiceUnsupported() {
+   executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
    tfw_cat --stdout --stderr
-   assertExitStatus '!=' 0
 }
 
 doc_EncryptedPayload="Add and extract an encrypted payload"

From 0dd1b302b52138048b9f1205b0d8d9c20292e3f0 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Thu, 28 Nov 2013 17:44:37 +1030
Subject: [PATCH 52/56] Rewrite Rhizome manifest parsing

Move validation checks into new function rhizome_manifest_validate()

Remove rhizome_manifest 'errors' field

Replace rhizome_manifest 'warnings' with 'malformed'

Replace rhizome_manifest 'manifest_bytes' with 'manifest_body_bytes' and
refactor to use 'manifest_all_bytes' in all manifest i/o

Refactor rhizome_manifest_verify() and reverse sense of return value to
match rhizome_manifest_validate()

New function rhizome_manifest_inspect() -- lightweight manifest parser
used when receiving Rhizome advertisements

New 'rhizomeops' test case for invalid manifest "service" field values,
now passes
---
 commandline.c           |  28 +-
 overlay_buffer.c        |  13 +-
 overlay_buffer.h        |   3 +-
 rhizome.c               |   9 +-
 rhizome.h               |  40 +-
 rhizome_bundle.c        | 792 +++++++++++++++++++++++-----------------
 rhizome_crypto.c        | 102 ++----
 rhizome_database.c      |  43 +--
 rhizome_direct.c        |  11 +-
 rhizome_direct_http.c   |   4 +-
 rhizome_fetch.c         |  13 +-
 rhizome_packetformats.c |  74 ++--
 tests/rhizomeops        |   9 +-
 tests/rhizomeprotocol   |   2 +
 14 files changed, 619 insertions(+), 524 deletions(-)

diff --git a/commandline.c b/commandline.c
index b5baeb0b..e497dcb7 100644
--- a/commandline.c
+++ b/commandline.c
@@ -1338,8 +1338,9 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
       DEBUGF("reading manifest from %s", manifestpath);
     /* Don't verify the manifest, because it will fail if it is incomplete.
        This is okay, because we fill in any missing bits and sanity check before
-       trying to write it out. */
-    if (rhizome_read_manifest_file(m, manifestpath, 0) == -1) {
+       trying to write it out. However, we do insist that whatever we load is
+       valid and not malformed. */
+    if (rhizome_read_manifest_file(m, manifestpath, 0) == -1 || m->malformed) {
       rhizome_manifest_free(m);
       keyring_free(keyring);
       return WHY("Manifest file could not be loaded -- not added to rhizome");
@@ -1584,22 +1585,19 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, struct cli_cont
   if ( cli_arg(parsed, "manifestpath", &manifestpath, NULL, "") == -1
     || cli_arg(parsed, "filepath", &filepath, NULL, "") == -1)
     return -1;
-  
   rhizome_manifest *m = rhizome_new_manifest();
   if (!m)
     return WHY("Out of manifests.");
-  
-  int ret=0;
-  if (rhizome_read_manifest_file(m, manifestpath, 0) == -1)
-    ret=-1;
-  // TODO why doesn't read manifest file set finalised???
-  m->finalised=1;
-  
-  if (ret==0 && rhizome_write_manifest_file(m, filepath, 1) == -1)
-    ret = -1;
-  
-  if (m)
-    rhizome_manifest_free(m);
+  int ret = -1;
+  if (   rhizome_read_manifest_file(m, manifestpath, 0) != -1
+      && rhizome_manifest_validate(m)
+      && rhizome_manifest_verify(m)
+  ) {
+    assert(m->finalised);
+    if (rhizome_write_manifest_file(m, filepath, 1) != -1)
+      ret = 0;
+  }
+  rhizome_manifest_free(m);
   return ret;
 }
 
diff --git a/overlay_buffer.c b/overlay_buffer.c
index 06686e3d..1d05dcb2 100644
--- a/overlay_buffer.c
+++ b/overlay_buffer.c
@@ -416,12 +416,17 @@ int test_offset(struct overlay_buffer *b,int start,int length)
   return 0;
 }
 
-int ob_getbyte(struct overlay_buffer *b, int ofs)
+// next byte without advancing
+int ob_peek(struct overlay_buffer *b)
 {
-  if (test_offset(b, ofs, 1))
+  if (test_offset(b, b->position, 1))
     return -1;
-  
-  return b->bytes[ofs];
+  return b->bytes[b->position];
+}
+
+void ob_skip(struct overlay_buffer *b, unsigned n)
+{
+  b->position += n;
 }
 
 int ob_get_bytes(struct overlay_buffer *b, unsigned char *buff, int len){
diff --git a/overlay_buffer.h b/overlay_buffer.h
index dd7ba4a0..cf69a4b9 100644
--- a/overlay_buffer.h
+++ b/overlay_buffer.h
@@ -95,7 +95,8 @@ void _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l);
 #define ob_append_rfs(b, l) _ob_append_rfs(__WHENCE__, b, l)
 
 // get one byte, -ve number indicates failure
-int ob_getbyte(struct overlay_buffer *b,int ofs);
+int ob_peek(struct overlay_buffer *b);
+void ob_skip(struct overlay_buffer *b, unsigned n);
 // get one byte from the current position, -ve number indicates failure
 int ob_get(struct overlay_buffer *b);
 int ob_get_bytes(struct overlay_buffer *b, unsigned char *buff, int len);
diff --git a/rhizome.c b/rhizome.c
index f063ee69..56378233 100644
--- a/rhizome.c
+++ b/rhizome.c
@@ -130,14 +130,11 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
   
   if (rhizome_read_manifest_file(m, manifest_path, buffer_len) == -1)
     return WHY("could not read manifest file");
-  if (rhizome_manifest_verify(m))
+  if (!rhizome_manifest_validate(m))
+    return WHY("manifest is invalid");
+  if (!rhizome_manifest_verify(m))
     return WHY("could not verify manifest");
   
-  /* Make sure we store signatures */
-  // TODO, why do we need this? Why isn't the state correct from rhizome_read_manifest_file? 
-  // This feels like a hack...
-  m->manifest_bytes=m->manifest_all_bytes;
-  
   /* Do we already have this manifest or newer? */
   int64_t dbVersion = -1;
   if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END) == -1)
diff --git a/rhizome.h b/rhizome.h
index 7b171a15..e2a284b9 100644
--- a/rhizome.h
+++ b/rhizome.h
@@ -124,7 +124,7 @@ extern time_ms_t rhizome_voice_timeout;
 typedef struct rhizome_signature {
   unsigned char signature[crypto_sign_edwards25519sha512batch_BYTES
 			  +crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES+1];
-  int signatureLength;
+  size_t signatureLength;
 } rhizome_signature;
 
 #define RHIZOME_BAR_BYTES 32
@@ -191,18 +191,15 @@ typedef struct rhizome_manifest
   unsigned char *signatories[MAX_MANIFEST_VARS];
   uint8_t signatureTypes[MAX_MANIFEST_VARS];
 
-  /* Imperfections.
-   *  - Errors involve the correctness of fields that are mandatory for proper
-   *    operation of the transport and storage layer.  A manifest with errors > 0
-   *    must not be stored, transmitted or supplied via any API.
-   *  - Warnings indicate a manifest that cannot be fully understood by this
-   *    version of Rhizome (probably from a future or a very old past version
-   *    of Rhizome).  During add or import (local injection), the manifest
-   *    should not be imported.  During extract or export (local) a warning or
-   *    error message should be logged.
+  /* Set to non-zero if a manifest has been parsed that cannot be fully
+   * understood by this version of Rhizome (probably from a future or a very
+   * old past version of Rhizome).  During add (local injection), the manifest
+   * should not be imported.  During extract (local decode) a warning or error
+   * message should be logged.  Manifests marked as malformed are still
+   * transported, imported and exported normally, as long as their signature is
+   * valid.
    */
-  unsigned short errors;
-  unsigned short warnings;
+  unsigned short malformed;
 
   /* Set non-zero after variables have been packed and signature blocks
    * appended.  All fields below may not be valid until the manifest has been
@@ -211,7 +208,7 @@ typedef struct rhizome_manifest
   bool_t finalised;
 
   /* Whether the manifest contains a signature that corresponds to the manifest
-   * id (ie public key).  Caches the result of 
+   * id (ie public key).
    */
   bool_t selfSigned;
 
@@ -219,6 +216,10 @@ typedef struct rhizome_manifest
    */
   bool_t dataFileUnlinkOnFree;
 
+  /* Set if the ID field (cryptoSignPublic) contains a bundle ID.
+   */
+  bool_t has_id;
+
   /* Set if the tail field is valid, ie, the bundle is a journal.
    */
   bool_t is_journal;
@@ -315,8 +316,8 @@ typedef struct rhizome_manifest
   unsigned group_count;
   char *groups[MAX_MANIFEST_VARS];
 
-  unsigned manifest_bytes;
-  unsigned manifest_all_bytes;
+  size_t manifest_body_bytes;
+  size_t manifest_all_bytes;
   unsigned char manifestdata[MAX_MANIFEST_BYTES];
   unsigned char manifesthash[crypto_hash_sha512_BYTES];
 
@@ -448,11 +449,20 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
 
 #define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1)
 
+struct rhizome_manifest_summary {
+  rhizome_bid_t bid;
+  int64_t version;
+  size_t body_len;
+};
+
+int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ);
+
 int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
 int rhizome_manifest_selfsign(rhizome_manifest *m);
 int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
 int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
 int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferPAndSize);
+int rhizome_manifest_validate(rhizome_manifest *m);
 int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out);
 
 void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m);
diff --git a/rhizome_bundle.c b/rhizome_bundle.c
index 8ecf5b51..9a7cb590 100644
--- a/rhizome_bundle.c
+++ b/rhizome_bundle.c
@@ -92,7 +92,7 @@ static const char *_rhizome_manifest_set(struct __sourceloc __whence, rhizome_ma
       m->finalised = 0;
       return ret;
     }
-  if (m->var_count >= MAX_MANIFEST_VARS)
+  if (m->var_count >= NELS(m->vars))
     return WHYNULL("no more manifest vars");
   if ((m->vars[m->var_count] = str_edup(var)) == NULL)
     return NULL;
@@ -344,72 +344,52 @@ void _rhizome_manifest_del_author(struct __sourceloc __whence, rhizome_manifest
   }
 }
 
+/* Compute the hash of the manifest's body, including the NUL byte that separates the body from
+ * the signature block, and verify that a signature is present and is correct.
+ *
+ * Returns 1 if the manifest signature is valid, ie, the signature is a self-signature using the
+ * manifest's own private key.  Sets the m->finalised flag to 1.
+ *
+ * Returns 0 if there are no signatures or if the signature block does not verify.
+ *
+ * Only call this function on manifests for which rhizome_manifest_validate(m) has returned true.
+ */
 int rhizome_manifest_verify(rhizome_manifest *m)
 {
-  unsigned end_of_text=0;
-
-  /* find end of manifest body and start of signatures */
-  while(m->manifestdata[end_of_text]&&end_of_text<m->manifest_all_bytes)
-    end_of_text++;
-  end_of_text++; /* include null byte in body for verification purposes */
-
-  /* Calculate hash of the text part of the file, as we need to couple this with
-     each signature block to */
-  crypto_hash_sha512(m->manifesthash,m->manifestdata,end_of_text);
-
-  /* Read signature blocks from file. */
-  unsigned ofs = end_of_text;
-  while(ofs<m->manifest_all_bytes) {
-    if (config.debug.rhizome)
-      DEBUGF("ofs=0x%x, m->manifest_bytes=0x%x", ofs,m->manifest_all_bytes);
-    if (rhizome_manifest_extract_signature(m, &ofs))
+  assert(m->manifest_body_bytes > 0);
+  assert(m->manifest_all_bytes > 0);
+  assert(m->manifest_body_bytes <= m->manifest_all_bytes);
+  if (m->manifest_body_bytes == m->manifest_all_bytes)
+    assert(m->manifestdata[m->manifest_body_bytes - 1] == '\0');
+  // Hash the body
+  crypto_hash_sha512(m->manifesthash, m->manifestdata, m->manifest_body_bytes);
+  // Read signature blocks
+  unsigned ofs = m->manifest_body_bytes;
+  while (ofs < m->manifest_all_bytes) {
+    if (rhizome_manifest_extract_signature(m, &ofs) == -1)
       break;
   }
+  assert(ofs <= m->manifest_all_bytes);
 
-  if (m->sig_count==0) {
-    WHYF("Manifest has zero valid signatures");
-    m->errors++;
+  // Make sure the first signatory's public key is the bundle ID
+  assert(m->has_id);
+  if (m->sig_count == 0) {
+    if (config.debug.rhizome)
+      DEBUG("Manifest has no signature blocks, but should have self-signature block");
+    m->selfSigned = 0;
+    return 0;
   }
-
-  /* Make sure that id variable is correct */
-  {
-    rhizome_bid_t bid;
-    const char *id = rhizome_manifest_get(m,"id");
-    if (!id) {
-      WHY("Manifest lacks 'id' field");
-      m->errors++;
-    } else if (str_to_rhizome_bid_t(&bid, id) == -1) {
-      WHY("Invalid manifest 'id' field");
-      m->errors++;
-    } else if (cmp_rhizome_bid_t(&bid, &m->cryptoSignPublic) != 0) {
-      WHYF("Manifest id field does not match cryptoSignPublic: id=%s, cryptoSignPublic=%s",
-	  alloca_tohex_rhizome_bid_t(bid),
-	  alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)
-	);
-      m->errors++;
-    } else if (m->sig_count == 0 || memcmp(m->signatories[0], bid.binary, sizeof bid.binary) != 0) {
-      if (config.debug.rhizome) {
-	if (m->sig_count>0) {
-	  DEBUGF("Manifest id variable does not match first signature block (signature key is %s)",
-		  alloca_tohex(m->signatories[0], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)
-		  );
-	} else {
-	  DEBUG("Manifest has no signature blocks, but should have self-signature block");
-	}
-      }
-      m->errors++;
-      m->selfSigned = 0;
-    } else
-      m->selfSigned = 1;
+  if (memcmp(m->signatories[0], m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary) != 0) {
+    if (config.debug.rhizome)
+      DEBUGF("Manifest id does not match first signature block (signature key is %s)",
+	      alloca_tohex(m->signatories[0], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)
+	    );
+    m->selfSigned = 0;
+    return 0;
   }
-
-  /* Mark as finalised, as it is all read and intact,
-     unless of course it has errors, or is lacking a self-signature. */
-  if (!m->errors) m->finalised=1;
-  else WHY("Verified a manifest that has errors, so marking as not finalised");
-
-  if (m->errors) return WHY("Manifest verification failed");
-  else return 0;
+  m->selfSigned = 1;
+  m->finalised = 1;
+  return 1;
 }
 
 ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_size)
@@ -425,259 +405,396 @@ ssize_t read_whole_file(const char *path, unsigned char *buffer, size_t buffer_s
   return ret;
 }
 
-int rhizome_manifest_parse(rhizome_manifest *m)
+static void rhizome_manifest_clear(rhizome_manifest *m)
 {
-  IN();
-  m->manifest_all_bytes=m->manifest_bytes;
-  m->var_count = 0;
+  while (m->var_count) {
+    --m->var_count;
+    free((char *) m->vars[m->var_count]);
+    free((char *) m->values[m->var_count]);
+    m->vars[m->var_count] = m->values[m->var_count] = NULL;
+  }
+  while (m->sig_count) {
+    --m->sig_count;
+    free(m->signatories[m->sig_count]);
+    m->signatories[m->sig_count] = NULL;
+  }
+  m->malformed = 0;
+  m->has_id = 0;
+  m->is_journal = 0;
   m->filesize = RHIZOME_SIZE_UNSET;
   m->tail = RHIZOME_SIZE_UNSET;
+  m->version = 0;
+  // TODO initialise more fields
+}
 
-  /* Parse out variables, signature etc */
-  int have_id = 0;
-  int have_version = 0;
-  int have_filehash = 0;
-
-  unsigned ofs = 0;
-  while (ofs < m->manifest_bytes && m->manifestdata[ofs]) {
-    char line[1024];
-    unsigned limit = ofs + sizeof line - 1;
-    if (limit > m->manifest_bytes)
-      limit = m->manifest_bytes;
-    char *p = line;
-    while (ofs < limit && !(m->manifestdata[ofs] == '\0' || m->manifestdata[ofs] == '\n' || m->manifestdata[ofs] == '\r'))
-	  *p++ = m->manifestdata[ofs++];
-    *p = '\0';
-    if (m->manifestdata[ofs] == '\r')
-      ++ofs;
-    if (m->manifestdata[ofs] == '\n')
-      ++ofs;
-    /* Ignore blank lines */
-    if (line[0] == '\0')
-      continue;
-    /* Ignore comment lines */
-    if (line[0] == '#' || line[0] == '!')
-      continue;
-    /* Parse field=value lines */
-    size_t linelen = p - line;
-    p = strchr(line, '=');
-    if (p == NULL || p == line) {
-      m->errors++;
-      WARNF("Malformed manifest line: %s", alloca_toprint(80, line, linelen));
-    } else {
-      *p++ = '\0';
-      char *var = line;
-      char *value = p;
-      if (rhizome_manifest_get(m, var)) {
-	if (config.debug.rejecteddata)
-	  DEBUGF("Ill formed manifest file, duplicate variable \"%s\"", var);
-	m->errors++;
-      } else if (m->var_count >= MAX_MANIFEST_VARS) {
-	if (config.debug.rejecteddata)
-	  WARN("Ill formed manifest file, too many variables");
-	m->errors++;
-      } else {
-	m->vars[m->var_count] = strdup(var);
-	m->values[m->var_count] = strdup(value);
-	/* The bundle ID is implicit in transit, but we need to store it in the manifest, so that
-	 * reimporting manifests on receiver nodes works easily.  We might implement something that
-	 * strips the id variable out of the manifest when sending it, or some other scheme to avoid
-	 * sending all the extra bytes.
-	 */
-	if (strcasecmp(var, "id") == 0) {
-	  have_id = 1;
-	  if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) == -1) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid manifest id: %s", value);
-	    m->errors++;
-	  } else {
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].id = %s", m->manifest_record_number, alloca_tohex_sid_t(m->cryptoSignPublic));
+int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ)
+{
+  const char *const end = buf + len;
+  int has_bid = 0;
+  int has_version = 0;
+  const char *begin = buf;
+  enum { Label, Value, Error } state = Label;
+  const char *p;
+  for (p = buf; state != Error && p < end && *p; ++p)
+    switch (state) {
+      case Label:
+	if (*p == '=') {
+	  if (p == begin)
+	    state = Error; // nil field name
+	  else {
+	    int *has = NULL;
+	    if (p == begin + 2 && strncmp(begin, "id", 2) == 0)
+	      has = &has_bid;
+	    else if (p == begin + 7 && strncmp(begin, "version", 7) == 0)
+	      has = &has_version;
+	    state = Value;
+	    if (has) {
+	      if (*has)
+		state = Error; // duplicate
+	      else {
+		*has = 1;
+		begin = p + 1;
+	      }
+	    }
 	  }
-	} else if (strcasecmp(var, "filehash") == 0) {
-	  have_filehash = 1;
-	  if (str_to_rhizome_filehash_t(&m->filehash, value) == -1) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid filehash: %s", value);
-	    m->errors++;
-	  } else {
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].filehash = %s", m->manifest_record_number, alloca_tohex_rhizome_filehash_t(m->filehash));
+	} else if (!(p == begin ? isalpha(*p) : isalnum(*p)))
+	  state = Error; // bad field name
+	break;
+      case Value:
+	if (*p == '\n') {
+	  const char *eol = p[-1] == '\r' ? p - 1 : p;
+	  if (has_bid == 1) {
+	    const char *e;
+	    if (strn_to_rhizome_bid_t(&summ->bid, begin, &e) == 0 && e == eol)
+	      has_bid = 2;
+	    else
+	      state = Error; // invalid "id" field
+	  } else if (has_version == 1) {
+	    const char *e;
+	    if (str_to_uint64(begin, 10, (uint64_t*)&summ->version, &e) && e == eol)
+	      has_version = 2;
+	    else
+	      state = Error; // invalid "version" field
 	  }
-	} else if (strcasecmp(var, "filesize") == 0) {
-	  uint64_t filesize;
-	  if (!str_to_uint64(value, 10, &filesize, NULL) || filesize == RHIZOME_SIZE_UNSET) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid filesize: %s", value);
-	    m->errors++;
-	  } else {
-	    m->filesize = filesize;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].filesize = %"PRIu64, m->manifest_record_number, m->filesize);
+	  if (state == Value) {
+	    state = Label;
+	    begin = p + 1;
 	  }
-	} else if (strcasecmp(var, "version") == 0) {
-	  have_version = 1;
-	  uint64_t version;
-	  if (!str_to_uint64(value, 10, &version, NULL)) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid version: %s", value);
-	    m->errors++;
-	  } else {
-	    m->version = version;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].version = %"PRIu64, m->manifest_record_number, m->version);
-	  }
-	// since rhizome *MUST* be able to carry future manifest versions
-	// if any of these fields are not well formed, the manifest can still be imported and exported
-	// but the bundle should not be added or exported
-	} else if (strcasecmp(var, "tail") == 0) {
-	  uint64_t tail;
-	  if (!str_to_uint64(value, 10, &tail, NULL) || tail == RHIZOME_SIZE_UNSET) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid tail: %s", value);
-	    m->errors++;
-	  } else {
-	    m->tail = tail;
-	    m->is_journal = 1;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].tail = %"PRIu64, m->manifest_record_number, m->tail);
-	  }
-	} else if (strcasecmp(var, "BK") == 0) {
-	  if (str_to_rhizome_bk_t(&m->bundle_key, value) == -1) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid BK: %s", value);
-	    m->warnings++;
-	  } else {
-	    m->has_bundle_key = 1;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].BK = %s", m->manifest_record_number, alloca_tohex_rhizome_bk_t(m->bundle_key));
-	  }
-	} else if (strcasecmp(var, "service") == 0) {
-	  if (rhizome_str_is_manifest_service(value)) {
-	    m->service = m->values[m->var_count]; // will be free()d when vars[] and values[] are free()d
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].service = %s", m->manifest_record_number, alloca_str_toprint(m->service));
-	  } else {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid service: %s", value);
-	    m->warnings++;
-	  }
-	} else if (strcasecmp(var, "date") == 0) {
-	  int64_t date;
-	  if (!str_to_int64(value, 10, &date, NULL)) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid date: %s", value);
-	    m->warnings++;
-	  } else {
-	    m->date = date;
-	    m->has_date = 1;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].date = %"PRItime_ms_t, m->manifest_record_number, m->date);
-	  }
-	} else if (strcasecmp(var, "sender") == 0) {
-	  if (str_to_sid_t(&m->sender, value) == -1) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid sender: %s", value);
-	    m->warnings++;
-	  } else {
-	    m->has_sender = 1;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].sender = %s", m->manifest_record_number, alloca_tohex_sid_t(m->sender));
-	  }
-	} else if (strcasecmp(var, "recipient") == 0) {
-	  if (str_to_sid_t(&m->recipient, value) == -1) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid recipient: %s", value);
-	    m->warnings++;
-	  } else {
-	    m->has_recipient = 1;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].recipient = %s", m->manifest_record_number, alloca_tohex_sid_t(m->recipient));
-	  }
-	} else if (strcasecmp(var, "name") == 0) {
-	  if (value[0] == '\0') {
-	    if (config.debug.rejecteddata)
-	      WARN("Empty name");
-	    //m->warnings++;  TODO Meshms code should set a name for its "conversations" bundle
-	  }
-	  m->name = m->values[m->var_count]; // will be free()d when vars[] and values[] are free()d
-	  if (config.debug.rhizome_manifest)
-	    DEBUGF("PARSE manifest[%d].name = %s", m->manifest_record_number, alloca_str_toprint(m->name));
-	} else if (strcasecmp(var, "crypt") == 0) {
-	  if (!(strcmp(value, "0") == 0 || strcmp(value, "1") == 0)) {
-	    if (config.debug.rejecteddata)
-	      DEBUGF("Invalid crypt: %s", value);
-	    m->warnings++;
-	  } else {
-	    m->payloadEncryption = (value[0] == '1') ? PAYLOAD_ENCRYPTED : PAYLOAD_CLEAR;
-	    if (config.debug.rhizome_manifest)
-	      DEBUGF("PARSE manifest[%d].crypt = %u", m->manifest_record_number, m->payloadEncryption == PAYLOAD_ENCRYPTED ? 1 : 0);
-	  }
-	} else {
-	  // An unknown field is not an error... older rhizome nodes must carry newer manifests.
-	  if (config.debug.rhizome_manifest)
-	    DEBUGF("SKIP manifest[%d].%s = %s", m->manifest_record_number, var, alloca_str_toprint(value));
 	}
-	m->var_count++;
-      }
+	break;
+      default:
+	abort();
     }
-  }
-  /* The null byte gets included in the check sum */
-  if (ofs < m->manifest_bytes)
-    ++ofs;
+  if (p < end && *p == '\0')
+    ++p;
+  summ->body_len = p - buf;
+  return state == Label && has_bid == 2 && has_version == 2;
+}
 
-  /* Remember where the text ends */
-  unsigned end_of_text = ofs;
-  m->manifest_bytes = end_of_text;
+/* Parse a Rhizome text manifest from its internal buffer up to and including the terminating NUL
+ * character which marks the start of the signature block.
+ *
+ * Prior to calling, the caller must set up m->manifest_all_bytes to the length of the manifest
+ * text, including the signature block, and set m->manifestdata[0..m->manifest_all_bytes-1] to
+ * contain the manifest text and signature block to be parsed.
+ *
+ * A "well formed" manifest consists of a series of zero or more lines with the form:
+ *
+ *	LABEL "=" VALUE [ CR ] LF
+ *
+ * where LABEL matches the regular expression [A-Za-z][A-Za-z0-9]* (identifier without underscore)
+ *       VALUE is any value that does not contain NUL, CR or LF (leading and trailing spaces are
+ *	       not stripped from VALUE)
+ *       NUL is ASCII 0
+ *       CR is ASCII 13
+ *       LF is ASCII 10
+ *
+ * Unpacks all parsed field labels and string values into the m->vars[] and m->values[] arrays, as
+ * pointers to malloc(3)ed NUL terminated strings, in the order they appear, and sets m->var_count
+ * to the number of fields unpacked.  Sets m->manifest_body_bytes to the number of bytes in the text
+ * portion up to and including the optional NUL that starts the signature block (if present).
+ *
+ * Returns 1 if the manifest is not well formed (syntax violation), any essential field is
+ * malformed, or if there are any duplicate fields.  In this case the m->vars[] and m->values[]
+ * arrays are not set and the manifest is returned to the state it was in prior to calling.
+ *
+ * Returns 0 if the manifest is well formed, if there are no duplicate fields, and if all essential
+ * fields are valid.  Counts invalid non-essential fields and unrecognised fields in m->malformed.
+ *
+ * Returns -1 if there is an unrecoverable error (eg, malloc(3) returns NULL, out of memory).
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+static int rhizome_manifest_parse(rhizome_manifest *m)
+{
+  IN();
+  assert(m->manifest_all_bytes <= sizeof m->manifestdata);
+  assert(m->manifest_body_bytes == 0);
+  assert(m->var_count == 0);
+  assert(!m->malformed);
+  assert(!m->has_id);
+  assert(!m->is_journal);
+  assert(m->filesize == RHIZOME_SIZE_UNSET);
+  assert(m->tail == RHIZOME_SIZE_UNSET);
+  assert(m->version == 0);
 
-  // verify that all required fields are consistent.
-  if (!have_id) {
-    if (config.debug.rejecteddata)
-      DEBUG("Missing manifest id field");
-    m->errors++;
-  }
-  if (!have_version) {
-    if (config.debug.rejecteddata)
-      DEBUG("Missing version field");
-    m->errors++;
-  }
-  if (m->filesize == RHIZOME_SIZE_UNSET) {
-    if (config.debug.rejecteddata)
-      DEBUG("Missing filesize field");
-    m->errors++;
-  }
-  if (!have_filehash && m->filesize > 0) {
-    if (config.debug.rejecteddata)
-      DEBUG("Missing filehash field");
-    m->errors++;
-  }
-  if (have_filehash && m->filesize == 0) {
-    if (config.debug.rejecteddata)
-      DEBUG("Spurious filehash field");
-    m->errors++;
-  }
+  unsigned has_invalid_essential = 0;
+  unsigned has_duplicate = 0;
 
-  // warn if expected fields are missing
-  if (m->service == NULL) {
-    if (config.debug.rejecteddata)
-      DEBUG("Missing service field");
-    m->warnings++;
+  const char *const end = (const char *)m->manifestdata + m->manifest_all_bytes;
+  const char *p;
+  unsigned line_number = 0;
+  for (p = (const char *)m->manifestdata; p < end && *p; ++p) {
+    ++line_number;
+    if (!isalpha(*p)) {
+      if (config.debug.rhizome_manifest)
+	DEBUGF("Invalid manifest field name at line %u: %s", line_number, alloca_toprint(20, p, end - p));
+      break;
+    }
+    const char *const plabel = p++;
+    while (p < end && isalnum(*p))
+      ++p;
+    if (*p != '=') {
+      if (config.debug.rhizome_manifest)
+	DEBUGF("Invalid manifest field name at line %u: %s", line_number, alloca_toprint(-1, plabel, p - plabel + 1));
+      break;
+    }
+    const char *const pvalue = ++p;
+    while (p < end && *p && *p != '\n')
+      ++p;
+    if (p >= end || *p != '\n') {
+      if (config.debug.rhizome_manifest)
+	DEBUGF("Missing manifest newline at line %u: %s", line_number, alloca_toprint(-1, plabel, p - plabel));
+      break;
+    }
+    const char *const eol = (p > pvalue && p[-1] == '\r') ? p - 1 : p;
+    if (m->var_count >= NELS(m->vars)) {
+      if (config.debug.rhizome_manifest)
+	DEBUGF("Manifest field limit reached at line %u", line_number);
+      break;
+    }
+    assert(pvalue - plabel - 1 > 0);
+    const char *label = strn_edup(plabel, pvalue - plabel - 1);
+    const char *value = strn_edup(pvalue, eol - pvalue);
+    if (label == NULL || value == NULL) {
+      free((char *)label);
+      free((char *)value);
+      RETURN(-1);
+    }
+    enum { FIELD_UNKNOWN, FIELD_OK, FIELD_DUPLICATE, FIELD_MALFORMED, FIELD_INVALID } status = FIELD_UNKNOWN;
+    if (rhizome_manifest_get(m, label))
+      status = FIELD_DUPLICATE;
+    else if (strcasecmp(label, "id") == 0) {
+      if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) != -1) {
+	status = FIELD_OK;
+	m->has_id = 1;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].id = %s", m->manifest_record_number, alloca_tohex_sid_t(m->cryptoSignPublic));
+      } else
+	status = FIELD_INVALID;
+    }
+    else if (strcasecmp(label, "version") == 0) {
+      uint64_t version;
+      if (str_to_uint64(value, 10, &version, NULL) && version != 0) {
+	status = FIELD_OK;
+	m->version = version;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].version = %"PRIu64, m->manifest_record_number, m->version);
+      } else
+	status = FIELD_INVALID;
+    }
+    else if (strcasecmp(label, "filehash") == 0) {
+      if (str_to_rhizome_filehash_t(&m->filehash, value) != -1 && !rhizome_filehash_t_is_zero(m->filehash)) {
+	status = FIELD_OK;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].filehash = %s", m->manifest_record_number, alloca_tohex_rhizome_filehash_t(m->filehash));
+      } else
+	status = FIELD_INVALID;
+    }
+    else if (strcasecmp(label, "filesize") == 0) {
+      uint64_t filesize;
+      if (str_to_uint64(value, 10, &filesize, NULL) && filesize != RHIZOME_SIZE_UNSET) {
+	status = FIELD_OK;
+	m->filesize = filesize;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].filesize = %"PRIu64, m->manifest_record_number, m->filesize);
+      } else
+	status = FIELD_INVALID;
+    }
+    else if (strcasecmp(label, "tail") == 0) {
+      uint64_t tail;
+      if (str_to_uint64(value, 10, &tail, NULL) && tail != RHIZOME_SIZE_UNSET) {
+	status = FIELD_OK;
+	m->tail = tail;
+	m->is_journal = 1;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].tail = %"PRIu64, m->manifest_record_number, m->tail);
+      } else
+	status = FIELD_INVALID;
+    }
+    // Since rhizome MUST be able to carry future manifest versions, if any of the following fields
+    // are not well formed, they are simply not unpacked into their respective struct elements and
+    // treated as an unrecognised field.  The m->malformed flag is set so that the application API
+    // layer can refuse to add (or export?) the bundle.
+    else if (strcasecmp(label, "BK") == 0) {
+      if (str_to_rhizome_bk_t(&m->bundle_key, value) != -1) {
+	status = FIELD_OK;
+	m->has_bundle_key = 1;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].BK = %s", m->manifest_record_number, alloca_tohex_rhizome_bk_t(m->bundle_key));
+      } else
+	status = FIELD_MALFORMED;
+    }
+    else if (strcasecmp(label, "service") == 0) {
+      if (rhizome_str_is_manifest_service(value)) {
+	status = FIELD_OK;
+	m->service = value; // will be free()d when vars[] and values[] are free()d
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].service = %s", m->manifest_record_number, alloca_str_toprint(m->service));
+      } else
+	status = FIELD_MALFORMED;
+    }
+    else if (strcasecmp(label, "date") == 0) {
+      int64_t date;
+      if (str_to_int64(value, 10, &date, NULL)) {
+	status = FIELD_OK;
+	m->date = date;
+	m->has_date = 1;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].date = %"PRItime_ms_t, m->manifest_record_number, m->date);
+      } else
+	status = FIELD_MALFORMED;
+    }
+    else if (strcasecmp(label, "sender") == 0) {
+      if (str_to_sid_t(&m->sender, value) != -1) {
+	status = FIELD_OK;
+	m->has_sender = 1;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].sender = %s", m->manifest_record_number, alloca_tohex_sid_t(m->sender));
+      } else
+	status = FIELD_MALFORMED;
+    }
+    else if (strcasecmp(label, "recipient") == 0) {
+      if (str_to_sid_t(&m->recipient, value) != -1) {
+	status = FIELD_OK;
+	m->has_recipient = 1;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].recipient = %s", m->manifest_record_number, alloca_tohex_sid_t(m->recipient));
+      } else
+	status = FIELD_MALFORMED;
+    }
+    else if (strcasecmp(label, "name") == 0) {
+      status = FIELD_OK;
+      m->name = value; // will be free()d when vars[] and values[] are free()d
+      if (config.debug.rhizome_manifest)
+	DEBUGF("PARSE manifest[%d].name = %s", m->manifest_record_number, alloca_str_toprint(m->name));
+    }
+    else if (strcasecmp(label, "crypt") == 0) {
+      if (strcmp(value, "0") == 0 || strcmp(value, "1") == 0) {
+	status = FIELD_OK;
+	m->payloadEncryption = (value[0] == '1') ? PAYLOAD_ENCRYPTED : PAYLOAD_CLEAR;
+	if (config.debug.rhizome_manifest)
+	  DEBUGF("PARSE manifest[%d].crypt = %u", m->manifest_record_number, m->payloadEncryption == PAYLOAD_ENCRYPTED ? 1 : 0);
+      } else
+	status = FIELD_MALFORMED;
+    }
+    const char *reason = NULL;
+    switch (status) {
+      case FIELD_OK:
+	m->vars[m->var_count] = label;
+	m->values[m->var_count] = value;
+	++m->var_count;
+	break;
+      case FIELD_DUPLICATE:
+	++has_duplicate;
+	reason = "duplicate";
+	break;
+      case FIELD_INVALID:
+	++has_invalid_essential;
+	reason = "invalid";
+	break;
+      case FIELD_UNKNOWN:
+	++m->malformed;
+	reason = "unsupported";
+	break;
+      case FIELD_MALFORMED:
+	++m->malformed;
+	reason = "invalid";
+	break;
+      default:
+	abort();
+    }
+    if (reason) {
+      if (config.debug.rhizome_manifest)
+	DEBUGF("SKIP manifest[%d].%s = %s (%s)", m->manifest_record_number, label, alloca_str_toprint(value), reason);
+      free((char *)label);
+      free((char *)value);
+    }
+    assert(p < end);
+    assert(*p == '\n');
   }
-  if (!m->has_date) {
-    if (config.debug.rejecteddata)
-      DEBUG("Missing date field");
-    m->warnings++;
+  if ((p < end && *p) || has_invalid_essential || has_duplicate) {
+    rhizome_manifest_clear(m);
+    RETURN(1);
   }
-
-  if (m->errors || m->warnings) {
-    if (config.debug.rejecteddata)
-      dump("manifest body", m->manifestdata, (size_t) m->manifest_bytes);
+  // The null byte is included in the body (and checksum), not the signature block
+  if (p < end) {
+    assert(*p == '\0');
+    ++p;
   }
-
+  m->manifest_body_bytes = p - (const char *)m->manifestdata;
   RETURN(0);
   OUT();
 }
 
+/* Return 1 if all necessary fields are present, 0 if not.  Increment m->malformed if any
+ * unnecessary fields are missing.
+ */
+int rhizome_manifest_validate(rhizome_manifest *m)
+{
+  int ret = 1;
+  if (!m->has_id) {
+    if (config.debug.rhizome_manifest)
+      DEBUG("Missing 'id' field");
+    ret = 0;
+  }
+  if (m->version == 0) {
+    if (config.debug.rhizome_manifest)
+      DEBUG("Missing 'version' field");
+    ret = 0;
+  }
+  if (m->filesize == RHIZOME_SIZE_UNSET) {
+    if (config.debug.rhizome_manifest)
+      DEBUG("Missing 'filesize' field");
+    ret = 0;
+  }
+  if (rhizome_filehash_t_is_zero(m->filehash)) {
+    if (m->filesize > 0) {
+      if (config.debug.rhizome_manifest)
+	DEBUG("Missing 'filehash' field");
+      ret = 0;
+    }
+  } else {
+    if (m->filesize == 0) {
+      if (config.debug.rhizome_manifest)
+	DEBUG("Spurious 'filehash' field");
+      ret = 0;
+    }
+  }
+  // warn if expected fields are missing
+  if (m->service == NULL) {
+    if (config.debug.rhizome_manifest)
+      DEBUG("Missing 'service' field");
+    ++m->malformed;
+  }
+  if (!m->has_date) {
+    if (config.debug.rhizome_manifest)
+      DEBUG("Missing 'date' field");
+    ++m->malformed;
+  }
+  return ret;
+}
+
 int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferP)
 {
   if (!m)
@@ -686,15 +803,19 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t
     return WHY("Buffer too big");
 
   if (bufferP) {
-    m->manifest_bytes=bufferP;
-    memcpy(m->manifestdata, filename, m->manifest_bytes);
+    m->manifest_all_bytes=bufferP;
+    memcpy(m->manifestdata, filename, m->manifest_all_bytes);
   } else {
     ssize_t bytes = read_whole_file(filename, m->manifestdata, sizeof m->manifestdata);
     if (bytes == -1)
       return -1;
-    m->manifest_bytes = bytes;
+    m->manifest_all_bytes = (size_t) bytes;
+  }
+  switch (rhizome_manifest_parse(m)) {
+    case 0: return 0;
+    case -1: return -1;
+    default: return WHY("Invalid manifest");
   }
-  return rhizome_manifest_parse(m);
 }
 
 int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out)
@@ -804,8 +925,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
   if (config.debug.manifests) _log_manifest_trace(__whence, __FUNCTION__);
 
   // Set global defaults for a manifest (which are not zero)
-  m->filesize = RHIZOME_SIZE_UNSET;
-  m->tail = RHIZOME_SIZE_UNSET;
+  rhizome_manifest_clear(m);
 
   return m;
 }
@@ -829,17 +949,7 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
 	);
 
   /* Free variable and signature blocks. */
-  unsigned i;
-  for(i=0;i<m->var_count;i++) {
-    free((char *) m->vars[i]);
-    free((char *) m->values[i]);
-    m->vars[i] = m->values[i] = NULL;
-  }
-  for(i=0;i<m->sig_count;i++) {
-    free(m->signatories[i]);
-    m->signatories[i] = NULL;
-  }
-
+  rhizome_manifest_clear(m);
   if (m->dataFileName) {
     if (m->dataFileUnlinkOnFree && unlink(m->dataFileName) == -1)
       WARNF_perror("unlink(%s)", alloca_str_toprint(m->dataFileName));
@@ -856,48 +966,54 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
   return;
 }
 
-/* Convert variable list to string, complaining if it ends up
-   too long.
-   Signatures etc will be added later. */
+/* Convert variable list into manifest text body and compute the hash.  Do not sign.
+ */
 int rhizome_manifest_pack_variables(rhizome_manifest *m)
 {
+  assert(m->var_count <= NELS(m->vars));
+  strbuf sb = strbuf_local((char*)m->manifestdata, sizeof m->manifestdata);
   unsigned i;
-  unsigned ofs = 0;
-  for(i=0;i<m->var_count;i++)
-    {
-      if ((ofs+strlen(m->vars[i])+1+strlen(m->values[i])+1+1)>MAX_MANIFEST_BYTES)
-	return WHY("Manifest variables too long in total to fit in MAX_MANIFEST_BYTES");
-      snprintf((char *)&m->manifestdata[ofs],MAX_MANIFEST_BYTES-ofs,"%s=%s\n",
-	       m->vars[i],m->values[i]);
-      ofs+=strlen((char *)&m->manifestdata[ofs]);
-    }
-  m->manifestdata[ofs++]=0x00;
-  m->manifest_bytes=ofs;
-  if (config.debug.rhizome) DEBUG("Repacked variables in manifest.");
-  m->manifest_all_bytes=ofs;
-
-  /* Recalculate hash */
-  crypto_hash_sha512(m->manifesthash,m->manifestdata,m->manifest_bytes);
-
+  for (i = 0; i < m->var_count; ++i) {
+    strbuf_puts(sb, m->vars[i]);
+    strbuf_putc(sb, '=');
+    strbuf_puts(sb, m->values[i]);
+    strbuf_putc(sb, '\n');
+  }
+  if (strbuf_overrun(sb))
+    return WHYF("Manifest overflow: body of %zu bytes exceeds limit of %zu", strbuf_count(sb) + 1, sizeof m->manifestdata);
+  m->manifest_body_bytes = strbuf_len(sb) + 1;
+  if (config.debug.rhizome)
+    DEBUGF("Repacked variables into manifest: %zu bytes", m->manifest_body_bytes);
+  m->manifest_all_bytes = m->manifest_body_bytes;
+  m->selfSigned = 0;
   return 0;
 }
 
-/* Sign this manifest using our it's own BID secret key.
-   TODO: need a third-party signing primitive, eg, to support signing with SAS.
+/* Sign this manifest using it's own BID secret key.  Manifest must not already be signed.
+ * Manifest body hash must already be computed.
  */
 int rhizome_manifest_selfsign(rhizome_manifest *m)
 {
+  assert(m->manifest_body_bytes > 0);
+  assert(m->manifest_body_bytes <= sizeof m->manifestdata);
+  assert(m->manifestdata[m->manifest_body_bytes - 1] == '\0');
+  assert(m->manifest_body_bytes == m->manifest_all_bytes); // no signature yet
   if (!m->haveSecret)
     return WHY("Need private key to sign manifest");
+  crypto_hash_sha512(m->manifesthash, m->manifestdata, m->manifest_body_bytes);
   rhizome_signature sig;
   if (rhizome_sign_hash(m, &sig) == -1)
     return WHY("rhizome_sign_hash() failed");
+  assert(sig.signatureLength > 0);
   /* Append signature to end of manifest data */
-  if (sig.signatureLength + m->manifest_bytes > MAX_MANIFEST_BYTES)
-    return WHY("Manifest plus signatures is too long");
-  bcopy(&sig.signature[0], &m->manifestdata[m->manifest_bytes], sig.signatureLength);
-  m->manifest_bytes += sig.signatureLength;
-  m->manifest_all_bytes = m->manifest_bytes;
+  if (sig.signatureLength + m->manifest_body_bytes > sizeof m->manifestdata)
+    return WHYF("Manifest overflow: body %zu + signature %zu bytes exceeds limit of %zu",
+		m->manifest_body_bytes,
+		sig.signatureLength,
+		sizeof m->manifestdata
+	      );
+  bcopy(sig.signature, m->manifestdata + m->manifest_body_bytes, sig.signatureLength);
+  m->manifest_all_bytes = m->manifest_body_bytes + sig.signatureLength;
   return 0;
 }
 
diff --git a/rhizome_crypto.c b/rhizome_crypto.c
index 3055204f..d2e29102 100644
--- a/rhizome_crypto.c
+++ b/rhizome_crypto.c
@@ -434,7 +434,7 @@ int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk,
   int mLen = crypto_hash_sha512_BYTES;
   int r = crypto_sign_edwards25519sha512batch(signatureBuffer, &sigLen, &hash[0], mLen, sk);
   if (r)
-    RETURN(WHY("crypto_sign_edwards25519sha512batch() failed."));
+    RETURN(WHY("crypto_sign_edwards25519sha512batch() failed"));
   /* Here we use knowledge of the internal structure of the signature block
      to remove the hash, since that is implicitly transported, thus reducing the
      actual signature size down to 64 bytes.
@@ -457,7 +457,7 @@ typedef struct manifest_signature_block_cache {
 #define SIG_CACHE_SIZE 1024
 manifest_signature_block_cache sig_cache[SIG_CACHE_SIZE];
 
-int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char *sig,int sig_len)
+int rhizome_manifest_lookup_signature_validity(const unsigned char *hash, const unsigned char *sig, int sig_len)
 {
   IN();
   unsigned int slot=0;
@@ -505,73 +505,45 @@ int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char
 int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs)
 {
   IN();
-  if (!m)
-    RETURN(WHY("NULL pointer passed in as manifest"));
-  if (config.debug.rhizome)
-    DEBUGF("m->manifest_all_bytes=%u m->manifest_bytes=%u *ofs=%u", m->manifest_all_bytes, m->manifest_bytes, *ofs);
-
-  if ((*ofs) >= m->manifest_all_bytes)
-    RETURN(0);
-
+  if (config.debug.rhizome_manifest)
+    DEBUGF("*ofs=%u m->manifest_all_bytes=%zu", *ofs, m->manifest_all_bytes);
+  assert((*ofs) < m->manifest_all_bytes);
+  const unsigned char *sig = m->manifestdata + *ofs;
   uint8_t sigType = m->manifestdata[*ofs];
   uint8_t len = (sigType << 2) + 4 + 1;
-
-  /* Each signature type is required to have a different length to detect it.
-     At present only crypto_sign_edwards25519sha512batch() signatures are
-     supported. */
-  int r;
-  if (m->sig_count<MAX_MANIFEST_VARS)
-    switch(sigType) 
-      {
-      case 0x17: /* crypto_sign_edwards25519sha512batch() */
-	/* Reconstitute signature block */
-	r=rhizome_manifest_lookup_signature_validity
-	  (m->manifesthash,&m->manifestdata[(*ofs)+1],96);
-#ifdef DEPRECATED
-	unsigned char sigBuf[256];
-	unsigned char verifyBuf[256];
-	unsigned char publicKey[256];
-	bcopy(&m->manifestdata[(*ofs)+1],&sigBuf[0],64);
-	bcopy(&m->manifesthash[0],&sigBuf[64],crypto_hash_sha512_BYTES);
-	/* Get public key of signatory */
-	bcopy(&m->manifestdata[(*ofs)+1+64],&publicKey[0],crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
-	unsigned long long mlen=0;
-	int r=crypto_sign_edwards25519sha512batch_open(verifyBuf,&mlen,&sigBuf[0],128, publicKey);
-#endif
-	if (r) {
-	  (*ofs)+=len;
-	  m->errors++;
-	  RETURN(WHY("Error in signature block (verification failed)."));
-	} else {
-	  /* Signature block passes, so add to list of signatures */
-	  m->signatureTypes[m->sig_count]=len;
-	  m->signatories[m->sig_count]
-	    =malloc(crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
-	  if(!m->signatories[m->sig_count]) {
-	    (*ofs)+=len;
-	    RETURN(WHY("malloc() failed when reading signature block"));
-	  }
-	  bcopy(&m->manifestdata[(*ofs)+1+64],m->signatories[m->sig_count],
-		crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
-	  m->sig_count++;
-	  if (config.debug.rhizome) DEBUG("Signature passed.");
-	}
-	break;
-      default:
-	(*ofs)+=len;
-	m->errors++;
-	RETURN(WHYF("Encountered illegal or malformed signature block (unknown type=0x%02x @ offset 0x%x)",sigType,(*ofs)-len));
-      }
-  else
+  if (*ofs + len > m->manifest_all_bytes) {
+    WARNF("Invalid signature at offset %u: type=%#02x gives len=%u that overruns manifest size",
+	*ofs, sigType, len);
+    RETURN(1);
+  }
+  *ofs += len;
+  assert (m->sig_count <= NELS(m->signatories));
+  if (m->sig_count == NELS(m->signatories)) {
+    WARN("Too many signature blocks in manifest");
+    RETURN(2);
+  }
+  switch (sigType) {
+    case 0x17: // crypto_sign_edwards25519sha512batch()
     {
-      (*ofs)+=len;
-      WHY("Too many signature blocks in manifest.");
-      m->errors++;
+      assert(len == 97);
+      /* Reconstitute signature block */
+      int r = rhizome_manifest_lookup_signature_validity(m->manifesthash, sig + 1, 96);
+      if (r) {
+	WARN("Signature verification failed");
+	RETURN(4);
+      }
+      m->signatureTypes[m->sig_count] = len;
+      if ((m->signatories[m->sig_count] = emalloc(crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)) == NULL)
+	RETURN(-1);
+      bcopy(sig + 1 + 64, m->signatories[m->sig_count], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
+      m->sig_count++;
+      if (config.debug.rhizome)
+	DEBUG("Signature verified");
+      RETURN(0);
     }
-
-  (*ofs)+=len;
-  RETURN(0);
-  OUT();
+  }
+  WARNF("Unsupported signature at ofs=%u: type=%#02x", sig - m->manifestdata, sigType);
+  RETURN(3);
 }
 
 // add value to nonce, with the same result regardless of CPU endian order
diff --git a/rhizome_database.c b/rhizome_database.c
index 348e1e77..d7a269e1 100644
--- a/rhizome_database.c
+++ b/rhizome_database.c
@@ -164,31 +164,29 @@ void sqlite_log(void *ignored, int result, const char *msg)
   WARNF("Sqlite: %d %s", result, msg);
 }
 
-void verify_bundles(){
+void verify_bundles()
+{
   // assume that only the manifest itself can be trusted
   // fetch all manifests and reinsert them.
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   // This cursor must be ordered descending as re-inserting the manifests will give them a new higher manifest id.
   // If we didn't, we'd get stuck in an infinite loop.
   sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;");
-  while(sqlite_step_retry(&retry, statement)==SQLITE_ROW){
+  while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
     sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
     const void *manifest = sqlite3_column_blob(statement, 1);
     size_t manifest_length = sqlite3_column_bytes(statement, 1);
     rhizome_manifest *m=rhizome_new_manifest();
-    int ret=0;
-    ret = rhizome_read_manifest_file(m, manifest, manifest_length);
-    if (ret==0 && m->errors)
-      ret=-1;
-    if (ret==0)
-      ret=rhizome_manifest_verify(m);
-    if (ret==0){
-      m->finalised=1;
-      m->manifest_bytes=m->manifest_all_bytes;
-      // store it again, to ensure it is valid and stored correctly with matching file content.
-      ret=rhizome_store_bundle(m);
+    int ret = -1;
+    if (   rhizome_read_manifest_file(m, manifest, manifest_length) == 0
+	&& rhizome_manifest_validate(m)
+	&& rhizome_manifest_verify(m)
+    ) {
+      assert(m->finalised);
+      // Store it again, to ensure that MANIFESTS columns are up to date.
+      ret = rhizome_store_bundle(m);
     }
-    if (ret!=0){
+    if (ret) {
       if (config.debug.rhizome)
 	DEBUGF("Removing invalid manifest entry @%lld", rowid);
       sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END);
@@ -1386,7 +1384,7 @@ int rhizome_store_bundle(rhizome_manifest *m)
 	  "?,?,?,?,?,?,?,?,?,?,?,?,?"
 	");",
 	RHIZOME_BID_T, &m->cryptoSignPublic,
-	STATIC_BLOB, m->manifestdata, m->manifest_bytes,
+	STATIC_BLOB, m->manifestdata, m->manifest_all_bytes,
 	INT64, m->version,
 	INT64, (int64_t) now,
 	STATIC_BLOB, bar, RHIZOME_BAR_BYTES,
@@ -1604,7 +1602,9 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
     rhizome_manifest *m = c->manifest = rhizome_new_manifest();
     if (m == NULL)
       RETURN(-1);
-    if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
+    if (   rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1
+	|| !rhizome_manifest_validate(m)
+    ) {
       WHYF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
       continue;
     }
@@ -1710,7 +1710,6 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
     strbuf_puts(b, " AND recipient = ?");
   if (strbuf_overrun(b))
     return WHYF("SQL command too long: %s", strbuf_str(b));
-
   int ret = 0;
   sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
   sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->filesize, STATIC_TEXT, m->service, END);
@@ -1739,11 +1738,13 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
     const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
     const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
     size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
-    if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) {
+    if (   rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1
+	|| !rhizome_manifest_validate(blob_m)
+    ) {
       WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
       goto next;
     }
-    if (rhizome_manifest_verify(blob_m)) {
+    if (!rhizome_manifest_verify(blob_m)) {
       WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
       goto next;
     }
@@ -1781,8 +1782,8 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
   const char *q_author = (const char *) sqlite3_column_text(statement, 4);
   size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
   uint64_t q_rowid = sqlite3_column_int64(statement, 5);
-  if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1)
-    return WHYF("Manifest %s exists but is invalid", q_id);
+  if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1 || !rhizome_manifest_validate(m))
+    return WHYF("Manifest bid=%s in database but invalid", q_id);
   if (q_author) {
     sid_t author;
     if (str_to_sid_t(&author, q_author) == -1)
diff --git a/rhizome_direct.c b/rhizome_direct.c
index 02d91833..7d982195 100644
--- a/rhizome_direct.c
+++ b/rhizome_direct.c
@@ -442,11 +442,12 @@ rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,int pref
       if (!manifestblob) goto error;
 
       rhizome_manifest *m=rhizome_new_manifest();
-      if (rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1)
-	{
-	  rhizome_manifest_free(m);
-	  goto error;
-	}
+      if (   rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1
+	  || !rhizome_manifest_validate(m)
+      ) {
+	rhizome_manifest_free(m);
+	goto error;
+      }
       
       DEBUGF("Read manifest");
       sqlite3_blob_close(blob);
diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c
index 400ab40a..b84a5430 100644
--- a/rhizome_direct_http.c
+++ b/rhizome_direct_http.c
@@ -257,7 +257,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
     if (config.debug.rhizome)
       DEBUGF("Import sans-manifest appeared to succeed");
     /* Respond with the manifest that was added. */
-    http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes);
+    http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_all_bytes);
     /* clean up after ourselves */
     if (mout && mout != m)
       rhizome_manifest_free(mout);
@@ -658,7 +658,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
 	  "\r\n";
 	/* Work out what the content length should be */
 	if (config.debug.rhizome_tx)
-	  DEBUGF("manifest_all_bytes=%u, manifest_bytes=%u", m->manifest_all_bytes, m->manifest_bytes);
+	  DEBUGF("manifest_all_bytes=%u, manifest_body_bytes=%u", m->manifest_all_bytes, m->manifest_body_bytes);
 	assert(m->filesize != RHIZOME_SIZE_UNSET);
 	size_t content_length =
 	    strlen(template2) - 2 /* minus 2 for the "%s" that gets replaced */
diff --git a/rhizome_fetch.c b/rhizome_fetch.c
index 7f1d5d74..f01d080e 100644
--- a/rhizome_fetch.c
+++ b/rhizome_fetch.c
@@ -464,10 +464,9 @@ int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int
 static int rhizome_import_received_bundle(struct rhizome_manifest *m)
 {
   m->finalised = 1;
-  m->manifest_bytes = m->manifest_all_bytes; // store the signatures too
   if (config.debug.rhizome_rx) {
     DEBUGF("manifest len=%u has %u signatories. Associated filesize=%"PRIu64" bytes", 
-	   m->manifest_bytes, m->sig_count, m->filesize);
+	   m->manifest_all_bytes, m->sig_count, m->filesize);
     dump("manifest", m->manifestdata, m->manifest_all_bytes);
   }
   return rhizome_add_manifest(m, m->ttl - 1 /* TTL */);
@@ -887,7 +886,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
 
   assert(m->filesize != RHIZOME_SIZE_UNSET);
   if (m->filesize == 0) {
-    if (rhizome_manifest_verify(m) != 0) {
+    if (!rhizome_manifest_verify(m)) {
       WHY("Error verifying manifest when considering for import");
       /* Don't waste time looking at this manifest again for a while */
       rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
@@ -923,7 +922,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
 	    rhizome_manifest_free(m);
 	    RETURN(0);
 	  }
-	  if (!m->selfSigned && rhizome_manifest_verify(m)) {
+	  if (!m->selfSigned && !rhizome_manifest_verify(m)) {
 	    WHY("Error verifying manifest when considering queuing for import");
 	    /* Don't waste time looking at this manifest again for a while */
 	    rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
@@ -949,7 +948,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
     RETURN(1);
   }
 
-  if (!m->selfSigned && rhizome_manifest_verify(m)) {
+  if (!m->selfSigned && !rhizome_manifest_verify(m)) {
     WHY("Error verifying manifest when considering queuing for import");
     /* Don't waste time looking at this manifest again for a while */
     rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
@@ -1306,7 +1305,9 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
        call schedule queued items. */
     rhizome_manifest *m = rhizome_new_manifest();
     if (m) {
-      if (rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1) {
+      if (   rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1
+	  || !rhizome_manifest_validate(m)
+      ) {
 	DEBUGF("Couldn't read manifest");
 	rhizome_manifest_free(m);
       } else {
diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c
index 01af7c16..461959f1 100644
--- a/rhizome_packetformats.c
+++ b/rhizome_packetformats.c
@@ -294,7 +294,6 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
   int ad_frame_type=ob_get(f->payload);
   struct sockaddr_in httpaddr = context->addr;
   httpaddr.sin_port = htons(RHIZOME_HTTP_PORT);
-  size_t manifest_length;
   rhizome_manifest *m=NULL;
 
   int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);
@@ -305,73 +304,66 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
   
   if (ad_frame_type & HAS_MANIFESTS){
     /* Extract whole manifests */
-    while(f->payload->position < f->payload->sizeLimit) {
-      if (ob_getbyte(f->payload, f->payload->position)==0xff){
-	f->payload->position++;
+    while (ob_remaining(f->payload) > 0) {
+      if (ob_peek(f->payload) == 0xff) {
+	ob_skip(f->payload, 1);
 	break;
       }
-	
-      manifest_length = ob_get_ui16(f->payload);
+
+      size_t manifest_length = ob_get_ui16(f->payload);
       if (manifest_length==0) continue;
       
       unsigned char *data = ob_get_bytes_ptr(f->payload, manifest_length);
       if (!data) {
 	WHYF("Illegal manifest length field in rhizome advertisement frame %zu vs %d", 
-	     manifest_length, f->payload->sizeLimit - f->payload->position);
+	     manifest_length, ob_remaining(f->payload));
 	break;
       }
 
-      /* Read manifest without verifying signatures (which would waste lots of
-	 energy, everytime we see a manifest that we already have).
-	 In fact, it would be better here to do a really rough and ready parser
-	 to get the id and version fields out, and avoid the memory copies that
-	 otherwise happen. 
-	 But we do need to make sure that at least one signature is there.
-      */	
-      m = rhizome_new_manifest();
-      if (!m) {
-	WHY("Out of manifests");
+      // Briefly inspect the manifest to see if it looks interesting.
+      struct rhizome_manifest_summary summ;
+      if (!rhizome_manifest_inspect((char *)data, manifest_length, &summ)) {
+	if (config.debug.rhizome_ads)
+	  DEBUG("Ignoring manifest that looks malformed");
 	goto next;
       }
       
-      if (rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1) {
-	WHY("Error parsing manifest body");
-	goto next;
-      }
-      
-      /* trim manifest ID to a prefix for ease of debugging
-	 (that is the only use of this */
-      if (config.debug.rhizome_ads){
-	DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
-      }
+      if (config.debug.rhizome_ads)
+	DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(summ.bid), summ.version);
 
-      /* Crude signature presence test */
-      if (m->manifest_bytes >= m->manifest_all_bytes){
-	// no signature was found when parsing?
-	/* ignore the announcement, but don't ignore other people
-	   offering the same manifest */
+      // If it looks like there is no signature at all, ignore the announcement but don't brown-list
+      // the manifest ID, so that we will still process other offers of the same manifest with
+      // signatures.
+      if (summ.body_len == manifest_length) {
 	if (config.debug.rhizome_ads)
 	  DEBUG("Ignoring manifest announcment with no signature");
 	goto next;
       }
 
-      if (rhizome_ignore_manifest_check(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary)){
+      if (rhizome_ignore_manifest_check(summ.bid.binary, sizeof summ.bid.binary)){
 	/* Ignoring manifest that has caused us problems recently */
 	if (config.debug.rhizome_ads)
-	  DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
+	  DEBUGF("Ignoring manifest with errors bid=%s", alloca_tohex_rhizome_bid_t(summ.bid));
 	goto next;
       }
 
-      if (m->errors > 0){
-	if (config.debug.rhizome_ads)
-	  DEBUG("Unverified manifest has errors - so not processing any further.");
-	/* Don't waste any time on this manifest in future attempts for at least
-	     a minute. */
+      // The manifest looks potentially interesting, so now do a full parse and validation.
+      if ((m = rhizome_new_manifest()) == NULL)
+	goto next;
+      if (   rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1
+	  || !rhizome_manifest_validate(m)
+      ) {
+	WARN("Malformed manifest");
+	// Don't attend to this manifest for at least a minute
 	rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
 	goto next;
       }
-      /* Manifest is okay, so see if it is worth storing */
-
+      assert(m->has_id);
+      assert(m->version != 0);
+      assert(cmp_rhizome_bid_t(&m->cryptoSignPublic, &summ.bid) == 0);
+      assert(m->version == summ.version);
+      assert(m->manifest_body_bytes == summ.body_len);
+      
       // are we already fetching this bundle [or later]?
       rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
       if (mf && mf->version >= m->version)
diff --git a/tests/rhizomeops b/tests/rhizomeops
index 615357ff..2c464bd5 100755
--- a/tests/rhizomeops
+++ b/tests/rhizomeops
@@ -545,10 +545,9 @@ setup_AddDeDuplicate() {
    assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
 }
 test_AddDeDuplicate() {
-   # Add first file again - nothing should change in its manifests, and it
-   # should appear that the add command succeeded (with perhaps some grumbling
-   # on stderr).
-   execute --exit-status=2 $servald rhizome add file $SIDB1 file1 file1.manifestA
+   # Add first file again - should return a "duplicate" status code and nothing
+   # should change in its manifests.
+   execute --exit-status=2 --stderr $servald rhizome add file $SIDB1 file1 file1.manifestA
    assert [ -s file1.manifestA ]
    assert_stdout_add_file file1
    extract_stdout_secret file1_dup_secret
@@ -558,7 +557,7 @@ test_AddDeDuplicate() {
    assert diff file1.manifest file1.manifestA
    assert [ $file1_secret = $file1_dup_secret ]
    # Repeat for second file.
-   execute --exit-status=2 $servald rhizome add file $SIDB1 file2 file2.manifestA
+   execute --exit-status=2 --stderr $servald rhizome add file $SIDB1 file2 file2.manifestA
    assert [ -s file2.manifestA ]
    assert_stdout_add_file file2
    extract_stdout_secret file2_dup_secret
diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index ff9a58f3..497d8c8d 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -43,6 +43,8 @@ configure_servald_server() {
       set debug.rhizome on \
       set debug.httpd on \
       set debug.rhizome_httpd on \
+      set debug.rhizome_manifest on \
+      set debug.rhizome_ads on \
       set debug.rhizome_tx on \
       set debug.rhizome_rx on \
       set server.respawn_on_crash off \

From 6bff87f7e5926d5699dcc734d10744fde37c712d Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Sun, 1 Dec 2013 05:49:50 +1030
Subject: [PATCH 53/56] Increase 'rhizomeprotocol' test case timeout

The 50% MDP packet loss test case is now likely to take more than 60
seconds to transfer the bundles, probably because of stricter manifest
syntax
---
 tests/rhizomeprotocol | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/rhizomeprotocol b/tests/rhizomeprotocol
index 7b521484..bf025861 100755
--- a/tests/rhizomeprotocol
+++ b/tests/rhizomeprotocol
@@ -59,7 +59,7 @@ setup_common() {
 }
 
 receive_and_update_bundle() {
-   wait_until bundle_received_by $BID:$VERSION +B
+   wait_until "$@" bundle_received_by $BID:$VERSION +B
    set_instance +B
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=0 file1
@@ -67,7 +67,7 @@ receive_and_update_bundle() {
    set_instance +A
    rhizome_update_file file1 file2
    set_instance +B
-   wait_until bundle_received_by $BID:$VERSION +B
+   wait_until "$@" bundle_received_by $BID:$VERSION +B
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=0 file2
    assert_rhizome_received file2
@@ -767,7 +767,7 @@ setup_SimulatedRadio2() {
    foreach_instance +A +B start_radio_instance
 }
 test_SimulatedRadio2() {
-   receive_and_update_bundle
+   receive_and_update_bundle --timeout=120
 }
 teardown_SimulatedRadio2() {
    teardown

From 2bb0d2047a661a85d27569e46aa9aca347d878a3 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 2 Dec 2013 11:12:31 +1030
Subject: [PATCH 54/56] Enforce valid Rhizome 'service' and 'name' strings

---
 dataformats.c    | 13 +++++++++++++
 dataformats.h    |  1 +
 rhizome_bundle.c |  2 ++
 3 files changed, 16 insertions(+)

diff --git a/dataformats.c b/dataformats.c
index ed841ca2..1518ab80 100644
--- a/dataformats.c
+++ b/dataformats.c
@@ -171,6 +171,19 @@ int rhizome_str_is_manifest_service(const char *text)
   return *text == '\0';
 }
 
+/* A name cannot contain a LF because that is the Rhizome text manifest field terminator.  For the
+ * time being, CR is not allowed either, because the Rhizome field terminator includes an optional
+ * CR.  See rhizome_manifest_parse().
+ *
+ * @author Andrew Bettison <andrew@servalproject.com>
+ */
+int rhizome_str_is_manifest_name(const char *text)
+{
+  while (*text && *text != '\n' && *text != '\r')
+    ++text;
+  return *text == '\0';
+}
+
 int str_is_did(const char *did)
 {
   size_t len = 0;
diff --git a/dataformats.h b/dataformats.h
index 67afb746..a9abc661 100644
--- a/dataformats.h
+++ b/dataformats.h
@@ -15,6 +15,7 @@ int rhizome_str_is_bundle_crypt_key(const char *text);
 int rhizome_strn_is_file_hash(const char *text);
 int rhizome_str_is_file_hash(const char *text);
 int rhizome_str_is_manifest_service(const char *text);
+int rhizome_str_is_manifest_name(const char *text);
 
 void write_uint64(unsigned char *o,uint64_t v);
 void write_uint16(unsigned char *o,uint16_t v);
diff --git a/rhizome_bundle.c b/rhizome_bundle.c
index 77754411..e513f855 100644
--- a/rhizome_bundle.c
+++ b/rhizome_bundle.c
@@ -204,6 +204,7 @@ void _rhizome_manifest_del_bundle_key(struct __sourceloc __whence, rhizome_manif
 void _rhizome_manifest_set_service(struct __sourceloc __whence, rhizome_manifest *m, const char *service)
 {
   if (service) {
+    assert(rhizome_str_is_manifest_service(service));
     const char *v = rhizome_manifest_set(m, "service", service);
     assert(v); // TODO: remove known manifest fields from vars[]
     m->service = v;
@@ -223,6 +224,7 @@ void _rhizome_manifest_del_service(struct __sourceloc __whence, rhizome_manifest
 void _rhizome_manifest_set_name(struct __sourceloc __whence, rhizome_manifest *m, const char *name)
 {
   if (name) {
+    assert(rhizome_str_is_manifest_name(name));
     const char *v = rhizome_manifest_set(m, "name", name);
     assert(v); // TODO: remove known manifest fields from vars[]
     m->name = v;

From 9953949c6e792c5032373d103db6272072d59d97 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 2 Dec 2013 15:56:32 +1030
Subject: [PATCH 55/56] Improve test framework: forked commands inherit stdin

Remove the subshell parentheses around the command invoked using fork(),
so that the forked command inherits the file descriptors.  This allows
it to be used like "fork some command < input".
---
 testframework.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testframework.sh b/testframework.sh
index 903a7cc1..4e4035da 100644
--- a/testframework.sh
+++ b/testframework.sh
@@ -1651,7 +1651,7 @@ fork() {
    local _tfw_process_tmp="$_tfw_tmp/fork-$forkid"
    mkdir "$_tfw_process_tmp" || _tfw_fatalexit
    $_tfw_assert_noise && tfw_log "# fork[$forkid] START" $(shellarg "$@")
-   ( "$@" ) 6>"$_tfw_process_tmp/log.stdout" 1>&6 2>"$_tfw_process_tmp/log.stderr" 7>"$_tfw_process_tmp/log.xtrace" &
+   "$@" 6>"$_tfw_process_tmp/log.stdout" 1>&6 2>"$_tfw_process_tmp/log.stderr" 7>"$_tfw_process_tmp/log.xtrace" &
    _tfw_forked_pids[$forkid]=$!
    [ -n "$_tfw_forklabel" ] && eval _tfw_fork_label_$_tfw_forklabel=$forkid
    $_tfw_assert_noise && tfw_log "# fork[$forkid] ${_tfw_forklabel:+%$_tfw_forklabel }pid=$! STARTED"

From 4f0aa3a41ca3ae976a118c4446b2653c96325200 Mon Sep 17 00:00:00 2001
From: Andrew Bettison <andrew@servalproject.org>
Date: Mon, 2 Dec 2013 16:07:37 +1030
Subject: [PATCH 56/56] Improve test framework: include %label in fork log

---
 testframework.sh | 35 +++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/testframework.sh b/testframework.sh
index 4e4035da..7a26bbf5 100644
--- a/testframework.sh
+++ b/testframework.sh
@@ -127,6 +127,7 @@ declare -a _tfw_test_names=()
 declare -a _tfw_test_sourcefiles=()
 declare -a _tfw_job_pgids=()
 declare -a _tfw_forked_pids=()
+declare -a _tfw_forked_labels=()
 
 # The rest of this file is parsed for extended glob patterns.
 _tfw_shopt _tfw_orig_shopt -s extglob
@@ -1648,13 +1649,15 @@ fork() {
       shift
       [ -n "$_tfw_forkid" ] && error "fork label '%$_tfw_forklabel' already in use"
    fi
+   local desc="fork[$forkid]${_tfw_forklabel:+ %$_tfw_forklabel}"
    local _tfw_process_tmp="$_tfw_tmp/fork-$forkid"
    mkdir "$_tfw_process_tmp" || _tfw_fatalexit
-   $_tfw_assert_noise && tfw_log "# fork[$forkid] START" $(shellarg "$@")
+   $_tfw_assert_noise && tfw_log "# $desc START" $(shellarg "$@")
    "$@" 6>"$_tfw_process_tmp/log.stdout" 1>&6 2>"$_tfw_process_tmp/log.stderr" 7>"$_tfw_process_tmp/log.xtrace" &
    _tfw_forked_pids[$forkid]=$!
+   _tfw_forked_labels[$forkid]="$_tfw_forklabel"
    [ -n "$_tfw_forklabel" ] && eval _tfw_fork_label_$_tfw_forklabel=$forkid
-   $_tfw_assert_noise && tfw_log "# fork[$forkid] ${_tfw_forklabel:+%$_tfw_forklabel }pid=$! STARTED"
+   $_tfw_assert_noise && tfw_log "# $desc pid=$! STARTED"
 }
 
 fork_terminate() {
@@ -1664,7 +1667,7 @@ fork_terminate() {
    for arg; do
       _tfw_set_forklabel "$arg" || error "not a fork label '$arg'"
       [ -n "$_tfw_forkid" ] || error "no such fork: %$_tfw_forklabel"
-      _tfw_terminate $_tfw_forkid
+      _tfw_forkterminate $_tfw_forkid
    done
 }
 
@@ -1692,7 +1695,7 @@ fork_terminate_all() {
    $_tfw_assert_noise && tfw_log "# fork_terminate_all"
    local forkid
    for ((forkid=0; forkid < ${#_tfw_forked_pids[*]}; ++forkid)); do
-      _tfw_terminate $forkid
+      _tfw_forkterminate $forkid
    done
 }
 
@@ -1728,12 +1731,14 @@ _tfw_set_forklabel() {
    return 1
 }
 
-_tfw_terminate() {
+_tfw_forkterminate() {
    local forkid="$1"
    [ -z "$forkid" ] && return 1
    local pid=${_tfw_forked_pids[$forkid]}
+   local label=${_tfw_forked_labels[$forkid]}
+   local desc="fork[$forkid]${label:+ %$label}"
    [ -z "$pid" ] && return 1
-   $_tfw_assert_noise && tfw_log "# fork[$forkid] kill -TERM $pid"
+   $_tfw_assert_noise && tfw_log "# $desc kill -TERM $pid"
    kill -TERM $pid 2>/dev/null
 }
 
@@ -1741,30 +1746,32 @@ _tfw_forkwait() {
    local forkid="$1"
    [ -z "$forkid" ] && return 0
    local pid=${_tfw_forked_pids[$forkid]}
+   local label=${_tfw_forked_labels[$forkid]}
    [ -z "$pid" ] && return 0
    kill -0 $pid 2>/dev/null && return 1 # still running
    _tfw_forked_pids[$forkid]=
    wait $pid # should not block because process has exited
    local status=$?
-   $_tfw_assert_noise && tfw_log "# fork[$forkid] pid=$pid EXIT status=$status"
-   echo "++++++++++ fork[$forkid] log.stdout ++++++++++"
+   local desc="fork[$forkid]${label:+ %$label}"
+   $_tfw_assert_noise && tfw_log "# $desc pid=$pid EXIT status=$status"
+   echo "++++++++++ $desc log.stdout ++++++++++"
    cat $_tfw_tmp/fork-$forkid/log.stdout
    echo "++++++++++"
-   echo "++++++++++ fork[$forkid] log.stderr ++++++++++"
+   echo "++++++++++ $desc log.stderr ++++++++++"
    cat $_tfw_tmp/fork-$forkid/log.stderr
    echo "++++++++++"
    if $_tfw_trace; then
-      echo "++++++++++ fork[$forkid] log.xtrace ++++++++++"
+      echo "++++++++++ $desc log.xtrace ++++++++++"
       cat $_tfw_tmp/fork-$forkid/log.xtrace
       echo "++++++++++"
    fi
    case $status in
    0) ;;
    143) ;; # terminated with SIGTERM (probably from fork_terminate)
-   1) _tfw_fail "fork[$forkid] process exited with FAIL status";;
-   254) _tfw_error "fork[$forkid] process exited with ERROR status";;
-   255) _tfw_fatal "fork[$forkid] process exited with FATAL status";;
-   *) _tfw_error "fork[$forkid] process exited with status=$status";;
+   1) _tfw_fail "$desc process exited with FAIL status";;
+   254) _tfw_error "$desc process exited with ERROR status";;
+   255) _tfw_fatal "$desc process exited with FATAL status";;
+   *) _tfw_error "$desc process exited with status=$status";;
    esac
    return 0
 }