Add please explain message

- non-tick packets now include a poorly formed SELF_ANNOUNCE to correctly identify every packet sender

commandline.c

@@ -490,7 +490,7 @@ int app_dna_lookup(int argc, const char *const *argv, struct command_line_option
 	  }
 
 	  last_tx=now;
-	  interval+=interval;
+	  interval+=interval>>1;
 	}
       time_ms_t short_timeout=125;
       while(short_timeout>0) {

overlay_address.c

@@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "serval.h"
 #include "overlay_address.h"
 #include "overlay_buffer.h"
+#include "overlay_packet.h"
 
 #define MAX_BPIS 1024
 #define BPI_MASK 0x3ff
@@ -387,102 +388,212 @@ int overlay_address_append_self(overlay_interface *interface, struct overlay_buf
   return 0;
 }
 
-static int mark_full(struct subscriber *subscriber, void *context){
+static int add_explain_response(struct subscriber *subscriber, void *context){
-  subscriber->send_full=1;
+  struct decode_context *response = context;
+  if (!response->please_explain){
+    response->please_explain = calloc(sizeof(struct overlay_frame),1);
+    response->please_explain->payload=ob_new();
+    ob_limitsize(response->please_explain->payload, 1024);
+  }
+  // add the whole subscriber id to the payload, stop if we run out of space
+  DEBUGF("Adding full sid by way of explanation %s", alloca_tohex_sid(subscriber->sid));
+  if (ob_append_bytes(response->please_explain->payload, subscriber->sid, SID_SIZE))
+    return 1;
   return 0;
 }
 
-int find_subscr_buffer(struct overlay_buffer *b, int len, int create, struct subscriber **subscriber){
+int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int code, int len, int create, struct subscriber **subscriber){
   unsigned char *id = ob_get_bytes_ptr(b, len);
   if (!id)
     return WHY("Not enough space in buffer to parse address");
-  if (!subscriber)
+  
-    return WHY("Expected subscriber");
+  if (!subscriber){
+    WARN("Could not resolve address, no buffer supplied");
+    context->invalid_addresses=1;
+    return 0;
+  }
+  
   *subscriber=find_subscriber(id, len, create);
   
   if (!*subscriber){
-    INFOF("Abbreviation %s not found", alloca_tohex(id, len));
+    context->invalid_addresses=1;
     
-    // If the abbreviation is too short, mark any subscribers that match to send full SID's
+    // generate a please explain in the passed in context
-    walk_tree(&root, 0, id, len, id, len, mark_full, NULL);
     
-    // HACK, imperfect... better to send a sas key request
+    // add the abbreviation you told me about
-    // always send my full sid when we fail to resolve an abbreviation
+    if (!context->please_explain){
-    // they may not know us either
+      context->please_explain = calloc(sizeof(struct overlay_frame),1);
-    if (my_subscriber)
+      context->please_explain->payload=ob_new();
-      my_subscriber->send_full = 1;
+      ob_limitsize(context->please_explain->payload, 1024);
-    return 1;
+    }
+    
+    // And I'll tell you about any subscribers I know that match this abbreviation, 
+    // so you don't try to use an abbreviation that's too short in future.
+    walk_tree(&root, 0, id, len, id, len, add_explain_response, context);
+    
+    INFOF("Asking for explanation of %s", alloca_tohex(id, len));
+    if (code>=0)
+      ob_append_byte(context->please_explain->payload, code);
+    ob_append_bytes(context->please_explain->payload, id, len);
+    
+  }else{
+    previous=*subscriber;
+    previous_broadcast=NULL;
   }
-  previous=*subscriber;
-  previous_broadcast=NULL;
   return 0;
 }
 
 // returns 0 = success, -1 = fatal parsing error, 1 = unable to identify address
-int overlay_address_parse(struct overlay_buffer *b, struct broadcast *broadcast, struct subscriber **subscriber)
+int overlay_address_parse(struct decode_context *context, struct overlay_buffer *b, struct broadcast *broadcast, struct subscriber **subscriber)
 {
   int code = ob_getbyte(b,b->position);
+  if (code<0)
+    return -1;
+  
   switch(code){
     case OA_CODE_BROADCAST:
       b->position++;
       if (subscriber)
 	*subscriber=NULL;
       
-      if (!broadcast)
+      if (!broadcast){
-	return WHY("No broadcast structure for receiving broadcast address");
+	context->invalid_addresses=1;
-      
+      }else{
-      ob_get_bytes(b, broadcast->id, BROADCAST_LEN);
+	ob_get_bytes(b, broadcast->id, BROADCAST_LEN);
-      previous=NULL;
+      }
       previous_broadcast=broadcast;
+      previous=NULL;
       return 0;
       
     case OA_CODE_SELF:
       b->position++;
-      if (!subscriber)
+      if (!subscriber){
-	return WHY("Expected subscriber");
+	WARN("Could not resolve address, no buffer supplied");
-      if (!sender){
+	context->invalid_addresses=1;
+      }else if (!sender){
 	INFO("Could not resolve address, sender has not been set");
-	return 1;
+	context->invalid_addresses=1;
+      }else{
+	*subscriber=sender;
+	previous=sender;
       }
-      *subscriber=sender;
-      previous=sender;
       return 0;
       
     case OA_CODE_PREVIOUS:
       b->position++;
-      if (!subscriber)
+      
-	return WHY("Expected subscriber");
       // previous may be null, if the previous address was a broadcast. 
       // In this case we want the subscriber to be null as well and not report an error,
-      if (previous)
-	*subscriber=previous;
       
-      // not an error if broadcast is NULL, as the previous OA_CODE_BROADCAST address must have been valid.
+      if (!subscriber){
-      else if (previous_broadcast){
+	WARN("Could not resolve address, no buffer supplied");
+	context->invalid_addresses=1;
+      }else if (previous){
+	*subscriber=previous;
+      }else if (previous_broadcast){
+	*subscriber=NULL;
+	// not an error if broadcast is NULL, as the previous OA_CODE_BROADCAST address must have been valid.
 	if (broadcast)
 	  bcopy(previous_broadcast->id, broadcast->id, BROADCAST_LEN);
       }else{
 	INFO("Unable to decode previous address");
-	return 1;
+	context->invalid_addresses=1;
       }
       return 0;
       
     case OA_CODE_PREFIX3:
       b->position++;
-      return find_subscr_buffer(b,3,0,subscriber);
+      return find_subscr_buffer(context, b, code, 3,0,subscriber);
       
     case OA_CODE_PREFIX7:
       b->position++;
-      return find_subscr_buffer(b,7,0,subscriber);
+      return find_subscr_buffer(context, b, code, 7,0,subscriber);
       
     case OA_CODE_PREFIX11:
       b->position++;
-      return find_subscr_buffer(b,11,0,subscriber);
+      return find_subscr_buffer(context, b, code, 11,0,subscriber);
   }
   
+  // we must assume that we wont be able to understand the rest of the packet
   if (code<=0x0f)
     return WHYF("Unsupported abbreviation code %d", code);
-  return find_subscr_buffer(b,SID_SIZE,1,subscriber);
+  
+  return find_subscr_buffer(context, b, -1, SID_SIZE,1,subscriber);
+}
+
+// once we've finished parsing a packet, complete and send a please explain if required.
+int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination){
+  if (!context->please_explain)
+    return 0;
+  
+  context->please_explain->type = OF_TYPE_PLEASEEXPLAIN;
+  
+  if (source)
+    context->please_explain->source = source;
+  else
+    context->please_explain->source = my_subscriber;
+  
+  if (destination){
+    context->please_explain->destination = destination;
+    context->please_explain->ttl=64;
+  }else{
+    context->please_explain->ttl=2;// how will this work with olsr??
+    overlay_broadcast_generate_address(&context->please_explain->broadcast_id);
+  }
+  
+  DEBUGF("Queued please explain");
+  if (!overlay_payload_enqueue(OQ_MESH_MANAGEMENT, context->please_explain))
+    return 0;
+  op_free(context->please_explain);
+  return 0;
+}
+
+// process an incoming request for explanation of subscriber abbreviations
+int process_explain(struct overlay_frame *frame){
+  struct overlay_buffer *b=frame->payload;
+  
+  struct decode_context context={
+    .please_explain=NULL,
+  };
+  
+  while(b->position < b->sizeLimit){
+    int code = ob_getbyte(b,b->position);
+    int len=SID_SIZE;
+    
+    switch(code){
+      case OA_CODE_PREFIX3:
+	len=3;
+	b->position++;
+	break;
+      case OA_CODE_PREFIX7:
+	len=7;
+	b->position++;
+	break;
+      case OA_CODE_PREFIX11:
+	len=11;
+	b->position++;
+	break;
+    }
+    
+    if (len==SID_SIZE && code<=0x0f)
+      return WHYF("Unsupported abbreviation code %d", code);
+    
+    unsigned char *sid = ob_get_bytes_ptr(b, len);
+    
+    if (len==SID_SIZE){
+      // This message is also used to inform people of previously unknown subscribers
+      // make sure we know this one
+      find_subscriber(sid,len,1);
+      INFOF("Now know about %s", alloca_tohex(sid, len));
+    }else{
+      // reply to the sender with all subscribers that match this abbreviation
+      INFOF("Sending responses for %s", alloca_tohex(sid, len));
+      walk_tree(&root, 0, sid, len, sid, len, add_explain_response, &context);
+    }
+  }
+  
+  send_please_explain(&context, frame->destination, frame->source);
+  return 0;
 }
 
 void overlay_address_clear(void){

overlay_address.h

@@ -99,6 +99,11 @@ struct broadcast{
   unsigned char id[BROADCAST_LEN];
 };
 
+struct decode_context{
+  int invalid_addresses;
+  struct overlay_frame *please_explain;
+};
+
 extern struct subscriber *my_subscriber;
 extern struct subscriber *directory_service;
 
@@ -109,13 +114,17 @@ int set_reachable(struct subscriber *subscriber, int reachable);
 int reachable_unicast(struct subscriber *subscriber, overlay_interface *interface, struct in_addr addr, int port);
 int load_subscriber_address(struct subscriber *subscriber);
 
+int process_explain(struct overlay_frame *frame);
 int overlay_broadcast_drop_check(struct broadcast *addr);
 int overlay_broadcast_generate_address(struct broadcast *addr);
 
 int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast);
 int overlay_address_append(struct overlay_buffer *b, struct subscriber *subscriber);
 int overlay_address_append_self(overlay_interface *interface, struct overlay_buffer *b);
-int overlay_address_parse(struct overlay_buffer *b, struct broadcast *broadcast, struct subscriber **subscriber);
+
+int overlay_address_parse(struct decode_context *context, struct overlay_buffer *b, struct broadcast *broadcast, struct subscriber **subscriber);
+int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination);
+
 void overlay_address_clear(void);
 void overlay_address_set_sender(struct subscriber *subscriber);
 

overlay_interface.c

@@ -968,7 +968,7 @@ overlay_queue_dump(overlay_txqueue *q)
 }
 
 static void
-overlay_init_packet(struct outgoing_packet *packet, overlay_interface *interface){
+overlay_init_packet(struct outgoing_packet *packet, overlay_interface *interface, int tick){
   packet->interface = interface;
   packet->i = (interface - overlay_interfaces);
   packet->dest=interface->broadcast_address;
@@ -977,6 +977,29 @@ overlay_init_packet(struct outgoing_packet *packet, overlay_interface *interface
   ob_append_bytes(packet->buffer,magic_header,4);
   
   overlay_address_clear();
+  
+  if (tick){
+    /* 1. Send announcement about ourselves, including one SID that we host if we host more than one SID
+     (the first SID we host becomes our own identity, saving a little bit of data here).
+     */
+    overlay_add_selfannouncement(packet->i, packet->buffer);
+  }else{
+    // add a badly formatted dummy self announce payload to tell people we sent this.
+    ob_append_byte(packet->buffer, OF_TYPE_SELFANNOUNCE);
+    ob_append_byte(packet->buffer, 1);
+    ob_append_rfs(packet->buffer, SID_SIZE + 2);
+    
+    /* from me, to me, via me 
+     (it's shorter than an actual broadcast, 
+     and receivers wont try to process it 
+     since its not going to have a payload body anyway) */
+    overlay_address_append_self(interface, packet->buffer);
+    overlay_address_set_sender(my_subscriber);
+    ob_append_byte(packet->buffer, OA_CODE_PREVIOUS);
+    ob_append_byte(packet->buffer, OA_CODE_PREVIOUS);
+    
+    ob_patch_rfs(packet->buffer, COMPUTE_RFS_LENGTH);
+  }
 }
 
 // update the alarm time and return 1 if changed
@@ -1070,7 +1093,7 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
 	{
 	  if (overlay_interfaces[i].state==INTERFACE_STATE_UP
 	      && !frame->broadcast_sent_via[i]){
-	    overlay_init_packet(packet, &overlay_interfaces[i]);
+	    overlay_init_packet(packet, &overlay_interfaces[i], 0);
 	    break;
 	  }
 	}
@@ -1081,7 +1104,7 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
 	  continue;
 	}
       }else{
-	overlay_init_packet(packet, next_hop->interface);
+	overlay_init_packet(packet, next_hop->interface, 0);
 	if (next_hop->reachable==REACHABLE_UNICAST){
 	  packet->dest = next_hop->address;
 	  packet->unicast=1;
@@ -1212,17 +1235,11 @@ overlay_tick_interface(int i, time_ms_t now) {
     RETURN(0);
   }
 
+  if (debug&DEBUG_OVERLAYINTERFACES) DEBUGF("Ticking interface #%d",i);
+  
   // initialise the packet buffer
   bzero(&packet, sizeof(struct outgoing_packet));
-  overlay_init_packet(&packet, &overlay_interfaces[i]);
+  overlay_init_packet(&packet, &overlay_interfaces[i], 1);
-  
-  if (debug&DEBUG_OVERLAYINTERFACES) DEBUGF("Ticking interface #%d",i);
-
-  /* 1. Send announcement about ourselves, including one SID that we host if we host more than one SID
-     (the first SID we host becomes our own identity, saving a little bit of data here).
-  */
-  if (overlay_add_selfannouncement(i,packet.buffer) == -1)
-    return WHY("tick failed");
   
   /* Add advertisements for ROUTES */
   overlay_route_add_advertisements(packet.interface, packet.buffer);

overlay_olsr.c

@@ -113,6 +113,9 @@ static void parse_frame(struct overlay_buffer *buff){
   struct overlay_frame frame;
   u_int8_t addr_len;
   struct in_addr *addr;
+  struct decode_context context={
+    .please_explain=NULL,
+  };
   
   memset(&frame,0,sizeof(struct overlay_frame));
   // parse the incoming olsr header
@@ -132,11 +135,11 @@ static void parse_frame(struct overlay_buffer *buff){
   addr = (struct in_addr *)ob_get_bytes_ptr(buff, addr_len);
   
   // read source subscriber
-  if (overlay_address_parse(buff, NULL, &frame.source))
+  if (overlay_address_parse(&context, buff, NULL, &frame.source))
-    return;
+    goto end;
   
-  if (!frame.source)
+  if (context.invalid_addresses)
-    return;
+    goto end;
   
   if (frame.source->reachable==REACHABLE_NONE){
     // locate the interface we should send outgoing unicast packets to
@@ -149,8 +152,11 @@ static void parse_frame(struct overlay_buffer *buff){
   
   // read source broadcast id
   // assume each packet may arrive multiple times due to routing loops between servald overlay and olsr.
-  if (overlay_address_parse(buff, &frame.broadcast_id, NULL))
+  if (overlay_address_parse(&context, buff, &frame.broadcast_id, NULL))
-    return;
+    goto end;
+  
+  if (context.invalid_addresses)
+    goto end;
   
   frame.modifiers=ob_get(buff);
   
@@ -163,6 +169,8 @@ static void parse_frame(struct overlay_buffer *buff){
   overlay_saw_mdp_containing_frame(&frame, gettime_ms());
   
   // TODO relay this packet to other non-olsr networks.
+end:
+  send_please_explain(&context, my_subscriber, frame.source);
 }
 
 static void olsr_read(struct sched_ent *alarm){

overlay_packetformats.c

@@ -52,6 +52,9 @@ int process_incoming_frame(time_ms_t now, struct overlay_interface *interface, s
     case OF_TYPE_DATA_VOICE:
       overlay_saw_mdp_containing_frame(f,now);
       break;
+    case OF_TYPE_PLEASEEXPLAIN:
+      process_explain(f);
+      break;
     default:
       return WHYF("Support for f->type=0x%x not yet implemented",f->type);
       break;
@@ -151,6 +154,11 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
   */
 
   struct overlay_frame f;
+  struct subscriber *sender=NULL;
+  struct decode_context context={
+    .please_explain=NULL,
+  };
+  
   time_ms_t now = gettime_ms();
   struct overlay_buffer *b = ob_static(packet, len);
   ob_limitsize(b, len);
@@ -182,6 +190,8 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
   
   /* Skip magic bytes and version */
   while(b->position < b->sizeLimit){
+    context.invalid_addresses=0;
+    
     int flags = ob_get(b);
     
     /* Get normal form of packet type and modifiers */
@@ -213,7 +223,6 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
       break;
     }
 
-    int payload_start=b->position;
     int next_payload = b->position + payload_len;
     
     /* Always attempt to resolve all of the addresses in a packet, or we could fail to understand an important payload 
@@ -226,16 +235,19 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
      */
     
     struct subscriber *nexthop=NULL;
+    bzero(f.broadcast_id.id, BROADCAST_LEN);
     
-    // if we can't parse one of the addresses, skip processing the payload
+    // if the structure of the addresses looks wrong, stop immediately
-    if (overlay_address_parse(b, &f.broadcast_id, &nexthop)
+    if (overlay_address_parse(&context, b, &f.broadcast_id, &nexthop)
-	|| overlay_address_parse(b, NULL, &f.destination)
+	|| overlay_address_parse(&context, b, NULL, &f.destination)
-	|| overlay_address_parse(b, NULL, &f.source)){
+	|| overlay_address_parse(&context, b, NULL, &f.source)){
-      WHYF("Parsing failed for type %x", f.type);
-      dump(NULL, b->bytes + payload_start, payload_len);
       goto next;
     }
     
+    // if we can't understand one of the addresses, skip processing the payload
+    if (context.invalid_addresses)
+      goto next;
+
     if (debug&DEBUG_OVERLAYFRAMES){
       DEBUGF("Received payload type %x, len %d", f.type, next_payload - b->position);
       DEBUGF("Payload from %s", alloca_tohex_sid(f.source->sid));
@@ -245,7 +257,19 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
     }
 
     if (f.type==OF_TYPE_SELFANNOUNCE){
+      sender = f.source;
       overlay_address_set_sender(f.source);
+      
+      // if this is a dummy announcement for a node that isn't in our routing table
+      if (f.destination && 
+	(f.source->reachable == REACHABLE_NONE || f.source->reachable == REACHABLE_UNICAST) && 
+	(!f.source->node) &&
+	(interface->fileP || recvaddr->sa_family==AF_INET)){
+	  struct sockaddr_in *addr=(struct sockaddr_in *)recvaddr;
+	  
+	  // mark this subscriber as reachable directly via unicast.
+	  reachable_unicast(f.source, interface, addr->sin_addr, ntohs(addr->sin_port));
+      }
     }
     
     // ignore any payload we sent
@@ -269,18 +293,6 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
       goto next;
     }
     
-    // HACK, change to packet transmitter when packet format includes that.
-    if (f.type!=OF_TYPE_SELFANNOUNCE && 
-	(f.source->reachable == REACHABLE_NONE || f.source->reachable == REACHABLE_UNICAST)&& 
-	(!f.source->node) &&
-	(interface->fileP || recvaddr->sa_family==AF_INET)){
-      struct sockaddr_in *addr=(struct sockaddr_in *)recvaddr;
-      
-      // mark this subscriber as reachable directly via unicast.
-      reachable_unicast(f.source, interface, addr->sin_addr, ntohs(addr->sin_port));
-    }
-    
-    
     f.payload = ob_slice(b, b->position, next_payload - b->position);
     if (!f.payload){
       WHY("Payload length is longer than remaining packet size");
@@ -313,6 +325,8 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
   }
   
   ob_free(b);
+  
+  send_please_explain(&context, my_subscriber, sender);
   return 0;
 }
 

overlay_route.c

@@ -367,9 +367,10 @@ int overlay_route_node_can_hear_me(struct subscriber *subscriber, int sender_int
      3. Update score of how reliably we can hear this node */
 
   /* Get neighbour structure */
-  struct overlay_neighbour *neh=overlay_route_get_neighbour_structure(subscriber->node,1 /* create if necessary */);
+  
+  struct overlay_neighbour *neh=overlay_route_get_neighbour_structure(get_node(subscriber, 1),1 /* create if necessary */);
   if (!neh)
-    return -1;
+    return WHY("Unable to create neighbour structure");
   
   int obs_index=neh->most_recent_observation_id;
   int merge=0;
@@ -567,7 +568,7 @@ int overlay_route_recalc_neighbour_metrics(struct overlay_neighbour *n, time_ms_
   IN();
 
   if (!n->node)
-    RETURN(0);
+    RETURN(WHY("Neighbour is not a node"));
 
   if (debug&DEBUG_OVERLAYROUTING)
     DEBUGF("Updating neighbour metrics for %s", alloca_tohex_sid(n->node->subscriber->sid));
@@ -884,7 +885,7 @@ int overlay_route_dump()
    taking into account the age of the most recent observation */
 int overlay_route_tick_neighbour(int neighbour_id, time_ms_t now)
 {
-  if (neighbour_id>0)
+  if (neighbour_id>0 && overlay_neighbours[neighbour_id].node)
     if (overlay_route_recalc_neighbour_metrics(&overlay_neighbours[neighbour_id],now)) 
       WHY("overlay_route_recalc_neighbour_metrics() failed");