Rework reachable link flags for simpler bitmask tests

commandline.c

@@ -1757,29 +1757,19 @@ int app_route_print(int argc, const char *const *argv, struct command_line_optio
       cli_printf(alloca_tohex_sid(p->sid));
       cli_delim(":");
       
-      switch (p->reachable){
+      if (p->reachable==REACHABLE_NONE)
-	case REACHABLE_NONE:
 	cli_printf("NONE");
-	  break;
+      if (p->reachable & REACHABLE_SELF)
-	case REACHABLE_SELF:
+	cli_printf("SELF ");
-	  cli_printf("SELF");
+      if (p->reachable & REACHABLE_ASSUMED)
-	  break;
+	cli_printf("ASSUMED ");
-	case REACHABLE_DIRECT:
+      if (p->reachable & REACHABLE_BROADCAST)
-	  cli_printf("DIRECT");
+	cli_printf("BROADCAST ");
-	  break;
+      if (p->reachable & REACHABLE_UNICAST)
-	case REACHABLE_INDIRECT:
+	cli_printf("UNICAST ");
-	  cli_printf("INDIRECT");
+      if (p->reachable & REACHABLE_INDIRECT)
-	  break;
+	cli_printf("INDIRECT ");
-	case REACHABLE_UNICAST:
+      
-	  cli_printf("UNICAST");
-	  break;
-	case REACHABLE_BROADCAST:
-	  cli_printf("BROADCAST");
-	  break;
-	default:
-	  cli_printf("%d",p->reachable);
-	  break;
-      }
       cli_delim(":");
       cli_printf(alloca_tohex_sid(p->neighbour));
       cli_delim("\n");

directory_client.c

@@ -100,7 +100,7 @@ static void directory_update(struct sched_ent *alarm){
   load_directory_config();
   
   if (directory_service){
-    if (subscriber_is_reachable(directory_service) != REACHABLE_NONE){
+    if (subscriber_is_reachable(directory_service) & REACHABLE){
       directory_send_keyring(directory_service);
       
       unschedule(alarm);

overlay_address.c

@@ -36,6 +36,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #define BPI_MASK 0x3ff
 static struct broadcast bpilist[MAX_BPIS];
 
+#define OA_CODE_SELF 0xff
+#define OA_CODE_PREVIOUS 0xfe
+
 // each node has 16 slots based on the next 4 bits of a subscriber id
 // each slot either points to another tree node or a struct subscriber.
 struct tree_node{
@@ -166,18 +169,18 @@ int subscriber_is_reachable(struct subscriber *subscriber){
       ret = REACHABLE_NONE;
     
     // avoid infinite recursion...
-    else if (subscriber->next_hop->reachable!=REACHABLE_DIRECT && 
+    else if (!(subscriber->next_hop->reachable & REACHABLE_DIRECT))
-	subscriber->next_hop->reachable!=REACHABLE_UNICAST)
       ret = REACHABLE_NONE;
     else{
       int r = subscriber_is_reachable(subscriber->next_hop);
-      if (r!=REACHABLE_DIRECT && r!= REACHABLE_UNICAST)
+      if (r&REACHABLE_ASSUMED)
+	ret = REACHABLE_NONE;
+      else if (!(r & REACHABLE_DIRECT))
 	ret = REACHABLE_NONE;
     }
   }
   
-  if (ret==REACHABLE_DIRECT || 
+  if (ret & REACHABLE_DIRECT){
-      ret==REACHABLE_UNICAST){
     // make sure the interface is still up
     if (!subscriber->interface)
       ret=REACHABLE_NONE;
@@ -185,14 +188,6 @@ int subscriber_is_reachable(struct subscriber *subscriber){
       ret=REACHABLE_NONE;
   }
   
-  // after all of that, should we use a default route?
-  if (ret==REACHABLE_NONE &&
-      directory_service && 
-      subscriber!=directory_service &&
-      subscriber_is_reachable(directory_service)!=REACHABLE_NONE){
-    ret = REACHABLE_DEFAULT_ROUTE;
-  }
-  
   return ret;
 }
 
@@ -211,9 +206,6 @@ int set_reachable(struct subscriber *subscriber, int reachable){
 	break;
       case REACHABLE_SELF:
 	break;
-      case REACHABLE_DIRECT:
-	DEBUGF("REACHABLE DIRECTLY sid=%s", alloca_tohex_sid(subscriber->sid));
-	break;
       case REACHABLE_INDIRECT:
 	DEBUGF("REACHABLE INDIRECTLY sid=%s", alloca_tohex_sid(subscriber->sid));
 	break;
@@ -227,14 +219,13 @@ int set_reachable(struct subscriber *subscriber, int reachable){
   }
 
   /* Pre-emptively send a sas request */
-  if (!subscriber->sas_valid && reachable!=REACHABLE_SELF && reachable!=REACHABLE_NONE && reachable!=REACHABLE_BROADCAST)
+  if (!subscriber->sas_valid && reachable&REACHABLE)
     keyring_send_sas_request(subscriber);
 
   // Hacky layering violation... send our identity to a directory service
   if (subscriber==directory_service &&
-      (old_value==REACHABLE_NONE||old_value==REACHABLE_BROADCAST) &&
+      (!(old_value&REACHABLE)) &&
-      (reachable!=REACHABLE_NONE&&reachable!=REACHABLE_BROADCAST)
+      reachable&REACHABLE)
-      )
     directory_registration();
   
   return 0;
@@ -242,7 +233,7 @@ int set_reachable(struct subscriber *subscriber, int reachable){
 
 // mark the subscriber as reachable via reply unicast packet
 int reachable_unicast(struct subscriber *subscriber, overlay_interface *interface, struct in_addr addr, int port){
-  if (subscriber->reachable!=REACHABLE_NONE && subscriber->reachable!=REACHABLE_UNICAST)
+  if (subscriber->reachable&REACHABLE)
     return WHYF("Subscriber %s is already reachable", alloca_tohex_sid(subscriber->sid));
   
   if (subscriber->node)
@@ -335,11 +326,16 @@ int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadca
 // append an appropriate abbreviation into the address
 int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber)
 {
+  if (!subscriber)
+    return WHY("No address supplied");
+  
   if (context && subscriber==context->sender){
-    ob_append_byte(b, OA_CODE_SELF);
+    if (ob_append_byte(b, OA_CODE_SELF))
+      return -1;
     
   }else if(context && subscriber==context->previous){
-    ob_append_byte(b, OA_CODE_PREVIOUS);
+    if (ob_append_byte(b, OA_CODE_PREVIOUS))
+      return -1;
     
   }else{
     int len=SID_SIZE;
@@ -352,8 +348,10 @@ int overlay_address_append(struct decode_context *context, struct overlay_buffer
       if (len>SID_SIZE)
 	len=SID_SIZE;
     }
-    ob_append_byte(b, len);
+    if (ob_append_byte(b, len))
-    ob_append_bytes(b, subscriber->sid, len);
+      return -1;
+    if (ob_append_bytes(b, subscriber->sid, len))
+      return -1;
   }
   if (context)
     context->previous = subscriber;
@@ -468,7 +466,9 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc
   else
     context->please_explain->source = my_subscriber;
   
-  if (destination && destination->reachable!=REACHABLE_NONE){
+  context->please_explain->source->send_full=1;
+  
+  if (destination && (destination->reachable & REACHABLE)){
     context->please_explain->destination = destination;
     context->please_explain->ttl=64;
   }else{

overlay_address.h

@@ -25,26 +25,22 @@
 // not reachable
 #define REACHABLE_NONE 0
 
-// immediate neighbour
-#define REACHABLE_DIRECT 1
-
-// reachable via unicast packet
-#define REACHABLE_UNICAST 2
-
-// packets must be routed
-#define REACHABLE_INDIRECT 3
-
-// packets can probably be flooded to this peer with ttl=2
-// (temporary state for new peers before path discovery has finished)
-#define REACHABLE_BROADCAST 4
-
 // this subscriber is in our keystore
-#define REACHABLE_SELF 5
+#define REACHABLE_SELF (1<<0)
 
-#define REACHABLE_DEFAULT_ROUTE 6
+// immediate neighbour broadcast packet
+#define REACHABLE_BROADCAST (1<<1)
 
-#define OA_CODE_SELF 0xff
+// reachable directly via unicast packet
-#define OA_CODE_PREVIOUS 0xfe
+#define REACHABLE_UNICAST (1<<2)
+
+// packets must be routed via next_hop
+#define REACHABLE_INDIRECT (1<<3)
+
+#define REACHABLE_ASSUMED (1<<4)
+
+#define REACHABLE_DIRECT (REACHABLE_BROADCAST|REACHABLE_UNICAST)
+#define REACHABLE (REACHABLE_DIRECT|REACHABLE_INDIRECT)
 
 #define BROADCAST_LEN 8
 

overlay_mdp.c

@@ -892,9 +892,7 @@ static int search_subscribers(struct subscriber *subscriber, void *context){
   }
   
   if (response->mode == MDP_ADDRLIST_MODE_ROUTABLE_PEERS && 
-      (subscriber->reachable != REACHABLE_DIRECT && 
+      (!(subscriber->reachable &REACHABLE))){
-       subscriber->reachable != REACHABLE_INDIRECT && 
-       subscriber->reachable != REACHABLE_UNICAST)){
     return 0;
   }
     

overlay_packet.h

@@ -38,7 +38,6 @@ struct overlay_frame {
   /* Mark which interfaces the frame has been sent on,
    so that we can ensure that broadcast frames get sent
    exactly once on each interface */
-  int sendBroadcast;
   unsigned char broadcast_sent_via[OVERLAY_MAX_INTERFACES];
   struct broadcast broadcast_id;
   
@@ -49,6 +48,7 @@ struct overlay_frame {
   
   /* IPv4 node frame was received from (if applicable) */
   struct sockaddr *recvaddr;
+  overlay_interface *interface;
   
   /* Actual payload */
   struct overlay_buffer *payload;

overlay_packetformats.c

@@ -187,6 +187,7 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
   
   bzero(&f,sizeof(struct overlay_frame));
   
+  f.interface = interface;
   if (recvaddr->sa_family==AF_INET){
     f.recvaddr=recvaddr; 
     if (debug&DEBUG_OVERLAYFRAMES)
@@ -220,6 +221,8 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
     int flags = ob_get(b);
     
     if (flags & PAYLOAD_FLAG_SENDER_SAME){
+      if (!context.sender)
+	context.invalid_addresses=1;
       f.source = context.sender;
     }else{
       if (overlay_address_parse(&context, b, &f.source))
@@ -335,7 +338,7 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
     context.sender->address = *addr;
     
     // if this is a dummy announcement for a node that isn't in our routing table
-    if ((context.sender->reachable == REACHABLE_NONE || context.sender->reachable == REACHABLE_UNICAST) && 
+    if (context.sender->reachable == REACHABLE_NONE && 
 	(!context.sender->node) &&
 	(interface->fileP || recvaddr->sa_family==AF_INET)){
       

overlay_payload.c

@@ -100,7 +100,7 @@ int overlay_frame_append_payload(struct decode_context *context, overlay_interfa
   
   if (overlay_frame_build_header(context, headers,
 			     p->queue, p->type, p->modifiers, p->ttl,
-			     (p->sendBroadcast?&p->broadcast_id:NULL), next_hop, 
+			     (p->destination?NULL:&p->broadcast_id), next_hop, 
 			     p->destination, p->source))
     goto cleanup;
   

overlay_queue.c

@@ -163,7 +163,7 @@ int overlay_payload_enqueue(struct overlay_frame *p)
   
   if (p->destination){
     int r = subscriber_is_reachable(p->destination);
-    if (r == REACHABLE_SELF || r == REACHABLE_NONE)
+    if (!(r&REACHABLE))
       return WHYF("Cannot send %x packet, destination %s is %s", p->type, alloca_tohex_sid(p->destination->sid), r==REACHABLE_SELF?"myself":"unreachable");
   }
   
@@ -211,8 +211,6 @@ int overlay_payload_enqueue(struct overlay_frame *p)
     // just drop it now
     if (drop)
       return -1;
-    
-    p->sendBroadcast=1;
   }
   
   struct overlay_frame *l=queue->last;
@@ -262,7 +260,7 @@ overlay_calc_queue_time(overlay_txqueue *queue, struct overlay_frame *frame){
   time_ms_t send_time;
   
   // ignore packet if the destination is currently unreachable
-  if (frame->destination && subscriber_is_reachable(frame->destination)==REACHABLE_NONE)
+  if (frame->destination && (!(subscriber_is_reachable(frame->destination)&REACHABLE)))
     return 0;
   
   // when is the next packet from this queue due?
@@ -306,46 +304,57 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
     struct subscriber *next_hop = frame->destination;
     
     if (next_hop){
-      switch(subscriber_is_reachable(next_hop)){
+      // Where do we need to route this payload next?
-	case REACHABLE_NONE:
+      
+      int r = subscriber_is_reachable(next_hop);
+      
+      // first, should we try to bounce this payload off the directory service?
+      if (r==REACHABLE_NONE && 
+	  directory_service && 
+	  next_hop!=directory_service){
+	next_hop=directory_service;
+	r=subscriber_is_reachable(directory_service);
+      }
+      
+      // do we need to route via a neighbour?
+      if (r&REACHABLE_INDIRECT){
+	next_hop = next_hop->next_hop;
+	r = subscriber_is_reachable(next_hop);
+      }
+      
+      if (!(r&REACHABLE_DIRECT))
 	goto skip;
       
-	case REACHABLE_INDIRECT:
+      // ignore resend logic for unicast packets, where wifi gives better resilience
-	  next_hop=next_hop->next_hop;
+      if (r&REACHABLE_UNICAST)
-	  frame->sendBroadcast=0;
+	frame->send_copies=1;
-	  break;
       
-	case REACHABLE_DEFAULT_ROUTE:
+      if (packet->buffer){
-	  next_hop=directory_service;
+	// is this packet going our way?
-	  frame->sendBroadcast=0;
+	if(packet->interface != next_hop->interface)
-	  break;
+	  goto skip;
 	
-	case REACHABLE_DIRECT:
+	if ((r&REACHABLE_BROADCAST) && packet->unicast)
-	case REACHABLE_UNICAST:
+	  goto skip;
-	  frame->sendBroadcast=0;
-	  break;
 	
-	case REACHABLE_BROADCAST:
+	if ((r&REACHABLE_UNICAST) && 
-	  if (!frame->sendBroadcast){
+	    (!packet->unicast || packet->dest.sin_addr.s_addr != next_hop->address.sin_addr.s_addr))
-	    if (frame->ttl>2)
+	  goto skip;
-	      frame->ttl=2;
+	    
-	    frame->sendBroadcast=1;
+      }else{
-	    if (is_all_matching(frame->broadcast_id.id, BROADCAST_LEN, 0)){
+	// start a new packet buffer.
-	      overlay_broadcast_generate_address(&frame->broadcast_id);
+	overlay_init_packet(packet, next_hop->interface, 0);
-	      // mark it as already seen so we don't immediately retransmit it
+	if(r&REACHABLE_UNICAST){
-	      overlay_broadcast_drop_check(&frame->broadcast_id);
+	  packet->unicast_subscriber = next_hop;
-	    }
+	  packet->dest = next_hop->address;
-	    int i;
+	  packet->unicast=1;
-	    for(i=0;i<OVERLAY_MAX_INTERFACES;i++)
-	      frame->broadcast_sent_via[i]=0;
-	  }
-	  break;
 	}
       }
-    
+    }else{
-    if (!packet->buffer){
+      if (packet->buffer){
-      // use the interface of the first payload we find
+	if (frame->broadcast_sent_via[packet->i])
-      if (frame->sendBroadcast){
+	  goto skip;
+      }else{
 	// find an interface that we haven't broadcast on yet
 	int i;
 	for(i=0;i<OVERLAY_MAX_INTERFACES;i++)
@@ -358,41 +367,17 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
 	}
 	
 	if (!packet->buffer){
-	  // oh dear, why is this broadcast still in the queue?
+	  // huh, we don't need to send it anywhere?
 	  frame = overlay_queue_remove(queue, frame);
 	  continue;
 	}
-      }else{
-	overlay_init_packet(packet, next_hop->interface, 0);
-	if (next_hop->reachable==REACHABLE_UNICAST){
-	  packet->unicast_subscriber = next_hop;
-	  packet->dest = next_hop->address;
-	  packet->unicast=1;
-	}
-      }
-      
-    }else{
-      // make sure this payload can be sent via this interface
-      if (frame->sendBroadcast){
-	if (frame->broadcast_sent_via[packet->i]){
-	  goto skip;
-	}
-      }else{
-	if(packet->interface != next_hop->interface)
-	  goto skip;
-	if (next_hop->reachable==REACHABLE_DIRECT && packet->unicast)
-	  goto skip;
-	if (next_hop->reachable==REACHABLE_UNICAST && 
-	    ((!packet->unicast) ||
-	     packet->dest.sin_addr.s_addr != next_hop->address.sin_addr.s_addr))
-	  goto skip;
       }
     }
     
     if (debug&DEBUG_OVERLAYFRAMES){
       DEBUGF("Sending payload type %x len %d for %s via %s", frame->type, ob_position(frame->payload),
 	     frame->destination?alloca_tohex_sid(frame->destination->sid):"All",
-	     frame->sendBroadcast?alloca_tohex(frame->broadcast_id.id, BROADCAST_LEN):alloca_tohex_sid(next_hop->sid));
+	     next_hop?alloca_tohex_sid(next_hop->sid):alloca_tohex(frame->broadcast_id.id, BROADCAST_LEN));
     }
     
     if (overlay_frame_append_payload(&packet->context, packet->interface, frame, next_hop, packet->buffer))
@@ -406,7 +391,11 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
     // mark the payload as sent
     int keep_payload = 0;
     
-    if (frame->sendBroadcast){
+    if (next_hop){
+      frame->send_copies --;
+      if (frame->send_copies>0)
+	keep_payload=1;
+    }else{
       int i;
       frame->broadcast_sent_via[packet->i]=1;
       
@@ -419,11 +408,6 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
 	    break;
 	  }
       }
-    }else{
-      frame->send_copies --;
-      // ignore resend logic for unicast packets, where wifi gives better resilience
-      if (frame->send_copies>0 && !packet->unicast)
-	keep_payload=1;
     }
     
     if (!keep_payload){

overlay_route.c

@@ -237,9 +237,11 @@ int overlay_route_ack_selfannounce(struct overlay_frame *f,
   /* set source to ourselves */
   out->source = my_subscriber;
 
-  /* Try to use broadcast if we don't have a route yet */
+  /* Assume immediate neighbour via broadcast packet if we don't have a route yet */
-  if (out->destination->reachable == REACHABLE_NONE)
+  if (out->destination->reachable == REACHABLE_NONE){
-    set_reachable(out->destination, REACHABLE_BROADCAST);
+    out->destination->interface=f->interface;
+    set_reachable(out->destination, REACHABLE_ASSUMED|REACHABLE_BROADCAST);
+  }
   
   /* Set the time in the ack. Use the last sequence number we have seen
      from this neighbour, as that may be helpful information for that neighbour
@@ -444,12 +446,15 @@ int overlay_route_recalc_node_metrics(overlay_node *n, time_ms_t now)
   int best_observation=-1;
   int reachable = REACHABLE_NONE;
   
-  // TODO expiry timer since last self announce
-  if (n->subscriber->reachable==REACHABLE_BROADCAST)
-    reachable = REACHABLE_BROADCAST;
   overlay_interface *interface=NULL;
   struct subscriber *next_hop=NULL;
   
+  // TODO assumption timeout...
+  if (n->subscriber->reachable&REACHABLE_ASSUMED){
+    reachable=n->subscriber->reachable;
+    interface=n->subscriber->interface;
+  }
+  
   if (n->neighbour_id)
   {
     /* Node is also a direct neighbour, so check score that way */
@@ -466,7 +471,7 @@ int overlay_route_recalc_node_metrics(overlay_node *n, time_ms_t now)
       {
 	best_score=neighbour->scores[i];
 	best_observation=-1;
-	reachable=REACHABLE_DIRECT;
+	reachable=REACHABLE_BROADCAST;
 	interface = &overlay_interfaces[i];
       }
     }
@@ -475,7 +480,7 @@ int overlay_route_recalc_node_metrics(overlay_node *n, time_ms_t now)
   if (best_score<=0){
     for(o=0;o<OVERLAY_MAX_OBSERVATIONS;o++)
       {
-	if (n->observations[o].observed_score && n->observations[o].sender->reachable==REACHABLE_DIRECT)
+	if (n->observations[o].observed_score && n->observations[o].sender->reachable&REACHABLE)
 	  {
 	    int discounted_score=n->observations[o].observed_score;
 	    discounted_score-=(now-n->observations[o].rx_time)/1000;
@@ -509,7 +514,7 @@ int overlay_route_recalc_node_metrics(overlay_node *n, time_ms_t now)
     case REACHABLE_INDIRECT:
       n->subscriber->next_hop = next_hop;
       break;
-    case REACHABLE_DIRECT:
+    case REACHABLE_BROADCAST:
       n->subscriber->interface = interface;
       n->subscriber->address = interface->broadcast_address;
       break;

tests/routing

@@ -60,7 +60,8 @@ setup_single_link() {
    setup_servald
    assert_no_servald_processes
    foreach_instance +A +B create_single_identity
-   start_servald_instances +A +B
+   foreach_instance +A +B add_interface dummy1
+   foreach_instance +A +B start_routing_instance
 }
 
 doc_single_link="Start 2 instances on one link"
@@ -70,6 +71,25 @@ test_single_link() {
    tfw_cat --stdout --stderr
 }
 
+setup_multiple_nodes() {
+   setup_servald
+   assert_no_servald_processes
+   foreach_instance +A +B +C +D create_single_identity
+   foreach_instance +A +B +C +D add_interface dummy1
+   foreach_instance +A +B +C +D start_routing_instance
+}
+
+doc_multiple_nodes="Multiple nodes on one link"
+test_multiple_nodes() {
+   set_instance +A
+   executeOk_servald mdp ping $SIDB 3
+   tfw_cat --stdout --stderr
+   executeOk_servald mdp ping $SIDC 3
+   tfw_cat --stdout --stderr
+   executeOk_servald mdp ping $SIDD 3
+   tfw_cat --stdout --stderr
+}
+
 setup_multihop_linear() {
    setup_servald
    assert_no_servald_processes
@@ -82,7 +102,6 @@ setup_multihop_linear() {
 
 doc_multihop_linear="Start 4 instances in a linear arrangement"
 test_multihop_linear() {
-   wait_until --sleep=0.25 instances_see_each_other +A +B +C +D
    set_instance +A
    executeOk_servald mdp ping $SIDD 3
    tfw_cat --stdout --stderr