Reworked Rhizome advertisement frames to include whole manifests

inline where possible.

overlay.c

@@ -196,7 +196,7 @@ int overlay_frame_process(int interface,overlay_frame *f)
       /* Great, we have the address, so we can get on with things */
       break;
     case OA_PLEASEEXPLAIN:
-      return WHY("Address cannot be resolved -- aborting packet processing.\n");
+      return WHY("Address cannot be resolved -- aborting packet processing.");
       /* XXX Should send a please explain to get this address resolved. */
       break;
     case OA_UNSUPPORTED:

rhizome.c

@@ -1390,14 +1390,18 @@ int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs)
 }
 
 int bundles_available=-1;
-int bundle_offset=0;
+int bundle_offset[2]={0,0};
 int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
 {
+  int pass;
   int bytes=e->sizeLimit-e->length;
-  int overhead=1+1+3+1+1+1; /* maximum overhead */
+  int overhead=1+8+1+3+1+1+1; /* maximum overhead */
-  int slots=(bytes-overhead)/8;
+  int slots=(bytes-overhead)/RHIZOME_BAR_BYTES;
   if (slots>30) slots=30;
   int slots_used=0;
+  int bytes_used=0;
+  int bytes_available=bytes-overhead;
+  int bundles_advertised=0;
 
   if (slots<1) return WHY("No room for node advertisements");
 
@@ -1416,8 +1420,12 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
   ob_append_byte(e,OA_CODE_PREVIOUS);
   ob_append_byte(e,OA_CODE_SELF);
 
-  /* Version of rhizome advert block */
+  /* Randomly choose whether to advertise manifests or BARs first. */
-  ob_append_byte(e,1);
+  int skipmanifests=random()&1;
+  /* Version of rhizome advert block:
+     1 = manifests then BARs,
+     2 = BARs only */
+  ob_append_byte(e,1+skipmanifests);
 
   /* XXX Should add priority bundles here.
      XXX Should prioritise bundles for subscribed groups, Serval-authorised files
@@ -1432,14 +1440,26 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
     WHY("Group handling not completely thought out here yet.");
 
   /* Get number of bundles available if required */
-  if (bundles_available==-1||(bundle_offset>=bundles_available)) {
   bundles_available=sqlite_exec_int64("SELECT COUNT(BAR) FROM MANIFESTS;");
-    bundle_offset=0;
+  if (bundles_available==-1||(bundle_offset[0]>=bundles_available)) 
-  }
+    bundle_offset[0]=0;
+  if (bundles_available==-1||(bundle_offset[1]>=bundles_available)) 
+    bundle_offset[1]=0;
   
+  for(pass=skipmanifests;pass<2;pass++)
+    {
       sqlite3_stmt *statement;
       char query[1024];
-  snprintf(query,1024,"SELECT BAR,ROWID FROM MANIFESTS LIMIT %d,%d",bundle_offset,slots);
+      switch(pass) {
+      case 0: /* Full manifests */
+	snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS LIMIT %d,%d",
+		 bundle_offset[pass],slots);
+	break;
+      case 1: /* BARs */
+	snprintf(query,1024,"SELECT BAR,ROWID FROM MANIFESTS LIMIT %d,%d",
+		 bundle_offset[pass],slots);
+	break;
+      }      
 
       switch (sqlite3_prepare_v2(rhizome_db,query,-1,&statement,NULL))
 	{
@@ -1453,37 +1473,71 @@ int overlay_rhizome_add_advertisements(int interface_number,overlay_buffer *e)
 	  WHY(sqlite3_errmsg(rhizome_db));
 	  return WHY("Could not prepare sql statement for fetching BARs for advertisement.");
 	}
-  while((slots_used<slots)&&(sqlite3_step(statement)==SQLITE_ROW)&&
+      while((bytes_used<bytes_available)&&(sqlite3_step(statement)==SQLITE_ROW)&&
 	    (e->length+RHIZOME_BAR_BYTES<=e->sizeLimit))
 	{
 	  sqlite3_blob *blob;
 	  int column_type=sqlite3_column_type(statement, 0);
 	  switch(column_type) {
 	  case SQLITE_BLOB:
-	if (sqlite3_blob_open(rhizome_db,"main","manifests","bar",
+	    if (sqlite3_blob_open(rhizome_db,"main","manifests",
+				  pass?"bar":"manifest",
 				  sqlite3_column_int64(statement,1) /* rowid */,
 				  0 /* read only */,&blob)!=SQLITE_OK)
 	      {
 		WHY("Couldn't open blob");
 		continue;
 	      }
-	ob_makespace(e,RHIZOME_BAR_BYTES);
+	    int blob_bytes=sqlite3_blob_bytes(blob);
-	if (sqlite3_blob_read(blob,&e->bytes[e->length],RHIZOME_BAR_BYTES,0)
+	    if (pass&&(blob_bytes!=RHIZOME_BAR_BYTES)) {
+	      if (debug&DEBUG_RHIZOME) 
+		fprintf(stderr,"Found a BAR that is the wrong size - ignoring\n");
+	      continue;
+	    }
+	    
+	    /* Only include manifests that are <=1KB inline.
+	       Longer ones are only advertised by BAR */
+	    if (blob_bytes>1024) continue;
+
+	    int overhead=0;
+	    if (!pass) overhead=2;
+	    if (ob_makespace(e,overhead+blob_bytes)) {
+	      if (debug&DEBUG_RHIZOME) 
+		fprintf(stderr,"Stopped cramming %s into Rhizome advertisement frame.\n",
+			pass?"BARs":"manifests");
+	      break;
+	    }
+	    if (!pass) {
+	      /* put manifest length field and manifest ID */
+	      e->bytes[e->length]=(blob_bytes>>8)&0xff;
+	      e->bytes[e->length+1]=(blob_bytes>>0)&0xff;
+	      if (debug&DEBUG_RHIZOME)
+		fprintf(stderr,"length bytes written at offset 0x%x\n",e->length);
+	    }
+	    if (sqlite3_blob_read(blob,&e->bytes[e->length+overhead],blob_bytes,0)
 		!=SQLITE_OK) {
-	  WHY("Couldn't read from blob");
+	      if (debug&DEBUG_RHIZOME) WHY("Couldn't read from blob");
 	      sqlite3_blob_close(blob);
 	      continue;
 	    }
-	e->length+=RHIZOME_BAR_BYTES;
+	    e->length+=overhead+blob_bytes;
-	slots_used++;
+	    bytes_used+=overhead+blob_bytes;
+	    bundles_advertised++;
 	    
 	    sqlite3_blob_close(blob);
 	  }
 	}
-
-  if (debug&DEBUG_RHIZOME) printf("Appended %d rhizome advertisements to packet.\n",slots_used);
-  e->bytes[rfs_offset]=1+8+1+1+1+RHIZOME_BAR_BYTES*slots_used;
       sqlite3_finalize(statement);
+      if (!pass) 
+	{
+	  /* Mark end of whole manifests by writing 0xff, which is more than the MSB
+	     of a manifest's length is allowed to be. */
+	  ob_append_byte(e,0xff);
+	}
+    }
+  
+  if (debug&DEBUG_RHIZOME) printf("Appended %d rhizome advertisements to packet.\n",bundles_advertised);
+  e->bytes[rfs_offset]=1+8+1+1+1+bytes_used;
 
   return 0;
 }

serval_packetvisualise.c

@@ -280,11 +280,82 @@ int isOverlayPacket(FILE *f,unsigned char *packet,int *ofs,int len)
 	  break;
 	case 0x50: /* rhizome advertisement */
 	  {
-	    int i,j;
+	    int i,j,k;
-	    fprintf(f,"%sRhizome bundle advertisement record (BAR) announcements, version %d\n",indent(8),frame[0]);
+	    int rhizome_ad_frame_type=frame[0];
-	    if (frame[0]>1) fprintf(f,"%sWARNING: Version is newer than I understand.\n",
+	    fprintf(f,"%sRhizome bundle advertisement frame, version %d\n",indent(8),rhizome_ad_frame_type);
-				    indent(10));
+	    if (rhizome_ad_frame_type>2) 
-	    for(i=1;i<(frame_len-31);i+=32) {
+	      fprintf(f,"%sWARNING: Version is newer than I understand.\n",indent(10));
+	    i=1;
+	    if (rhizome_ad_frame_type==1) {
+	      /* Frame contains whole manifest(s) */
+	      fprintf(f,"%sBundle Manifest(s) (i=%d, frame_len=%d):\n",
+		      indent(8),i,frame_len);
+	      while(i<frame_len)
+		{		  
+		  int manifest_len=(frame[i]<<8)+frame[i+1];
+		  /* Check for end of manifests */
+		  if (manifest_len>=0xff00) { i+=1; break; }
+		  else i+=2;
+		  /* find manifest self-signature block */
+		  for(j=0;j<manifest_len;j++) if (frame[i+j]==0) { j++; break;}
+		  fprintf(f,"%smanifest id (from first signature block) = ",indent(10));
+		  for(k=0;k<32;k++) fprintf(f,"%02X",frame[i+j+k+1+64]);
+		  fprintf(f,"\n");
+		  /* Print manifest text body */
+		  int column=0;
+		  fprintf(f,"%sManifest variables:\n",indent(12));
+		  for(k=0;k<(j-1);k++) {		    
+		    if (!column) { fprintf(f,"%s",indent(14)); column=14; }
+		    switch(frame[i+k]) {
+		    case '\r': /* ignore CR */
+		    case '\n': /* LF */
+		      column=0;
+		      /* fall through */
+		    default:
+		      fprintf(f,"%c",frame[i+k]);
+		    }
+		  }
+		  /* Print manifest signature blocks */
+		  
+		  fprintf(f,"%sManifest signature blocks\n",indent(12));
+		  for(;j<manifest_len;)
+		    {
+		      int sigLen=frame[i+j];
+		      switch(sigLen) {
+		      case 0x61: /* cryptosign signature */
+			fprintf(f,"%sNaCl CryptoSign Generated Signature\n",indent(14));
+			fprintf(f,"%sPublic key of signatory = ",indent(16));
+			for(k=0;k<32;k++) fprintf(f,"%02X",frame[i+j+1+64+k]);
+			fprintf(f,"\n");
+			fprintf(f,"%sSignature data:",indent(16));
+			for(k=0;k<64;k++)
+			  {
+			    if (!(k&0xf)) fprintf(f,"\n%s",indent(18-1));
+			    fprintf(f," %02X",frame[i+j+1+k]);
+			  }
+			fprintf(f,"\n");
+			break;
+		      case 0:
+			sigLen=1;
+		      default:
+			fprintf(f,"%sUnknown Signature Type 0x%02x\n",indent(14),frame[i+j]);
+			fprintf(f,"%sSignature data:",indent(16));
+			for(k=0;k<(sigLen-1);k++)
+			  {
+			    if (!(k&0xf)) fprintf(f,"\n%s",indent(18-1));
+			    fprintf(f," %02X",frame[i+j+1+k]);
+			  }
+			fprintf(f,"\n");			
+			break;
+		      }
+		      j+=sigLen;
+		    }
+		  i+=manifest_len;
+		}
+	    }
+
+	    fprintf(f,"%sBundle Advertisement Records (BARs):\n",indent(8));
+	    for(;i<(frame_len-31);i+=32) {
 	      fprintf(f,"%smanifest id = %02X%02X%02X%02X%02X%02X%02X%02X*\n",
 		      indent(10),frame[i],frame[i+1],frame[i+2],frame[i+3],
 		      frame[i+4],frame[i+5],frame[i+6],frame[i+7]);