From 4e6ef2e69e692fb5add33ae41b0701ed4b276d81 Mon Sep 17 00:00:00 2001 From: gardners Date: Tue, 3 Jan 2012 16:35:02 +1030 Subject: [PATCH] Various fixes to Rhizome web server and BAR generation. Can now present a set of BARs via HTTP. --- dna.c | 2 + rhizome.c | 103 +++++++++++++++++++++++++++++++++++++++++++++---- rhizome.h | 11 +++++- rhizome_http.c | 58 +++++++++++++++++++++------- 4 files changed, 153 insertions(+), 21 deletions(-) diff --git a/dna.c b/dna.c index b0a54405..f3e28279 100644 --- a/dna.c +++ b/dna.c @@ -336,6 +336,8 @@ int main(int argc,char **argv) manifest */ return rhizome_bundle_import(optarg, NULL /* no groups - XXX should allow them */, + 255 /* ttl - XXX should read from somewhere, + e.g., bar if being imported */, 0 /* int verifyP */, 1 /* int checkFileP */, 1 /* int signP */); diff --git a/rhizome.c b/rhizome.c index 023d2df1..7ccb9c23 100644 --- a/rhizome.c +++ b/rhizome.c @@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "mphlr.h" #include "rhizome.h" +#include long long rhizome_space=0; char *rhizome_datastore_path=NULL; @@ -77,7 +78,7 @@ int rhizome_opendb() fprintf(stderr,"SQLite could not create GROUPLIST table: %s\n",sqlite3_errmsg(rhizome_db)); exit(1); } - if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, manifest blob, version integer,inserttime integer);",NULL,NULL,NULL)) + if (sqlite3_exec(rhizome_db,"CREATE TABLE IF NOT EXISTS MANIFESTS(id text not null primary key, manifest blob, version integer,inserttime integer, bar blob);",NULL,NULL,NULL)) { fprintf(stderr,"SQLite could not create MANIFESTS table: %s\n",sqlite3_errmsg(rhizome_db)); exit(1); @@ -213,7 +214,7 @@ int rhizome_make_space(int group_priority, long long bytes) /* XXX Get rid of any higher priority files that are not relevant in this time or location */ /* Couldn't make space */ - return WHY("Not implemented"); + return WHY("Incomplete"); } /* Drop the specified file from storage, and any manifests that reference it, @@ -293,7 +294,7 @@ int rhizome_manifest_priority(char *id) The file should be included in the specified rhizome groups, if possible. (some groups may be closed groups that we do not have the private key for.) */ -int rhizome_bundle_import(char *bundle,char *groups[], +int rhizome_bundle_import(char *bundle,char *groups[], int ttl, int verifyP, int checkFileP, int signP) { char filename[1024]; @@ -308,6 +309,9 @@ int rhizome_bundle_import(char *bundle,char *groups[], if (!m) return WHY("Could not read manifest file."); char hexhash[SHA512_DIGEST_STRING_LENGTH]; + /* work out time to live */ + if (ttl<0) ttl=0; if (ttl>254) ttl=254; m->ttl=ttl; + /* Keep associated file name handy for later */ m->dataFileName=strdup(filename); struct stat stat; @@ -536,6 +540,19 @@ long long rhizome_manifest_get_ll(rhizome_manifest *m,char *var) return -1; } +double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value) +{ + int i; + + if (!m) return default_value; + + for(i=0;ivar_count;i++) + if (!strcmp(m->vars[i],var)) + return strtod(m->values[i],NULL); + return default_value; +} + + int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value) { int i; @@ -718,6 +735,55 @@ int rhizome_update_file_priority(char *fileid) return 0; } +int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar) +{ + /* BAR = Bundle Advertisement Record. + Basically a 32byte precis of a given manifest, that includes version, time-to-live + and geographic bounding box information that is used to help manage flooding of + bundles. + + 64 bits - manifest ID prefix. + 56 bits - low 56 bits of version number. + 8 bits - TTL of bundle in hops. + 64 bits - length of associated file. + 16 bits - min latitude (-90 - +90). + 16 bits - min longitude (-180 - +180). + 16 bits - max latitude (-90 - +90). + 16 bits - max longitude (-180 - +180). + */ + + if (!m) return WHY("null manifest passed in"); + + int i; + + /* Manifest prefix */ + for(i=0;i<8;i++) bar[i]=m->cryptoSignPublic[i]; + /* Version */ + for(i=0;i<7;i++) bar[8+6-i]=(m->version>>(8*i))&0xff; + /* TTL */ + if (m->ttl>0) bar[15]=m->ttl-1; else bar[15]=0; + /* file length */ + for(i=0;i<8;i++) bar[16+7-i]=(m->fileLength>>(8*i))&0xff; + /* geo bounding box */ + double minLat=rhizome_manifest_get_double(m,"min_lat",-90); + if (minLat<-90) minLat=-90; if (minLat>90) minLat=90; + double minLong=rhizome_manifest_get_double(m,"min_long",-180); + if (minLong<-180) minLong=-180; if (minLong>180) minLong=180; + double maxLat=rhizome_manifest_get_double(m,"max_lat",+90); + if (maxLat<-90) maxLat=-90; if (maxLat>90) maxLat=90; + double maxLong=rhizome_manifest_get_double(m,"max_long",+180); + if (maxLong<-180) maxLong=-180; if (maxLong>180) maxLong=180; + + unsigned short v; + v=(minLat+90)*(65535/180); bar[24]=(v>>8)&0xff; bar[25]=(v>>0)&0xff; + v=(minLong+180)*(65535/360); bar[26]=(v>>8)&0xff; bar[27]=(v>>0)&0xff; + v=(maxLat+90)*(65535/180); bar[28]=(v>>8)&0xff; bar[29]=(v>>0)&0xff; + v=(maxLong+180)*(65535/360); bar[30]=(v>>8)&0xff; bar[31]=(v>>0)&0xff; + + return 0; +} + + /* Store the specified manifest into the sqlite database. We assume that sufficient space has been made for us. @@ -784,17 +850,22 @@ int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename) } /* Store manifest */ + WHY("*** Writing into manifests table"); snprintf(sqlcmd,1024, - "INSERT INTO MANIFESTS(id,manifest,version,inserttime) VALUES('%s',?,%lld,%lld);", + "INSERT INTO MANIFESTS(id,manifest,version,inserttime,bar) VALUES('%s',?,%lld,%lld,?);", manifestid,m->version,overlay_time_in_ms()); if (m->haveSecret) { if (rhizome_store_keypair_bytes(m->cryptoSignPublic,m->cryptoSignSecret)) - return WHY("Failed to store key pair."); + { + WHY("*** Insert into manifests failed (-1)."); + return WHY("Failed to store key pair."); + } } else { /* We don't have the secret for this manifest, so only allow updates if the self-signature is valid */ if (!m->selfSigned) { + WHY("*** Insert into manifests failed (-2)."); return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt."); } } @@ -802,7 +873,8 @@ int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename) sqlite3_stmt *statement; if (sqlite3_prepare_v2(rhizome_db,sqlcmd,strlen(sqlcmd)+1,&statement,&cmdtail) != SQLITE_OK) { - sqlite3_finalize(statement); + sqlite3_finalize(statement); + WHY("*** Insert into manifests failed."); return WHY(sqlite3_errmsg(rhizome_db)); } @@ -810,11 +882,28 @@ int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename) if (sqlite3_bind_blob(statement,1,m->manifestdata,m->manifest_bytes,SQLITE_TRANSIENT)!=SQLITE_OK) { sqlite3_finalize(statement); + WHY("*** Insert into manifests failed (2)."); return WHY(sqlite3_errmsg(rhizome_db)); } - if (rhizome_finish_sqlstatement(statement)) + /* Bind BAR to data field */ + unsigned char bar[RHIZOME_BAR_BYTES]; + rhizome_manifest_to_bar(m,bar); + + if (sqlite3_bind_blob(statement,2,bar,RHIZOME_BAR_BYTES,SQLITE_TRANSIENT) + !=SQLITE_OK) + { + sqlite3_finalize(statement); + WHY("*** Insert into manifests failed (3)."); + return WHY(sqlite3_errmsg(rhizome_db)); + } + + if (rhizome_finish_sqlstatement(statement)) { + WHY("*** Insert into manifests failed (4)."); return WHY("SQLite3 failed to insert row for manifest"); + } + else + WHY("*** Insert into manifests apparently worked."); /* Create relationship between file and manifest */ long long r=sqlite_exec_int64("INSERT INTO FILEMANIFESTS(manifestid,fileid) VALUES('%s','%s');", diff --git a/rhizome.h b/rhizome.h index a1bd5521..b9f35dcc 100644 --- a/rhizome.h +++ b/rhizome.h @@ -89,6 +89,8 @@ typedef struct rhizome_signature { int signatureLength; } rhizome_signature; +#define RHIZOME_BAR_BYTES 32 + #define MAX_MANIFEST_VARS 256 #define MAX_MANIFEST_BYTES 8192 typedef struct rhizome_manifest { @@ -125,6 +127,9 @@ typedef struct rhizome_manifest { All fields below may not be valid until the manifest has been finalised */ int finalised; + /* time-to-live in hops of this manifest. */ + int ttl; + /* When finalised, we keep the filehash and maximum priority due to any group membership handy */ long long fileLength; @@ -171,7 +176,8 @@ int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); int rhizome_store_file(char *file,char *hash,int priortity); char *rhizome_safe_encode(unsigned char *in,int len); int rhizome_finish_sqlstatement(sqlite3_stmt *statement); -int rhizome_bundle_import(char *bundle,char *groups[],int verifyP, int checkFileP, int signP); +int rhizome_bundle_import(char *bundle,char *groups[],int ttl, + int verifyP, int checkFileP, int signP); int rhizome_manifest_finalise(rhizome_manifest *m,int signP); char *rhizome_bytes_to_hex(unsigned char *in,int byteCount); int rhizome_hex_to_bytes(char *in,unsigned char *out,int hexChars); @@ -188,3 +194,6 @@ long long sqlite_exec_int64(char *sqlformat,...); int rhizome_server_http_response_header(rhizome_http_request *r,int result, char *mime_type,unsigned long long bytes); int rhizome_server_sql_query_fill_buffer(int rn,rhizome_http_request *r); +double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value); +int chartonybl(int c); +int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs); diff --git a/rhizome_http.c b/rhizome_http.c index 00d37797..664b8313 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -292,7 +292,7 @@ int hexFilter(char *s) } int rhizome_server_sql_query_http_response(int rn,rhizome_http_request *r, - char *column,char *query_body, + char *column,char *table,char *query_body, int bytes_per_row,int dehexP) { /* Run the provided SQL query progressively and return the values of the first @@ -346,11 +346,13 @@ int rhizome_server_sql_query_http_response(int rn,rhizome_http_request *r, WHY("no function yet exists to obtain our public key?"); /* build templated query */ - snprintf(query,1024,"SELECT %s %s",column,query_body); + snprintf(query,1024,"SELECT %s,rowid %s",column,query_body); query[1023]=0; bcopy(query,r->source,1024); r->source_index=0; r->source_flags=dehexP; + r->blob_column=strdup(column); + r->blob_table=strdup(table); printf("buffer_length=%d\n",r->buffer_length); @@ -360,6 +362,8 @@ int rhizome_server_sql_query_http_response(int rn,rhizome_http_request *r, int rhizome_server_sql_query_fill_buffer(int rn,rhizome_http_request *r) { + unsigned char blob_value[r->source_record_size*2+1]; + printf("populating with sql rows at offset %d\n",r->buffer_length); if (r->source_index>=r->source_count) { @@ -396,17 +400,45 @@ int rhizome_server_sql_query_fill_buffer(int rn,rhizome_http_request *r) { r->source_index++; - if (sqlite3_column_count(statement)!=1) { + if (sqlite3_column_count(statement)!=2) { sqlite3_finalize(statement); return WHY("sqlite3 returned multiple columns for a single column query"); } + sqlite3_blob *blob; const unsigned char *value; - if (sqlite3_column_type(statement, 0)==SQLITE_TEXT) - value=sqlite3_column_text(statement, 0); - else + int column_type=sqlite3_column_type(statement, 0); + switch(column_type) { + case SQLITE_TEXT: value=sqlite3_column_text(statement, 0); break; + case SQLITE_BLOB: + printf("table='%s',col='%s',rowid=%lld\n", + r->blob_table,r->blob_column, + sqlite3_column_int64(statement,1)); + if (sqlite3_blob_open(rhizome_db,"main",r->blob_table,r->blob_column, + sqlite3_column_int64(statement,1) /* rowid */, + 0 /* read only */,&blob)!=SQLITE_OK) + { + WHY("Couldn't open blob"); + continue; + } + if (sqlite3_blob_read(blob,&blob_value[0], + /* copy number of bytes based on whether we need to + de-hex the string or not */ + r->source_record_size*(1+(r->source_flags&1)),0) + !=SQLITE_OK) { + WHY("Couldn't read from blob"); + sqlite3_blob_close(blob); + continue; + } + WHY("Did read blob"); + value=blob_value; + sqlite3_blob_close(blob); + break; + default: /* improper column type, so don't include in report */ + WHY("Bad column type"); + printf("colunnt_type=%d\n",column_type); continue; - + } if (r->source_flags&1) { /* hex string to be converted */ int i; @@ -448,21 +480,21 @@ int rhizome_server_parse_http_request(int rn,rhizome_http_request *r) { /* Return the list of known groups */ printf("get /rhizome/groups (list of groups)\n"); - rhizome_server_sql_query_http_response(rn,r,"id","from groups",32,1); + rhizome_server_sql_query_http_response(rn,r,"id","groups","from groups",32,1); } else if (!strncasecmp(r->request,"GET /rhizome/files HTTP/1.", strlen("GET /rhizome/files HTTP/1."))) { /* Return the list of known files */ printf("get /rhizome/files (list of files)\n"); - rhizome_server_sql_query_http_response(rn,r,"id","from files",32,1); + rhizome_server_sql_query_http_response(rn,r,"id","files","from files",32,1); } - else if (!strncasecmp(r->request,"GET /rhizome/manifests HTTP/1.", - strlen("GET /rhizome/manifests HTTP/1."))) + else if (!strncasecmp(r->request,"GET /rhizome/bars HTTP/1.", + strlen("GET /rhizome/bars HTTP/1."))) { /* Return the list of known files */ - printf("get /rhizome/manifests (list of manifests)\n"); - rhizome_server_sql_query_http_response(rn,r,"id","from manifests",32,1); + printf("get /rhizome/bars (list of BARs)\n"); + rhizome_server_sql_query_http_response(rn,r,"bar","manifests","from manifests",32,0); } else if (sscanf(r->request,"GET /rhizome/file/%s HTTP/1.", id)==1)