From 3e4b20f5dfaa162d32ed95dcda739c147247548d Mon Sep 17 00:00:00 2001 From: gardners Date: Sat, 8 Sep 2012 20:00:14 +0930 Subject: [PATCH] rhizome_direct_get_bars() now allows selections to be based on size of associated data in a bundle, so that we can synchronise small things first. Also preliminary work on making a general cursor-type wrapper function for get_bars() so that it is easy for any rhizome direct transport driver to iterate over the known bundles in a rhizome datastore. #9 --- rhizome.h | 1 + rhizome_direct.c | 53 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/rhizome.h b/rhizome.h index 908c3424..1b0c3f82 100644 --- a/rhizome.h +++ b/rhizome.h @@ -391,6 +391,7 @@ int rhizome_direct_process_post_multipart_bytes (rhizome_http_request *r,const char *bytes,int count); int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES], unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES], + long long size_low,long long size_high, unsigned char *bars_out, int bars_requested); diff --git a/rhizome_direct.c b/rhizome_direct.c index 4f7491ce..bacf45c2 100644 --- a/rhizome_direct.c +++ b/rhizome_direct.c @@ -583,19 +583,45 @@ int app_rhizome_direct_sync(int argc, const char *const *argv, /* Start synchronising from the beginning of the BID address space */ memset(bid_low,0x00,RHIZOME_BAR_BYTES); - int bars_stuffed=rhizome_direct_get_bars(bid_low,bid_high,bars_out, + /* Loop through until no bars are returned */ + int bars_stuffed=1; + while(bars_stuffed>0) + { + bars_stuffed=rhizome_direct_get_bars(bid_low,bid_high, + 0,999999999999LL, + bars_out, sizeof(bars_out)/RHIZOME_BAR_BYTES); - DEBUGF("Receied %d BARs",bars_stuffed); - dump("BARs",bars_out,RHIZOME_BAR_BYTES*bars_stuffed); + DEBUGF("Received %d BARs",bars_stuffed); + dump("BARs",bars_out,RHIZOME_BAR_BYTES*bars_stuffed); + + /* Continue from next BID */ + bcopy(bid_high,bid_low,RHIZOME_MANIFEST_ID_BYTES); + int i; + for(i=RHIZOME_BAR_BYTES-1;i>=0;i--) + { + bid_low[i]++; + if (bid_low[i]) break; + } + if (i<0) break; + dump("bid_low",bid_low,RHIZOME_MANIFEST_ID_BYTES); + + } + return -1; } + + /* Read upto the next BARs from the Rhizome database, beginning from the first BAR that corresponds to a manifest with BID>=. Sets to the highest BID for which a BAR was returned. Return value is the number of BARs written into . + Only returns BARs for bundles within the specified size range. + This is used by the cursor wrapper function that passes over all of the + BARs in prioritised order. + XXX Once the rhizome database gets big, we will need to make sure that we have suitable indexes. It is tempting to just pack BARs by row_id, but the far end needs them in an orderly manner so that @@ -604,6 +630,7 @@ int app_rhizome_direct_sync(int argc, const char *const *argv, */ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES], unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES], + long long size_low,long long size_high, unsigned char *bars_out, int bars_requested) { @@ -613,8 +640,10 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE char query[1024]; snprintf(query,1024, "SELECT COUNT(BAR) FROM MANIFESTS" - " WHERE ID>='%s' ORDER BY BAR LIMIT %d;", - alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES),bars_requested); + " WHERE ID>='%s' AND FILESIZE BETWEEN %lld AND %lld ORDER BY BAR LIMIT %d;", + alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES), + size_low,size_high, + bars_requested); long long tmp = 0; if (sqlite_exec_int64_retry(&retry, &tmp, query) != 1) @@ -625,8 +654,11 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE snprintf(query,1024, "SELECT BAR,ROWID,ID FROM MANIFESTS" - " WHERE ID>='%s' ORDER BY BAR LIMIT %d;", - alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES),bars_requested); + " WHERE ID>='%s' AND FILESIZE BETWEEN %lld AND %lld" + " ORDER BY BAR LIMIT %d;", + alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES), + size_low,size_high, + bars_requested); sqlite3_stmt *statement=sqlite_prepare(query); sqlite3_blob *blob=NULL; @@ -668,8 +700,9 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE /* Remember the BID so that we cant write it into bid_high so that the caller knows how far we got. */ - bcopy((const char *)sqlite3_column_text(statement, 0), - bid_high,RHIZOME_MANIFEST_ID_BYTES); + fromhex(bid_high, + (const char *)sqlite3_column_text(statement, 2), + RHIZOME_MANIFEST_ID_BYTES); bars_written++; break; @@ -683,8 +716,6 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE sqlite3_finalize(statement); statement = NULL; - DEBUGF("Last manifest ID was '%s'",bid_high); - return bars_written; }