ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-02-24 02:41:18 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'sqlbind' into 'development'

Browse Source 
Issue #69: Fixed SQL injection vulnerabilities (for good?) by replacing
sprintf(3)-style SQL statement composition with a varargs parameter
binding syntax.

Introduced rhizome_bid_t type and used it everywhere instead of unsigned
char [RHIZOME_MANIFEST_ID_BYTES].  Bundle IDs are now stored and passed
around internally almost entirely in binary form.
This commit is contained in:
Andrew Bettison 2013-10-04 11:59:03 +09:30
commit 3758b038ab
20 changed files with 961 additions and 662 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
commandline.c
View File

@ -1336,10 +1336,15 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
rhizome_manifest_free(m);
return WHY("Manifest file could not be loaded -- not added to rhizome");
}
} else if(manifestid && *manifestid) {
} else if (manifestid && *manifestid) {
if (config.debug.rhizome)
DEBUGF("Reading manifest from database");
if (rhizome_retrieve_manifest(manifestid, m)){
rhizome_bid_t bid;
if (str_to_rhizome_bid_t(&bid, manifestid) == -1) {
rhizome_manifest_free(m);
return WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestid));
}
if (rhizome_retrieve_manifest(&bid, m)){
rhizome_manifest_free(m);
return WHY("Existing manifest could not be loaded -- not added to rhizome");
}
@ -1398,10 +1403,8 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
cli_put_string(context, service, "\n");
}
{
char bid[RHIZOME_MANIFEST_ID_STRLEN + 1];
rhizome_bytes_to_hex_upper(mout->cryptoSignPublic, bid, RHIZOME_MANIFEST_ID_BYTES);
cli_field_name(context, "manifestid", ":");
cli_put_string(context, bid, "\n");
cli_put_string(context, alloca_tohex_rhizome_bid_t(mout->cryptoSignPublic), "\n");
}
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
@ -1508,7 +1511,7 @@ int app_rhizome_import_bundle(const struct cli_parsed *parsed, struct cli_contex
}
{
cli_field_name(context, "manifestid", ":");
cli_put_string(context, alloca_tohex(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES), "\n");
cli_put_string(context, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), "\n");
}
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
@ -1596,17 +1599,15 @@ int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_context *cont
} else {
if (!manifestid)
return WHY("missing <manifestid> argument");
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
rhizome_bid_t bid;
if (str_to_rhizome_bid_t(&bid, manifestid) == -1)
return WHY("Invalid manifest ID");
char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
if (cli_arg(parsed, "bundle", NULL, NULL, NULL) == 0)
ret = rhizome_delete_bundle(manifestIdUpper);
ret = rhizome_delete_bundle(&bid);
else if (cli_arg(parsed, "manifest", NULL, NULL, NULL) == 0)
ret = rhizome_delete_manifest(manifestIdUpper);
ret = rhizome_delete_manifest(&bid);
else if (cli_arg(parsed, "payload", NULL, NULL, NULL) == 0)
ret = rhizome_delete_payload(manifestIdUpper);
ret = rhizome_delete_payload(&bid);
else
return WHY("unrecognised command");
}
@ -1656,13 +1657,10 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
int ret=0;
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
if (fromhexstr(manifest_id, manifestid, RHIZOME_MANIFEST_ID_BYTES) == -1)
rhizome_bid_t bid;
if (str_to_rhizome_bid_t(&bid, manifestid) == -1)
return WHY("Invalid manifest ID");
char manifestIdUpper[RHIZOME_MANIFEST_ID_STRLEN + 1];
tohex(manifestIdUpper, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
// treat empty string the same as null
if (bskhex && !*bskhex)
bskhex=NULL;
@ -1675,7 +1673,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
if (m==NULL)
return WHY("Out of manifests");
ret = rhizome_retrieve_manifest(manifestIdUpper, m);
ret = rhizome_retrieve_manifest(&bid, m);
if (ret==0){
// ignore errors
@ -1683,7 +1681,7 @@ int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_context *con
const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
cli_field_name(context, "service", ":"); cli_put_string(context, blob_service, "\n");
cli_field_name(context, "manifestid", ":"); cli_put_string(context, manifestIdUpper, "\n");
cli_field_name(context, "manifestid", ":"); cli_put_string(context, alloca_tohex_rhizome_bid_t(bid), "\n");
cli_field_name(context, "version", ":"); cli_put_long(context, m->version, "\n");
cli_field_name(context, "inserttime", ":"); cli_put_long(context, m->inserttime, "\n");
if (m->haveSecret) {

1
conf_schema.h
View File

@ -260,6 +260,7 @@ ATOM(bool_t, slipdecode, 0, boolean,, "")
ATOM(bool_t, slipbytestream, 0, boolean,, "")
ATOM(bool_t, packetconstruction, 0, boolean,, "")
ATOM(bool_t, rhizome, 0, boolean,, "")
ATOM(bool_t, rhizome_bind, 0, boolean,, "")
ATOM(bool_t, rhizome_tx, 0, boolean,, "")
ATOM(bool_t, rhizome_rx, 0, boolean,, "")
ATOM(bool_t, rhizome_ads, 0, boolean,, "")

28
dataformats.c
View File

@ -22,8 +22,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "str.h"
#include <ctype.h>
int str_to_sid_t(sid_t *sid, const char *hex)
int cmp_sid_t(const sid_t *a, const sid_t *b)
{
return memcmp(a, b, sizeof a->binary);
}
int str_to_sid_t(sid_t *sid, const char *hex) {
if (strcmp(hex, "broadcast") == 0) {
*sid = SID_BROADCAST;
return 0;
@ -68,6 +72,28 @@ int strn_is_subscriber_id(const char *sid, size_t *lenp)
return 0;
}
int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b)
{
return memcmp(a, b, sizeof a->binary);
}
int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex)
{
return fromhexstr(bid->binary, hex, sizeof bid->binary);
}
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp)
{
rhizome_bid_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
*bid = tmp;
if (endp)
*endp = hex + sizeof bid->binary * 2;
return 0;
}
int rhizome_strn_is_manifest_id(const char *id)
{
return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN);

136
meshms.c
View File

@ -11,7 +11,7 @@
// the manifest details for one half of a conversation
struct ply{
char bundle_id[RHIZOME_MANIFEST_ID_STRLEN+1];
rhizome_bid_t bundle_id;
uint64_t version;
uint64_t tail;
uint64_t size;
@ -90,13 +90,14 @@ static int get_my_conversation_bundle(const sid_t *my_sid, rhizome_manifest *m)
return 0;
}
static struct conversations *add_conv(struct conversations **conv, const sid_t *them){
struct conversations **ptr=conv;
static struct conversations *add_conv(struct conversations **conv, const sid_t *them)
{
struct conversations **ptr = conv;
while(*ptr){
int cmp = memcmp((*ptr)->them.binary, them, sizeof((*ptr)->them));
if (cmp==0)
int cmp = cmp_sid_t(&(*ptr)->them, them);
if (cmp == 0)
break;
if (cmp<0)
if (cmp < 0)
ptr = &(*ptr)->_left;
else
ptr = &(*ptr)->_right;
@ -104,60 +105,64 @@ static struct conversations *add_conv(struct conversations **conv, const sid_t *
if (!*ptr){
*ptr = emalloc_zero(sizeof(struct conversations));
if (*ptr)
memcpy((*ptr)->them.binary, them->binary, sizeof((*ptr)->them));
(*ptr)->them = *them;
}
return *ptr;
}
// find matching conversations
// if their_sid_hex == my_sid_hex, return all conversations with any recipient
static int get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv){
const char *my_sid_hex = alloca_tohex_sid(my_sid->binary);
const char *their_sid_hex = alloca_tohex_sid(their_sid?their_sid->binary:my_sid->binary);
// if their_sid == my_sid, return all conversations with any recipient
static int get_database_conversations(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry,
"SELECT id, version, filesize, tail, sender, recipient "
"FROM manifests "
"WHERE service = '"RHIZOME_SERVICE_MESHMS2"' "
"AND (sender=?1 or recipient=?1) "
"AND (sender=?2 or recipient=?2)");
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, version, filesize, tail, sender, recipient"
" FROM manifests"
" WHERE service = ?3"
" AND (sender=?1 or recipient=?1)"
" AND (sender=?2 or recipient=?2)",
SID_T, my_sid,
SID_T, their_sid ? their_sid : my_sid,
STATIC_TEXT, RHIZOME_SERVICE_MESHMS2,
END
);
if (!statement)
return -1;
int ret = sqlite3_bind_text(statement, 1, my_sid_hex, -1, SQLITE_STATIC);
if (ret!=SQLITE_OK)
goto end;
ret = sqlite3_bind_text(statement, 2, their_sid_hex, -1, SQLITE_STATIC);
if (ret!=SQLITE_OK)
goto end;
if (config.debug.meshms)
if (config.debug.meshms) {
const char *my_sid_hex = alloca_tohex_sid_t(*my_sid);
const char *their_sid_hex = alloca_tohex_sid_t(*(their_sid ? their_sid : my_sid));
DEBUGF("Looking for conversations for %s, %s", my_sid_hex, their_sid_hex);
}
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
const char *id = (const char *)sqlite3_column_text(statement, 0);
const char *id_hex = (const char *)sqlite3_column_text(statement, 0);
long long version = sqlite3_column_int64(statement, 1);
long long size = sqlite3_column_int64(statement, 2);
long long tail = sqlite3_column_int64(statement, 3);
const char *sender = (const char *)sqlite3_column_text(statement, 4);
const char *recipient = (const char *)sqlite3_column_text(statement, 5);
const char *them = recipient;
if (strcasecmp(them, my_sid_hex)==0)
them=sender;
sid_t their_sid;
fromhex(their_sid.binary, them, sizeof(their_sid));
if (config.debug.meshms)
DEBUGF("found id %s, sender %s, recipient %s", id, sender, recipient);
DEBUGF("found id %s, sender %s, recipient %s", id_hex, sender, recipient);
rhizome_bid_t bid;
if (str_to_rhizome_bid_t(&bid, id_hex) == -1) {
WHYF("invalid Bundle ID hex: %s -- skipping", alloca_str_toprint(id_hex));
continue;
}
const char *them = recipient;
sid_t their_sid;
if (str_to_sid_t(&their_sid, them) == -1) {
WHYF("invalid SID hex: %s -- skipping", alloca_str_toprint(them));
continue;
}
if (cmp_sid_t(&their_sid, my_sid) == 0) {
them = sender;
if (str_to_sid_t(&their_sid, them) == -1) {
WHYF("invalid SID hex: %s -- skipping", alloca_str_toprint(them));
continue;
}
}
struct conversations *ptr = add_conv(conv, &their_sid);
if (!ptr)
goto end;
break;
struct ply *p;
if (them==sender){
ptr->found_their_ply=1;
@ -166,20 +171,13 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si
ptr->found_my_ply=1;
p=&ptr->my_ply;
}
strncpy(p->bundle_id, id, RHIZOME_MANIFEST_ID_STRLEN+1);
p->bundle_id = bid;
p->version = version;
p->tail = tail;
p->size = size;
}
end:
if (ret!=SQLITE_OK){
WHYF("Query failed: %s", sqlite3_errmsg(rhizome_db));
free_conversations(*conv);
*conv=NULL;
}
sqlite3_finalize(statement);
return (ret==SQLITE_OK)?0:-1;
return 0;
}
static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid){
@ -201,12 +199,10 @@ static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_m
rhizome_manifest_set(m, "sender", my_sidhex);
rhizome_manifest_set(m, "recipient", their_sidhex);
rhizome_manifest_set_ll(m, "tail", m->journalTail);
if (rhizome_fill_manifest(m, NULL, my_sid, NULL))
return -1;
rhizome_manifest_get(m, "id", conv->my_ply.bundle_id, sizeof(conv->my_ply.bundle_id));
conv->found_my_ply=1;
conv->my_ply.bundle_id = m->cryptoSignPublic;
conv->found_my_ply = 1;
return 0;
}
@ -216,14 +212,15 @@ static int append_footer(unsigned char *buffer, char type, int payload_len){
return 2;
}
static int ply_read_open(struct ply_read *ply, const char *id, rhizome_manifest *m){
static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome_manifest *m)
{
if (config.debug.meshms)
DEBUGF("Opening ply %s", id);
if (rhizome_retrieve_manifest(id, m))
DEBUGF("Opening ply %s", alloca_tohex_rhizome_bid_t(*bid));
if (rhizome_retrieve_manifest(bid, m))
return -1;
int ret = rhizome_open_decrypt_read(m, NULL, &ply->read);
if (ret>0)
WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_bid(m->cryptoSignPublic), m->version);
WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (ret)
return ret;
ply->read.offset = ply->read.length = m->fileLength;
@ -304,7 +301,7 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv,
goto end;
if (conv->found_my_ply){
if (rhizome_retrieve_manifest(conv->my_ply.bundle_id, m))
if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m))
goto end;
// set the author of the manifest as we should already know that
bcopy(my_sid->binary, m->author, sizeof(m->author));
@ -354,10 +351,10 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){
DEBUG("Locating their last message");
// find the offset of their last message
if (rhizome_retrieve_manifest(conv->their_ply.bundle_id, m_theirs))
if (rhizome_retrieve_manifest(&conv->their_ply.bundle_id, m_theirs))
goto end;
if (ply_read_open(&ply, conv->their_ply.bundle_id, m_theirs))
if (ply_read_open(&ply, &conv->their_ply.bundle_id, m_theirs))
goto end;
ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_MESSAGE);
@ -389,10 +386,10 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){
m_ours = rhizome_new_manifest();
if (!m_ours)
goto end;
if (rhizome_retrieve_manifest(conv->my_ply.bundle_id, m_ours))
if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m_ours))
goto end;
if (ply_read_open(&ply, conv->my_ply.bundle_id, m_ours))
if (ply_read_open(&ply, &conv->my_ply.bundle_id, m_ours))
goto end;
ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_ACK);
@ -464,7 +461,8 @@ static int update_conversations(const sid_t *my_sid, struct conversations *conv)
// read our cached conversation list from our rhizome payload
// if we can't load the existing data correctly, just ignore it.
static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct conversations **conv){
static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid, struct conversations **conv)
{
if (m->haveSecret==NEW_BUNDLE_ID)
return 0;
@ -491,7 +489,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
break;
if (config.debug.meshms)
DEBUGF("Reading existing conversation for %s", alloca_tohex_sid(sid.binary));
if (their_sid && memcmp(sid.binary, their_sid->binary, sizeof(sid)))
if (their_sid && cmp_sid_t(&sid, their_sid) != 0)
continue;
struct conversations *ptr = add_conv(conv, &sid);
if (!ptr)
@ -773,7 +771,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
m_ours = rhizome_new_manifest();
if (!m_ours)
goto end;
if (ply_read_open(&read_ours, conv->my_ply.bundle_id, m_ours))
if (ply_read_open(&read_ours, &conv->my_ply.bundle_id, m_ours))
goto end;
uint64_t their_last_ack=0;
@ -784,7 +782,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
m_theirs = rhizome_new_manifest();
if (!m_theirs)
goto end;
if (ply_read_open(&read_theirs, conv->their_ply.bundle_id, m_theirs))
if (ply_read_open(&read_theirs, &conv->their_ply.bundle_id, m_theirs))
goto end;
// find their last ACK so we know if messages have been received
@ -881,7 +879,7 @@ end:
static int mark_read(struct conversations *conv, const sid_t *their_sid, const char *offset_str){
int ret=0;
if (conv){
int cmp = their_sid?memcmp(conv->them.binary, their_sid->binary, sizeof(sid_t)):0;
int cmp = their_sid ? cmp_sid_t(&conv->them, their_sid) : 0;
if (!their_sid || cmp<0){
ret+=mark_read(conv->_left, their_sid, offset_str);
}

2
monitor.c
View File

@ -577,7 +577,7 @@ int monitor_announce_bundle(rhizome_manifest *m)
char msg[1024];
int len = snprintf(msg,1024,"\n*%d:BUNDLE:%s\n",
m->manifest_all_bytes,
alloca_tohex_bid(m->cryptoSignPublic));
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
bcopy(m->manifestdata, &msg[len], m->manifest_all_bytes);
len+=m->manifest_all_bytes;
msg[len++]='\n';

45
overlay_mdp_services.c
View File

@ -29,7 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "crypto.h"
#include "log.h"
int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength)
int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength)
{
IN();
if (!is_rhizome_mdp_server_running())
@ -38,7 +38,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t
RETURN(WHYF("Invalid block length %d", blockLength));
if (config.debug.rhizome_tx)
DEBUGF("Requested blocks for %s @%"PRIx64" bitmap %x", alloca_tohex_bid(id), fileOffset, bitmap);
DEBUGF("Requested blocks for %s @%"PRIx64" bitmap %x", alloca_tohex_rhizome_bid_t(*bid), fileOffset, bitmap);
overlay_mdp_frame reply;
bzero(&reply,sizeof(reply));
@ -69,7 +69,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t
reply.out.queue=OQ_OPPORTUNISTIC;
reply.out.payload[0]='B'; // reply contains blocks
// include 16 bytes of BID prefix for identification
bcopy(id, &reply.out.payload[1], 16);
bcopy(bid->binary, &reply.out.payload[1], 16);
// and version of manifest (in the correct byte order)
// bcopy(&version, &reply.out.payload[1+16], sizeof(uint64_t));
write_uint64(&reply.out.payload[1+16],version);
@ -87,7 +87,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t
write_uint64(&reply.out.payload[1+16+8], offset);
int bytes_read = rhizome_read_cached(id, version, gettime_ms()+5000, offset, &reply.out.payload[1+16+8+8], blockLength);
int bytes_read = rhizome_read_cached(bid, version, gettime_ms()+5000, offset, &reply.out.payload[1+16+8+8], blockLength);
if (bytes_read<=0)
break;
@ -108,16 +108,12 @@ int rhizome_mdp_send_block(struct subscriber *dest, unsigned char *id, uint64_t
int overlay_mdp_service_rhizomerequest(struct overlay_frame *frame, overlay_mdp_frame *mdp)
{
uint64_t version=
read_uint64(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES]);
uint64_t fileOffset=
read_uint64(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES+8]);
uint32_t bitmap=
read_uint32(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8]);
uint16_t blockLength=
read_uint16(&mdp->out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8+4]);
return rhizome_mdp_send_block(frame->source, &mdp->out.payload[0], version, fileOffset, bitmap, blockLength);
const rhizome_bid_t *bidp = (const rhizome_bid_t *) &mdp->out.payload[0];
uint64_t version = read_uint64(&mdp->out.payload[sizeof bidp->binary]);
uint64_t fileOffset = read_uint64(&mdp->out.payload[sizeof bidp->binary + 8]);
uint32_t bitmap = read_uint32(&mdp->out.payload[sizeof bidp->binary + 8 + 8]);
uint16_t blockLength = read_uint16(&mdp->out.payload[sizeof bidp->binary + 8 + 8 + 4]);
return rhizome_mdp_send_block(frame->source, bidp, version, fileOffset, bitmap, blockLength);
}
int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp)
@ -353,30 +349,23 @@ end:
RETURN(ret);
}
static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp){
static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp)
{
int offset=0;
char id_hex[RHIZOME_MANIFEST_ID_STRLEN];
while (offset<mdp->out.payload_length){
unsigned char *bar=&mdp->out.payload[offset];
tohex(id_hex, &bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES);
strcat(id_hex, "%");
while (offset<mdp->out.payload_length) {
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
return WHY("Unable to allocate manifest");
if (!rhizome_retrieve_manifest(id_hex, m)){
unsigned char *bar = &mdp->out.payload[offset];
if (!rhizome_retrieve_manifest_by_prefix(&bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES, m)){
rhizome_advertise_manifest(frame->source, m);
// pre-emptively send the payload if it will fit in a single packet
if (m->fileLength > 0 && m->fileLength <= 1024){
rhizome_mdp_send_block(frame->source, m->cryptoSignPublic, m->version,
0, 0, m->fileLength);
}
if (m->fileLength > 0 && m->fileLength <= 1024)
rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->fileLength);
}
rhizome_manifest_free(m);
offset+=RHIZOME_BAR_BYTES;
}
return 0;
}

20
rhizome.c
View File

@ -139,8 +139,7 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
/* Do we already have this manifest or newer? */
int64_t dbVersion = -1;
const char *id=rhizome_manifest_get(m, "id", NULL, 0);
if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id='%s';", id) == -1)
if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END) == -1)
return WHY("Select failure");
if (dbVersion>=m->version)
@ -229,7 +228,7 @@ int rhizome_manifest_bind_id(rhizome_manifest *m_in)
return WHYF("Failed to obtain RS for %s to calculate BK",
alloca_tohex_sid(m_in->author));
}
if (!rhizome_secret2bk(m_in->cryptoSignPublic,rs,rs_len,bkbytes,m_in->cryptoSignSecret)) {
if (!rhizome_secret2bk(&m_in->cryptoSignPublic, rs, rs_len, bkbytes, m_in->cryptoSignSecret)) {
char bkhex[RHIZOME_BUNDLE_KEY_STRLEN + 1];
(void) tohex(bkhex, bkbytes, RHIZOME_BUNDLE_KEY_BYTES);
if (config.debug.rhizome) DEBUGF("set BK=%s", bkhex);
@ -261,15 +260,12 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
}
/* If the manifest already has an ID */
char id[SID_STRLEN + 1];
if (!rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1))
/* no manifest ID */
if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic))
return WHY("Manifest does not have an ID");
str_toupper_inplace(id);
/* Discard the new manifest unless it is newer than the most recent known version with the same ID */
int64_t storedversion = -1;
switch (sqlite_exec_int64(&storedversion, "SELECT version from manifests where id='%s';", id)) {
switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m_in->cryptoSignPublic, END)) {
case -1:
return WHY("Select failed");
case 0:
@ -281,7 +277,7 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
if (m_in->version < storedversion)
return WHY("Newer version exists");
if (m_in->version == storedversion)
return WHYF("Already have %s:%"PRId64", not adding", id, m_in->version);
return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m_in->cryptoSignPublic), m_in->version);
break;
default:
return WHY("Select found too many rows!");
@ -291,12 +287,6 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
return rhizome_store_bundle(m_in);
}
/* Update an existing Rhizome bundle */
int rhizome_bundle_push_update(char *id,long long version,unsigned char *data,int appendP)
{
return WHY("Not implemented");
}
/* When voice traffic is being carried, we need to throttle Rhizome down
to a more sensible level. Or possibly even supress it entirely.
*/

145
rhizome.h
View File

@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
# endif
#endif
// TODO Rename MANIFEST_ID to BUNDLE_ID
#define RHIZOME_MANIFEST_ID_BYTES crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES
#define RHIZOME_MANIFEST_ID_STRLEN (RHIZOME_MANIFEST_ID_BYTES * 2)
#define RHIZOME_BUNDLE_KEY_BYTES (crypto_sign_edwards25519sha512batch_SECRETKEYBYTES-crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)
@ -51,6 +52,19 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#define RHIZOME_HTTP_PORT 4110
#define RHIZOME_HTTP_PORT_MAX 4150
typedef struct rhizome_bid_binary {
unsigned char binary[RHIZOME_MANIFEST_ID_BYTES];
} rhizome_bid_t;
#define RHIZOME_BID_ZERO ((rhizome_bid_t){{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}})
#define RHIZOME_BID_MAX ((rhizome_bid_t){{0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff}})
#define rhizome_bid_t_is_zero(bid) is_all_matching((bid).binary, sizeof (*(rhizome_bid_t*)0).binary, 0)
#define rhizome_bid_t_is_max(bid) is_all_matching((bid).binary, sizeof (*(rhizome_bid_t*)0).binary, 0xff)
#define alloca_tohex_rhizome_bid_t(bid) alloca_tohex((bid).binary, sizeof (*(rhizome_bid_t*)0).binary)
int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b);
int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex);
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp);
typedef struct rhizome_bk_binary {
unsigned char binary[RHIZOME_BUNDLE_KEY_BYTES];
} rhizome_bk_t;
@ -106,7 +120,7 @@ typedef struct rhizome_manifest {
The filename as distributed on Rhizome will be the public key
of this pair, thus ensuring that noone can tamper with a bundle
except the creator. */
unsigned char cryptoSignPublic[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES];
rhizome_bid_t cryptoSignPublic;
unsigned char cryptoSignSecret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES];
/* Whether we have the secret for this manifest on hand */
int haveSecret;
@ -227,8 +241,6 @@ int rhizome_str_is_bundle_crypt_key(const char *text);
int rhizome_strn_is_file_hash(const char *text);
int rhizome_str_is_file_hash(const char *text);
#define alloca_tohex_bid(bid) alloca_tohex((bid), RHIZOME_MANIFEST_ID_BYTES)
int http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
typedef struct sqlite_retry_state {
@ -247,7 +259,7 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(const char *id,int maximum_priority);
int rhizome_manifest_priority(sqlite_retry_state *retry, const char *id);
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bufferPAndSize);
int rhizome_hash_file(rhizome_manifest *m, const char *filename,char *hash_out);
char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen);
@ -294,35 +306,71 @@ int (*sqlite_set_tracefunc(int (*newfunc)()))();
int is_debug_rhizome();
int is_debug_rhizome_ads();
sqlite3_stmt *_sqlite_prepare(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
sqlite3_stmt *_sqlite_prepare_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, strbuf stmt);
enum sqlbind_type {
END = 0xbabecafe,
NUL = 0xbeef, // (no arg)
INT, // int value
INT_TOSTR, // int value
UINT_TOSTR, // unsigned value
INT64, // int64_t value
INT64_TOSTR, // int64_t value
UINT64_TOSTR, // uint64_t value
TEXT, // const char *text,
TEXT_LEN, // const char *text, int bytes
STATIC_TEXT, // const char *text,
STATIC_TEXT_LEN, // const char *text, int bytes
STATIC_BLOB, // const void *blob, int bytes
ZEROBLOB, // int bytes
SID_T, // const sid_t *sidp
RHIZOME_BID_T, // const rhizome_bid_t *bidp
FILEHASH_T, // const unsigned char hash_binary[RHIZOME_FILEHASH_BYTES]
TOHEX, // const unsigned char *binary, unsigned bytes
TEXT_TOUPPER, // const char *text,
TEXT_LEN_TOUPPER, // const char *text, unsigned bytes
INDEX = 0xfade0000, // INDEX|INT, int index, ...
NAMED = 0xdead0000 // NAMED|INT, const char *label, ...
};
sqlite3_stmt *_sqlite_prepare(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext);
int _sqlite_bind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, ...);
int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement, va_list ap);
sqlite3_stmt *_sqlite_prepare_bind(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...);
int _sqlite_retry(struct __sourceloc, sqlite_retry_state *retry, const char *action);
void _sqlite_retry_done(struct __sourceloc, sqlite_retry_state *retry, const char *action);
int _sqlite_step_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, const char *sqlformat, ...);
int _sqlite_exec_void_loglevel(struct __sourceloc, int log_level, const char *sqlformat, ...);
int _sqlite_exec_void_retry(struct __sourceloc, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_void_retry_loglevel(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqlformat, ...);
int _sqlite_exec_int64(struct __sourceloc, int64_t *result, const char *sqlformat, ...);
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, int64_t *result, const char *sqlformat, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqlformat, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqlformat, va_list ap);
int _sqlite_step(struct __sourceloc, int log_level, sqlite_retry_state *retry, sqlite3_stmt *statement);
int _sqlite_exec_void(struct __sourceloc, int log_level, const char *sqltext, ...);
int _sqlite_exec_void_retry(struct __sourceloc, int log_level, sqlite_retry_state *retry, const char *sqltext, ...);
int _sqlite_exec_int64(struct __sourceloc, int64_t *result, const char *sqltext, ...);
int _sqlite_exec_int64_retry(struct __sourceloc, sqlite_retry_state *retry, int64_t *result, const char *sqltext, ...);
int _sqlite_exec_strbuf(struct __sourceloc, strbuf sb, const char *sqltext, ...);
int _sqlite_exec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, ...);
int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, strbuf sb, const char *sqltext, va_list ap);
#define sqlite_prepare(rs,fmt,...) _sqlite_prepare(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_prepare_loglevel(ll,rs,sb) _sqlite_prepare_loglevel(__WHENCE__, (ll), (rs), (sb))
#define sqlite_retry(rs,action) _sqlite_retry(__WHENCE__, (rs), (action))
#define sqlite_retry_done(rs,action) _sqlite_retry_done(__WHENCE__, (rs), (action))
#define sqlite_step(stmt) _sqlite_step_retry(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt))
#define sqlite_step_retry(rs,stmt) _sqlite_step_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt))
#define sqlite_exec_void(fmt,...) _sqlite_exec_void(__WHENCE__, (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_loglevel(ll,fmt,...) _sqlite_exec_void_loglevel(__WHENCE__, (ll), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry(rs,fmt,...) _sqlite_exec_void_retry(__WHENCE__, (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_void_retry_loglevel(ll,rs,fmt,...) _sqlite_exec_void_retry_loglevel(__WHENCE__, (ll), (rs), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64(res,fmt,...) _sqlite_exec_int64(__WHENCE__, (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_int64_retry(rs,res,fmt,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,fmt,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (fmt), ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,fmt,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (fmt), ##__VA_ARGS__)
// The 'arg' arguments in the following macros appear to be unnecessary, but
// they serve a very useful purpose, so don't remove them! They ensure that
// programmers do not forget the bind args, of which there must be at least
// one, even if it is only 'END' to make no bindings at all.
#define sqlite_prepare(rs,sql) _sqlite_prepare(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql))
#define sqlite_prepare_loglevel(ll,rs,sql) _sqlite_prepare(__WHENCE__, (ll), (rs), (sql))
#define sqlite_prepare_bind(rs,sql,arg,...) _sqlite_prepare_bind(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_prepare_bind_loglevel(ll,rs,sql,arg,...) _sqlite_prepare_bind(__WHENCE__, (ll), (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_bind(rs,stmt,arg,...) _sqlite_bind(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt), arg, ##__VA_ARGS__)
#define sqlite_bind_loglevel(ll,rs,stmt,arg,...) _sqlite_bind(__WHENCE__, (ll), (rs), (stmt), arg, ##__VA_ARGS__)
#define sqlite_retry(rs,action) _sqlite_retry(__WHENCE__, (rs), (action))
#define sqlite_retry_done(rs,action) _sqlite_retry_done(__WHENCE__, (rs), (action))
#define sqlite_exec(stmt) _sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt))
#define sqlite_exec_retry(rs,stmt) _sqlite_exec(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt))
#define sqlite_exec_retry_loglevel(ll,rs,stmt) _sqlite_exec(__WHENCE__, (ll), (rs), (stmt))
#define sqlite_step(stmt) _sqlite_step(__WHENCE__, LOG_LEVEL_ERROR, NULL, (stmt))
#define sqlite_step_retry(rs,stmt) _sqlite_step(__WHENCE__, LOG_LEVEL_ERROR, (rs), (stmt))
#define sqlite_exec_void(sql,arg,...) _sqlite_exec_void(__WHENCE__, LOG_LEVEL_ERROR, (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_void_loglevel(ll,sql,arg,...) _sqlite_exec_void(__WHENCE__, (ll), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_void_retry(rs,sql,arg,...) _sqlite_exec_void_retry(__WHENCE__, LOG_LEVEL_ERROR, (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_void_retry_loglevel(ll,rs,sql,arg,...) _sqlite_exec_void_retry(__WHENCE__, (ll), (rs), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_int64(res,sql,arg,...) _sqlite_exec_int64(__WHENCE__, (res), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_int64_retry(rs,res,sql,arg,...) _sqlite_exec_int64_retry(__WHENCE__, (rs), (res), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_strbuf(sb,sql,arg,...) _sqlite_exec_strbuf(__WHENCE__, (sb), (sql), arg, ##__VA_ARGS__)
#define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__)
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
@ -336,11 +384,12 @@ int rhizome_is_manifest_interesting(rhizome_manifest *m);
int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
const char *sender_sid, const char *recipient_sid,
int limit, int offset, char count_rows);
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest *m);
int rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m);
int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m);
int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
int rhizome_delete_bundle(const char *manifestid);
int rhizome_delete_manifest(const char *manifestid);
int rhizome_delete_payload(const char *manifestid);
int rhizome_delete_bundle(const rhizome_bid_t *bidp);
int rhizome_delete_manifest(const rhizome_bid_t *bidp);
int rhizome_delete_payload(const rhizome_bid_t *bidp);
int rhizome_delete_file(const char *fileid);
#define RHIZOME_DONTVERIFY 0
@ -350,25 +399,25 @@ int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int monitor_announce_bundle(rhizome_manifest *m);
int rhizome_find_secret(const unsigned char *authorSid, int *rs_len, const unsigned char **rs);
int rhizome_bk_xor_stream(
const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES],
const rhizome_bid_t *bidp,
const unsigned char *rs,
const size_t rs_len,
unsigned char *xor_stream,
int xor_stream_byte_count);
int rhizome_bk2secret(rhizome_manifest *m,
const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES],
const rhizome_bid_t *bidp,
const unsigned char *rs, const size_t rs_len,
/* The BK need only be the length of the secret half of the secret key */
const unsigned char bkin[RHIZOME_BUNDLE_KEY_BYTES],
unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]
);
int rhizome_secret2bk(
const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES],
const rhizome_bid_t *bidp,
const unsigned char *rs, const size_t rs_len,
/* The BK need only be the length of the secret half of the secret key */
unsigned char bkout[RHIZOME_BUNDLE_KEY_BYTES],
const unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]
);
);
unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m);
int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk);
@ -536,7 +585,7 @@ struct http_response {
const char * body;
};
int rhizome_received_content(unsigned char *bidprefix,uint64_t version,
int rhizome_received_content(const unsigned char *bidprefix,uint64_t version,
uint64_t offset,int count,unsigned char *bytes,
int type);
int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
@ -563,16 +612,16 @@ int is_rhizome_http_server_running();
typedef struct rhizome_direct_bundle_cursor {
/* Where the current fill started */
int64_t start_size_high;
unsigned char start_bid_low[RHIZOME_MANIFEST_ID_BYTES];
rhizome_bid_t start_bid_low;
/* Limit of where this cursor may traverse */
int64_t limit_size_high;
unsigned char limit_bid_high[RHIZOME_MANIFEST_ID_BYTES];
rhizome_bid_t limit_bid_high;
int64_t size_low;
int64_t size_high;
unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES];
unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES];
rhizome_bid_t bid_low;
rhizome_bid_t bid_high;
unsigned char *buffer;
int buffer_size;
int buffer_used;
@ -591,10 +640,10 @@ int rhizome_direct_bundle_iterator_unpickle_range(rhizome_direct_bundle_cursor *
int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,
int max_bars);
void rhizome_direct_bundle_iterator_free(rhizome_direct_bundle_cursor **c);
int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES],
unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES],
int rhizome_direct_get_bars(const rhizome_bid_t *bid_low,
rhizome_bid_t *bid_high,
int64_t size_low, int64_t size_high,
const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES],
const rhizome_bid_t *bid_max,
unsigned char *bars_out,
int bars_requested);
int rhizome_direct_process_post_multipart_bytes
@ -720,11 +769,11 @@ int rhizome_read_close(struct rhizome_read *read);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
int rhizome_dump_file(const char *id, const char *filepath, int64_t *length);
int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout,
int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, int length);
int rhizome_cache_close();
int rhizome_database_filehash_from_id(const char *id, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]);
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, char hash[SHA512_DIGEST_STRING_LENGTH]);
int overlay_mdp_service_rhizome_sync(struct overlay_frame *frame, overlay_mdp_frame *mdp);

8
rhizome_bundle.c
View File

@ -51,15 +51,15 @@ int rhizome_manifest_verify(rhizome_manifest *m)
/* Make sure that id variable is correct */
{
unsigned char manifest_id[RHIZOME_MANIFEST_ID_BYTES];
rhizome_bid_t bid;
char *id = rhizome_manifest_get(m,"id",NULL,0);
if (!id) {
WARN("Manifest lacks 'id' field");
m->errors++;
} else if (fromhexstr(manifest_id, id, RHIZOME_MANIFEST_ID_BYTES) == -1) {
} else if (str_to_rhizome_bid_t(&bid, id) == -1) {
WARN("Invalid manifest 'id' field");
m->errors++;
} else if (m->sig_count == 0 || memcmp(m->signatories[0], manifest_id, RHIZOME_MANIFEST_ID_BYTES) != 0) {
} else if (m->sig_count == 0 || memcmp(m->signatories[0], bid.binary, sizeof bid.binary) != 0) {
if (config.debug.rhizome) {
if (m->sig_count>0) {
DEBUGF("Manifest id variable does not match first signature block (signature key is %s)",
@ -159,7 +159,7 @@ int rhizome_manifest_parse(rhizome_manifest *m)
if (strcasecmp(var, "id") == 0) {
have_id = 1;
if (fromhexstr(m->cryptoSignPublic, value, RHIZOME_MANIFEST_ID_BYTES) == -1) {
if (str_to_rhizome_bid_t(&m->cryptoSignPublic, value) == -1) {
if (config.debug.rejecteddata)
WARNF("Invalid manifest id: %s", value);
m->errors++;

68
rhizome_crypto.c
View File

@ -38,16 +38,16 @@ unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m)
int rhizome_manifest_createid(rhizome_manifest *m)
{
if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic,m->cryptoSignSecret))
if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic.binary, m->cryptoSignSecret))
return WHY("Failed to create keypair for manifest ID.");
rhizome_manifest_set(m, "id", alloca_tohex_bid(m->cryptoSignPublic));
rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
m->haveSecret = NEW_BUNDLE_ID;
return 0;
}
struct signing_key{
unsigned char Private[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES];
unsigned char Public[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES];
rhizome_bid_t Public;
};
/* generate a keypair from a given seed string */
@ -58,7 +58,7 @@ static int generate_keypair(const char *seed, struct signing_key *key)
// The first 256 bits of the hash will be used as the private key of the BID.
bcopy(hash, key->Private, sizeof(key->Private));
if (crypto_sign_compute_public_key(key->Private, key->Public))
if (crypto_sign_compute_public_key(key->Private, key->Public.binary))
return WHY("Could not generate public key");
return 0;
}
@ -71,17 +71,15 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
if (generate_keypair(seed, &key))
return -1;
char *id = alloca_tohex_bid(key.Public);
int ret=rhizome_retrieve_manifest(id, m);
int ret=rhizome_retrieve_manifest(&key.Public, m);
if (ret<0)
return -1;
m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID;
bcopy(key.Public, m->cryptoSignPublic, sizeof(m->cryptoSignPublic));
bcopy(key.Private, m->cryptoSignSecret, sizeof(m->cryptoSignSecret));
m->cryptoSignPublic = key.Public;
bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret);
if (ret>0)
rhizome_manifest_set(m, "id", alloca_tohex_bid(m->cryptoSignPublic));
rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
return ret;
}
@ -93,7 +91,7 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
* @author Paul Gardner-Stephen <paul@servalproject.org>
*/
int rhizome_bk_xor_stream(
const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES],
const rhizome_bid_t *bidp,
const unsigned char *rs,
const size_t rs_len,
unsigned char *xor_stream,
@ -107,7 +105,7 @@ int rhizome_bk_xor_stream(
int combined_len = rs_len + crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES;
unsigned char buffer[combined_len];
bcopy(&rs[0], &buffer[0], rs_len);
bcopy(&bid[0], &buffer[rs_len], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
bcopy(&bidp->binary[0], &buffer[rs_len], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
unsigned char hash[crypto_hash_sha512_BYTES];
crypto_hash_sha512(hash,buffer,combined_len);
bcopy(hash,xor_stream,xor_stream_byte_count);
@ -121,7 +119,7 @@ int rhizome_bk_xor_stream(
second half of the secret key. The public key is the BID, so this simplifies
the BK<-->SECRET conversion processes. */
int rhizome_bk2secret(rhizome_manifest *m,
const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES],
const rhizome_bid_t *bidp,
const unsigned char *rs, const size_t rs_len,
/* The BK need only be the length of the secret half of the secret key */
const unsigned char bkin[RHIZOME_BUNDLE_KEY_BYTES],
@ -130,7 +128,7 @@ int rhizome_bk2secret(rhizome_manifest *m,
{
IN();
unsigned char xor_stream[RHIZOME_BUNDLE_KEY_BYTES];
if (rhizome_bk_xor_stream(bid,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES))
if (rhizome_bk_xor_stream(bidp,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES))
RETURN(WHY("rhizome_bk_xor_stream() failed"));
int i;
@ -140,16 +138,16 @@ int rhizome_bk2secret(rhizome_manifest *m,
secret[i] = bkin[i] ^ xor_stream[i];
/* Copy BID as public-key part of secret key */
for(;i!=crypto_sign_edwards25519sha512batch_SECRETKEYBYTES;++i)
secret[i]=bid[i-RHIZOME_BUNDLE_KEY_BYTES];
secret[i] = bidp->binary[i - RHIZOME_BUNDLE_KEY_BYTES];
bzero(xor_stream, sizeof xor_stream);
RETURN(rhizome_verify_bundle_privatekey(m,secret,bid));
RETURN(rhizome_verify_bundle_privatekey(m, secret, bidp->binary));
OUT();
}
int rhizome_secret2bk(
const unsigned char bid[crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES],
const rhizome_bid_t *bidp,
const unsigned char *rs, const size_t rs_len,
/* The BK need only be the length of the secret half of the secret key */
unsigned char bkout[RHIZOME_BUNDLE_KEY_BYTES],
@ -158,7 +156,7 @@ int rhizome_secret2bk(
{
IN();
unsigned char xor_stream[RHIZOME_BUNDLE_KEY_BYTES];
if (rhizome_bk_xor_stream(bid,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES))
if (rhizome_bk_xor_stream(bidp,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES))
RETURN(WHY("rhizome_bk_xor_stream() failed"));
int i;
@ -268,8 +266,7 @@ int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk)
const unsigned char *rs;
result = rhizome_find_secret(m->author, &rs_len, &rs);
if (result==0)
result = rhizome_bk2secret(m,m->cryptoSignPublic,rs,rs_len,
bkBytes,m->cryptoSignSecret);
result = rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret);
}
if (result == 0 && bsk && !rhizome_is_bk_none(bsk)){
@ -280,10 +277,9 @@ int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk)
}
}else if(bsk && !rhizome_is_bk_none(bsk)){
bcopy(m->cryptoSignPublic, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof(m->cryptoSignPublic));
bcopy(m->cryptoSignPublic.binary, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof m->cryptoSignPublic.binary);
bcopy(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES);
if (rhizome_verify_bundle_privatekey(m,m->cryptoSignSecret,
m->cryptoSignPublic))
if (rhizome_verify_bundle_privatekey(m, m->cryptoSignSecret, m->cryptoSignPublic.binary))
result=5;
else
result=0;
@ -363,8 +359,7 @@ int rhizome_find_bundle_author(rhizome_manifest *m)
RETURN(WHYF("invalid Rhizome Secret: length=%d", rs_len));
const unsigned char *rs = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key;
if (!rhizome_bk2secret(m,m->cryptoSignPublic,rs,rs_len,
bkBytes,m->cryptoSignSecret)) {
if (!rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret)) {
m->haveSecret=EXISTING_BUNDLE_ID;
if (memcmp(m->author, authorSid, sizeof m->author)){
@ -373,11 +368,12 @@ int rhizome_find_bundle_author(rhizome_manifest *m)
DEBUGF("found bundle author sid=%s", alloca_tohex_sid(m->author));
// if this bundle is already in the database, update the author.
if (m->inserttime){
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
if (sqlite_exec_void("UPDATE MANIFESTS SET author='%s' WHERE id='%s';", alloca_tohex_sid(m->author), id) == -1)
WARN("Error updating MANIFESTS author column");
}
if (m->inserttime)
sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
"UPDATE MANIFESTS SET author = ? WHERE id = ?;",
SID_T, (sid_t*)m->author,
RHIZOME_BID_T, &m->cryptoSignPublic,
END);
}
RETURN(0); // bingo
@ -407,11 +403,11 @@ int rhizome_verify_bundle_privatekey(rhizome_manifest *m,
crypto_sign_compute_public_key(sk,pk);
for (i = 0;i < 32;++i)
if (pkin[i] != pk[i]) {
if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic)
if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary)
m->haveSecret=0;
RETURN(-1);
}
if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic) {
if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) {
if (config.debug.rhizome)
DEBUGF("We have the private key for this bundle.");
m->haveSecret=EXISTING_BUNDLE_ID;
@ -427,7 +423,7 @@ int rhizome_sign_hash(rhizome_manifest *m,
if (!m->haveSecret && rhizome_extract_privatekey_required(m, NULL))
RETURN(-1);
int ret=rhizome_sign_hash_with_key(m,m->cryptoSignSecret,m->cryptoSignPublic,out);
int ret=rhizome_sign_hash_with_key(m, m->cryptoSignSecret, m->cryptoSignPublic.binary, out);
RETURN(ret);
OUT();
}
@ -692,10 +688,10 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
// journal bundles must always have the same nonce, regardless of version.
// otherwise, generate nonce from version#bundle id#version;
unsigned char raw_nonce[8+8+sizeof(m->cryptoSignPublic)];
unsigned char raw_nonce[8 + 8 + sizeof m->cryptoSignPublic.binary];
write_uint64(&raw_nonce[0], m->journalTail>=0?0:m->version);
bcopy(m->cryptoSignPublic, &raw_nonce[8], sizeof(m->cryptoSignPublic));
write_uint64(&raw_nonce[8+sizeof(m->cryptoSignPublic)], m->journalTail>=0?0:m->version);
bcopy(m->cryptoSignPublic.binary, &raw_nonce[8], sizeof m->cryptoSignPublic.binary);
write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->journalTail>=0?0:m->version);
unsigned char hash[crypto_hash_sha512_BYTES];

691
rhizome_database.c
View File

File diff suppressed because it is too large Load Diff

154
rhizome_direct.c
View File

@ -191,22 +191,19 @@ int rhizome_direct_continue_sync_request(rhizome_direct_sync_request *r)
if (r->cursor->size_high>=r->cursor->limit_size_high)
{
DEBUG("Out of bins");
if (memcmp(r->cursor->bid_low,r->cursor->limit_bid_high,
RHIZOME_MANIFEST_ID_BYTES)>=0)
{
DEBUG("out of BIDs");
/* Sync has finished.
The transport may have initiated one or more transfers, so
we cannot declare the sync complete until we know the transport
has finished transferring. */
if (!r->bundle_transfers_in_progress)
{
/* seems that all is done */
DEBUG("All done");
return rhizome_direct_conclude_sync_request(r);
} else
DEBUG("Stuck on in-progress transfers");
} else
if (cmp_rhizome_bid_t(&r->cursor->bid_low, &r->cursor->limit_bid_high) >= 0) {
DEBUG("out of BIDs");
/* Sync has finished.
The transport may have initiated one or more transfers, so
we cannot declare the sync complete until we know the transport
has finished transferring. */
if (!r->bundle_transfers_in_progress) {
/* seems that all is done */
DEBUG("All done");
return rhizome_direct_conclude_sync_request(r);
} else
DEBUG("Stuck on in-progress transfers");
} else
DEBUGF("bid_low<limit_bid_high");
}
@ -411,24 +408,19 @@ rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,int pref
Easiest way is to select with a BID range. We could instead have an extra
database column with the prefix.
*/
assert(prefix_length>=0);
assert(prefix_length<=RHIZOME_MANIFEST_ID_BYTES);
unsigned char low[RHIZOME_MANIFEST_ID_BYTES];
unsigned char high[RHIZOME_MANIFEST_ID_BYTES];
memset(low,0x00,RHIZOME_MANIFEST_ID_BYTES);
memset(high,0xff,RHIZOME_MANIFEST_ID_BYTES);
bcopy(bid_prefix,low,prefix_length);
bcopy(bid_prefix,high,prefix_length);
char query[1024];
snprintf(query,1024,"SELECT MANIFEST,ROWID FROM MANIFESTS WHERE ID>='%s' AND ID<='%s'",
alloca_tohex(low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(high,RHIZOME_MANIFEST_ID_BYTES));
rhizome_bid_t low = RHIZOME_BID_ZERO;
rhizome_bid_t high = RHIZOME_BID_MAX;
assert(prefix_length >= 0);
assert(prefix_length <= sizeof(rhizome_bid_t));
bcopy(bid_prefix, low.binary, prefix_length);
bcopy(bid_prefix, high.binary, prefix_length);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, query);
sqlite3_blob *blob=NULL;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT manifest, rowid FROM MANIFESTS WHERE id >= ? AND id <= ?",
RHIZOME_BID_T, &low,
RHIZOME_BID_T, &high,
END);
sqlite3_blob *blob=NULL;
if (sqlite_step_retry(&retry, statement) == SQLITE_ROW)
{
int ret;
@ -564,9 +556,8 @@ rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size)
void rhizome_direct_bundle_iterator_unlimit(rhizome_direct_bundle_cursor *r)
{
assert(r!=NULL);
r->limit_size_high=1LL<<48LL;
memset(r->limit_bid_high,0xff,RHIZOME_MANIFEST_ID_BYTES);
r->limit_bid_high = RHIZOME_BID_MAX;
return;
}
@ -599,13 +590,13 @@ int rhizome_direct_bundle_iterator_pickle_range(rhizome_direct_bundle_cursor *r,
v=r->start_size_high;
while(v>1) { ltwov++; v=v>>1; }
pickled[0]=ltwov;
for(v=0;v<4;v++) pickled[1+v]=r->start_bid_low[v];
for(v=0;v<4;v++) pickled[1+v]=r->start_bid_low.binary[v];
v=r->size_high;
DEBUGF("pickling size_high=%"PRId64,r->size_high);
ltwov=0;
while(v>1) { ltwov++; v=v>>1; }
pickled[1+4]=ltwov;
for(v=0;v<4;v++) pickled[1+4+1+v]=r->bid_high[v];
for(v=0;v<4;v++) pickled[1+4+1+v]=r->bid_high.binary[v];
return 1+4+1+4;
}
@ -626,13 +617,13 @@ int rhizome_direct_bundle_iterator_unpickle_range(rhizome_direct_bundle_cursor *
r->size_high=1LL<<pickled[0];
r->size_low=(r->size_high/2)+1;
if (r->size_high<=1024) r->size_low=0;
for(v=0;v<4;v++) r->bid_low[v]=pickled[1+v];
for(;v<RHIZOME_MANIFEST_ID_BYTES;v++) r->bid_low[v]=0x00;
r->bid_low = RHIZOME_BID_ZERO;
for (v=0;v<4;v++) r->bid_low.binary[v]=pickled[1+v];
/* Get end of range */
r->limit_size_high=1LL<<pickled[1+4];
for(v=0;v<4;v++) r->limit_bid_high[v]=pickled[1+4+1+v];
for(;v<RHIZOME_MANIFEST_ID_BYTES;v++) r->limit_bid_high[v]=0xff;
r->limit_bid_high = RHIZOME_BID_MAX;
for (v=0;v<4;v++) r->limit_bid_high.binary[v]=pickled[1+4+1+v];
return 0;
}
@ -654,7 +645,7 @@ int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,int max_
*/
/* This is the only information required to remember where we started: */
c->start_size_high=c->size_high;
bcopy(c->bid_low,c->start_bid_low,RHIZOME_MANIFEST_ID_BYTES);
c->start_bid_low = c->bid_low;
c->buffer_offset_bytes=1+4+1+4; /* space for pickled cursor range */
/* -1 is magic value for fill right up */
@ -675,17 +666,15 @@ int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,int max_
If we are not yet at the bundle data size limit, then any bundle is okay.
If we are at the bundle data size limit, then we need to honour
c->limit_bid_high. */
unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES];
if (c->size_high==c->limit_size_high)
bcopy(c->limit_bid_high,bid_max,RHIZOME_MANIFEST_ID_BYTES);
rhizome_bid_t bid_max;
if (c->size_high == c->limit_size_high)
bid_max = c->limit_bid_high;
else
memset(bid_max,0xff,RHIZOME_MANIFEST_ID_BYTES);
int stuffed_now=rhizome_direct_get_bars(c->bid_low,c->bid_high,
c->size_low,c->size_high,
bid_max,
&c->buffer[c->buffer_used
+c->buffer_offset_bytes],
bid_max = RHIZOME_BID_MAX;
int stuffed_now=rhizome_direct_get_bars(&c->bid_low, &c->bid_high,
c->size_low, c->size_high,
&bid_max,
&c->buffer[c->buffer_used + c->buffer_offset_bytes],
stuffable);
bundles_stuffed+=stuffed_now;
c->buffer_used+=RHIZOME_BAR_BYTES*stuffed_now;
@ -696,20 +685,19 @@ int rhizome_direct_bundle_iterator_fill(rhizome_direct_bundle_cursor *c,int max_
if (c->size_high<=1024) c->size_low=0;
DEBUGF("size=%"PRId64"..%"PRId64,c->size_low,c->size_high);
/* Record that we covered to the end of that size bin */
memset(c->bid_high,0xff,RHIZOME_MANIFEST_ID_BYTES);
c->bid_high = RHIZOME_BID_MAX;
if (c->size_high>c->limit_size_high)
memset(c->bid_low,0xff,RHIZOME_MANIFEST_ID_BYTES);
c->bid_low = RHIZOME_BID_MAX;
else
memset(c->bid_low,0x00,RHIZOME_MANIFEST_ID_BYTES);
c->bid_low = RHIZOME_BID_ZERO;
} else {
/* Continue from next BID */
bcopy(c->bid_high,c->bid_low,RHIZOME_MANIFEST_ID_BYTES);
c->bid_low = c->bid_high;
int i;
for(i=RHIZOME_BAR_BYTES-1;i>=0;i--)
{
c->bid_low[i]++;
if (c->bid_low[i]) break;
}
for(i=RHIZOME_BAR_BYTES-1;i>=0;i--) {
if (++c->bid_low.binary[i])
break;
}
if (i<0) break;
}
}
@ -743,31 +731,28 @@ void rhizome_direct_bundle_iterator_free(rhizome_direct_bundle_cursor **c)
it is possible to make provably complete comparison of the contents
of the respective rhizome databases.
*/
int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTES],
unsigned char bid_high[RHIZOME_MANIFEST_ID_BYTES],
int rhizome_direct_get_bars(const rhizome_bid_t *bidp_low,
rhizome_bid_t *bidp_high,
int64_t size_low, int64_t size_high,
const unsigned char bid_max[RHIZOME_MANIFEST_ID_BYTES],
const rhizome_bid_t *bidp_max,
unsigned char *bars_out,
int bars_requested)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
char query[1024];
snprintf(query,1024,
"SELECT BAR,ROWID,ID,FILESIZE FROM MANIFESTS"
" WHERE"
" FILESIZE BETWEEN %" PRId64 " AND %" PRId64
" AND ID>='%s' AND ID<='%s'"
// The following formulation doesn't remove the weird returning of
// bundles with out of range filesize values
// " WHERE ID>='%s' AND ID<='%s' AND FILESIZE > %lld AND FILESIZE < %lld"
" ORDER BY BAR LIMIT %d;",
size_low, size_high,
alloca_tohex(bid_low,RHIZOME_MANIFEST_ID_BYTES),
alloca_tohex(bid_max,RHIZOME_MANIFEST_ID_BYTES),
bars_requested);
sqlite3_stmt *statement=sqlite_prepare(&retry, query);
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT bar, rowid, id, filesize FROM MANIFESTS"
" WHERE filesize BETWEEN ? AND ? AND id >= ? AND id <= ?"
" ORDER BY bar LIMIT ?;",
INT64, size_low,
INT64, size_high,
RHIZOME_BID_T, bidp_low,
RHIZOME_BID_T, bidp_max,
INT, bars_requested,
// The following formulation doesn't remove the weird returning of
// bundles with out of range filesize values
// " WHERE id >= ? AND id <= ? AND filesize > ? AND filesize < ?"
END);
sqlite3_blob *blob=NULL;
int bars_written=0;
@ -784,8 +769,7 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE
int ret;
int64_t filesize = sqlite3_column_int64(statement, 3);
if (filesize<size_low||filesize>size_high) {
DEBUGF("WEIRDNESS ALERT: filesize=%"PRId64", but query was: %s",
filesize,query);
DEBUGF("WEIRDNESS ALERT: filesize=%"PRId64", but query was: %s", filesize, sqlite3_sql(statement));
break;
}
int64_t rowid = sqlite3_column_int64(statement, 1);
@ -813,9 +797,7 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE
/* Remember the BID so that we cant write it into bid_high so that the
caller knows how far we got. */
fromhex(bid_high,
(const char *)sqlite3_column_text(statement, 2),
RHIZOME_MANIFEST_ID_BYTES);
str_to_rhizome_bid_t(bidp_high, (const char *)sqlite3_column_text(statement, 2));
bars_written++;
break;
@ -831,4 +813,4 @@ int rhizome_direct_get_bars(const unsigned char bid_low[RHIZOME_MANIFEST_ID_BYTE
return bars_written;
}

15
rhizome_direct_http.c
View File

@ -884,17 +884,14 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* Start by getting the manifest, which is the main thing we need, and also
gives us the information we need for sending any associated file. */
rhizome_manifest
*m=rhizome_direct_get_manifest(&actionlist[i+1],
RHIZOME_BAR_PREFIX_BYTES);
if (!m) {
rhizome_manifest *m = rhizome_direct_get_manifest(&actionlist[i+1], RHIZOME_BAR_PREFIX_BYTES);
if (m == NULL) {
WHY("This should never happen. The manifest exists, but when I went looking for it, it doesn't appear to be there.");
goto next_item;
}
/* Get filehash and size from manifest if present */
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
DEBUGF("bundle id = '%s'",id);
DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
const char *hash = rhizome_manifest_get(m, "filehash", NULL, 0);
DEBUGF("bundle file hash = '%s'",hash);
long long filesize = rhizome_manifest_get_ll(m, "filesize");
@ -965,7 +962,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* send file contents */
{
char filehash[SHA512_DIGEST_STRING_LENGTH];
if (rhizome_database_filehash_from_id(id, version, filehash)<=0)
if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, filehash) == -1)
goto closeit;
struct rhizome_read read;
@ -1046,10 +1043,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
c->size_high,c->limit_size_high);
DEBUGF("c->buffer_size=%d",c->buffer_size);
r->cursor->size_low=c->limit_size_high;
bcopy(c->limit_bid_high,r->cursor->bid_low,4);
/* Set tail of BID to all high, as we assume the far end has returned all
BIDs with the specified prefix. */
memset(&r->cursor->bid_low[4],0xff,RHIZOME_MANIFEST_ID_BYTES);
r->cursor->bid_low = RHIZOME_BID_MAX;
bcopy(c->limit_bid_high.binary, r->cursor->bid_low.binary, 4);
}
rhizome_direct_bundle_iterator_free(&c);
#endif

71
rhizome_fetch.c
View File

@ -76,7 +76,7 @@ struct rhizome_fetch_slot {
int manifest_bytes;
/* MDP transport specific elements */
unsigned char bid[RHIZOME_MANIFEST_ID_BYTES];
rhizome_bid_t bid;
int64_t bidVersion;
int prefix_length;
int mdpIdleTimeout;
@ -263,7 +263,7 @@ static struct rhizome_fetch_slot *fetch_search_slot(const unsigned char *id, int
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
if (q->active.state != RHIZOME_FETCH_FREE &&
memcmp(id, q->active.manifest->cryptoSignPublic, prefix_length) == 0)
memcmp(id, q->active.manifest->cryptoSignPublic.binary, prefix_length) == 0)
return &q->active;
}
return NULL;
@ -279,7 +279,7 @@ static struct rhizome_fetch_candidate *fetch_search_candidate(const unsigned cha
struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
if (!c->manifest)
continue;
if (memcmp(c->manifest->cryptoSignPublic, id, prefix_length))
if (memcmp(c->manifest->cryptoSignPublic.binary, id, prefix_length))
continue;
return c;
}
@ -478,8 +478,8 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
slot->write_state.blob_rowid=-1;
if (slot->manifest) {
bcopy(slot->manifest->cryptoSignPublic,slot->bid,RHIZOME_MANIFEST_ID_BYTES);
slot->prefix_length=RHIZOME_MANIFEST_ID_BYTES;
slot->bid = slot->manifest->cryptoSignPublic;
slot->prefix_length = sizeof slot->bid.binary;
slot->bidVersion=slot->manifest->version;
/* Don't provide a filename, because we will stream the file straight into
the database. */
@ -493,8 +493,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
// if we're fetching a journal bundle, work out how many bytes we have of a previous version
// and therefore what range of bytes we should ask for
slot->previous = rhizome_new_manifest();
const char *id = rhizome_manifest_get(slot->manifest, "id", NULL, 0);
if (rhizome_retrieve_manifest(id, slot->previous)){
if (rhizome_retrieve_manifest(&slot->manifest->cryptoSignPublic, slot->previous)){
rhizome_manifest_free(slot->previous);
slot->previous=NULL;
@ -519,7 +518,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
RETURN(-1);
} else {
strbuf r = strbuf_local(slot->request, sizeof slot->request);
strbuf_sprintf(r, "GET /rhizome/manifestbyprefix/%s HTTP/1.0\r\n\r\n", alloca_tohex(slot->bid, slot->prefix_length));
strbuf_sprintf(r, "GET /rhizome/manifestbyprefix/%s HTTP/1.0\r\n\r\n", alloca_tohex(slot->bid.binary, slot->prefix_length));
if (strbuf_overrun(r))
RETURN(WHY("request overrun"));
slot->request_len = strbuf_len(r);
@ -631,8 +630,6 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
if (slot->state != RHIZOME_FETCH_FREE)
RETURN(SLOTBUSY);
const char *bid = alloca_tohex_bid(m->cryptoSignPublic);
/* Do the quick rejection tests first, before the more expensive ones,
like querying the database for manifests.
@ -650,7 +647,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
if (config.debug.rhizome_rx)
DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRId64" peerip=%s",
slotno(slot),
bid,
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
m->version,
m->fileLength,
alloca_sockaddr(peerip, sizeof(struct sockaddr_in))
@ -671,7 +668,7 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
* being published faster than we can fetch them.
*/
{
struct rhizome_fetch_slot *as = fetch_search_slot(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES);
struct rhizome_fetch_slot *as = fetch_search_slot(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
if (as){
const rhizome_manifest *am = as->manifest;
if (am->version < m->version) {
@ -746,7 +743,7 @@ rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip,
slot->peer_ipandport = *peerip;
slot->manifest = NULL;
bcopy(peersid,slot->peer_sid,SID_SIZE);
bcopy(prefix,slot->bid,prefix_length);
bcopy(prefix, slot->bid.binary, prefix_length);
slot->prefix_length=prefix_length;
/* Don't stream into a file blob in the database, because it is a manifest.
@ -857,11 +854,11 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
RETURN(0);
}
const char *bid = alloca_tohex_bid(m->cryptoSignPublic);
int priority=100; /* normal priority */
if (config.debug.rhizome_rx)
DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:", bid, m->version, m->fileLength, priority);
DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:",
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->fileLength, priority);
if (!rhizome_is_manifest_interesting(m)) {
if (config.debug.rhizome_rx)
@ -872,7 +869,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
if (config.debug.rhizome_rx) {
int64_t stored_version;
if (sqlite_exec_int64(&stored_version, "select version from manifests where id='%s'", bid) > 0)
if (sqlite_exec_int64(&stored_version, "SELECT version FROM MANIFESTS WHERE id = ?", RHIZOME_BID_T, &m->cryptoSignPublic, END) > 0)
DEBUGF(" is new (have version %"PRId64")", stored_version);
}
@ -880,8 +877,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
if (rhizome_manifest_verify(m) != 0) {
WHY("Error verifying manifest when considering for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m->cryptoSignPublic,
crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES, 60000);
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
rhizome_manifest_free(m);
RETURN(-1);
}
@ -908,7 +904,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
for (j = 0; j < q->candidate_queue_size; ) {
struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
if (c->manifest) {
if (memcmp(m->cryptoSignPublic, c->manifest->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES) == 0) {
if (cmp_rhizome_bid_t(&m->cryptoSignPublic, &c->manifest->cryptoSignPublic) == 0) {
if (c->manifest->version >= m->version) {
rhizome_manifest_free(m);
RETURN(0);
@ -916,8 +912,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
if (!m->selfSigned && rhizome_manifest_verify(m)) {
WHY("Error verifying manifest when considering queuing for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m->cryptoSignPublic,
crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES, 60000);
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
rhizome_manifest_free(m);
RETURN(-1);
}
@ -943,8 +938,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
if (!m->selfSigned && rhizome_manifest_verify(m)) {
WHY("Error verifying manifest when considering queuing for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m->cryptoSignPublic,
crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES, 60000);
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
rhizome_manifest_free(m);
RETURN(-1);
}
@ -966,7 +960,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
break;
DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%lld", i, j,
c->manifest,
alloca_tohex_bid(c->manifest->cryptoSignPublic),
alloca_tohex_rhizome_bid_t(c->manifest->cryptoSignPublic),
c->priority,
(long long) c->manifest->fileLength
);
@ -1080,8 +1074,8 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot)
mdp.packetTypeAndFlags=MDP_TX;
mdp.out.queue=OQ_ORDINARY;
mdp.out.payload_length=RHIZOME_MANIFEST_ID_BYTES+8+8+4+2;
bcopy(slot->bid,&mdp.out.payload[0],RHIZOME_MANIFEST_ID_BYTES);
mdp.out.payload_length= sizeof slot->bid.binary + 8 + 8 + 4 + 2;
bcopy(slot->bid.binary, &mdp.out.payload[0], sizeof slot->bid.binary);
uint32_t bitmap=0;
int requests=32;
@ -1099,11 +1093,11 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot)
}
offset+=slot->mdpRXBlockLength;
}
write_uint64(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES], slot->bidVersion);
write_uint64(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES+8], slot->write_state.file_offset);
write_uint32(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8], bitmap);
write_uint16(&mdp.out.payload[RHIZOME_MANIFEST_ID_BYTES+8+8+4], slot->mdpRXBlockLength);
write_uint64(&mdp.out.payload[sizeof slot->bid.binary], slot->bidVersion);
write_uint64(&mdp.out.payload[sizeof slot->bid.binary + 8], slot->write_state.file_offset);
write_uint32(&mdp.out.payload[sizeof slot->bid.binary + 8 + 8], bitmap);
write_uint16(&mdp.out.payload[sizeof slot->bid.binary + 8 + 8 + 4], slot->mdpRXBlockLength);
if (config.debug.rhizome_tx)
DEBUGF("src sid=%s, dst sid=%s, mdpRXWindowStart=0x%"PRIx64", slot->bidVersion=0x%"PRIx64,
@ -1297,7 +1291,7 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
rhizome_manifest_free(m);
} else {
if (config.debug.rhizome_rx){
DEBUGF("All looks good for importing manifest id=%s", alloca_tohex_bid(m->cryptoSignPublic));
DEBUGF("All looks good for importing manifest id=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
dump("slot->peerip",&slot->peer_ipandport,sizeof(slot->peer_ipandport));
dump("slot->peersid",&slot->peer_sid,sizeof(slot->peer_sid));
}
@ -1307,11 +1301,16 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
}
}
if (config.debug.rhizome_rx)
if (config.debug.rhizome_rx) {
time_ms_t now = gettime_ms();
time_ms_t interval = now - slot->start_time;
if (interval <= 0)
interval = 1;
DEBUGF("Closing rhizome fetch slot = 0x%p. Received %lld bytes in %lldms (%lldKB/sec).",
slot,(long long)slot->write_state.file_offset,
(long long)gettime_ms()-slot->start_time,
(long long)(slot->write_state.file_offset)/(gettime_ms()-slot->start_time));
(long long)interval,
(long long)((slot->write_state.file_offset) / interval));
}
rhizome_fetch_close(slot);
RETURN(-1);
@ -1355,7 +1354,7 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer
OUT();
}
int rhizome_received_content(unsigned char *bidprefix,
int rhizome_received_content(const unsigned char *bidprefix,
uint64_t version, uint64_t offset,
int count,unsigned char *bytes,int type)
{

12
rhizome_http.c
View File

@ -497,13 +497,13 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde
{
if (!is_rhizome_http_enabled())
return 1;
char id_hex[RHIZOME_MANIFEST_ID_STRLEN+1];
strncpy(id_hex, remainder, sizeof id_hex -1);
str_toupper_inplace(id_hex);
strcat(id_hex, "%");
rhizome_bid_t prefix;
const char *endp = NULL;
unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp);
if (endp == NULL || *endp != '\0' || prefix_len < 1)
return 1; // not found
rhizome_manifest *m = rhizome_new_manifest();
int ret = rhizome_retrieve_manifest(id_hex, m);
int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m);
if (ret==0)
rhizome_server_http_response(r, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
rhizome_manifest_free(m);

46
rhizome_packetformats.c
View File

@ -84,7 +84,7 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
/* Manifest prefix */
for(i=0;i<RHIZOME_BAR_PREFIX_BYTES;i++)
bar[RHIZOME_BAR_PREFIX_OFFSET+i]=m->cryptoSignPublic[i];
bar[RHIZOME_BAR_PREFIX_OFFSET+i]=m->cryptoSignPublic.binary[i];
/* file length */
bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->fileLength);
/* Version */
@ -134,38 +134,43 @@ uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar)
return bidprefix;
}
static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, const char *sql, int64_t *last_rowid){
int count=0;
sqlite3_stmt *statement=sqlite_prepare(retry, sql, *last_rowid);
static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, const char *sql, int64_t *last_rowid)
{
sqlite3_stmt *statement = sqlite_prepare(retry, sql);
if (statement == NULL)
return -1;
int params = sqlite3_bind_parameter_count(statement);
switch (params) {
case 0: break;
case 1:
if (sqlite_bind(retry, statement, INT64, *last_rowid, END) == -1)
return -1;
break;
default:
return WHYF("query has invalid number of parameters (%d): %s", params, sqlite3_sql(statement));
}
int count = 0;
while(sqlite_step_retry(retry, statement) == SQLITE_ROW) {
count++;
if (sqlite3_column_type(statement, 0)!=SQLITE_BLOB)
continue;
const void *data = sqlite3_column_blob(statement, 0);
int blob_bytes = sqlite3_column_bytes(statement, 0);
int64_t rowid = sqlite3_column_int64(statement, 1);
if (blob_bytes!=RHIZOME_BAR_BYTES) {
if (config.debug.rhizome_ads)
DEBUG("Found a BAR that is the wrong size - ignoring");
continue;
}
if (ob_append_bytes(e, (unsigned char *)data, blob_bytes)){
// out of room
count--;
break;
}
*last_rowid=rowid;
}
if (statement)
sqlite3_finalize(statement);
return count;
}
@ -191,7 +196,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){
goto end;
/* Get number of bundles available */
if (sqlite_exec_int64_retry(&retry, &bundles_available, "SELECT COUNT(BAR) FROM MANIFESTS;") != 1){
if (sqlite_exec_int64_retry(&retry, &bundles_available, "SELECT COUNT(BAR) FROM MANIFESTS;", END) != 1){
WHY("Could not count BARs for advertisement");
goto end;
}
@ -221,7 +226,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){
bundle_last_rowid=rowid;
count = append_bars(frame->payload, &retry,
"SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < %lld ORDER BY ROWID DESC LIMIT 17",
"SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < ? ORDER BY ROWID DESC LIMIT 17",
&bundle_last_rowid);
if (count<17)
bundle_last_rowid=INT64_MAX;
@ -263,7 +268,7 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){
if (overlay_payload_enqueue(frame)) goto error;
if (config.debug.rhizome_ads)
DEBUGF("Advertising manifest %s %"PRId64" to %s",
alloca_tohex_bid(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid(dest->sid):"broadcast");
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid(dest->sid):"broadcast");
return 0;
error:
@ -334,7 +339,7 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
(that is the only use of this */
if (config.debug.rhizome_ads){
long long version = rhizome_manifest_get_ll(m, "version");
DEBUGF("manifest id=%s version=%lld", alloca_tohex_bid(m->cryptoSignPublic), version);
DEBUGF("manifest id=%s version=%lld", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), version);
}
/* Crude signature presence test */
@ -347,10 +352,10 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
goto next;
}
if (rhizome_ignore_manifest_check(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES)){
if (rhizome_ignore_manifest_check(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary)){
/* Ignoring manifest that has caused us problems recently */
if (config.debug.rhizome_ads)
DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_bid(m->cryptoSignPublic));
DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
goto next;
}
@ -359,14 +364,13 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
DEBUG("Unverified manifest has errors - so not processing any further.");
/* Don't waste any time on this manifest in future attempts for at least
a minute. */
rhizome_queue_ignore_manifest(m->cryptoSignPublic,
RHIZOME_MANIFEST_ID_BYTES, 60000);
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
goto next;
}
/* Manifest is okay, so see if it is worth storing */
// are we already fetching this bundle [or later]?
rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES);
rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
if (mf && mf->version >= m->version)
goto next;

136
rhizome_store.c
View File

@ -8,12 +8,13 @@
int rhizome_exists(const char *fileHash)
{
int64_t gotfile = 0;
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE ID='%s' and datavalid=1;", fileHash) != 1)
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE id = ? and datavalid = 1;", TEXT_TOUPPER, fileHash, END) != 1)
return 0;
return gotfile;
}
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority){
int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int64_t file_length, int priority)
{
write->blob_fd=-1;
if (expectedFileHash){
@ -24,15 +25,16 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
}else{
write->id_known=0;
}
time_ms_t now = gettime_ms();
static uint64_t last_id=0;
write->temp_id = gettime_ms();
if (write->temp_id<last_id)
write->temp_id=last_id+1;
last_id=write->temp_id;
write->temp_id = now;
if (write->temp_id < last_id)
write->temp_id = last_id + 1;
last_id = write->temp_id;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
return WHY("Failed to begin transaction");
/*
@ -45,10 +47,15 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
*/
sqlite3_stmt *statement = NULL;
int ret=sqlite_exec_void_retry(&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) "
"VALUES('%"PRId64"',%"PRId64",%d,0,%"PRId64");",
write->temp_id, file_length, priority, gettime_ms());
int ret = sqlite_exec_void_retry(
&retry,
"INSERT OR REPLACE INTO FILES(id,length,highestpriority,datavalid,inserttime) VALUES(?,?,?,0,?);",
UINT64_TOSTR, write->temp_id,
INT64, file_length,
INT, priority,
INT64, now,
END
);
if (ret==-1)
goto insert_row_fail;
@ -61,8 +68,7 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
}
if (config.debug.externalblobs)
DEBUGF("Attempting to put blob for %"PRId64" in %s",
write->temp_id,blob_path);
DEBUGF("Attempting to put blob for id='%"PRId64"' in %s", write->temp_id, blob_path);
write->blob_fd=open(blob_path, O_CREAT | O_TRUNC | O_WRONLY, 0664);
if (write->blob_fd<0)
@ -72,45 +78,39 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
DEBUGF("Writing to new blob file %s (fd=%d)", blob_path, write->blob_fd);
}else{
statement = sqlite_prepare(&retry,"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES('%"PRId64"',?)",write->temp_id);
if (!statement) {
WHYF("Failed to insert into fileblobs: %s", sqlite3_errmsg(rhizome_db));
statement = sqlite_prepare_bind(
&retry,
"INSERT OR REPLACE INTO FILEBLOBS(id,data) VALUES(?,?)",
UINT64_TOSTR, write->temp_id,
ZEROBLOB, (int)file_length,
END);
if (statement == NULL)
goto insert_row_fail;
}
/* Bind appropriate sized zero-filled blob to data field */
if (sqlite3_bind_zeroblob(statement, 1, file_length) != SQLITE_OK) {
WHYF("sqlite3_bind_zeroblob() failed: %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(statement));
goto insert_row_fail;
}
/* Do actual insert, and abort if it fails */
int rowcount = 0;
int stepcode;
while ((stepcode = _sqlite_step_retry(__WHENCE__, LOG_LEVEL_ERROR, &retry, statement)) == SQLITE_ROW)
while ((stepcode = sqlite_step_retry(&retry, statement)) == SQLITE_ROW)
++rowcount;
if (rowcount)
WARNF("void query unexpectedly returned %d row%s", rowcount, rowcount == 1 ? "" : "s");
if (!sqlite_code_ok(stepcode)){
insert_row_fail:
WHYF("Failed to insert row for fileid=%"PRId64, write->temp_id);
WHYF("Failed to insert row for id='%"PRId64"'", write->temp_id);
if (statement) sqlite3_finalize(statement);
sqlite_exec_void_retry(&retry, "ROLLBACK;");
sqlite_exec_void_retry(&retry, "ROLLBACK;", END);
return -1;
}
sqlite3_finalize(statement);
statement=NULL;
/* Get rowid for inserted row, so that we can modify the blob */
write->blob_rowid = sqlite3_last_insert_rowid(rhizome_db);
if (config.debug.rhizome_rx)
DEBUGF("Got rowid %"PRId64" for %"PRId64, write->blob_rowid, write->temp_id);
DEBUGF("Got rowid=%"PRId64" for id='%"PRId64"'", write->blob_rowid, write->temp_id);
}
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1){
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1){
if (write->blob_fd>=0){
if (config.debug.externalblobs)
DEBUGF("Cancel write to fd %d", write->blob_fd);
@ -167,7 +167,7 @@ static int write_get_lock(struct rhizome_write *write_state){
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// use an explicit transaction so we can delay I/O failures until COMMIT so they can be retried.
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
return -1;
while(1){
@ -243,7 +243,7 @@ static int write_release_lock(struct rhizome_write *write_state){
sqlite3_errmsg(rhizome_db));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1)
ret=-1;
}
write_state->sql_blob=NULL;
@ -477,21 +477,27 @@ int rhizome_finish_write(struct rhizome_write *write)
rhizome_remove_file_datainvalid(&retry, write->id);
if (rhizome_exists(write->id)) {
// we've already got that payload, delete the new copy
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%"PRId64"';", write->temp_id);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%"PRId64"';", write->temp_id);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", UINT64_TOSTR, write->temp_id, END);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", UINT64_TOSTR, write->temp_id, END);
if (config.debug.rhizome)
DEBUGF("File id='%s' already present, removed id='%"PRId64"'", write->id, write->temp_id);
} else {
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
goto dbfailure;
// delete any half finished records
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILEBLOBS WHERE id='%s';",hash_out);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILEBLOBS WHERE id = ?;", STATIC_TEXT, write->id, END);
sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry, "DELETE FROM FILES WHERE id = ?;", STATIC_TEXT, write->id, END);
if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%"PRId64"'",
hash_out, gettime_ms(), write->temp_id) == -1)
if (sqlite_exec_void_retry(
&retry,
"UPDATE FILES SET id = ?, inserttime = ?, datavalid = 1 WHERE id = ?",
STATIC_TEXT, write->id,
INT64, gettime_ms(),
UINT64_TOSTR, write->temp_id,
END
) == -1
)
goto dbfailure;
if (fd>=0){
@ -501,7 +507,7 @@ int rhizome_finish_write(struct rhizome_write *write)
WHYF("Failed to generate file path");
goto dbfailure;
}
if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, hash_out)){
if (!FORM_RHIZOME_DATASTORE_PATH(dest_path, write->id)){
WHYF("Failed to generate file path");
goto dbfailure;
}
@ -512,13 +518,17 @@ int rhizome_finish_write(struct rhizome_write *write)
}
}else{
if (sqlite_exec_void_retry(&retry,
"UPDATE FILEBLOBS SET id='%s' WHERE rowid=%lld",
hash_out, write->blob_rowid) == -1){
goto dbfailure;
}
if (sqlite_exec_void_retry(
&retry,
"UPDATE FILEBLOBS SET id = ? WHERE rowid = ?",
STATIC_TEXT, write->id,
INT64, write->blob_rowid,
END
) == -1
)
goto dbfailure;
}
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1)
goto dbfailure;
if (config.debug.rhizome)
DEBUGF("Stored file %s", write->id);
@ -527,7 +537,7 @@ int rhizome_finish_write(struct rhizome_write *write)
return 0;
dbfailure:
sqlite_exec_void_retry(&retry, "ROLLBACK;");
sqlite_exec_void_retry(&retry, "ROLLBACK;", END);
failure:
rhizome_fail_write(write);
return -1;
@ -630,7 +640,7 @@ static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, stru
return -1;
if (config.debug.rhizome)
DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_bid(m->cryptoSignPublic), m->version);
DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
write->crypt=1;
if (m->journalTail>0)
@ -687,9 +697,9 @@ int rhizome_open_read(struct rhizome_read *read, const char *fileid)
if (sqlite_exec_int64(&read->blob_rowid,
"SELECT FILEBLOBS.rowid "
"FROM FILEBLOBS, FILES "
"WHERE FILEBLOBS.id = FILES.id "
"AND FILES.id = '%s' "
"AND FILES.datavalid != 0", read->id) == -1)
"WHERE FILEBLOBS.id = FILES.id"
" AND FILES.id = ?"
" AND FILES.datavalid != 0", STATIC_TEXT, read->id, END) == -1)
return -1;
if (read->blob_rowid != -1) {
read->length = -1; // discover the length on opening the db BLOB
@ -872,18 +882,18 @@ int rhizome_read_close(struct rhizome_read *read)
struct cache_entry{
struct cache_entry *_left;
struct cache_entry *_right;
unsigned char bundle_id[RHIZOME_MANIFEST_ID_BYTES];
rhizome_bid_t bundle_id;
uint64_t version;
struct rhizome_read read_state;
time_ms_t expires;
};
struct cache_entry *root;
static struct cache_entry ** find_entry_location(struct cache_entry **ptr, unsigned char *bundle_id, uint64_t version)
static struct cache_entry ** find_entry_location(struct cache_entry **ptr, const rhizome_bid_t *bundle_id, uint64_t version)
{
while(*ptr){
struct cache_entry *entry = *ptr;
int cmp = memcmp(bundle_id, entry->bundle_id, sizeof entry->bundle_id);
int cmp = cmp_rhizome_bid_t(bundle_id, &entry->bundle_id);
if (cmp==0){
if (entry->version==version)
break;
@ -921,7 +931,7 @@ static time_ms_t close_entries(struct cache_entry **entry, time_ms_t timeout)
// re-add both children to the tree
*entry=left;
if (right){
entry = find_entry_location(entry, right->bundle_id, right->version);
entry = find_entry_location(entry, &right->bundle_id, right->version);
*entry=right;
}
}else{
@ -970,19 +980,17 @@ int rhizome_cache_count()
}
// read a block of data, caching meta data for reuse
int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t timeout,
int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, int length)
{
// look for a cached entry
struct cache_entry **ptr = find_entry_location(&root, bundle_id, version);
struct cache_entry **ptr = find_entry_location(&root, bidp, version);
struct cache_entry *entry = *ptr;
// if we don't have one yet, create one and open it
if (!entry){
char *id_str = alloca_tohex_bid(bundle_id);
char filehash[SHA512_DIGEST_STRING_LENGTH];
if (rhizome_database_filehash_from_id(id_str, version, filehash)<=0)
if (rhizome_database_filehash_from_id(bidp, version, filehash) == -1)
return -1;
entry = emalloc_zero(sizeof(struct cache_entry));
@ -991,7 +999,7 @@ int rhizome_read_cached(unsigned char *bundle_id, uint64_t version, time_ms_t ti
free(entry);
return WHYF("Payload %s not found", filehash);
}
bcopy(bundle_id, entry->bundle_id, sizeof(entry->bundle_id));
entry->bundle_id = *bidp;
entry->version = version;
*ptr = entry;
}
@ -1055,7 +1063,7 @@ static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizom
return WHY("Unable to decrypt bundle, valid key not found");
}
if (config.debug.rhizome)
DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_bid(m->cryptoSignPublic), m->version);
DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (m->journalTail>0)
read_state->tail = m->journalTail;

1
serval.h
View File

@ -552,6 +552,7 @@ typedef struct sid_binary {
#define alloca_tohex_sid_t(sid) alloca_tohex((sid).binary, sizeof (*(sid_t*)0).binary)
int cmp_sid_t(const sid_t *a, const sid_t *b);
int str_to_sid_t(sid_t *sid, const char *hex);
int strn_to_sid_t(sid_t *sid, const char *hex, const char **endp);

2
tests/rhizomeops
View File

@ -577,7 +577,7 @@ test_AddMismatched() {
doc_AddUpdateSameVersion="Add new payload to existing manifest with same version fails"
setup_AddUpdateSameVersion() {
setup_AddDuplicate
setup_AddDeDuplicate
cp file1.manifest file1_2.manifest
strip_signatures file1_2.manifest
$SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d' file1_2.manifest

4
tests/rhizomeprotocol
View File

@ -710,7 +710,7 @@ test_DirectPull() {
set_instance +B
executeOk_servald rhizome direct pull
tfw_cat --stdout --stderr
assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A2:$VERSION_A2 --stderr
assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3
@ -732,7 +732,7 @@ test_DirectSync() {
set_instance +B
executeOk_servald rhizome direct sync
tfw_cat --stdout --stderr
assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A2:$VERSION_A2 --stderr
assert bundle_received_by $BID_A1:$VERSION_A1 $BID_A2:$VERSION_A2 $BID_A3:$VERSION_A3 --stderr
set_instance +A
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 fileA1 fileA2 fileA3 --fromhere=0 fileB1 fileB2 fileB3