From ecb79d818fc5e5023e4020534b620266b71f9863 Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Tue, 13 Sep 2016 14:14:21 +0930 Subject: [PATCH] Refactor storage of binary values to enable reuse --- headerfiles.mk | 1 + monitor.c | 3 +- nibble_tree.c | 168 ++++++++++++++++++++++++++++++++++++++++++ nibble_tree.h | 73 +++++++++++++++++++ overlay_address.c | 181 +++++++++------------------------------------- overlay_address.h | 13 ++-- overlay_mdp.c | 6 +- rhizome_sync.c | 6 +- route_link.c | 5 +- sourcefiles.mk | 1 + 10 files changed, 299 insertions(+), 158 deletions(-) create mode 100644 nibble_tree.c create mode 100644 nibble_tree.h diff --git a/headerfiles.mk b/headerfiles.mk index f65a7fac..85ae83ea 100644 --- a/headerfiles.mk +++ b/headerfiles.mk @@ -14,6 +14,7 @@ HDRS= fifo.h \ instance.h \ meshms.h \ message_ply.h \ + nibble_tree.h \ serval_types.h \ serval.h \ server.h \ diff --git a/monitor.c b/monitor.c index 4cab74af..8e0f365a 100644 --- a/monitor.c +++ b/monitor.c @@ -397,8 +397,9 @@ static void monitor_announce_peer(struct subscriber *subscriber, int prior_reach } DEFINE_TRIGGER(link_change, monitor_announce_peer); -static int monitor_announce_all_peers(struct subscriber *subscriber, void *UNUSED(context)) +static int monitor_announce_all_peers(void **record, void *UNUSED(context)) { + struct subscriber *subscriber = *record; if (subscriber->reachable&REACHABLE) monitor_announce_peer(subscriber, REACHABLE_NONE); return 0; diff --git a/nibble_tree.c b/nibble_tree.c new file mode 100644 index 00000000..370f5601 --- /dev/null +++ b/nibble_tree.c @@ -0,0 +1,168 @@ +/* +Serval DNA +Copyright (C) 2016 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include +#include +#include +#include +#include "mem.h" +#include "nibble_tree.h" + +static uint8_t get_nibble(const uint8_t *binary, int pos) +{ + uint8_t byte = binary[pos>>1]; + if (!(pos&1)) + byte=byte>>4; + return byte&0xF; +} + +enum tree_error_reason tree_find(struct tree_root *root, void **result, const uint8_t *binary, size_t bin_length, + tree_create_callback create_node, void *context) +{ + assert(bin_length <= root->binary_length); + struct tree_node *ptr = &root->_root_node; + + if (result) + *result = NULL; + + unsigned pos=0; + while(1) { + if (pos>>1 >= bin_length) + return TREE_NOT_UNIQUE; + + uint8_t nibble = get_nibble(binary, pos++); + void *node_ptr = ptr->tree_nodes[nibble]; + + if (ptr->is_tree & (1<binary_length){ + node_ptr = create_node(context, binary, bin_length); + if (!node_ptr) + return TREE_ERROR; + struct tree_record *tree_record = (struct tree_record *)node_ptr; + // TODO assert(memcmp(tree_record->binary, binary, bin_length) == 0))? + tree_record ->tree_depth = pos*4; + if (result) + *result = node_ptr; + ptr->tree_nodes[nibble] = node_ptr; + return TREE_FOUND; + } + return TREE_NOT_FOUND; + + }else{ + struct tree_record *tree_record = (struct tree_record *)node_ptr; + + // check that the remaining bytes of the value are the same + if (memcmp(tree_record->binary, binary, bin_length) == 0){ + if (result) + *result = node_ptr; + return TREE_FOUND; + } + + if (!create_node) + return TREE_NOT_FOUND; + + // no match? we need to bump this leaf node down a level so we can create a new record + struct tree_node *new_node = (struct tree_node *) emalloc_zero(sizeof(struct tree_node)); + if (!new_node) + return TREE_ERROR; + + ptr->tree_nodes[nibble] = new_node; + ptr->is_tree |= (1<binary, pos); + tree_record->tree_depth = (pos+1)*4; + ptr->tree_nodes[nibble] = node_ptr; + } + } +} + +static int walk(struct tree_node *node, unsigned pos, + uint8_t *empty, const uint8_t *binary, size_t bin_length, + walk_callback callback, void *context){ + unsigned i=0, e=16; + int ret=0; + *empty=1; + + if (binary){ + assert(pos*2 < bin_length); + uint8_t n = get_nibble(binary, pos); + for(;itree_nodes[i]){ + *empty=0; + break; + } + } + } + + for (;iis_tree & (1<tree_nodes[i], pos+1, &child_empty, binary, bin_length, callback, context); + if (child_empty){ + free(node->tree_nodes[i]); + node->tree_nodes[i]=NULL; + node->is_tree&=~(1<tree_nodes[i]){ + ret = callback(&node->tree_nodes[i], context); + } + if (ret) + return ret; + if (node->tree_nodes[i]) + *empty=0; + // stop comparing the start sid after looking at the first branch of the tree + binary=NULL; + } + + return ret; +} + +int tree_walk(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context) +{ + assert(!binary || bin_length <= root->binary_length); + uint8_t ignore; + return walk(&root->_root_node, 0, &ignore, binary, bin_length, callback, context); +} + +int tree_walk_prefix(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context) +{ + assert(bin_length <= root->binary_length); + struct tree_node *node = &root->_root_node; + unsigned pos=0; + for (; node && posis_tree & (1<tree_nodes[i]; + if (tree_record && memcmp(tree_record->binary, binary, bin_length)==0){ + return callback(&node->tree_nodes[i], context); + } + return 0; + } + node = node->tree_nodes[i]; + } + uint8_t ignore; + return walk(node, pos+1, &ignore, NULL, 0, callback, context); +} diff --git a/nibble_tree.h b/nibble_tree.h new file mode 100644 index 00000000..6fd8210a --- /dev/null +++ b/nibble_tree.h @@ -0,0 +1,73 @@ +/* +Serval DNA +Copyright (C) 2016 Serval Project Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef __SERVAL_DNA__NIBBLE_TREE_H +#define __SERVAL_DNA__NIBBLE_TREE_H + +struct tree_record{ + // number of bits of the binary value, to uniquely identify this record within the tree's current contents + unsigned tree_depth; + uint8_t binary[0]; +}; + +// each node has 16 slots based on the next 4 bits of the binary value +// each slot either points to another tree node or a data record +struct tree_node{ + // bit flags for the type of object each element points to + uint16_t is_tree; + void *tree_nodes[16]; +}; + +struct tree_root{ + size_t binary_length; + struct tree_node _root_node; +}; + +enum tree_error_reason { + TREE_NOT_UNIQUE = -3, + TREE_NOT_FOUND = -2, + TREE_ERROR = -1, + TREE_FOUND = 0 +}; + + +// allocate a new record and return it +// the returned memory buffer *must* begin with the same memory layout as struct tree_record +typedef void* (*tree_create_callback) (void *context, const uint8_t *binary, size_t bin_length); + +// find the record related to the given binary value +// if not found, the supplied not_found function will be called +// if the callback returns a non-null value it will be inserted into the tree +// returns either the current depth in the tree or a tree_error_reason +enum tree_error_reason tree_find(struct tree_root *root, void **result, const uint8_t *binary, size_t bin_length, + tree_create_callback create_node, void *context); + +// callback function for walking the tree +// return 0 to continue enumeration, anything else to stop +// set (*record) to null to indicate that memory has been released and the node should be removed from the tree +typedef int (*walk_callback) (void **record, void *context); + +// walk the tree, calling walk_callback for each node. +// if binary & bin_length have been supplied, skip all records <= this binary value +int tree_walk(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context); + +// walk the tree where nodes match the prefix binary / bin_length +int tree_walk_prefix(struct tree_root *root, const uint8_t *binary, size_t bin_length, walk_callback callback, void *context); + +#endif \ No newline at end of file diff --git a/overlay_address.c b/overlay_address.c index 9c54c0db..50c59032 100644 --- a/overlay_address.c +++ b/overlay_address.c @@ -48,19 +48,7 @@ static struct broadcast bpilist[MAX_BPIS]; #define OA_CODE_P2P_ME 0xfc #define OA_CODE_SIGNKEY 0xfb // full sign key of an identity, from which a SID can be derived -// each node has 16 slots based on the next 4 bits of a subscriber id -// each slot either points to another tree node or a struct subscriber. -struct tree_node{ - // bit flags for the type of object each element points to - uint16_t is_tree; - - union{ - struct tree_node *tree_nodes[16]; - struct subscriber *subscribers[16]; - }; -}; - -static __thread struct tree_node root; +static __thread struct tree_root root={.binary_length=SID_SIZE}; static __thread struct subscriber *my_subscriber=NULL; @@ -88,16 +76,9 @@ void release_my_subscriber(){ my_subscriber = NULL; } -static unsigned char get_nibble(const unsigned char *sidp, int pos) -{ - unsigned char byte = sidp[pos>>1]; - if (!(pos&1)) - byte=byte>>4; - return byte&0xF; -} - -static void free_subscriber(struct subscriber *subscriber) +static int free_node(void **record, void *UNUSED(context)) { + struct subscriber *subscriber = (struct subscriber *)*record; if (subscriber->link_state || subscriber->destination) FATAL("Can't free a subscriber that is being used in routing"); if (subscriber->sync_state) @@ -105,22 +86,8 @@ static void free_subscriber(struct subscriber *subscriber) if (subscriber->identity) FATAL("Can't free a subscriber that is unlocked in the keyring"); free(subscriber); -} - -static void free_children(struct tree_node *parent) -{ - int i; - for (i=0;i<16;i++){ - if (parent->is_tree & (1<tree_nodes[i]); - free(parent->tree_nodes[i]); - parent->tree_nodes[i]=NULL; - }else if(parent->subscribers[i]){ - free_subscriber(parent->subscribers[i]); - parent->subscribers[i]=NULL; - } - } - parent->is_tree=0; + *record=NULL; + return 0; } void free_subscribers() @@ -129,117 +96,35 @@ void free_subscribers() // who knows where subscriber ptr's may have leaked to. if (serverMode) FATAL("Freeing subscribers from a running daemon is not supported"); - free_children(&root); + tree_walk(&root, NULL, 0, free_node, NULL); +} + +static void *create_subscriber(void *UNUSED(context), const uint8_t *binary, size_t bin_length) +{ + assert(bin_length == SID_SIZE); + struct subscriber *ret = (struct subscriber *) emalloc_zero(sizeof(struct subscriber)); + if (ret){ + ret->sid = *(const sid_t *)binary; + DEBUGF(subscriber, "Stored %s", alloca_tohex_sid_t(ret->sid)); + } + return ret; } // find a subscriber struct from a whole or abbreviated subscriber id -struct subscriber *_find_subscriber(struct __sourceloc __whence, const unsigned char *sidp, int len, int create) +struct subscriber *find_subscriber(const uint8_t *sidp, int len, int create) { - IN(); - struct tree_node *ptr = &root; - int pos=0; - if (len!=SID_SIZE) - create =0; - struct subscriber *ret = NULL; - do { - unsigned char nibble = get_nibble(sidp, pos++); - if (ptr->is_tree & (1<tree_nodes[nibble]; - }else if(!ptr->subscribers[nibble]){ - // subscriber is not yet known - if (create && (ret = (struct subscriber *) emalloc_zero(sizeof(struct subscriber)))) { - ptr->subscribers[nibble] = ret; - ret->sid = *(const sid_t *)sidp; - ret->abbreviate_len = pos; - DEBUGF(subscriber, "Storing %s, abbrev_len=%d", alloca_tohex_sid_t(ret->sid), ret->abbreviate_len); - } - goto done; - }else{ - // there's a subscriber in this slot, does it match the rest of the sid we've been given? - ret = ptr->subscribers[nibble]; - if (memcmp(ret->sid.binary, sidp, len) == 0) - goto done; - // if we need to insert this subscriber, we have to make a new tree node first - if (!create) { - if (len != SID_SIZE) - DEBUGF(subscriber, "Prefix %s is not unique", alloca_tohex(sidp, len)); - ret = NULL; - goto done; - } - // create a new tree node and move the existing subscriber into it - struct tree_node *new = (struct tree_node *) emalloc_zero(sizeof(struct tree_node)); - if (new == NULL) { - ret = NULL; - goto done; - } - ptr->tree_nodes[nibble] = new; - ptr->is_tree |= (1<sid.binary, pos); - ptr->subscribers[nibble] = ret; - ret->abbreviate_len = pos + 1; - DEBUGF(subscriber, "Bumped %s, abbrev_len=%d (+ %p)", alloca_tohex_sid_t(ret->sid), ret->abbreviate_len, ptr); - // then go around the loop again to compare the next nibble against the sid until we find an empty slot. - } - } while(pos < len*2); -done: - RETURN(ret); -} - -/* - Walk the subscriber tree, calling the callback function for each subscriber. - if start is a valid pointer, the first entry returned will be after this subscriber - if the callback returns non-zero, the process will stop. - */ -static int walk_tree(struct tree_node *node, int pos, - const struct subscriber *start, - int(*callback)(struct subscriber *, void *), void *context){ - int i=0, e=16; - - if (start) - i=get_nibble(start->sid.binary, pos); - - for (;iis_tree & (1<tree_nodes[i], pos+1, start, callback, context)) - return 1; - }else if(node->subscribers[i]){ - if (callback(node->subscribers[i], context)) - return 1; - } - // stop comparing the start sid after looking at the first branch of the tree - start=NULL; - } - return 0; -} - -// walk the sub-tree for all subscribers that exactly match this id/len prefix. -static void prefix_matches(uint8_t *id, unsigned len, - int(*callback)(struct subscriber *, void *), void *context) -{ - struct tree_node *node = &root; - unsigned pos=0; - DEBUGF(subscriber, "Looking for %s", alloca_tohex(id, len)); - for (; node && posis_tree & (1<is_tree & (1<subscribers[i] && memcmp(node->subscribers[i]->sid.binary, id, len)==0) - callback(node->subscribers[i], context); - return; - } - node = node->tree_nodes[i]; - } - DEBUGF(subscriber, "Walking from %p", node); - walk_tree(node, pos+1, NULL, callback, context); + struct subscriber *result; + tree_find(&root, (void**)&result, sidp, len, create && len == SID_SIZE ? create_subscriber : NULL, NULL); + // ignore return code, just return the result + return result; } /* walk the tree, starting at start inclusive, calling the supplied callback function */ -void enum_subscribers(struct subscriber *start, int(*callback)(struct subscriber *, void *), void *context) +void enum_subscribers(struct subscriber *start, walk_callback callback, void *context) { - walk_tree(&root, 0, start, callback, context); + tree_walk(&root, start?start->sid.binary:NULL, SID_SIZE, callback, context); } // generate a new random broadcast address @@ -305,7 +190,9 @@ void overlay_address_append(struct decode_context *context, struct overlay_buffe ob_append_bytes(b, subscriber->sid.binary, SID_SIZE); subscriber->send_full=0; }else{ - int len=(subscriber->abbreviate_len+2)/2; + // always send 8-12 extra bits to disambiguate abbreviations + int len=(subscriber->tree_depth >> 3) + 1; + // add another 8 bits for our own packet headers if (context && (context->flags & DECODE_FLAG_ENCODING_HEADER)) len++; if (len>SID_SIZE) @@ -318,8 +205,9 @@ void overlay_address_append(struct decode_context *context, struct overlay_buffe context->previous = subscriber; } -static int add_explain_response(struct subscriber *subscriber, void *context) +static int add_explain_response(void **record, void *context) { + struct subscriber *subscriber = *record; struct decode_context *response = context; // only explain a SID once every half second. time_ms_t now = gettime_ms(); @@ -389,7 +277,7 @@ static int find_subscr_buffer(struct decode_context *context, struct overlay_buf context->flags|=DECODE_FLAG_INVALID_ADDRESS; if (context->flags & DECODE_FLAG_DONT_EXPLAIN){ - DEBUGF(subscriber, "Ignoring prefix %s", alloca_tohex(id, len)); + DEBUGF(subscriber, "Ignoring unknown prefix %s", alloca_tohex(id, len)); }else{ // generate a please explain in the passed in context @@ -403,7 +291,7 @@ static int find_subscr_buffer(struct decode_context *context, struct overlay_buf // And I'll tell you about any subscribers I know that match this abbreviation, // so you don't try to use an abbreviation that's too short in future. - prefix_matches(id, len, add_explain_response, context); + tree_walk_prefix(&root, id, len, add_explain_response, context); DEBUGF(subscriber, "Asking for explanation of %s", alloca_tohex(id, len)); ob_append_byte(context->please_explain->payload, len); @@ -568,7 +456,10 @@ int process_explain(struct overlay_frame *frame) int len = ob_get(b); switch (len){ case OA_CODE_P2P_YOU: - add_explain_response(get_my_subscriber(), &context); + { + void *sid = get_my_subscriber(); + add_explain_response(&sid, &context); + } break; case OA_CODE_SIGNKEY: decode_sid_from_signkey(b, NULL); @@ -591,7 +482,7 @@ int process_explain(struct overlay_frame *frame) uint8_t *sid = ob_get_bytes_ptr(b, len); // reply to the sender with all subscribers that match this abbreviation DEBUGF(subscriber, "Sending explain responses for %s", alloca_tohex(sid, len)); - prefix_matches(sid, len, add_explain_response, &context); + tree_walk_prefix(&root, sid, len, add_explain_response, &context); } } } diff --git a/overlay_address.h b/overlay_address.h index 8dbe0c2e..901be9b7 100644 --- a/overlay_address.h +++ b/overlay_address.h @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "constants.h" #include "os.h" // for time_ms_t #include "socket.h" +#include "nibble_tree.h" // not reachable #define REACHABLE_NONE 0 @@ -52,10 +53,11 @@ struct overlay_buffer; // This structure supports both our own routing protocol which can store calculation details in *node // or IP4 addresses reachable via any other kind of normal layer3 routing protocol, eg olsr struct subscriber{ + // minimum abbreviation length, in bits. + // Note this must be here to match the memory layout of struct tree_record + unsigned tree_depth; sid_t sid; - // minimum abbreviation length, in 4bit nibbles. - int abbreviate_len; - + int max_packet_version; // link state routing information @@ -119,10 +121,9 @@ struct subscriber *get_my_subscriber(); void release_my_subscriber(); extern __thread struct subscriber *directory_service; -struct subscriber *_find_subscriber(struct __sourceloc, const unsigned char *sid, int len, int create); -#define find_subscriber(sid, len, create) _find_subscriber(__WHENCE__, sid, len, create) +struct subscriber *find_subscriber(const uint8_t *sid, int len, int create); -void enum_subscribers(struct subscriber *start, int(*callback)(struct subscriber *, void *), void *context); +void enum_subscribers(struct subscriber *start, walk_callback callback, void *context); int set_reachable(struct subscriber *subscriber, struct network_destination *destination, struct subscriber *next_hop, int hop_count, struct subscriber *prior_hop); struct network_destination *load_subscriber_address(struct subscriber *subscriber); diff --git a/overlay_mdp.c b/overlay_mdp.c index 9d2c8d4c..e7083c59 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -1010,7 +1010,8 @@ static int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *c OUT(); } -static int search_subscribers(struct subscriber *subscriber, void *context){ +static int search_subscribers(void **record, void *context){ + struct subscriber *subscriber=*record; struct overlay_mdp_addrlist *response = context; if (response->mode == MDP_ADDRLIST_MODE_SELF && subscriber->reachable != REACHABLE_SELF){ @@ -1087,8 +1088,9 @@ static void send_route(struct subscriber *subscriber, struct socket_address *cli ob_free(b); } -static int routing_table(struct subscriber *subscriber, void *context) +static int routing_table(void **record, void *context) { + struct subscriber *subscriber = *record; if (subscriber->reachable != REACHABLE_NONE){ struct routing_state *state = (struct routing_state *)context; send_route(subscriber, state->client, state->header); diff --git a/rhizome_sync.c b/rhizome_sync.c index 0cc333de..bb06013a 100644 --- a/rhizome_sync.c +++ b/rhizome_sync.c @@ -83,8 +83,9 @@ void rhizome_sync_status_html(struct strbuf *b, struct subscriber *subscriber) state->bars_skipped); } -static int sync_status(struct subscriber *subscriber, void *UNUSED(context)) +static int sync_status(void **record, void *UNUSED(context)) { + struct subscriber *subscriber = *record; if (!subscriber->sync_state) return 0; struct rhizome_sync *state=subscriber->sync_state; @@ -219,8 +220,9 @@ static void rhizome_sync_send_requests(struct subscriber *subscriber, struct rhi } } -static int sync_bundle_inserted(struct subscriber *subscriber, void *context) +static int sync_bundle_inserted(void **record, void *context) { + struct subscriber *subscriber = *record; const rhizome_bar_t *bar = context; if (!subscriber->sync_state) return 0; diff --git a/route_link.c b/route_link.c index 59ea867d..492cf1d3 100644 --- a/route_link.c +++ b/route_link.c @@ -167,7 +167,7 @@ struct link_state{ }; DEFINE_ALARM(link_send); -static int append_link(struct subscriber *subscriber, void *context); +static int append_link(void **record, void *context); static int neighbour_find_best_link(struct neighbour *n); struct neighbour *neighbours=NULL; @@ -496,8 +496,9 @@ static int append_link_state(struct overlay_buffer *payload, char flags, return 0; } -static int append_link(struct subscriber *subscriber, void *context) +static int append_link(void **record, void *context) { + struct subscriber *subscriber = *record; if (subscriber == get_my_subscriber()) return 0; diff --git a/sourcefiles.mk b/sourcefiles.mk index 48089ed0..5f5a3b46 100644 --- a/sourcefiles.mk +++ b/sourcefiles.mk @@ -81,6 +81,7 @@ SERVAL_DAEMON_SOURCES = \ overlay_mdp_services.c \ mdp_filter.c \ msp_server.c \ + nibble_tree.c \ overlay_olsr.c \ overlay_packetformats.c \ overlay_payload.c \