diff --git a/Makefile.linux b/Makefile.linux index 4f48d222b..794de5e39 100644 --- a/Makefile.linux +++ b/Makefile.linux @@ -7,7 +7,7 @@ DEFS=-DZT_ARCH="$(ARCH)" -DZT_OSNAME="linux" -DZT_TRACE LIBS= # Uncomment for a release optimized build -CFLAGS=-Wall -O3 -fno-unroll-loops -fstack-protector -pthread $(INCLUDES) -DNDEBUG $(DEFS) +CFLAGS=-Wall -O3 -fno-unroll-loops -fvisibility=hidden -fstack-protector -pthread $(INCLUDES) -DNDEBUG $(DEFS) STRIP=strip --strip-all # Uncomment for a debug build diff --git a/ext/kissdb/README.md b/ext/kissdb/README.md deleted file mode 100644 index c47affefd..000000000 --- a/ext/kissdb/README.md +++ /dev/null @@ -1,70 +0,0 @@ -kissdb -====== - -(Keep It) Simple Stupid Database - -KISSDB is about the simplest key/value store you'll ever see, anywhere. -It's written in plain vanilla C using only the standard string and FILE -I/O functions, and should port to just about anything with a disk or -something that acts like one. - -It stores keys and values of fixed length in a stupid-simple file format -based on fixed-size hash tables. If a hash collision occurrs, a new "page" -of hash table is appended to the database. The format is append-only. -There is no delete. Puts that replace an existing value, however, will not -grow the file as they will overwrite the existing entry. - -Hash table size is a space/speed trade-off parameter. Larger hash tables -will reduce collisions and speed things up a bit, at the expense of memory -and disk space. A good size is usually about 1/2 the average number of -entries you expect. - -Features: - - * Tiny, compiles to ~4k on an x86_64 Linux system - * Small memory footprint (only caches hash tables) - * Very space-efficient (on disk) if small hash tables are used - * Makes a decent effort to be robust on power loss - * Pretty respectably fast, especially given its simplicity - * 64-bit, file size limit is 2^64 bytes - * Ports to anything with a C compiler and stdlib/stdio - * Public domain - -Limitations: - - * Fixed-size keys and values, must recreate and copy to change any init size parameter - * Add/update only, no delete - * Iteration is supported but key order is undefined - * No search for subsets of keys/values - * No indexes - * No transactions - * No special recovery features if a database gets corrupted - * No built-in thread-safety (guard it with a mutex in MT code) - * No built-in caching of data (only hash tables are cached for lookup speed) - * No endian-awareness (currently), so big-endian DBs won't read on little-endian machines - -Alternative key/value stores and embedded databases: - - * [MDB](http://symas.com/mdb/) uses mmap() and is very fast (not quite as tiny/simple/portable) - * [CDB](http://cr.yp.to/cdb.html) is also minimal and fast, probably the closest thing to this (but has a 4gb size limit) - * [Kyoto Cabinet](http://fallabs.com/kyotocabinet/) is very fast, full-featured, and modern (license required for commercial use) - * [SQLite](http://www.sqlite.org/) gives you a complete embedded SQL server (public domain, very mature, much larger) - * Others include GDBM, NDBM, Berkeley DB, etc. Use your Googles. :) - -KISSDB is good if you want space-efficient relatively fast write-once/read-many storage -of keys mapped to values. It's not a good choice if you need searches, indexes, delete, -structured storage, or widely varying key/value sizes. It's also probably not a good -choice if you need a long-lived database for critical data, since it lacks recovery -features and is brittle if its internals are modified. It would be better for a cache -of data that can be restored or "re-learned," such as keys, Bitcoin transactions, nodes -on a peer-to-peer network, log analysis results, rendered web pages, session cookies, -auth tokens, etc. - -KISSDB is in the public domain. One reason it was written was the -poverty of simple key/value databases with wide open licensing. Even old -ones like GDBM have GPL, not LGPL, licenses. - -See comments in kissdb.h for documentation. Makefile can be used to build -a test program on systems with gcc. - -Author: Adam Ierymenko / ZeroTier Networks LLC diff --git a/ext/kissdb/SPEC.txt b/ext/kissdb/SPEC.txt deleted file mode 100644 index f90b11f13..000000000 --- a/ext/kissdb/SPEC.txt +++ /dev/null @@ -1,60 +0,0 @@ ------ - -KISSDB file format (version 2) -Author: Adam Ierymenko - ------ - -In keeping with the goal of minimalism the file format is very simple, the -sort of thing that would be given as an example in an introductory course in -data structures. It's a basic hash table that adds additional pages of hash -table entries on collision. - -It consists of a 28 byte header followed by a series of hash tables and data. -All integer values are stored in the native word order of the target -architecture (in the future the code might be fixed to make everything -little-endian if anyone cares about that). - -The header consists of the following fields: - -[0-3] magic numbers: (ASCII) 'K', 'd', 'B', KISSDB_VERSION (currently 2) -[4-11] 64-bit hash table size in entries -[12-19] 64-bit key size in bytes -[20-27] 64-bit value size in bytes - -Hash tables are arrays of [hash table size + 1] 64-bit integers. The extra -entry, if nonzero, is the offset in the file of the next hash table, forming -a linked list of hash tables across the file. - -Immediately following the header, the first hash table will be written when -the first key/value is added. The algorithm for adding new entries is as -follows: - -(1) The key is hashed using a 64-bit variant of the DJB2 hash function, and - this is taken modulo hash table size to get a bucket number. -(2) Hash tables are checked in order, starting with the first hash table, - until a zero (empty) bucket is found. If one is found, skip to step (4). -(3) If no empty buckets are found in any hash table, a new table is appended - to the file and the final pointer in the previous hash table is set to - its offset. (In the code the update of the next hash table pointer in - the previous hash table happens last, after the whole write is complete, - to avoid corruption on power loss.) -(4) The key and value are appended, in order with no additional meta-data, - to the database file. Before appending the offset in the file stream - where they will be stored is saved. After appending, this offset is - written to the empty hash table bucket we chose in steps 2/3. Hash table - updates happen last to avoid corruption if the write does not complete. - -Lookup of a key/value pair occurs as follows: - -(1) The key is hashed and taken modulo hash table size to get a bucket - number. -(2) If this bucket's entry in the hash table is nonzero, the key at the - offset specified by this bucket is compared to the key being looked up. - If they are equal, the value is read and returned. -(3) If the keys are not equal, the next hash table is checked and step (2) - is repeated. If an empty bucket is encountered or if we run out of hash - tables, the key was not found. - -To update an existing value, its location is looked up and the value portion -of the entry is rewritten. diff --git a/ext/kissdb/kissdb.c b/ext/kissdb/kissdb.c deleted file mode 100644 index e5ee99795..000000000 --- a/ext/kissdb/kissdb.c +++ /dev/null @@ -1,447 +0,0 @@ -/* (Keep It) Simple Stupid Database - * - * Written by Adam Ierymenko - * KISSDB is in the public domain and is distributed with NO WARRANTY. */ - -/* Compile with KISSDB_TEST to build as a test program. */ - -/* Note: big-endian systems will need changes to implement byte swapping - * on hash table file I/O. Or you could just use it as-is if you don't care - * that your database files will be unreadable on little-endian systems. */ - -#define _FILE_OFFSET_BITS 64 - -#include "kissdb.h" - -#include -#include -#include - -#ifdef _WIN32 -#define fseeko _fseeki64 -#define ftello _ftelli64 -#endif - -#define KISSDB_HEADER_SIZE ((sizeof(uint64_t) * 3) + 4) - -/* djb2 hash function */ -static uint64_t KISSDB_hash(const void *b,unsigned long len) -{ - unsigned long i; - uint64_t hash = 5381; - for(i=0;if = (FILE *)0; - fopen_s(&db->f,path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); -#else - db->f = fopen(path,((mode == KISSDB_OPEN_MODE_RWREPLACE) ? "w+b" : (((mode == KISSDB_OPEN_MODE_RDWR)||(mode == KISSDB_OPEN_MODE_RWCREAT)) ? "r+b" : "rb"))); -#endif - if (!db->f) { - if (mode == KISSDB_OPEN_MODE_RWCREAT) { -#ifdef _WIN32 - db->f = (FILE *)0; - fopen_s(&db->f,path,"w+b"); -#else - db->f = fopen(path,"w+b"); -#endif - } - if (!db->f) - return KISSDB_ERROR_IO; - } - - if (fseeko(db->f,0,SEEK_END)) { - fclose(db->f); - return KISSDB_ERROR_IO; - } - if (ftello(db->f) < KISSDB_HEADER_SIZE) { - /* write header if not already present */ - if ((hash_table_size)&&(key_size)&&(value_size)) { - if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp2[0] = 'K'; tmp2[1] = 'd'; tmp2[2] = 'B'; tmp2[3] = KISSDB_VERSION; - if (fwrite(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp = hash_table_size; - if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp = key_size; - if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - tmp = value_size; - if (fwrite(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - fflush(db->f); - } else { - fclose(db->f); - return KISSDB_ERROR_INVALID_PARAMETERS; - } - } else { - if (fseeko(db->f,0,SEEK_SET)) { fclose(db->f); return KISSDB_ERROR_IO; } - if (fread(tmp2,4,1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if ((tmp2[0] != 'K')||(tmp2[1] != 'd')||(tmp2[2] != 'B')||(tmp2[3] != KISSDB_VERSION)) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if (!tmp) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - hash_table_size = (unsigned long)tmp; - if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if (!tmp) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - key_size = (unsigned long)tmp; - if (fread(&tmp,sizeof(uint64_t),1,db->f) != 1) { fclose(db->f); return KISSDB_ERROR_IO; } - if (!tmp) { - fclose(db->f); - return KISSDB_ERROR_CORRUPT_DBFILE; - } - value_size = (unsigned long)tmp; - } - - db->hash_table_size = hash_table_size; - db->key_size = key_size; - db->value_size = value_size; - db->hash_table_size_bytes = sizeof(uint64_t) * (hash_table_size + 1); /* [hash_table_size] == next table */ - - httmp = malloc(db->hash_table_size_bytes); - if (!httmp) { - fclose(db->f); - return KISSDB_ERROR_MALLOC; - } - db->num_hash_tables = 0; - db->hash_tables = (uint64_t *)0; - while (fread(httmp,db->hash_table_size_bytes,1,db->f) == 1) { - hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); - if (!hash_tables_rea) { - KISSDB_close(db); - free(httmp); - return KISSDB_ERROR_MALLOC; - } - db->hash_tables = hash_tables_rea; - - memcpy(((uint8_t *)db->hash_tables) + (db->hash_table_size_bytes * db->num_hash_tables),httmp,db->hash_table_size_bytes); - ++db->num_hash_tables; - if (httmp[db->hash_table_size]) { - if (fseeko(db->f,httmp[db->hash_table_size],SEEK_SET)) { - KISSDB_close(db); - free(httmp); - return KISSDB_ERROR_IO; - } - } else break; - } - free(httmp); - - return 0; -} - -void KISSDB_close(KISSDB *db) -{ - if (db->hash_tables) - free(db->hash_tables); - if (db->f) - fclose(db->f); - memset(db,0,sizeof(KISSDB)); -} - -int KISSDB_get(KISSDB *db,const void *key,void *vbuf) -{ - uint8_t tmp[4096]; - const uint8_t *kptr; - unsigned long klen,i; - uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; - uint64_t offset; - uint64_t *cur_hash_table; - long n; - - cur_hash_table = db->hash_tables; - for(i=0;inum_hash_tables;++i) { - offset = cur_hash_table[hash]; - if (offset) { - if (fseeko(db->f,offset,SEEK_SET)) - return KISSDB_ERROR_IO; - - kptr = (const uint8_t *)key; - klen = db->key_size; - while (klen) { - n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); - if (n > 0) { - if (memcmp(kptr,tmp,n)) - goto get_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long)n; - } else return 1; /* not found */ - } - - if (fread(vbuf,db->value_size,1,db->f) == 1) - return 0; /* success */ - else return KISSDB_ERROR_IO; - } else return 1; /* not found */ -get_no_match_next_hash_table: - cur_hash_table += db->hash_table_size + 1; - } - - return 1; /* not found */ -} - -int KISSDB_put(KISSDB *db,const void *key,const void *value) -{ - uint8_t tmp[4096]; - const uint8_t *kptr; - unsigned long klen,i; - uint64_t hash = KISSDB_hash(key,db->key_size) % (uint64_t)db->hash_table_size; - uint64_t offset; - uint64_t htoffset,lasthtoffset; - uint64_t endoffset; - uint64_t *cur_hash_table; - uint64_t *hash_tables_rea; - long n; - - lasthtoffset = htoffset = KISSDB_HEADER_SIZE; - cur_hash_table = db->hash_tables; - for(i=0;inum_hash_tables;++i) { - offset = cur_hash_table[hash]; - if (offset) { - /* rewrite if already exists */ - if (fseeko(db->f,offset,SEEK_SET)) - return KISSDB_ERROR_IO; - - kptr = (const uint8_t *)key; - klen = db->key_size; - while (klen) { - n = (long)fread(tmp,1,(klen > sizeof(tmp)) ? sizeof(tmp) : klen,db->f); - if (n > 0) { - if (memcmp(kptr,tmp,n)) - goto put_no_match_next_hash_table; - kptr += n; - klen -= (unsigned long)n; - } - } - - if (fwrite(value,db->value_size,1,db->f) == 1) { - fflush(db->f); - return 0; /* success */ - } else return KISSDB_ERROR_IO; - } else { - /* add if an empty hash table slot is discovered */ - if (fseeko(db->f,0,SEEK_END)) - return KISSDB_ERROR_IO; - endoffset = ftello(db->f); - - if (fwrite(key,db->key_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - if (fwrite(value,db->value_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - - if (fseeko(db->f,htoffset + (sizeof(uint64_t) * hash),SEEK_SET)) - return KISSDB_ERROR_IO; - if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) - return KISSDB_ERROR_IO; - cur_hash_table[hash] = endoffset; - - fflush(db->f); - - return 0; /* success */ - } -put_no_match_next_hash_table: - lasthtoffset = htoffset; - htoffset = cur_hash_table[db->hash_table_size]; - cur_hash_table += (db->hash_table_size + 1); - } - - /* if no existing slots, add a new page of hash table entries */ - if (fseeko(db->f,0,SEEK_END)) - return KISSDB_ERROR_IO; - endoffset = ftello(db->f); - - hash_tables_rea = realloc(db->hash_tables,db->hash_table_size_bytes * (db->num_hash_tables + 1)); - if (!hash_tables_rea) - return KISSDB_ERROR_MALLOC; - db->hash_tables = hash_tables_rea; - cur_hash_table = &(db->hash_tables[(db->hash_table_size + 1) * db->num_hash_tables]); - memset(cur_hash_table,0,db->hash_table_size_bytes); - - cur_hash_table[hash] = endoffset + db->hash_table_size_bytes; /* where new entry will go */ - - if (fwrite(cur_hash_table,db->hash_table_size_bytes,1,db->f) != 1) - return KISSDB_ERROR_IO; - - if (fwrite(key,db->key_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - if (fwrite(value,db->value_size,1,db->f) != 1) - return KISSDB_ERROR_IO; - - if (db->num_hash_tables) { - if (fseeko(db->f,lasthtoffset + (sizeof(uint64_t) * db->hash_table_size),SEEK_SET)) - return KISSDB_ERROR_IO; - if (fwrite(&endoffset,sizeof(uint64_t),1,db->f) != 1) - return KISSDB_ERROR_IO; - db->hash_tables[((db->hash_table_size + 1) * (db->num_hash_tables - 1)) + db->hash_table_size] = endoffset; - } - - ++db->num_hash_tables; - - fflush(db->f); - - return 0; /* success */ -} - -void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi) -{ - dbi->db = db; - dbi->h_no = 0; - dbi->h_idx = 0; -} - -int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf) -{ - uint64_t offset; - - if ((dbi->h_no < dbi->db->num_hash_tables)&&(dbi->h_idx < dbi->db->hash_table_size)) { - while (!(offset = dbi->db->hash_tables[((dbi->db->hash_table_size + 1) * dbi->h_no) + dbi->h_idx])) { - if (++dbi->h_idx >= dbi->db->hash_table_size) { - dbi->h_idx = 0; - if (++dbi->h_no >= dbi->db->num_hash_tables) - return 0; - } - } - if (fseeko(dbi->db->f,offset,SEEK_SET)) - return KISSDB_ERROR_IO; - if (fread(kbuf,dbi->db->key_size,1,dbi->db->f) != 1) - return KISSDB_ERROR_IO; - if (fread(vbuf,dbi->db->value_size,1,dbi->db->f) != 1) - return KISSDB_ERROR_IO; - if (++dbi->h_idx >= dbi->db->hash_table_size) { - dbi->h_idx = 0; - ++dbi->h_no; - } - return 1; - } - - return 0; -} - -#ifdef KISSDB_TEST - -#include - -int main(int argc,char **argv) -{ - uint64_t i,j; - uint64_t v[8]; - KISSDB db; - KISSDB_Iterator dbi; - char got_all_values[10000]; - int q; - - printf("Opening new empty database test.db...\n"); - - if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RWREPLACE,1024,8,sizeof(v))) { - printf("KISSDB_open failed\n"); - return 1; - } - - printf("Adding and then re-getting 10000 64-byte values...\n"); - - for(i=0;i<10000;++i) { - for(j=0;j<8;++j) - v[j] = i; - if (KISSDB_put(&db,&i,v)) { - printf("KISSDB_put failed (%"PRIu64")\n",i); - return 1; - } - memset(v,0,sizeof(v)); - if ((q = KISSDB_get(&db,&i,v))) { - printf("KISSDB_get (1) failed (%"PRIu64") (%d)\n",i,q); - return 1; - } - for(j=0;j<8;++j) { - if (v[j] != i) { - printf("KISSDB_get (1) failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - } - - printf("Getting 10000 64-byte values...\n"); - - for(i=0;i<10000;++i) { - if ((q = KISSDB_get(&db,&i,v))) { - printf("KISSDB_get (2) failed (%"PRIu64") (%d)\n",i,q); - return 1; - } - for(j=0;j<8;++j) { - if (v[j] != i) { - printf("KISSDB_get (2) failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - } - - printf("Closing and re-opening database in read-only mode...\n"); - - KISSDB_close(&db); - - if (KISSDB_open(&db,"test.db",KISSDB_OPEN_MODE_RDONLY,1024,8,sizeof(v))) { - printf("KISSDB_open failed\n"); - return 1; - } - - printf("Getting 10000 64-byte values...\n"); - - for(i=0;i<10000;++i) { - if ((q = KISSDB_get(&db,&i,v))) { - printf("KISSDB_get (3) failed (%"PRIu64") (%d)\n",i,q); - return 1; - } - for(j=0;j<8;++j) { - if (v[j] != i) { - printf("KISSDB_get (3) failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - } - - printf("Iterator test...\n"); - - KISSDB_Iterator_init(&db,&dbi); - i = 0xdeadbeef; - memset(got_all_values,0,sizeof(got_all_values)); - while (KISSDB_Iterator_next(&dbi,&i,&v) > 0) { - if (i < 10000) - got_all_values[i] = 1; - else { - printf("KISSDB_Iterator_next failed, bad data (%"PRIu64")\n",i); - return 1; - } - } - for(i=0;i<10000;++i) { - if (!got_all_values[i]) { - printf("KISSDB_Iterator failed, missing value index %"PRIu64"\n",i); - return 1; - } - } - - KISSDB_close(&db); - - printf("All tests OK!\n"); - - return 0; -} - -#endif diff --git a/ext/kissdb/kissdb.h b/ext/kissdb/kissdb.h deleted file mode 100644 index b4e87954d..000000000 --- a/ext/kissdb/kissdb.h +++ /dev/null @@ -1,171 +0,0 @@ -/* (Keep It) Simple Stupid Database - * - * Written by Adam Ierymenko - * KISSDB is in the public domain and is distributed with NO WARRANTY. */ - -#ifndef ___KISSDB_H -#define ___KISSDB_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Version: 2 - * - * This is the file format identifier, and changes any time the file - * format changes. The code version will be this dot something, and can - * be seen in tags in the git repository. - */ -#define KISSDB_VERSION 2 - -/** - * KISSDB database state - * - * These fields can be read by a user, e.g. to look up key_size and - * value_size, but should never be changed. - */ -typedef struct { - unsigned long hash_table_size; - unsigned long key_size; - unsigned long value_size; - unsigned long hash_table_size_bytes; - unsigned long num_hash_tables; - uint64_t *hash_tables; - FILE *f; -} KISSDB; - -/** - * I/O error or file not found - */ -#define KISSDB_ERROR_IO -1 - -/** - * Out of memory - */ -#define KISSDB_ERROR_MALLOC -2 - -/** - * Invalid paramters (e.g. missing _size paramters on init to create database) - */ -#define KISSDB_ERROR_INVALID_PARAMETERS -3 - -/** - * Database file appears corrupt - */ -#define KISSDB_ERROR_CORRUPT_DBFILE -4 - -/** - * Open mode: read only - */ -#define KISSDB_OPEN_MODE_RDONLY 1 - -/** - * Open mode: read/write - */ -#define KISSDB_OPEN_MODE_RDWR 2 - -/** - * Open mode: read/write, create if doesn't exist - */ -#define KISSDB_OPEN_MODE_RWCREAT 3 - -/** - * Open mode: truncate database, open for reading and writing - */ -#define KISSDB_OPEN_MODE_RWREPLACE 4 - -/** - * Open database - * - * The three _size parameters must be specified if the database could - * be created or re-created. Otherwise an error will occur. If the - * database already exists, these parameters are ignored and are read - * from the database. You can check the struture afterwords to see what - * they were. - * - * @param db Database struct - * @param path Path to file - * @param mode One of the KISSDB_OPEN_MODE constants - * @param hash_table_size Size of hash table in 64-bit entries (must be >0) - * @param key_size Size of keys in bytes - * @param value_size Size of values in bytes - * @return 0 on success, nonzero on error - */ -extern int KISSDB_open( - KISSDB *db, - const char *path, - int mode, - unsigned long hash_table_size, - unsigned long key_size, - unsigned long value_size); - -/** - * Close database - * - * @param db Database struct - */ -extern void KISSDB_close(KISSDB *db); - -/** - * Get an entry - * - * @param db Database struct - * @param key Key (key_size bytes) - * @param vbuf Value buffer (value_size bytes capacity) - * @return -1 on I/O error, 0 on success, 1 on not found - */ -extern int KISSDB_get(KISSDB *db,const void *key,void *vbuf); - -/** - * Put an entry (overwriting it if it already exists) - * - * In the already-exists case the size of the database file does not - * change. - * - * @param db Database struct - * @param key Key (key_size bytes) - * @param value Value (value_size bytes) - * @return -1 on I/O error, 0 on success - */ -extern int KISSDB_put(KISSDB *db,const void *key,const void *value); - -/** - * Cursor used for iterating over all entries in database - */ -typedef struct { - KISSDB *db; - unsigned long h_no; - unsigned long h_idx; -} KISSDB_Iterator; - -/** - * Initialize an iterator - * - * @param db Database struct - * @param i Iterator to initialize - */ -extern void KISSDB_Iterator_init(KISSDB *db,KISSDB_Iterator *dbi); - -/** - * Get the next entry - * - * The order of entries returned by iterator is undefined. It depends on - * how keys hash. - * - * @param Database iterator - * @param kbuf Buffer to fill with next key (key_size bytes) - * @param vbuf Buffer to fill with next value (value_size bytes) - * @return 0 if there are no more entries, negative on error, positive if an kbuf/vbuf have been filled - */ -extern int KISSDB_Iterator_next(KISSDB_Iterator *dbi,void *kbuf,void *vbuf); - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/node/Identity.hpp b/node/Identity.hpp index dfa1c7a99..cb911b92f 100644 --- a/node/Identity.hpp +++ b/node/Identity.hpp @@ -256,13 +256,15 @@ public: p += ZT_ADDRESS_LENGTH; if (b[p++] != IDENTITY_TYPE_C25519) - throw std::invalid_argument("Identity: deserialize(): unsupported identity type"); + throw std::invalid_argument("unsupported identity type"); memcpy(_publicKey.data,b.field(p,_publicKey.size()),_publicKey.size()); p += _publicKey.size(); - unsigned int privateKeyLength = b[p++]; - if ((privateKeyLength)&&(privateKeyLength == ZT_C25519_PRIVATE_KEY_LEN)) { + unsigned int privateKeyLength = (unsigned int)b[p++]; + if (privateKeyLength) { + if (privateKeyLength != ZT_C25519_PRIVATE_KEY_LEN) + throw std::invalid_argument("invalid private key"); _privateKey = new C25519::Private(); memcpy(_privateKey->data,b.field(p,ZT_C25519_PRIVATE_KEY_LEN),ZT_C25519_PRIVATE_KEY_LEN); p += ZT_C25519_PRIVATE_KEY_LEN; diff --git a/node/Node.cpp b/node/Node.cpp index 71d8b0971..8b2815e20 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -380,6 +380,7 @@ Node::ReasonForTermination Node::run() // Clean up some obsolete files if present -- this will be removed later Utils::rm((_r->homePath + ZT_PATH_SEPARATOR_S + "status")); Utils::rm((_r->homePath + ZT_PATH_SEPARATOR_S + "thisdeviceismine")); + Utils::rm((_r->homePath + ZT_PATH_SEPARATOR_S + "peer.db")); // Make sure networks.d exists #ifdef __WINDOWS__ @@ -407,7 +408,7 @@ Node::ReasonForTermination Node::run() _r->mc = new Multicaster(); _r->sw = new Switch(_r); _r->demarc = new Demarc(_r); - _r->topology = new Topology(_r,(_r->homePath + ZT_PATH_SEPARATOR_S + "peer.db").c_str()); + _r->topology = new Topology(_r); _r->sysEnv = new SysEnv(_r); try { _r->nc = new NodeConfig(_r,configAuthToken.c_str(),impl->controlPort); diff --git a/node/Peer.cpp b/node/Peer.cpp index 57396375a..d30a1d31f 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -34,12 +34,13 @@ Peer::Peer() : _id(), _ipv4p(), _ipv6p(), + _lastUsed(0), _lastUnicastFrame(0), _lastMulticastFrame(0), + _lastAnnouncedTo(0), _vMajor(0), _vMinor(0), - _vRevision(0), - _dirty(false) + _vRevision(0) { } @@ -48,12 +49,13 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity) _id(peerIdentity), _ipv4p(), _ipv6p(), + _lastUsed(0), _lastUnicastFrame(0), _lastMulticastFrame(0), + _lastAnnouncedTo(0), _vMajor(0), _vMinor(0), - _vRevision(0), - _dirty(true) + _vRevision(0) { if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) throw std::runtime_error("new peer identity key agreement failed"); @@ -72,16 +74,12 @@ void Peer::onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const I _lastAnnouncedTo = now; _r->sw->announceMulticastGroups(SharedPtr(this)); } - - _dirty = true; } if (verb == Packet::VERB_FRAME) { _lastUnicastFrame = now; - _dirty = true; } else if (verb == Packet::VERB_MULTICAST_FRAME) { _lastMulticastFrame = now; - _dirty = true; } } @@ -90,7 +88,6 @@ bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,u if ((_ipv6p.isActive(now))||((!(_ipv4p.addr))&&(_ipv6p.addr))) { if (_r->demarc->send(_ipv6p.localPort,_ipv6p.addr,data,len,-1)) { _ipv6p.lastSend = now; - _dirty = true; return true; } } @@ -98,7 +95,6 @@ bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,u if (_ipv4p.addr) { if (_r->demarc->send(_ipv4p.localPort,_ipv4p.addr,data,len,-1)) { _ipv4p.lastSend = now; - _dirty = true; return true; } } @@ -112,14 +108,12 @@ bool Peer::sendFirewallOpener(const RuntimeEnvironment *_r,uint64_t now) if (_ipv4p.addr) { if (_r->demarc->send(_ipv4p.localPort,_ipv4p.addr,"\0",1,ZT_FIREWALL_OPENER_HOPS)) { _ipv4p.lastFirewallOpener = now; - _dirty = true; sent = true; } } if (_ipv6p.addr) { if (_r->demarc->send(_ipv6p.localPort,_ipv6p.addr,"\0",1,ZT_FIREWALL_OPENER_HOPS)) { _ipv6p.lastFirewallOpener = now; - _dirty = true; sent = true; } } @@ -132,14 +126,12 @@ bool Peer::sendPing(const RuntimeEnvironment *_r,uint64_t now) if (_ipv4p.addr) { if (_r->sw->sendHELLO(SharedPtr(this),_ipv4p.localPort,_ipv4p.addr)) { _ipv4p.lastSend = now; - _dirty = true; sent = true; } } if (_ipv6p.addr) { if (_r->sw->sendHELLO(SharedPtr(this),_ipv6p.localPort,_ipv6p.addr)) { _ipv6p.lastSend = now; - _dirty = true; sent = true; } } @@ -151,11 +143,9 @@ void Peer::setPathAddress(const InetAddress &addr,bool fixed) if (addr.isV4()) { _ipv4p.addr = addr; _ipv4p.fixed = fixed; - _dirty = true; } else if (addr.isV6()) { _ipv6p.addr = addr; _ipv6p.fixed = fixed; - _dirty = true; } } @@ -165,15 +155,12 @@ void Peer::clearFixedFlag(InetAddress::AddressType t) case InetAddress::TYPE_NULL: _ipv4p.fixed = false; _ipv6p.fixed = false; - _dirty = true; break; case InetAddress::TYPE_IPV4: _ipv4p.fixed = false; - _dirty = true; break; case InetAddress::TYPE_IPV6: _ipv6p.fixed = false; - _dirty = true; break; } } diff --git a/node/Peer.hpp b/node/Peer.hpp index 2106a7872..6c123746d 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -48,24 +48,6 @@ #include "NonCopyable.hpp" #include "Mutex.hpp" -/** - * Max length of serialized peer record - */ -#define ZT_PEER_MAX_SERIALIZED_LENGTH ( \ - ZT_PEER_SECRET_KEY_LENGTH + \ - ZT_IDENTITY_MAX_BINARY_SERIALIZED_LENGTH + \ - ( ( \ - (sizeof(uint64_t) * 4) + \ - sizeof(uint16_t) + \ - 1 + \ - sizeof(uint16_t) + \ - 16 + \ - 1 \ - ) * 2) + \ - (sizeof(uint64_t) * 3) + \ - (sizeof(uint16_t) * 3) \ -) - namespace ZeroTier { /** @@ -99,6 +81,16 @@ public: Peer(const Identity &myIdentity,const Identity &peerIdentity) throw(std::runtime_error); + /** + * @return Time peer record was last used in any way + */ + inline uint64_t lastUsed() const throw() { return _lastUsed; } + + /** + * @param now New time of last use + */ + inline void setLastUsed(uint64_t now) throw() { _lastUsed = now; } + /** * @return This peer's ZT address (short for identity().address()) */ @@ -254,10 +246,8 @@ public: { if (addr == _ipv4p.addr) { _ipv4p.latency = latency; - _dirty = true; } else if (addr == _ipv6p.addr) { _ipv6p.latency = latency; - _dirty = true; } } @@ -357,33 +347,16 @@ public: return std::string("?"); } - /** - * Get and reset dirty flag - * - * @return Previous value of dirty flag before reset - */ - inline bool getAndResetDirty() - throw() - { - bool d = _dirty; - _dirty = false; - return d; - } - - /** - * @return Current value of dirty flag - */ - inline bool dirty() const throw() { return _dirty; } - template inline void serialize(Buffer &b) throw(std::out_of_range) { - b.append((unsigned char)3); // version + b.append((unsigned char)4); // version b.append(_key,sizeof(_key)); _id.serialize(b,false); _ipv4p.serialize(b); _ipv6p.serialize(b); + b.append(_lastUsed); b.append(_lastUnicastFrame); b.append(_lastMulticastFrame); b.append(_lastAnnouncedTo); @@ -398,13 +371,14 @@ public: { unsigned int p = startAt; - if (b[p++] != 3) + if (b[p++] != 4) throw std::invalid_argument("Peer: deserialize(): version mismatch"); memcpy(_key,b.field(p,sizeof(_key)),sizeof(_key)); p += sizeof(_key); p += _id.deserialize(b,p); p += _ipv4p.deserialize(b,p); p += _ipv6p.deserialize(b,p); + _lastUsed = b.template at(p); p += sizeof(uint64_t); _lastUnicastFrame = b.template at(p); p += sizeof(uint64_t); _lastMulticastFrame = b.template at(p); p += sizeof(uint64_t); _lastAnnouncedTo = b.template at(p); p += sizeof(uint64_t); @@ -412,8 +386,6 @@ public: _vMinor = b.template at(p); p += sizeof(uint16_t); _vRevision = b.template at(p); p += sizeof(uint16_t); - _dirty = false; - return (p - startAt); } @@ -538,14 +510,12 @@ private: WanPath _ipv4p; WanPath _ipv6p; + uint64_t _lastUsed; uint64_t _lastUnicastFrame; uint64_t _lastMulticastFrame; uint64_t _lastAnnouncedTo; unsigned int _vMajor,_vMinor,_vRevision; - // Fields below this line are not persisted with serialize() --------------- - - bool _dirty; AtomicCounter __refCount; }; diff --git a/node/Topology.cpp b/node/Topology.cpp index 3414812a3..8add2f4ef 100644 --- a/node/Topology.cpp +++ b/node/Topology.cpp @@ -33,32 +33,14 @@ namespace ZeroTier { -#define ZT_KISSDB_HASH_TABLE_SIZE 32768 -#define ZT_KISSDB_KEY_SIZE ZT_ADDRESS_LENGTH -#define ZT_KISSDB_VALUE_SIZE ZT_PEER_MAX_SERIALIZED_LENGTH - -Topology::Topology(const RuntimeEnvironment *renv,const char *dbpath) : +Topology::Topology(const RuntimeEnvironment *renv) : _r(renv), _amSupernode(false) { - if (KISSDB_open(&_dbm,dbpath,KISSDB_OPEN_MODE_RWCREAT,ZT_KISSDB_HASH_TABLE_SIZE,ZT_KISSDB_KEY_SIZE,ZT_KISSDB_VALUE_SIZE)) { - if (KISSDB_open(&_dbm,dbpath,KISSDB_OPEN_MODE_RWREPLACE,ZT_KISSDB_HASH_TABLE_SIZE,ZT_KISSDB_KEY_SIZE,ZT_KISSDB_VALUE_SIZE)) - throw std::runtime_error("unable to open peer database (rw/create)"); - } - - if ((_dbm.key_size != ZT_KISSDB_KEY_SIZE)||(_dbm.value_size != ZT_KISSDB_VALUE_SIZE)||(_dbm.hash_table_size != ZT_KISSDB_HASH_TABLE_SIZE)) { - KISSDB_close(&_dbm); - if (KISSDB_open(&_dbm,dbpath,KISSDB_OPEN_MODE_RWREPLACE,ZT_KISSDB_HASH_TABLE_SIZE,ZT_KISSDB_KEY_SIZE,ZT_KISSDB_VALUE_SIZE)) - throw std::runtime_error("unable to open peer database (recreate)"); - } - - Utils::lockDownFile(dbpath,false); // node.db caches secrets } Topology::~Topology() { - // Flush last changes to disk - clean(); } void Topology::setSupernodes(const std::map< Identity,std::vector > &sn) @@ -68,6 +50,7 @@ void Topology::setSupernodes(const std::map< Identity,std::vector > _supernodes = sn; _supernodeAddresses.clear(); _supernodePeers.clear(); + uint64_t now = Utils::now(); for(std::map< Identity,std::vector >::const_iterator i(sn.begin());i!=sn.end();++i) { if (i->first != _r->identity) { @@ -76,6 +59,7 @@ void Topology::setSupernodes(const std::map< Identity,std::vector > p = addPeer(SharedPtr(new Peer(_r->identity,i->first))); for(std::vector::const_iterator j(i->second.begin());j!=i->second.end();++j) p->setPathAddress(*j,true); + p->setLastUsed(now); _supernodePeers.push_back(p); } _supernodeAddresses.insert(i->first.address()); @@ -90,27 +74,11 @@ SharedPtr Topology::addPeer(const SharedPtr &peer) TRACE("BUG: addNewPeer() caught and ignored attempt to add peer for self"); throw std::logic_error("cannot add peer for self"); } - - SharedPtr actualPeer; - { - Mutex::Lock _l(_activePeers_m); - actualPeer = _activePeers.insert(std::pair< Address,SharedPtr >(peer->address(),peer)).first->second; - } - - uint64_t atmp[ZT_ADDRESS_LENGTH]; - actualPeer->address().copyTo(atmp,ZT_ADDRESS_LENGTH); - - Buffer b; - actualPeer->serialize(b); - b.zeroUnused(); - - _dbm_m.lock(); - if (KISSDB_put(&_dbm,atmp,b.data())) { - TRACE("error writing %s to peerdb",actualPeer->address().toString().c_str()); - } else actualPeer->getAndResetDirty(); - _dbm_m.unlock(); - - return actualPeer; + uint64_t now = Utils::now(); + Mutex::Lock _l(_activePeers_m); + SharedPtr p(_activePeers.insert(std::pair< Address,SharedPtr >(peer->address(),peer)).first->second); + p->setLastUsed(now); + return p; } SharedPtr Topology::getPeer(const Address &zta) @@ -119,34 +87,13 @@ SharedPtr Topology::getPeer(const Address &zta) TRACE("BUG: ignored attempt to getPeer() for self, returned NULL"); return SharedPtr(); } - - { - Mutex::Lock _l(_activePeers_m); - std::map< Address,SharedPtr >::const_iterator ap(_activePeers.find(zta)); - if ((ap != _activePeers.end())&&(ap->second)) - return ap->second; + uint64_t now = Utils::now(); + Mutex::Lock _l(_activePeers_m); + std::map< Address,SharedPtr >::const_iterator ap(_activePeers.find(zta)); + if ((ap != _activePeers.end())&&(ap->second)) { + ap->second->setLastUsed(now); + return ap->second; } - - unsigned char ztatmp[ZT_ADDRESS_LENGTH]; - zta.copyTo(ztatmp,ZT_ADDRESS_LENGTH); - - Buffer b(ZT_KISSDB_VALUE_SIZE); - _dbm_m.lock(); - if (!KISSDB_get(&_dbm,ztatmp,b.data())) { - _dbm_m.unlock(); - - SharedPtr p(new Peer()); - try { - p->deserialize(b,0); - Mutex::Lock _l(_activePeers_m); - _activePeers[zta] = p; - return p; - } catch ( ... ) { - TRACE("unexpected exception deserializing peer %s from peerdb",zta.toString().c_str()); - return SharedPtr(); - } - } else _dbm_m.unlock(); - return SharedPtr(); } @@ -183,8 +130,11 @@ skip_and_try_next_supernode: ++sn; } - if ((bestSupernode)||(strictAvoid)) + if (bestSupernode) { + bestSupernode->setLastUsed(now); return bestSupernode; + } else if (strictAvoid) + return SharedPtr(); for(std::vector< SharedPtr >::const_iterator sn=_supernodePeers.begin();sn!=_supernodePeers.end();++sn) { if ((*sn)->hasActiveDirectPath(now)) { @@ -203,36 +153,12 @@ skip_and_try_next_supernode: } if (bestSupernode) - return bestSupernode; - - return _supernodePeers[_r->prng->next32() % _supernodePeers.size()]; + bestSupernode->setLastUsed(now); + return bestSupernode; } void Topology::clean() { - TRACE("cleaning caches and flushing modified peers to disk..."); - - Mutex::Lock _l(_activePeers_m); - for(std::map< Address,SharedPtr >::iterator p(_activePeers.begin());p!=_activePeers.end();++p) { - if (p->second->getAndResetDirty()) { - try { - uint64_t atmp[ZT_ADDRESS_LENGTH]; - p->second->identity().address().copyTo(atmp,ZT_ADDRESS_LENGTH); - - Buffer b; - p->second->serialize(b); - b.zeroUnused(); - - _dbm_m.lock(); - if (KISSDB_put(&_dbm,atmp,b.data())) { - TRACE("error writing %s to peer.db",p->second->identity().address().toString().c_str()); - } - _dbm_m.unlock(); - } catch ( ... ) { - TRACE("unexpected exception flushing %s to peer.db",p->second->identity().address().toString().c_str()); - } - } - } } } // namespace ZeroTier diff --git a/node/Topology.hpp b/node/Topology.hpp index acae83d64..ac143b048 100644 --- a/node/Topology.hpp +++ b/node/Topology.hpp @@ -43,8 +43,6 @@ #include "InetAddress.hpp" #include "Utils.hpp" -#include "../ext/kissdb/kissdb.h" - namespace ZeroTier { class RuntimeEnvironment; @@ -55,7 +53,7 @@ class RuntimeEnvironment; class Topology { public: - Topology(const RuntimeEnvironment *renv,const char *dbpath); + Topology(const RuntimeEnvironment *renv); ~Topology(); /** @@ -283,9 +281,6 @@ private: // Set to true if my identity is in _supernodes volatile bool _amSupernode; - - KISSDB _dbm; - Mutex _dbm_m; }; } // namespace ZeroTier diff --git a/objects.mk b/objects.mk index 28d314348..547033f93 100644 --- a/objects.mk +++ b/objects.mk @@ -1,5 +1,4 @@ OBJS=\ - ext/kissdb/kissdb.o \ ext/lz4/lz4hc.o \ ext/lz4/lz4.o \ node/C25519.o \