diff --git a/commandline.c b/commandline.c index 5ff8416f..d85c4f88 100644 --- a/commandline.c +++ b/commandline.c @@ -18,7 +18,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include +#include +#include +#include #include "serval.h" +#include "rhizome.h" typedef struct command_line_option { int (*function)(int argc,char **argv,struct command_line_option *o); @@ -618,11 +622,50 @@ int app_server_get(int argc,char **argv,struct command_line_option *o) int app_rhizome_add(int argc, char **argv, struct command_line_option *o) { - char *manifestpath = cli_arg(argc, argv, o, "manifestfilepath", ""); - char *filepath = cli_arg(argc, argv, o, "filepath", ""); - printf("manifestpath = \"%s\"\n", manifestpath); - printf("filepath = \"%s\"\n", filepath); - return 0; + const char *filepath = cli_arg(argc, argv, o, "filepath", ""); + const char *manifestpath = cli_arg(argc, argv, o, "manifestpath", ""); + /* Ensure the Rhizome database exists and is open */ + if (create_serval_instance_dir() == -1) + return -1; + rhizome_datastore_path = serval_instancepath(); + rhizome_opendb(); + /* Create a new manifest that will represent the file, having a "name" value of the file's + * basename and a "date" value of right now */ + rhizome_manifest *m = rhizome_new_manifest(); + const char *name = strrchr(filepath, '/'); + name = name ? name + 1 : filepath; + rhizome_manifest_set(m, "name", name); + rhizome_manifest_set_ll(m, "date", overlay_gettime_ms()); + /* Add the manifest and its associated file to the Rhizome database, generating an "id" in the + * process */ + int ret = rhizome_add_manifest(m, filepath, + NULL, // no groups - XXX should allow them + 255, // ttl - XXX should read from somewhere + 0, // int verifyP + 1, // int checkFileP + 1 // int signP + ); + if (ret == -1) { + return WHY("Manifest not added to Rhizome database"); + } else { + /* If successfully added, overwrite the manifest file so that the Java component that is + * invoking this command can read it to obtain feedback on the result. */ + if (manifestpath[0] && rhizome_write_manifest_file(m, manifestpath) == -1) { + ret = WHY("Could not overwrite manifest file."); + } + } + rhizome_manifest_free(m); + return ret; +} + +int app_rhizome_list(int argc, char **argv, struct command_line_option *o) +{ + /* Create the instance directory if it does not yet exist */ + if (create_serval_instance_dir() == -1) + return -1; + rhizome_datastore_path = serval_instancepath(); + rhizome_opendb(); + rhizome_list_manifests(0, 0); } /* NULL marks ends of command structure. @@ -663,7 +706,9 @@ command_line_option command_line_options[]={ "Set specified configuration variable."}, {app_server_get,{"config","get","[]",NULL},0, "Get specified configuration variable."}, - {app_rhizome_add,{"rhizome","add","","[]",NULL},0, - "Add a manifest and file to Rhizome."}, + {app_rhizome_add,{"rhizome","add","file","","[]",NULL},0, + "Add a file to Rhizome and optionally write its manifest to the given path"}, + {app_rhizome_list,{"rhizome","list",NULL},0, + "List all manifests and files in Rhizome"}, {NULL,{NULL}} }; diff --git a/dna.c b/dna.c index 7500887f..6bfabc2b 100644 --- a/dna.c +++ b/dna.c @@ -609,13 +609,10 @@ int main(int argc,char **argv) rhizome_opendb(); /* Also set hlr file to be in the Rhizome directory, to save the need to specify it separately. */ - char temp[1024];temp[1023]=0; - snprintf(temp,1024,"%s/hlr.dat",optarg); - if (temp[1023]) { + char temp[1024]; + if (snprintf(temp, sizeof(temp), "%s/hlr.dat", optarg) >= sizeof(temp)) exit(WHY("Rhizome directory name too long.")); - } - hlr_file=strdup(temp); - + hlr_file = strdup(temp); break; case 'M': /* Distribute specified manifest and file pair using Rhizome. */ /* This option assumes that the manifest is locally produced, and will diff --git a/rhizome.c b/rhizome.c index c58e3865..965ffd50 100644 --- a/rhizome.c +++ b/rhizome.c @@ -36,72 +36,92 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. The file should be included in the specified rhizome groups, if possible. (some groups may be closed groups that we do not have the private key for.) */ + int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl, int verifyP, int checkFileP, int signP) { char filename[1024]; char manifestname[1024]; - char *buffer; + if (snprintf(filename, sizeof(filename), "%s/import/file.%s", rhizome_datastore_path, bundle) >= sizeof(filename) + || snprintf(manifestname, sizeof(manifestname), "%s/import/manifest.%s", rhizome_datastore_path, bundle) >= sizeof(manifestname)) { + return WHY("Manifest bundle name too long"); + } - snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0; - snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0; + /* Read manifest file if no manifest was given */ + rhizome_manifest *m = m_in; + if (!m_in) { + m = rhizome_read_manifest_file(manifestname, 0 /* file not buffer */, RHIZOME_VERIFY); + if (!m) + return WHY("Could not read manifest file."); + } else { + if (debug&DEBUG_RHIZOMESYNC) + fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP); + } - /* Open files */ - rhizome_manifest *m=m_in; - if (!m_in) - m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,RHIZOME_VERIFY); - else - if (debug&DEBUG_RHIZOMESYNC) fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP); + /* Add the manifest and its associated file to the Rhizome database. */ + int ret = rhizome_add_manifest(m, filename, groups, ttl, verifyP, checkFileP, signP); + unlink(filename); + if (ret == -1) { + unlink(manifestname); + } else { + /* >>> For testing, write manifest file back to disk and leave it there */ + // unlink(manifestname); + if (rhizome_write_manifest_file(m, manifestname)) + ret = WHY("Could not write manifest file."); + } - if (!m) return WHY("Could not read manifest file."); + /* If the manifest was allocated in this function, then this function is responsible for freeing + * it */ + if (!m_in) + rhizome_manifest_free(m); + + return ret; +} + +int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl, int verifyP, int checkFileP, int signP) +{ + char *buffer; char hexhash[SHA512_DIGEST_STRING_LENGTH]; - /* work out time to live */ - if (ttl<0) ttl=0; if (ttl>254) ttl=254; m->ttl=ttl; - /* Keep associated file name handy for later */ - m->dataFileName=strdup(filename); - if (checkFileP) { + m->dataFileName = strdup(filename); + + /* Store time to live */ + m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl; + + /* Check file is accessible and discover its length */ + if (checkFileP || verifyP) { struct stat stat; - if (lstat(filename,&stat)) { + if (lstat(filename,&stat)) return WHY("Could not stat() associated file"); - m->fileLength=stat.st_size; - } + m->fileLength = stat.st_size; } - if (checkFileP||signP) { - if (rhizome_hash_file(filename,hexhash)) { - rhizome_manifest_free(m); + if (checkFileP || signP) { + if (rhizome_hash_file(filename, hexhash)) return WHY("Could not hash file."); - } - memcpy(&m->fileHexHash[0],&hexhash[0],SHA512_DIGEST_STRING_LENGTH); - m->fileHashedP=1; + memcpy(&m->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH); + m->fileHashedP = 1; } - if (verifyP) - { - /* Make sure hashes match. - Make sure that no signature verification errors were spotted on loading. */ - int verifyErrors=0; - char *mhexhash; - if (checkFileP) { - if ((mhexhash=rhizome_manifest_get(m,"filehash",NULL,0))!=NULL) - if (strcmp(hexhash,mhexhash)) - verifyErrors++; - } - if (m->errors) - verifyErrors+=m->errors; - if (verifyErrors) { - rhizome_manifest_free(m); - unlink(manifestname); - unlink(filename); - return WHY("Errors encountered verifying bundle manifest"); - } + if (verifyP) { + /* Make sure hashes match. + Make sure that no signature verification errors were spotted on loading. */ + int verifyErrors=0; + char *mhexhash; + if (checkFileP) { + if ((mhexhash=rhizome_manifest_get(m,"filehash",NULL,0))!=NULL) + if (strcmp(hexhash,mhexhash)) + verifyErrors++; } - - if (!verifyP) { - if ((buffer=rhizome_manifest_get(m,"id",NULL,0))!=NULL) { + if (m->errors) + verifyErrors+=m->errors; + if (verifyErrors) + return WHY("Errors encountered verifying bundle manifest"); + } + else { + if (!(buffer = rhizome_manifest_get(m, "id", NULL, 0))) { /* No bundle id (256 bit random string being a public key in the NaCl CryptoSign crypto system), so create one, and keep the private key handy. */ printf("manifest does not have an id\n"); @@ -124,43 +144,34 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in rhizome_manifest_set(m,"filehash",hexhash); if (rhizome_manifest_get(m,"version",NULL,0)==NULL) /* Version not set, so set one */ - rhizome_manifest_set_ll(m,"version",overlay_gettime_ms()); - rhizome_manifest_set_ll(m,"first_byte",0); - rhizome_manifest_set_ll(m,"last_byte",rhizome_file_size(filename)); + rhizome_manifest_set_ll(m,"version", overlay_gettime_ms()); + rhizome_manifest_set_ll(m,"first_byte", 0); + rhizome_manifest_set_ll(m,"last_byte", m->fileLength); } /* Discard if it is older than the most recent known version */ - long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)); - if (storedversion>rhizome_manifest_get_ll(m,"version")) - { - rhizome_manifest_free(m); - return WHY("Newer version exists"); - } + long long storedversion = sqlite_exec_int64( + "SELECT version from manifests where id='%s';", + rhizome_bytes_to_hex(m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES) + ); + if (storedversion > rhizome_manifest_get_ll(m, "version")) + return WHY("Newer version exists"); /* Add group memberships */ - -int i; - if (groups) for(i=0;groups[i];i++) rhizome_manifest_add_group(m,groups[i]); + if (groups) { + int i; + for(i = 0; groups[i]; i++) + rhizome_manifest_add_group(m, groups[i]); + } - if (rhizome_manifest_finalise(m,signP)) { + if (rhizome_manifest_finalise(m,signP)) return WHY("Failed to finalise manifest.\n"); - } - - /* Write manifest back to disk */ - if (rhizome_write_manifest_file(m,manifestname)) { - rhizome_manifest_free(m); - return WHY("Could not write manifest file."); - } /* Okay, it is written, and can be put directly into the rhizome database now */ - int r=rhizome_store_bundle(m,filename); - if (!r) { - // XXX For testing unlink(manifestname); - unlink(filename); - return 0; - } + if (rhizome_store_bundle(m, filename) == -1) + return WHY("rhizome_store_bundle() failed."); - return WHY("rhizome_store_bundle() failed."); + return 0; } /* Update an existing Rhizome bundle */ diff --git a/rhizome.h b/rhizome.h index 4b37733a..417b1e16 100644 --- a/rhizome.h +++ b/rhizome.h @@ -160,16 +160,16 @@ extern sqlite3 *rhizome_db; int rhizome_opendb(); int rhizome_manifest_createid(rhizome_manifest *m); -int rhizome_write_manifest_file(rhizome_manifest *m,char *filename); +int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename); int rhizome_manifest_sign(rhizome_manifest *m); int rhizome_drop_stored_file(char *id,int maximum_priority); int rhizome_manifest_priority(char *id); -rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferPAndSize,int flags); -int rhizome_hash_file(char *filename,char *hash_out); +rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferPAndSize, int flags); +int rhizome_hash_file(const char *filename,char *hash_out); char *rhizome_manifest_get(rhizome_manifest *m,char *var,char *value_out,int maxlen); long long rhizome_manifest_get_ll(rhizome_manifest *m,char *var); int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value); -int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value); +int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value); long long rhizome_file_size(char *filename); void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line, rhizome_manifest *m); @@ -177,13 +177,15 @@ void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line rhizome_manifest *_rhizome_new_manifest(const char *file,const char *func,int line); #define rhizome_new_manifest() _rhizome_new_manifest(__FILE__,__FUNCTION__,__LINE__) int rhizome_manifest_pack_variables(rhizome_manifest *m); -int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename); +int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename); int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); -int rhizome_store_file(char *file,char *hash,int priortity); +int rhizome_store_file(const char *file,char *hash,int priortity); char *rhizome_safe_encode(unsigned char *in,int len); int rhizome_finish_sqlstatement(sqlite3_stmt *statement); int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl, int verifyP, int checkFileP, int signP); +int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl, + int verifyP, int checkFileP, int signP); int rhizome_manifest_finalise(rhizome_manifest *m,int signP); char *rhizome_bytes_to_hex(unsigned char *in,int byteCount); int rhizome_hex_to_bytes(char *in,unsigned char *out,int hexChars); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index 3ed93280..78faaa3b 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -21,7 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "rhizome.h" #include -rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flags) +rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferP, int flags) { if (bufferP>MAX_MANIFEST_BYTES) return NULL; @@ -30,12 +30,15 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag if (bufferP) { m->manifest_bytes=bufferP; - bcopy(filename,m->manifestdata,m->manifest_bytes); + memcpy(m->manifestdata, filename, m->manifest_bytes); } else { FILE *f=fopen(filename,"r"); - if (!f) { WHY("Could not open manifest file for reading."); - rhizome_manifest_free(m); return NULL; } + if (!f) { + WHYF("Could not open manifest file %s for reading.", filename); + rhizome_manifest_free(m); + return NULL; + } m->manifest_bytes = fread(m->manifestdata,1,MAX_MANIFEST_BYTES,f); fclose(f); } @@ -144,7 +147,7 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag return m; } -int rhizome_hash_file(char *filename,char *hash_out) +int rhizome_hash_file(const char *filename,char *hash_out) { /* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one go. Trouble is, we need to run Serval DNA on filesystems that lack mmap(), @@ -212,7 +215,7 @@ double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_ } -int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value) +int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value) { int i; @@ -245,18 +248,6 @@ int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value) return rhizome_manifest_set(m,var,svalue); } -long long rhizome_file_size(char *filename) -{ - FILE *f; - - /* XXX really should just use stat instead of opening the file */ - f=fopen(filename,"r"); - fseek(f,0,SEEK_END); - long long size=ftello(f); - fclose(f); - return size; -} - #define MAX_RHIZOME_MANIFESTS 16 rhizome_manifest manifests[MAX_RHIZOME_MANIFESTS]; char manifest_free[MAX_RHIZOME_MANIFESTS]; @@ -420,7 +411,7 @@ int rhizome_manifest_sign(rhizome_manifest *m) return 0; } -int rhizome_write_manifest_file(rhizome_manifest *m,char *filename) +int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename) { if (!m) return WHY("Manifest is null."); if (!m->finalised) return WHY("Manifest must be finalised before it can be written."); diff --git a/rhizome_database.c b/rhizome_database.c index 5c12cda7..1779c74b 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -290,7 +290,7 @@ int rhizome_drop_stored_file(char *id,int maximum_priority) We need to also need to create the appropriate row(s) in the MANIFESTS, FILES, FILEMANIFESTS and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well. */ -int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename) +int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename) { char sqlcmd[1024]; const char *cmdtail; @@ -500,13 +500,68 @@ char *rhizome_safe_encode(unsigned char *in,int len) return r; } +int rhizome_list_manifests(int limit, int offset) +{ + char sqlcmd[1024]; + int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id"); + if (n >= sizeof(sqlcmd)) + return WHY("SQL command too long"); + if (limit) { + n += snprintf(&sqlcmd[n], sizeof(sqlcmd) - n, " LIMIT %u", limit); + if (n >= sizeof(sqlcmd)) + return WHY("SQL command too long"); + } + if (offset) { + n += snprintf(&sqlcmd[n], sizeof(sqlcmd) - n, " OFFSET %u", offset); + if (n >= sizeof(sqlcmd)) + return WHY("SQL command too long"); + } + sqlite3_stmt *statement; + const char *cmdtail; + int ret = 0; + if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) { + sqlite3_finalize(statement); + ret = WHY(sqlite3_errmsg(rhizome_db)); + } else { + size_t rows = 0; + while (sqlite3_step(statement) == SQLITE_ROW) { + ++rows; + if (!( sqlite3_column_count(statement) == 7 + && sqlite3_column_type(statement, 0) == SQLITE_TEXT + && sqlite3_column_type(statement, 1) == SQLITE_INTEGER + && sqlite3_column_type(statement, 2) == SQLITE_INTEGER + && sqlite3_column_type(statement, 3) == SQLITE_TEXT + && sqlite3_column_type(statement, 4) == SQLITE_BLOB + && sqlite3_column_type(statement, 5) == SQLITE_INTEGER + && sqlite3_column_type(statement, 6) == SQLITE_INTEGER + )) { + ret = WHY("Incorrect statement column"); + break; + } + size_t filesize = sqlite3_column_int(statement, 1); + size_t manifestblobsize = sqlite3_column_bytes(statement, 4); + const char *manifestblob = (char *) sqlite3_column_blob(statement, 4); + printf("manifest blob = %s\n", manifestblob); + rhizome_manifest *m = rhizome_read_manifest_file(manifestblob, manifestblobsize, RHIZOME_VERIFY); + const char *name = rhizome_manifest_get(m, "name", NULL, 0); + printf("file id = %s\nfile length = %u\nfile datavalid = %u\nfile name = \"%s\"\n", + sqlite3_column_text(statement, 0), + filesize, + sqlite3_column_int(statement, 2), + name + ); + rhizome_manifest_free(m); + } + printf("Found %lu rows\n", (unsigned long) rows); + } + sqlite3_finalize(statement); + return ret; +} -/* The following function just stores the file (or silently returns if it already - exists). - The relationships of manifests to this file are the responsibility of the - caller. */ -int rhizome_store_file(char *file,char *hash,int priority) { - +/* The following function just stores the file (or silently returns if it already exists). + The relationships of manifests to this file are the responsibility of the caller. */ +int rhizome_store_file(const char *file,char *hash,int priority) +{ int fd=open(file,O_RDONLY); if (fd<0) return WHY("Could not open associated file"); @@ -695,5 +750,3 @@ int rhizome_update_file_priority(char *fileid) } return 0; } - -