Implement "rhizome add file" and "rhizome list"

This commit is contained in:
Andrew Bettison 2012-04-02 17:42:40 +09:30
parent 0b11389023
commit 74986a0c30
6 changed files with 219 additions and 120 deletions

View File

@ -18,7 +18,11 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <sys/stat.h>
#include <sys/time.h>
#include <math.h>
#include <string.h>
#include "serval.h"
#include "rhizome.h"
typedef struct command_line_option {
int (*function)(int argc,char **argv,struct command_line_option *o);
@ -618,11 +622,50 @@ int app_server_get(int argc,char **argv,struct command_line_option *o)
int app_rhizome_add(int argc, char **argv, struct command_line_option *o)
{
char *manifestpath = cli_arg(argc, argv, o, "manifestfilepath", "");
char *filepath = cli_arg(argc, argv, o, "filepath", "");
printf("manifestpath = \"%s\"\n", manifestpath);
printf("filepath = \"%s\"\n", filepath);
return 0;
const char *filepath = cli_arg(argc, argv, o, "filepath", "");
const char *manifestpath = cli_arg(argc, argv, o, "manifestpath", "");
/* Ensure the Rhizome database exists and is open */
if (create_serval_instance_dir() == -1)
return -1;
rhizome_datastore_path = serval_instancepath();
rhizome_opendb();
/* Create a new manifest that will represent the file, having a "name" value of the file's
* basename and a "date" value of right now */
rhizome_manifest *m = rhizome_new_manifest();
const char *name = strrchr(filepath, '/');
name = name ? name + 1 : filepath;
rhizome_manifest_set(m, "name", name);
rhizome_manifest_set_ll(m, "date", overlay_gettime_ms());
/* Add the manifest and its associated file to the Rhizome database, generating an "id" in the
* process */
int ret = rhizome_add_manifest(m, filepath,
NULL, // no groups - XXX should allow them
255, // ttl - XXX should read from somewhere
0, // int verifyP
1, // int checkFileP
1 // int signP
);
if (ret == -1) {
return WHY("Manifest not added to Rhizome database");
} else {
/* If successfully added, overwrite the manifest file so that the Java component that is
* invoking this command can read it to obtain feedback on the result. */
if (manifestpath[0] && rhizome_write_manifest_file(m, manifestpath) == -1) {
ret = WHY("Could not overwrite manifest file.");
}
}
rhizome_manifest_free(m);
return ret;
}
int app_rhizome_list(int argc, char **argv, struct command_line_option *o)
{
/* Create the instance directory if it does not yet exist */
if (create_serval_instance_dir() == -1)
return -1;
rhizome_datastore_path = serval_instancepath();
rhizome_opendb();
rhizome_list_manifests(0, 0);
}
/* NULL marks ends of command structure.
@ -663,7 +706,9 @@ command_line_option command_line_options[]={
"Set specified configuration variable."},
{app_server_get,{"config","get","[<variable>]",NULL},0,
"Get specified configuration variable."},
{app_rhizome_add,{"rhizome","add","<manifestfilepath>","[<filepath>]",NULL},0,
"Add a manifest and file to Rhizome."},
{app_rhizome_add,{"rhizome","add","file","<filepath>","[<manifestpath>]",NULL},0,
"Add a file to Rhizome and optionally write its manifest to the given path"},
{app_rhizome_list,{"rhizome","list",NULL},0,
"List all manifests and files in Rhizome"},
{NULL,{NULL}}
};

9
dna.c
View File

@ -609,13 +609,10 @@ int main(int argc,char **argv)
rhizome_opendb();
/* Also set hlr file to be in the Rhizome directory, to save the need to specify it
separately. */
char temp[1024];temp[1023]=0;
snprintf(temp,1024,"%s/hlr.dat",optarg);
if (temp[1023]) {
char temp[1024];
if (snprintf(temp, sizeof(temp), "%s/hlr.dat", optarg) >= sizeof(temp))
exit(WHY("Rhizome directory name too long."));
}
hlr_file=strdup(temp);
hlr_file = strdup(temp);
break;
case 'M': /* Distribute specified manifest and file pair using Rhizome. */
/* This option assumes that the manifest is locally produced, and will

157
rhizome.c
View File

@ -36,72 +36,92 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
The file should be included in the specified rhizome groups, if possible.
(some groups may be closed groups that we do not have the private key for.)
*/
int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
int verifyP, int checkFileP, int signP)
{
char filename[1024];
char manifestname[1024];
char *buffer;
if (snprintf(filename, sizeof(filename), "%s/import/file.%s", rhizome_datastore_path, bundle) >= sizeof(filename)
|| snprintf(manifestname, sizeof(manifestname), "%s/import/manifest.%s", rhizome_datastore_path, bundle) >= sizeof(manifestname)) {
return WHY("Manifest bundle name too long");
}
snprintf(filename,1024,"%s/import/file.%s",rhizome_datastore_path,bundle); filename[1023]=0;
snprintf(manifestname,1024,"%s/import/manifest.%s",rhizome_datastore_path,bundle); manifestname[1023]=0;
/* Read manifest file if no manifest was given */
rhizome_manifest *m = m_in;
if (!m_in) {
m = rhizome_read_manifest_file(manifestname, 0 /* file not buffer */, RHIZOME_VERIFY);
if (!m)
return WHY("Could not read manifest file.");
} else {
if (debug&DEBUG_RHIZOMESYNC)
fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP);
}
/* Open files */
rhizome_manifest *m=m_in;
if (!m_in)
m=rhizome_read_manifest_file(manifestname,0 /* file not buffer */,RHIZOME_VERIFY);
else
if (debug&DEBUG_RHIZOMESYNC) fprintf(stderr,"Importing direct from manifest structure hashP=%d\n",m->fileHashedP);
/* Add the manifest and its associated file to the Rhizome database. */
int ret = rhizome_add_manifest(m, filename, groups, ttl, verifyP, checkFileP, signP);
unlink(filename);
if (ret == -1) {
unlink(manifestname);
} else {
/* >>> For testing, write manifest file back to disk and leave it there */
// unlink(manifestname);
if (rhizome_write_manifest_file(m, manifestname))
ret = WHY("Could not write manifest file.");
}
if (!m) return WHY("Could not read manifest file.");
/* If the manifest was allocated in this function, then this function is responsible for freeing
* it */
if (!m_in)
rhizome_manifest_free(m);
return ret;
}
int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl, int verifyP, int checkFileP, int signP)
{
char *buffer;
char hexhash[SHA512_DIGEST_STRING_LENGTH];
/* work out time to live */
if (ttl<0) ttl=0; if (ttl>254) ttl=254; m->ttl=ttl;
/* Keep associated file name handy for later */
m->dataFileName=strdup(filename);
if (checkFileP) {
m->dataFileName = strdup(filename);
/* Store time to live */
m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
/* Check file is accessible and discover its length */
if (checkFileP || verifyP) {
struct stat stat;
if (lstat(filename,&stat)) {
if (lstat(filename,&stat))
return WHY("Could not stat() associated file");
m->fileLength=stat.st_size;
}
m->fileLength = stat.st_size;
}
if (checkFileP||signP) {
if (rhizome_hash_file(filename,hexhash)) {
rhizome_manifest_free(m);
if (checkFileP || signP) {
if (rhizome_hash_file(filename, hexhash))
return WHY("Could not hash file.");
}
memcpy(&m->fileHexHash[0],&hexhash[0],SHA512_DIGEST_STRING_LENGTH);
m->fileHashedP=1;
memcpy(&m->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH);
m->fileHashedP = 1;
}
if (verifyP)
{
/* Make sure hashes match.
Make sure that no signature verification errors were spotted on loading. */
int verifyErrors=0;
char *mhexhash;
if (checkFileP) {
if ((mhexhash=rhizome_manifest_get(m,"filehash",NULL,0))!=NULL)
if (strcmp(hexhash,mhexhash))
verifyErrors++;
}
if (m->errors)
verifyErrors+=m->errors;
if (verifyErrors) {
rhizome_manifest_free(m);
unlink(manifestname);
unlink(filename);
return WHY("Errors encountered verifying bundle manifest");
}
if (verifyP) {
/* Make sure hashes match.
Make sure that no signature verification errors were spotted on loading. */
int verifyErrors=0;
char *mhexhash;
if (checkFileP) {
if ((mhexhash=rhizome_manifest_get(m,"filehash",NULL,0))!=NULL)
if (strcmp(hexhash,mhexhash))
verifyErrors++;
}
if (!verifyP) {
if ((buffer=rhizome_manifest_get(m,"id",NULL,0))!=NULL) {
if (m->errors)
verifyErrors+=m->errors;
if (verifyErrors)
return WHY("Errors encountered verifying bundle manifest");
}
else {
if (!(buffer = rhizome_manifest_get(m, "id", NULL, 0))) {
/* No bundle id (256 bit random string being a public key in the NaCl CryptoSign crypto system),
so create one, and keep the private key handy. */
printf("manifest does not have an id\n");
@ -124,43 +144,34 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in
rhizome_manifest_set(m,"filehash",hexhash);
if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
/* Version not set, so set one */
rhizome_manifest_set_ll(m,"version",overlay_gettime_ms());
rhizome_manifest_set_ll(m,"first_byte",0);
rhizome_manifest_set_ll(m,"last_byte",rhizome_file_size(filename));
rhizome_manifest_set_ll(m,"version", overlay_gettime_ms());
rhizome_manifest_set_ll(m,"first_byte", 0);
rhizome_manifest_set_ll(m,"last_byte", m->fileLength);
}
/* Discard if it is older than the most recent known version */
long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';",rhizome_bytes_to_hex(m->cryptoSignPublic,crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES));
if (storedversion>rhizome_manifest_get_ll(m,"version"))
{
rhizome_manifest_free(m);
return WHY("Newer version exists");
}
long long storedversion = sqlite_exec_int64(
"SELECT version from manifests where id='%s';",
rhizome_bytes_to_hex(m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)
);
if (storedversion > rhizome_manifest_get_ll(m, "version"))
return WHY("Newer version exists");
/* Add group memberships */
int i;
if (groups) for(i=0;groups[i];i++) rhizome_manifest_add_group(m,groups[i]);
if (groups) {
int i;
for(i = 0; groups[i]; i++)
rhizome_manifest_add_group(m, groups[i]);
}
if (rhizome_manifest_finalise(m,signP)) {
if (rhizome_manifest_finalise(m,signP))
return WHY("Failed to finalise manifest.\n");
}
/* Write manifest back to disk */
if (rhizome_write_manifest_file(m,manifestname)) {
rhizome_manifest_free(m);
return WHY("Could not write manifest file.");
}
/* Okay, it is written, and can be put directly into the rhizome database now */
int r=rhizome_store_bundle(m,filename);
if (!r) {
// XXX For testing unlink(manifestname);
unlink(filename);
return 0;
}
if (rhizome_store_bundle(m, filename) == -1)
return WHY("rhizome_store_bundle() failed.");
return WHY("rhizome_store_bundle() failed.");
return 0;
}
/* Update an existing Rhizome bundle */

View File

@ -160,16 +160,16 @@ extern sqlite3 *rhizome_db;
int rhizome_opendb();
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_write_manifest_file(rhizome_manifest *m,char *filename);
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename);
int rhizome_manifest_sign(rhizome_manifest *m);
int rhizome_drop_stored_file(char *id,int maximum_priority);
int rhizome_manifest_priority(char *id);
rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferPAndSize,int flags);
int rhizome_hash_file(char *filename,char *hash_out);
rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferPAndSize, int flags);
int rhizome_hash_file(const char *filename,char *hash_out);
char *rhizome_manifest_get(rhizome_manifest *m,char *var,char *value_out,int maxlen);
long long rhizome_manifest_get_ll(rhizome_manifest *m,char *var);
int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value);
int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value);
int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value);
long long rhizome_file_size(char *filename);
void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line,
rhizome_manifest *m);
@ -177,13 +177,15 @@ void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line
rhizome_manifest *_rhizome_new_manifest(const char *file,const char *func,int line);
#define rhizome_new_manifest() _rhizome_new_manifest(__FILE__,__FUNCTION__,__LINE__)
int rhizome_manifest_pack_variables(rhizome_manifest *m);
int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename);
int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename);
int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
int rhizome_store_file(char *file,char *hash,int priortity);
int rhizome_store_file(const char *file,char *hash,int priortity);
char *rhizome_safe_encode(unsigned char *in,int len);
int rhizome_finish_sqlstatement(sqlite3_stmt *statement);
int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
int verifyP, int checkFileP, int signP);
int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl,
int verifyP, int checkFileP, int signP);
int rhizome_manifest_finalise(rhizome_manifest *m,int signP);
char *rhizome_bytes_to_hex(unsigned char *in,int byteCount);
int rhizome_hex_to_bytes(char *in,unsigned char *out,int hexChars);

View File

@ -21,7 +21,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "rhizome.h"
#include <stdlib.h>
rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flags)
rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferP, int flags)
{
if (bufferP>MAX_MANIFEST_BYTES) return NULL;
@ -30,12 +30,15 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag
if (bufferP) {
m->manifest_bytes=bufferP;
bcopy(filename,m->manifestdata,m->manifest_bytes);
memcpy(m->manifestdata, filename, m->manifest_bytes);
}
else {
FILE *f=fopen(filename,"r");
if (!f) { WHY("Could not open manifest file for reading.");
rhizome_manifest_free(m); return NULL; }
if (!f) {
WHYF("Could not open manifest file %s for reading.", filename);
rhizome_manifest_free(m);
return NULL;
}
m->manifest_bytes = fread(m->manifestdata,1,MAX_MANIFEST_BYTES,f);
fclose(f);
}
@ -144,7 +147,7 @@ rhizome_manifest *rhizome_read_manifest_file(char *filename,int bufferP,int flag
return m;
}
int rhizome_hash_file(char *filename,char *hash_out)
int rhizome_hash_file(const char *filename,char *hash_out)
{
/* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one
go. Trouble is, we need to run Serval DNA on filesystems that lack mmap(),
@ -212,7 +215,7 @@ double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_
}
int rhizome_manifest_set(rhizome_manifest *m,char *var,char *value)
int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value)
{
int i;
@ -245,18 +248,6 @@ int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value)
return rhizome_manifest_set(m,var,svalue);
}
long long rhizome_file_size(char *filename)
{
FILE *f;
/* XXX really should just use stat instead of opening the file */
f=fopen(filename,"r");
fseek(f,0,SEEK_END);
long long size=ftello(f);
fclose(f);
return size;
}
#define MAX_RHIZOME_MANIFESTS 16
rhizome_manifest manifests[MAX_RHIZOME_MANIFESTS];
char manifest_free[MAX_RHIZOME_MANIFESTS];
@ -420,7 +411,7 @@ int rhizome_manifest_sign(rhizome_manifest *m)
return 0;
}
int rhizome_write_manifest_file(rhizome_manifest *m,char *filename)
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename)
{
if (!m) return WHY("Manifest is null.");
if (!m->finalised) return WHY("Manifest must be finalised before it can be written.");

View File

@ -290,7 +290,7 @@ int rhizome_drop_stored_file(char *id,int maximum_priority)
We need to also need to create the appropriate row(s) in the MANIFESTS, FILES,
FILEMANIFESTS and GROUPMEMBERSHIPS tables, and possibly GROUPLIST as well.
*/
int rhizome_store_bundle(rhizome_manifest *m,char *associated_filename)
int rhizome_store_bundle(rhizome_manifest *m, const char *associated_filename)
{
char sqlcmd[1024];
const char *cmdtail;
@ -500,13 +500,68 @@ char *rhizome_safe_encode(unsigned char *in,int len)
return r;
}
int rhizome_list_manifests(int limit, int offset)
{
char sqlcmd[1024];
int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id");
if (n >= sizeof(sqlcmd))
return WHY("SQL command too long");
if (limit) {
n += snprintf(&sqlcmd[n], sizeof(sqlcmd) - n, " LIMIT %u", limit);
if (n >= sizeof(sqlcmd))
return WHY("SQL command too long");
}
if (offset) {
n += snprintf(&sqlcmd[n], sizeof(sqlcmd) - n, " OFFSET %u", offset);
if (n >= sizeof(sqlcmd))
return WHY("SQL command too long");
}
sqlite3_stmt *statement;
const char *cmdtail;
int ret = 0;
if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) {
sqlite3_finalize(statement);
ret = WHY(sqlite3_errmsg(rhizome_db));
} else {
size_t rows = 0;
while (sqlite3_step(statement) == SQLITE_ROW) {
++rows;
if (!( sqlite3_column_count(statement) == 7
&& sqlite3_column_type(statement, 0) == SQLITE_TEXT
&& sqlite3_column_type(statement, 1) == SQLITE_INTEGER
&& sqlite3_column_type(statement, 2) == SQLITE_INTEGER
&& sqlite3_column_type(statement, 3) == SQLITE_TEXT
&& sqlite3_column_type(statement, 4) == SQLITE_BLOB
&& sqlite3_column_type(statement, 5) == SQLITE_INTEGER
&& sqlite3_column_type(statement, 6) == SQLITE_INTEGER
)) {
ret = WHY("Incorrect statement column");
break;
}
size_t filesize = sqlite3_column_int(statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(statement, 4);
const char *manifestblob = (char *) sqlite3_column_blob(statement, 4);
printf("manifest blob = %s\n", manifestblob);
rhizome_manifest *m = rhizome_read_manifest_file(manifestblob, manifestblobsize, RHIZOME_VERIFY);
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
printf("file id = %s\nfile length = %u\nfile datavalid = %u\nfile name = \"%s\"\n",
sqlite3_column_text(statement, 0),
filesize,
sqlite3_column_int(statement, 2),
name
);
rhizome_manifest_free(m);
}
printf("Found %lu rows\n", (unsigned long) rows);
}
sqlite3_finalize(statement);
return ret;
}
/* The following function just stores the file (or silently returns if it already
exists).
The relationships of manifests to this file are the responsibility of the
caller. */
int rhizome_store_file(char *file,char *hash,int priority) {
/* The following function just stores the file (or silently returns if it already exists).
The relationships of manifests to this file are the responsibility of the caller. */
int rhizome_store_file(const char *file,char *hash,int priority)
{
int fd=open(file,O_RDONLY);
if (fd<0) return WHY("Could not open associated file");
@ -695,5 +750,3 @@ int rhizome_update_file_priority(char *fileid)
}
return 0;
}