ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-03-22 11:55:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Create journal append command

Browse Source
This commit is contained in:
Jeremy Lakeman 2013-07-05 16:24:50 +09:30
parent 2bc07a8eaa
commit e758e0130f
5 changed files with 256 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
commandline.c
View File

@ -1266,14 +1266,16 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *filepath, *manifestpath, *authorSidHex, *bskhex;
const char *filepath, *manifestpath, *manifestid, *authorSidHex, *bskhex;
cli_arg(parsed, "filepath", &filepath, NULL, "");
if (cli_arg(parsed, "author_sid", &authorSidHex, cli_optional_sid, "") == -1)
return -1;
cli_arg(parsed, "manifestpath", &manifestpath, NULL, "");
cli_arg(parsed, "manifestid", &manifestid, NULL, "");
if (cli_arg(parsed, "bsk", &bskhex, cli_optional_bundle_key, NULL) == -1)
return -1;
sid_t authorSid;
if (authorSidHex[0] && str_to_sid_t(&authorSid, authorSidHex) == -1)
return WHYF("invalid author_sid: %s", authorSidHex);
@ -1286,6 +1288,8 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
if (bskhex && fromhexstr(bsk.binary, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1)
return WHYF("invalid bsk: \"%s\"", bskhex);
int journal = strcasecmp(parsed->args[1], "journal")==0;
if (create_serval_instance_dir() == -1)
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
@ -1299,8 +1303,9 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
if (!m)
return WHY("Manifest struct could not be allocated -- not added to rhizome");
if (manifestpath[0] && access(manifestpath, R_OK) == 0) {
if (config.debug.rhizome) DEBUGF("reading manifest from %s", manifestpath);
if (manifestpath && *manifestpath && access(manifestpath, R_OK) == 0) {
if (config.debug.rhizome)
DEBUGF("reading manifest from %s", manifestpath);
/* Don't verify the manifest, because it will fail if it is incomplete.
This is okay, because we fill in any missing bits and sanity check before
trying to write it out. */
@ -1308,25 +1313,40 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
rhizome_manifest_free(m);
return WHY("Manifest file could not be loaded -- not added to rhizome");
}
} else if(manifestid && *manifestid) {
if (config.debug.rhizome)
DEBUGF("Reading manifest from database");
if (rhizome_retrieve_manifest(manifestid, m)){
rhizome_manifest_free(m);
return WHY("Existing manifest could not be loaded -- not added to rhizome");
}
} else {
if (config.debug.rhizome) DEBUGF("manifest file %s does not exist -- creating new manifest", manifestpath);
}
if (rhizome_stat_file(m, filepath)){
rhizome_manifest_free(m);
return -1;
if (config.debug.rhizome)
DEBUGF("Creating new manifest");
}
if (rhizome_fill_manifest(m, filepath, *authorSidHex?&authorSid:NULL, bskhex?&bsk:NULL)){
rhizome_manifest_free(m);
return -1;
}
if (m->fileLength){
if (rhizome_add_file(m, filepath)){
if (journal){
if (rhizome_append_journal_file(m, bskhex?&bsk:NULL, 0, filepath)){
rhizome_manifest_free(m);
return -1;
}
}else{
if (rhizome_stat_file(m, filepath)){
rhizome_manifest_free(m);
return -1;
}
if (m->fileLength){
if (rhizome_add_file(m, filepath)){
rhizome_manifest_free(m);
return -1;
}
}
}
rhizome_manifest *mout = NULL;
@ -1336,7 +1356,7 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co
return -1;
}
if (manifestpath[0]
if (manifestpath && *manifestpath
&& rhizome_write_manifest_file(mout, manifestpath, 0) == -1)
ret = WHY("Could not overwrite manifest file.");
const char *service = rhizome_manifest_get(mout, "service", NULL, 0);
@ -2283,9 +2303,11 @@ struct cli_schema command_line_options[]={
{app_rhizome_hash_file,{"rhizome","hash","file","<filepath>",NULL}, 0,
"Compute the Rhizome hash of a file"},
{app_rhizome_add_file,{"rhizome","add","file" KEYRING_PIN_OPTIONS,"<author_sid>","<filepath>","[<manifestpath>]","[<bsk>]",NULL}, 0,
"Add a file to Rhizome and optionally write its manifest to the given path"},
"Add a file to Rhizome and optionally write its manifest to the given path"},
{app_rhizome_add_file, {"rhizome", "journal", "append" KEYRING_PIN_OPTIONS, "<author_sid>", "<manifestid>", "<filepath>", "[<bsk>]", NULL}, 0,
"Append content to a journal bundle"},
{app_rhizome_import_bundle,{"rhizome","import","bundle","<filepath>","<manifestpath>",NULL}, 0,
"Import a payload/manifest pair into Rhizome"},
"Import a payload/manifest pair into Rhizome"},
{app_rhizome_list,{"rhizome","list" KEYRING_PIN_OPTIONS,
"[<service>]","[<name>]","[<sender_sid>]","[<recipient_sid>]","[<offset>]","[<limit>]",NULL}, 0,
"List all manifests and files in Rhizome"},

5
rhizome.h
View File

@ -679,6 +679,11 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath);
int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
int rhizome_add_file(rhizome_manifest *m, const char *filepath);
int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size);
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len);
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
const unsigned char *key, const unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const char *fileid, int hash);

2
rhizome_bundle.c
View File

@ -761,7 +761,7 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
}
int crypt = rhizome_manifest_get_ll(m,"crypt");
if (crypt==-1 && m->fileLength){
if (crypt==-1){
// no explicit crypt flag, should we encrypt this bundle?
char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);

216
rhizome_store.c
View File

@ -135,30 +135,27 @@ int rhizome_open_write(struct rhizome_write *write, char *expectedFileHash, int6
return 0;
}
/* Write write_state->buffer into the store
Note that we don't support random writes as the contents must be hashed in order
But we don't enforce linear writes yet. */
int rhizome_flush(struct rhizome_write *write_state){
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size){
IN();
/* Make sure we aren't being asked to write more data than we expected */
if (write_state->file_offset + write_state->data_size > write_state->file_length)
RETURN(WHYF("Too much content supplied, %d + %d > %d",
write_state->file_offset, write_state->data_size, write_state->file_length));
if (write_state->file_offset + data_size > write_state->file_length)
RETURN(WHYF("Too much content supplied, %lld + %d > %lld",
write_state->file_offset, data_size, write_state->file_length));
if (write_state->data_size<=0)
if (data_size<=0)
RETURN(WHY("No content supplied"));
if (write_state->crypt){
if (rhizome_crypt_xor_block(write_state->buffer, write_state->data_size,
write_state->file_offset, write_state->key, write_state->nonce))
if (rhizome_crypt_xor_block(buffer, data_size, write_state->file_offset, write_state->key, write_state->nonce))
RETURN(-1);
}
if (write_state->blob_fd>=0) {
int ofs=0;
// keep trying until all of the data is written.
while(ofs < write_state->data_size){
int r=write(write_state->blob_fd, write_state->buffer + ofs, write_state->data_size - ofs);
lseek(write_state->blob_fd, write_state->file_offset, SEEK_SET);
while(ofs < data_size){
int r=write(write_state->blob_fd, buffer + ofs, data_size - ofs);
if (r<0)
RETURN(WHY_perror("write"));
if (config.debug.externalblobs)
@ -182,7 +179,7 @@ int rhizome_flush(struct rhizome_write *write_state){
RETURN(-1);
}
ret=sqlite3_blob_write(blob, write_state->buffer, write_state->data_size,
ret=sqlite3_blob_write(blob, buffer, data_size,
write_state->file_offset);
if (sqlite_code_busy(ret))
@ -212,15 +209,24 @@ int rhizome_flush(struct rhizome_write *write_state){
}while(1);
}
SHA512_Update(&write_state->sha512_context, write_state->buffer, write_state->data_size);
write_state->file_offset+=write_state->data_size;
SHA512_Update(&write_state->sha512_context, buffer, data_size);
write_state->file_offset+=data_size;
if (config.debug.rhizome)
DEBUGF("Written %lld of %lld", write_state->file_offset, write_state->file_length);
write_state->data_size=0;
RETURN(0);
RETURN(data_size);
OUT();
}
/* Write write_state->buffer into the store
Note that we don't support random writes as the contents must be hashed in order
But we don't enforce linear writes yet. */
int rhizome_flush(struct rhizome_write *write_state){
int wrote = rhizome_write_buffer(write_state, write_state->buffer, write_state->data_size);
if (wrote == write_state->data_size)
write_state->data_size=0;
return wrote>=0?0:-1;
}
/* Expects file to be at least file_length in size, ignoring anything longer than that */
int rhizome_write_file(struct rhizome_write *write, const char *filename){
FILE *f = fopen(filename, "r");
@ -240,7 +246,7 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){
return -1;
}
write->data_size+=r;
DEBUGF("Read %d from file", r);
if (rhizome_flush(write)){
fclose(f);
return -1;
@ -430,6 +436,18 @@ int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
return 0;
}
static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_write *write)
{
write->crypt=1;
// if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
if (rhizome_derive_key(m, bsk))
return -1;
bcopy(m->payloadKey, write->key, sizeof(write->key));
bcopy(m->payloadNonce, write->nonce, sizeof(write->nonce));
return 0;
}
// import a file for a new bundle with an unknown file hash
// update the manifest with the details of the file
int rhizome_add_file(rhizome_manifest *m, const char *filepath)
@ -441,32 +459,27 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
if (rhizome_open_write(&write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT))
return -1;
write.crypt=m->payloadEncryption;
if (write.crypt){
// if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
if (rhizome_derive_key(m, NULL))
if (m->payloadEncryption){
if (rhizome_write_derive_key(m, NULL, &write))
return -1;
if (config.debug.rhizome)
DEBUGF("Encrypting file contents");
bcopy(m->payloadKey, write.key, sizeof(write.key));
bcopy(m->payloadNonce, write.nonce, sizeof(write.nonce));
}
if (rhizome_write_file(&write, filepath)){
rhizome_fail_write(&write);
return -1;
}
if (rhizome_finish_write(&write)){
rhizome_fail_write(&write);
return -1;
}
if (rhizome_finish_write(&write))
goto failure;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
rhizome_manifest_set(m, "filehash", m->fileHexHash);
return 0;
failure:
rhizome_fail_write(&write);
return -1;
}
/* Return -1 on error, 0 if file blob found, 1 if not found.
@ -678,3 +691,142 @@ int rhizome_dump_file(const char *id, const char *filepath, int64_t *length)
rhizome_read_close(&read_state);
return ret;
}
// pipe data from one payload to another
static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write, uint64_t length)
{
if (length > write->file_length - write->file_offset)
return WHY("Unable to pipe that much data");
while(length>0){
int size=write->buffer_size - write->data_size;
if (size > length)
size=length;
int r = rhizome_read(read, write->buffer + write->data_size, size);
if (r<0)
return r;
write->data_size+=r;
length -= r;
DEBUGF("Piping %d bytes", r);
if (rhizome_flush(write))
return -1;
}
return 0;
}
// open an existing journal bundle, advance the head pointer, duplicate the existing content and get ready to add more.
int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size)
{
int ret = 0;
if (advance_by!=0)
return WHY("Advancing a journal is not yet supported");
if (advance_by > m->fileLength)
return WHY("Cannot advance past the existing content");
uint64_t old_length = m->fileLength;
uint64_t copy_length = old_length - advance_by;
m->fileLength = m->fileLength + new_size - advance_by;
DEBUGF("Before %lld, advance %lld, new %lld = %lld, %lld", old_length, advance_by, new_size, copy_length, m->fileLength);
rhizome_manifest_set_ll(m, "filesize", m->fileLength);
m->version = m->fileLength;
rhizome_manifest_set_ll(m,"version",m->version);
ret = rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
if (ret)
goto failure;
if (copy_length>0){
DEBUGF("Copying %lld", copy_length);
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);
ret = rhizome_open_decrypt_read(m, bsk, &read_state, 1);
if (ret)
goto failure;
read_state.offset = advance_by;
ret = rhizome_pipe(&read_state, write, copy_length);
rhizome_read_close(&read_state);
if (ret)
goto failure;
}
if (m->payloadEncryption){
ret = rhizome_write_derive_key(m, bsk, write);
if (ret)
goto failure;
}
return 0;
failure:
if (ret)
rhizome_fail_write(write);
return ret;
}
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len)
{
struct rhizome_write write;
bzero(&write, sizeof write);
int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, len);
if (ret)
return -1;
if (buffer && len){
ret = rhizome_write_buffer(&write, buffer, len);
if (ret)
goto failure;
}
ret = rhizome_finish_write(&write);
if (ret)
goto failure;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
rhizome_manifest_set(m, "filehash", m->fileHexHash);
return 0;
failure:
if (ret)
rhizome_fail_write(&write);
return ret;
}
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename)
{
struct stat stat;
if (lstat(filename,&stat))
return WHYF("Could not stat() payload file '%s'",filename);
struct rhizome_write write;
bzero(&write, sizeof write);
int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, stat.st_size);
if (ret)
return -1;
if (stat.st_size){
ret = rhizome_write_file(&write, filename);
if (ret)
goto failure;
}
ret = rhizome_finish_write(&write);
if (ret)
goto failure;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
rhizome_manifest_set(m, "filehash", m->fileHexHash);
return 0;
failure:
if (ret)
rhizome_fail_write(&write);
return ret;
}

30
tests/rhizomeops
View File

@ -26,10 +26,16 @@ shopt -s extglob
setup_rhizome() {
set_instance +A
executeOk_servald config set debug.rhizome on
executeOk_servald config \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
create_single_identity
set_instance +B
executeOk_servald config set debug.rhizome on
executeOk_servald config \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
create_identities 4
assert [ $SIDB1 != $SIDA1 ]
assert [ $SIDB2 != $SIDA1 ]
@ -658,6 +664,26 @@ test_EncryptedPayload() {
assert ! diff file1 file1y
}
doc_JournalAdd="Create and append to a journal"
setup_JournalAdd() {
setup_servald
setup_rhizome
echo "Part One" > file1
echo "Part Two" > file2
cat file1 file2 > file
}
test_JournalAdd() {
executeOk_servald rhizome journal append $SIDB1 "" file1
tfw_cat --stdout --stderr
assert_stdout_add_file file1
extract_stdout_keyvalue BID 'manifestid' '[0-9A-F]\+'
executeOk_servald rhizome journal append $SIDB1 $BID file2
tfw_cat --stdout --stderr
executeOk_servald rhizome extract file $BID filex
tfw_cat --stdout --stderr
assert diff file filex
}
doc_MeshMSAddCreate="First add MeshMS creates manifest"
setup_MeshMSAddCreate() {
setup_servald