mirror of
https://github.com/servalproject/serval-dna.git
synced 2025-04-07 11:08:36 +00:00
Refactor "rhizome add file" and "rhizome journal append"
Put manifest creation logic into new rhizome_bundle_add_file() function, in preparation for implementing new HTTP POST /restful/rhizome/append request Several 'rhizomeops' tests fail
This commit is contained in:
Andrew Bettison
parent
67a6e1b9b6
commit
19119e759c
134
rhizome.c
134
rhizome.c
@ -93,6 +93,140 @@ int rhizome_fetch_delay_ms()
|
||||
return config.rhizome.fetch_delay_ms;
|
||||
}
|
||||
|
||||
/* Create a manifest structure to accompany adding a file to Rhizome or appending to a journal.
|
||||
* This function is used by all application-facing APIs (eg, CLI, RESTful HTTP). It differs from
|
||||
* the import operation in that if the caller does not supply a complete, signed manifest then this
|
||||
* operation will create it using the fields supplied. Also, the caller can supply a clear-text
|
||||
* payload with the 'crypt=1' field to cause it to be stored encrypted.
|
||||
*
|
||||
* - 'm' must point to a manifest structure into which any supplied partial manifest has already
|
||||
* been parsed. If the caller supplied no (partial) manifest at all, then the manifest 'm' will
|
||||
* be blank.
|
||||
*
|
||||
* - 'bsk' must point to a supplied bundle secret parameter, or NULL if none was supplied.
|
||||
*
|
||||
* - if 'appending' is true then the new bundle will be a journal bundle, otherwise it will be a
|
||||
* normal bundle. Any existing manifest must be consistent; eg, an append will fail if a bundle
|
||||
* with the same Bundle Id already exists in the store and is not a journal.
|
||||
*
|
||||
* - 'author' must point to a supplied author parameter, or NULL if none was supplied.
|
||||
*
|
||||
* - 'file_path' can point to a supplied payload file name (eg, if the payload was read from a named
|
||||
* file), or can be NULL. If not NULL, then the file's name will be used to fill in the 'name'
|
||||
* field of the manifest if it was not explicitly supplied in 'm' or in the existing manifest.
|
||||
*
|
||||
* If the add is successful, modifies '*mout' to point to the constructed Manifest, which might be
|
||||
* 'm' or might be another manifest, and returns NULL. It is the caller's responsibility to free
|
||||
* '*mout'.
|
||||
*
|
||||
* If the add fails for any reason, returns a pointer to a nul-terminated text string that describes
|
||||
* the reason, and does not alter '*mout'.
|
||||
*
|
||||
* @author Andrew Bettison <andrew@servalproject.com>
|
||||
*/
|
||||
const char * rhizome_bundle_add_file(int appending,
|
||||
rhizome_manifest *m,
|
||||
rhizome_manifest **mout,
|
||||
const rhizome_bk_t *bsk,
|
||||
const sid_t *author,
|
||||
const char *file_path
|
||||
)
|
||||
{
|
||||
const char *reason = NULL;
|
||||
rhizome_manifest *existing_manifest = NULL;
|
||||
rhizome_manifest *new_manifest = NULL;
|
||||
assert(m != NULL);
|
||||
// Caller must not supply a malformed manifest (but an invalid one is okay because missing
|
||||
// fields will be filled in, so we don't check validity here).
|
||||
assert(!m->malformed);
|
||||
if (m->has_id) {
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Reading manifest from database: id=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
|
||||
if ((existing_manifest = rhizome_new_manifest()) == NULL) {
|
||||
WHY(reason = "Manifest struct could not be allocated");
|
||||
goto error;
|
||||
}
|
||||
enum rhizome_bundle_status status = rhizome_retrieve_manifest(&m->cryptoSignPublic, existing_manifest);
|
||||
switch (status) {
|
||||
case RHIZOME_BUNDLE_STATUS_NEW:
|
||||
// No manifest with that bundle ID exists in the store, so we are building a bundle from
|
||||
// scratch.
|
||||
rhizome_manifest_free(existing_manifest);
|
||||
existing_manifest = NULL;
|
||||
break;
|
||||
case RHIZOME_BUNDLE_STATUS_SAME:
|
||||
// Found a manifest with the same bundle ID. Overwrite it with the supplied manifest.
|
||||
if (rhizome_manifest_overwrite(existing_manifest, m) == -1) {
|
||||
WHY(reason = "Existing manifest could not be overwritten");
|
||||
goto error;
|
||||
}
|
||||
new_manifest = existing_manifest;
|
||||
existing_manifest = NULL;
|
||||
break;
|
||||
case RHIZOME_BUNDLE_STATUS_BUSY:
|
||||
WHY(reason = "Existing manifest not retrieved due to Rhizome store locking");
|
||||
goto error;
|
||||
case RHIZOME_BUNDLE_STATUS_ERROR:
|
||||
WHY(reason = "Error retrieving existing manifest from Rhizome store");
|
||||
goto error;
|
||||
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
|
||||
case RHIZOME_BUNDLE_STATUS_OLD:
|
||||
case RHIZOME_BUNDLE_STATUS_INVALID:
|
||||
case RHIZOME_BUNDLE_STATUS_FAKE:
|
||||
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
|
||||
case RHIZOME_BUNDLE_STATUS_NO_ROOM:
|
||||
case RHIZOME_BUNDLE_STATUS_READONLY:
|
||||
FATALF("rhizome_retrieve_manifest() returned %s", rhizome_bundle_status_message(status));
|
||||
}
|
||||
}
|
||||
// If no existing bundle has been identified, we are building a bundle from scratch.
|
||||
if (!new_manifest) {
|
||||
new_manifest = m;
|
||||
// A new journal manifest needs these fields set so that the first append can succeed.
|
||||
if (appending) {
|
||||
rhizome_manifest_set_filesize(new_manifest, 0);
|
||||
rhizome_manifest_set_tail(new_manifest, 0);
|
||||
}
|
||||
}
|
||||
if (appending && !m->is_journal){
|
||||
// TODO: return a special status code for this case
|
||||
WHY(reason = "Cannot append to a non-journal");
|
||||
goto error;
|
||||
}
|
||||
if (!appending && m->is_journal) {
|
||||
// TODO: return a special status code for this case
|
||||
WHY(reason = "Cannot add a journal bundle (use append instead)");
|
||||
goto error;
|
||||
}
|
||||
if (bsk) {
|
||||
if (new_manifest->has_id) {
|
||||
if (!rhizome_apply_bundle_secret(new_manifest, bsk)) {
|
||||
WHY(reason = "Supplied bundle secret does not match Bundle Id");
|
||||
goto error;
|
||||
}
|
||||
} else {
|
||||
if (rhizome_new_bundle_from_secret(new_manifest, bsk) == -1) {
|
||||
WHY(reason = "Failed to create bundle from given secret");
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
}
|
||||
// TODO: one day there will be no default service, but for now if no service
|
||||
// is specified, it defaults to 'file' (file sharing).
|
||||
if (m->service == NULL)
|
||||
rhizome_manifest_set_service(new_manifest, RHIZOME_SERVICE_FILE);
|
||||
if ((reason = rhizome_fill_manifest(new_manifest, file_path, author ? author : NULL)) != NULL)
|
||||
goto error;
|
||||
*mout = new_manifest;
|
||||
return NULL;
|
||||
error:
|
||||
if (new_manifest && new_manifest != m && new_manifest != existing_manifest)
|
||||
rhizome_manifest_free(new_manifest);
|
||||
if (existing_manifest)
|
||||
rhizome_manifest_free(existing_manifest);
|
||||
return reason;
|
||||
}
|
||||
|
||||
/* Import a bundle from a pair of files, one containing the manifest and the optional other
|
||||
* containing the payload. The work is all done by rhizome_bundle_import() and
|
||||
* rhizome_store_manifest().
|
||||
|
||||
@ -426,6 +426,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc);
|
||||
|
||||
int rhizome_store_manifest(rhizome_manifest *m);
|
||||
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
|
||||
const char * rhizome_bundle_add_file(int appending, rhizome_manifest *m, rhizome_manifest **mout, const rhizome_bk_t *bsk, const sid_t *author, const char *file_path);
|
||||
int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath);
|
||||
|
||||
int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath);
|
||||
|
||||
114
rhizome_cli.c
114
rhizome_cli.c
@ -112,27 +112,34 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
|
||||
{
|
||||
if (config.debug.verbose)
|
||||
DEBUG_cli_parsed(parsed);
|
||||
const char *filepath, *manifestpath, *manifestid, *authorSidHex, *bsktext;
|
||||
const char *filepath, *manifestpath, *manifestIdHex, *authorSidHex, *bsktext;
|
||||
|
||||
int force_new = 0 == cli_arg(parsed, "--force-new", NULL, NULL, NULL);
|
||||
cli_arg(parsed, "filepath", &filepath, NULL, "");
|
||||
if (cli_arg(parsed, "author_sid", &authorSidHex, cli_optional_sid, "") == -1)
|
||||
return -1;
|
||||
cli_arg(parsed, "manifestpath", &manifestpath, NULL, "");
|
||||
cli_arg(parsed, "manifestid", &manifestid, NULL, "");
|
||||
cli_arg(parsed, "manifestid", &manifestIdHex, NULL, "");
|
||||
if (cli_arg(parsed, "bsk", &bsktext, cli_optional_bundle_secret_key, NULL) == -1)
|
||||
return -1;
|
||||
|
||||
sid_t authorSid;
|
||||
if (authorSidHex[0] && str_to_sid_t(&authorSid, authorSidHex) == -1)
|
||||
if (!authorSidHex || !*authorSidHex)
|
||||
authorSidHex = NULL;
|
||||
else if (str_to_sid_t(&authorSid, authorSidHex) == -1)
|
||||
return WHYF("invalid author_sid: %s", authorSidHex);
|
||||
|
||||
// treat empty string the same as null
|
||||
if (bsktext && !*bsktext)
|
||||
bsktext = NULL;
|
||||
rhizome_bid_t bid;
|
||||
if (!manifestIdHex || !*manifestIdHex)
|
||||
manifestIdHex = NULL;
|
||||
else if (str_to_rhizome_bid_t(&bid, manifestIdHex) == -1)
|
||||
return WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestIdHex));
|
||||
|
||||
rhizome_bk_t bsk;
|
||||
if (bsktext && str_to_rhizome_bsk_t(&bsk, bsktext) == -1)
|
||||
return WHYF("invalid bsk: \"%s\"", bsktext);
|
||||
if (!bsktext || !*bsktext)
|
||||
bsktext = NULL;
|
||||
else if (str_to_rhizome_bsk_t(&bsk, bsktext) == -1)
|
||||
return WHYF("invalid BSK: \"%s\"", bsktext);
|
||||
|
||||
unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi;
|
||||
struct field {
|
||||
@ -170,7 +177,7 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
|
||||
}
|
||||
}
|
||||
|
||||
int journal = strcasecmp(parsed->args[1], "journal")==0;
|
||||
int appending = strcasecmp(parsed->args[1], "journal")==0;
|
||||
|
||||
if (create_serval_instance_dir() == -1)
|
||||
return -1;
|
||||
@ -183,55 +190,25 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
|
||||
if (rhizome_opendb() == -1)
|
||||
goto finish;
|
||||
|
||||
/* Create a new manifest that will represent the file. If a manifest file was supplied, then read
|
||||
* it, otherwise create a blank manifest. */
|
||||
/* Create a manifest in memory that to accompany the added file. Initially the manifest is blank.
|
||||
* If a manifest file is supplied, then read and parse it, barfing if it contains any duplicate
|
||||
* fields or invalid values. If it successfully parses, then overwrite it with any command-line
|
||||
* manifest field settings, overriding the values parsed from the file. Barf if any of these new
|
||||
* values are malformed. We don't validate the resulting manifest, it order to allow the user to
|
||||
* supply an incomplete manifest. Any missing fields will be filled in later.
|
||||
*/
|
||||
if ((m = rhizome_new_manifest()) == NULL){
|
||||
ret = WHY("Manifest struct could not be allocated -- not added to rhizome");
|
||||
ret = WHY("Manifest struct could not be allocated -- not added");
|
||||
goto finish;
|
||||
}
|
||||
if (manifestpath && *manifestpath && access(manifestpath, R_OK) == 0) {
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("reading manifest from %s", manifestpath);
|
||||
/* Don't verify the manifest, because it will fail if it is incomplete.
|
||||
This is okay, because we fill in any missing bits and sanity check before
|
||||
trying to write it out. However, we do insist that whatever we load is
|
||||
parsed okay and not malformed. */
|
||||
if (rhizome_read_manifest_from_file(m, manifestpath) || m->malformed) {
|
||||
ret = WHY("Manifest file could not be loaded -- not added to rhizome");
|
||||
goto finish;
|
||||
}
|
||||
} else if (manifestid && *manifestid) {
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Reading manifest from database");
|
||||
rhizome_bid_t bid;
|
||||
if (str_to_rhizome_bid_t(&bid, manifestid) == -1) {
|
||||
ret = WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestid));
|
||||
goto finish;
|
||||
}
|
||||
if (rhizome_retrieve_manifest(&bid, m) != RHIZOME_BUNDLE_STATUS_SAME) {
|
||||
ret = WHY("Existing manifest could not be loaded -- not added to rhizome");
|
||||
goto finish;
|
||||
}
|
||||
} else {
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("Creating new manifest");
|
||||
if (journal) {
|
||||
rhizome_manifest_set_filesize(m, 0);
|
||||
rhizome_manifest_set_tail(m, 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (journal && !m->is_journal){
|
||||
// TODO: return a special status code for this case
|
||||
ret = WHY("Existing manifest is not a journal");
|
||||
goto finish;
|
||||
}
|
||||
if (!journal && m->is_journal) {
|
||||
// TODO: return a special status code for this case
|
||||
ret = WHY("Existing manifest is a journal");
|
||||
goto finish;
|
||||
}
|
||||
|
||||
if (nfields) {
|
||||
unsigned i;
|
||||
for (i = 0; i != nfields; ++i) {
|
||||
@ -269,26 +246,36 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
|
||||
}
|
||||
}
|
||||
|
||||
if (bsktext) {
|
||||
if (m->has_id) {
|
||||
if (!rhizome_apply_bundle_secret(m, &bsk)) {
|
||||
ret = WHY("Supplied bundle secret does not match Bundle Id");
|
||||
goto finish;
|
||||
}
|
||||
} else {
|
||||
if (rhizome_new_bundle_from_secret(m, &bsk) == -1) {
|
||||
ret = WHY("Failed to create bundle from given secret");
|
||||
goto finish;
|
||||
}
|
||||
/* If a manifest ID (bundle ID) was supplied on the command line, first ensure it does not
|
||||
* contradict any manifest ID present in the supplied manifest file, then insert it into the
|
||||
* manifest.
|
||||
*/
|
||||
if (manifestIdHex) {
|
||||
if (!m->has_id)
|
||||
rhizome_manifest_set_id(m, &bid);
|
||||
else if (cmp_rhizome_bid_t(&m->cryptoSignPublic, &bid) != 0) {
|
||||
ret = WHYF("manifestid=%s does not match manifest id=%s", manifestIdHex, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
|
||||
goto finish;
|
||||
}
|
||||
}
|
||||
if (m->service == NULL)
|
||||
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
|
||||
if (rhizome_fill_manifest(m, filepath, *authorSidHex ? &authorSid : NULL))
|
||||
goto finish;
|
||||
|
||||
/* Create an in-memory manifest for the file being added.
|
||||
*/
|
||||
rhizome_manifest *mout = NULL;
|
||||
if (rhizome_bundle_add_file(appending, m, &mout, bsktext ? &bsk : NULL, authorSidHex ? &authorSid : NULL, filepath) != NULL) {
|
||||
ret = WHY("Cannot create manifest -- not added");
|
||||
goto finish;
|
||||
}
|
||||
assert(mout != NULL);
|
||||
if (mout != m) {
|
||||
rhizome_manifest_free(m);
|
||||
m = mout;
|
||||
}
|
||||
mout = NULL;
|
||||
|
||||
// Insert the payload into the Rhizome store.
|
||||
enum rhizome_payload_status pstatus;
|
||||
if (journal){
|
||||
if (appending) {
|
||||
pstatus = rhizome_append_journal_file(m, 0, filepath);
|
||||
if (config.debug.rhizome)
|
||||
DEBUGF("rhizome_append_journal_file() returned %d %s", pstatus, rhizome_payload_status_message(pstatus));
|
||||
@ -335,7 +322,6 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
|
||||
}
|
||||
if (!pstatus_valid)
|
||||
FATALF("pstatus = %d", pstatus);
|
||||
rhizome_manifest *mout = NULL;
|
||||
if (status == RHIZOME_BUNDLE_STATUS_NEW) {
|
||||
if (!rhizome_manifest_validate(m) || m->malformed)
|
||||
status = RHIZOME_BUNDLE_STATUS_INVALID;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user