From 19119e759c57919677325b073954dc72f33fbae6 Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Mon, 16 Mar 2015 22:44:15 +1030 Subject: [PATCH] Refactor "rhizome add file" and "rhizome journal append" Put manifest creation logic into new rhizome_bundle_add_file() function, in preparation for implementing new HTTP POST /restful/rhizome/append request Several 'rhizomeops' tests fail --- rhizome.c | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++ rhizome.h | 1 + rhizome_cli.c | 114 +++++++++++++++++++----------------------- 3 files changed, 185 insertions(+), 64 deletions(-) diff --git a/rhizome.c b/rhizome.c index 9cc189a2..96a53711 100644 --- a/rhizome.c +++ b/rhizome.c @@ -93,6 +93,140 @@ int rhizome_fetch_delay_ms() return config.rhizome.fetch_delay_ms; } +/* Create a manifest structure to accompany adding a file to Rhizome or appending to a journal. + * This function is used by all application-facing APIs (eg, CLI, RESTful HTTP). It differs from + * the import operation in that if the caller does not supply a complete, signed manifest then this + * operation will create it using the fields supplied. Also, the caller can supply a clear-text + * payload with the 'crypt=1' field to cause it to be stored encrypted. + * + * - 'm' must point to a manifest structure into which any supplied partial manifest has already + * been parsed. If the caller supplied no (partial) manifest at all, then the manifest 'm' will + * be blank. + * + * - 'bsk' must point to a supplied bundle secret parameter, or NULL if none was supplied. + * + * - if 'appending' is true then the new bundle will be a journal bundle, otherwise it will be a + * normal bundle. Any existing manifest must be consistent; eg, an append will fail if a bundle + * with the same Bundle Id already exists in the store and is not a journal. + * + * - 'author' must point to a supplied author parameter, or NULL if none was supplied. + * + * - 'file_path' can point to a supplied payload file name (eg, if the payload was read from a named + * file), or can be NULL. If not NULL, then the file's name will be used to fill in the 'name' + * field of the manifest if it was not explicitly supplied in 'm' or in the existing manifest. + * + * If the add is successful, modifies '*mout' to point to the constructed Manifest, which might be + * 'm' or might be another manifest, and returns NULL. It is the caller's responsibility to free + * '*mout'. + * + * If the add fails for any reason, returns a pointer to a nul-terminated text string that describes + * the reason, and does not alter '*mout'. + * + * @author Andrew Bettison + */ +const char * rhizome_bundle_add_file(int appending, + rhizome_manifest *m, + rhizome_manifest **mout, + const rhizome_bk_t *bsk, + const sid_t *author, + const char *file_path + ) +{ + const char *reason = NULL; + rhizome_manifest *existing_manifest = NULL; + rhizome_manifest *new_manifest = NULL; + assert(m != NULL); + // Caller must not supply a malformed manifest (but an invalid one is okay because missing + // fields will be filled in, so we don't check validity here). + assert(!m->malformed); + if (m->has_id) { + if (config.debug.rhizome) + DEBUGF("Reading manifest from database: id=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + if ((existing_manifest = rhizome_new_manifest()) == NULL) { + WHY(reason = "Manifest struct could not be allocated"); + goto error; + } + enum rhizome_bundle_status status = rhizome_retrieve_manifest(&m->cryptoSignPublic, existing_manifest); + switch (status) { + case RHIZOME_BUNDLE_STATUS_NEW: + // No manifest with that bundle ID exists in the store, so we are building a bundle from + // scratch. + rhizome_manifest_free(existing_manifest); + existing_manifest = NULL; + break; + case RHIZOME_BUNDLE_STATUS_SAME: + // Found a manifest with the same bundle ID. Overwrite it with the supplied manifest. + if (rhizome_manifest_overwrite(existing_manifest, m) == -1) { + WHY(reason = "Existing manifest could not be overwritten"); + goto error; + } + new_manifest = existing_manifest; + existing_manifest = NULL; + break; + case RHIZOME_BUNDLE_STATUS_BUSY: + WHY(reason = "Existing manifest not retrieved due to Rhizome store locking"); + goto error; + case RHIZOME_BUNDLE_STATUS_ERROR: + WHY(reason = "Error retrieving existing manifest from Rhizome store"); + goto error; + case RHIZOME_BUNDLE_STATUS_DUPLICATE: + case RHIZOME_BUNDLE_STATUS_OLD: + case RHIZOME_BUNDLE_STATUS_INVALID: + case RHIZOME_BUNDLE_STATUS_FAKE: + case RHIZOME_BUNDLE_STATUS_INCONSISTENT: + case RHIZOME_BUNDLE_STATUS_NO_ROOM: + case RHIZOME_BUNDLE_STATUS_READONLY: + FATALF("rhizome_retrieve_manifest() returned %s", rhizome_bundle_status_message(status)); + } + } + // If no existing bundle has been identified, we are building a bundle from scratch. + if (!new_manifest) { + new_manifest = m; + // A new journal manifest needs these fields set so that the first append can succeed. + if (appending) { + rhizome_manifest_set_filesize(new_manifest, 0); + rhizome_manifest_set_tail(new_manifest, 0); + } + } + if (appending && !m->is_journal){ + // TODO: return a special status code for this case + WHY(reason = "Cannot append to a non-journal"); + goto error; + } + if (!appending && m->is_journal) { + // TODO: return a special status code for this case + WHY(reason = "Cannot add a journal bundle (use append instead)"); + goto error; + } + if (bsk) { + if (new_manifest->has_id) { + if (!rhizome_apply_bundle_secret(new_manifest, bsk)) { + WHY(reason = "Supplied bundle secret does not match Bundle Id"); + goto error; + } + } else { + if (rhizome_new_bundle_from_secret(new_manifest, bsk) == -1) { + WHY(reason = "Failed to create bundle from given secret"); + goto error; + } + } + } + // TODO: one day there will be no default service, but for now if no service + // is specified, it defaults to 'file' (file sharing). + if (m->service == NULL) + rhizome_manifest_set_service(new_manifest, RHIZOME_SERVICE_FILE); + if ((reason = rhizome_fill_manifest(new_manifest, file_path, author ? author : NULL)) != NULL) + goto error; + *mout = new_manifest; + return NULL; +error: + if (new_manifest && new_manifest != m && new_manifest != existing_manifest) + rhizome_manifest_free(new_manifest); + if (existing_manifest) + rhizome_manifest_free(existing_manifest); + return reason; +} + /* Import a bundle from a pair of files, one containing the manifest and the optional other * containing the payload. The work is all done by rhizome_bundle_import() and * rhizome_store_manifest(). diff --git a/rhizome.h b/rhizome.h index 921ed19a..babe587b 100644 --- a/rhizome.h +++ b/rhizome.h @@ -426,6 +426,7 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc); int rhizome_store_manifest(rhizome_manifest *m); int rhizome_store_file(rhizome_manifest *m,const unsigned char *key); +const char * rhizome_bundle_add_file(int appending, rhizome_manifest *m, rhizome_manifest **mout, const rhizome_bk_t *bsk, const sid_t *author, const char *file_path); int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath); int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath); diff --git a/rhizome_cli.c b/rhizome_cli.c index 7921f22e..57fb05fc 100644 --- a/rhizome_cli.c +++ b/rhizome_cli.c @@ -112,27 +112,34 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont { if (config.debug.verbose) DEBUG_cli_parsed(parsed); - const char *filepath, *manifestpath, *manifestid, *authorSidHex, *bsktext; + const char *filepath, *manifestpath, *manifestIdHex, *authorSidHex, *bsktext; int force_new = 0 == cli_arg(parsed, "--force-new", NULL, NULL, NULL); cli_arg(parsed, "filepath", &filepath, NULL, ""); if (cli_arg(parsed, "author_sid", &authorSidHex, cli_optional_sid, "") == -1) return -1; cli_arg(parsed, "manifestpath", &manifestpath, NULL, ""); - cli_arg(parsed, "manifestid", &manifestid, NULL, ""); + cli_arg(parsed, "manifestid", &manifestIdHex, NULL, ""); if (cli_arg(parsed, "bsk", &bsktext, cli_optional_bundle_secret_key, NULL) == -1) return -1; sid_t authorSid; - if (authorSidHex[0] && str_to_sid_t(&authorSid, authorSidHex) == -1) + if (!authorSidHex || !*authorSidHex) + authorSidHex = NULL; + else if (str_to_sid_t(&authorSid, authorSidHex) == -1) return WHYF("invalid author_sid: %s", authorSidHex); - // treat empty string the same as null - if (bsktext && !*bsktext) - bsktext = NULL; + rhizome_bid_t bid; + if (!manifestIdHex || !*manifestIdHex) + manifestIdHex = NULL; + else if (str_to_rhizome_bid_t(&bid, manifestIdHex) == -1) + return WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestIdHex)); + rhizome_bk_t bsk; - if (bsktext && str_to_rhizome_bsk_t(&bsk, bsktext) == -1) - return WHYF("invalid bsk: \"%s\"", bsktext); + if (!bsktext || !*bsktext) + bsktext = NULL; + else if (str_to_rhizome_bsk_t(&bsk, bsktext) == -1) + return WHYF("invalid BSK: \"%s\"", bsktext); unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi; struct field { @@ -170,7 +177,7 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont } } - int journal = strcasecmp(parsed->args[1], "journal")==0; + int appending = strcasecmp(parsed->args[1], "journal")==0; if (create_serval_instance_dir() == -1) return -1; @@ -183,55 +190,25 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont if (rhizome_opendb() == -1) goto finish; - /* Create a new manifest that will represent the file. If a manifest file was supplied, then read - * it, otherwise create a blank manifest. */ + /* Create a manifest in memory that to accompany the added file. Initially the manifest is blank. + * If a manifest file is supplied, then read and parse it, barfing if it contains any duplicate + * fields or invalid values. If it successfully parses, then overwrite it with any command-line + * manifest field settings, overriding the values parsed from the file. Barf if any of these new + * values are malformed. We don't validate the resulting manifest, it order to allow the user to + * supply an incomplete manifest. Any missing fields will be filled in later. + */ if ((m = rhizome_new_manifest()) == NULL){ - ret = WHY("Manifest struct could not be allocated -- not added to rhizome"); + ret = WHY("Manifest struct could not be allocated -- not added"); goto finish; } if (manifestpath && *manifestpath && access(manifestpath, R_OK) == 0) { if (config.debug.rhizome) DEBUGF("reading manifest from %s", manifestpath); - /* Don't verify the manifest, because it will fail if it is incomplete. - This is okay, because we fill in any missing bits and sanity check before - trying to write it out. However, we do insist that whatever we load is - parsed okay and not malformed. */ if (rhizome_read_manifest_from_file(m, manifestpath) || m->malformed) { ret = WHY("Manifest file could not be loaded -- not added to rhizome"); goto finish; } - } else if (manifestid && *manifestid) { - if (config.debug.rhizome) - DEBUGF("Reading manifest from database"); - rhizome_bid_t bid; - if (str_to_rhizome_bid_t(&bid, manifestid) == -1) { - ret = WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestid)); - goto finish; - } - if (rhizome_retrieve_manifest(&bid, m) != RHIZOME_BUNDLE_STATUS_SAME) { - ret = WHY("Existing manifest could not be loaded -- not added to rhizome"); - goto finish; - } - } else { - if (config.debug.rhizome) - DEBUGF("Creating new manifest"); - if (journal) { - rhizome_manifest_set_filesize(m, 0); - rhizome_manifest_set_tail(m, 0); - } } - - if (journal && !m->is_journal){ - // TODO: return a special status code for this case - ret = WHY("Existing manifest is not a journal"); - goto finish; - } - if (!journal && m->is_journal) { - // TODO: return a special status code for this case - ret = WHY("Existing manifest is a journal"); - goto finish; - } - if (nfields) { unsigned i; for (i = 0; i != nfields; ++i) { @@ -269,26 +246,36 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont } } - if (bsktext) { - if (m->has_id) { - if (!rhizome_apply_bundle_secret(m, &bsk)) { - ret = WHY("Supplied bundle secret does not match Bundle Id"); - goto finish; - } - } else { - if (rhizome_new_bundle_from_secret(m, &bsk) == -1) { - ret = WHY("Failed to create bundle from given secret"); - goto finish; - } + /* If a manifest ID (bundle ID) was supplied on the command line, first ensure it does not + * contradict any manifest ID present in the supplied manifest file, then insert it into the + * manifest. + */ + if (manifestIdHex) { + if (!m->has_id) + rhizome_manifest_set_id(m, &bid); + else if (cmp_rhizome_bid_t(&m->cryptoSignPublic, &bid) != 0) { + ret = WHYF("manifestid=%s does not match manifest id=%s", manifestIdHex, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic)); + goto finish; } } - if (m->service == NULL) - rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE); - if (rhizome_fill_manifest(m, filepath, *authorSidHex ? &authorSid : NULL)) - goto finish; + /* Create an in-memory manifest for the file being added. + */ + rhizome_manifest *mout = NULL; + if (rhizome_bundle_add_file(appending, m, &mout, bsktext ? &bsk : NULL, authorSidHex ? &authorSid : NULL, filepath) != NULL) { + ret = WHY("Cannot create manifest -- not added"); + goto finish; + } + assert(mout != NULL); + if (mout != m) { + rhizome_manifest_free(m); + m = mout; + } + mout = NULL; + + // Insert the payload into the Rhizome store. enum rhizome_payload_status pstatus; - if (journal){ + if (appending) { pstatus = rhizome_append_journal_file(m, 0, filepath); if (config.debug.rhizome) DEBUGF("rhizome_append_journal_file() returned %d %s", pstatus, rhizome_payload_status_message(pstatus)); @@ -335,7 +322,6 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont } if (!pstatus_valid) FATALF("pstatus = %d", pstatus); - rhizome_manifest *mout = NULL; if (status == RHIZOME_BUNDLE_STATUS_NEW) { if (!rhizome_manifest_validate(m) || m->malformed) status = RHIZOME_BUNDLE_STATUS_INVALID;