diff --git a/httpd.h b/httpd.h index 306287c8..0ffac1e2 100644 --- a/httpd.h +++ b/httpd.h @@ -108,16 +108,8 @@ typedef struct httpd_request /* For receiving RESTful Rhizome insert request */ struct { - // If this is really a (journal) append request - bool_t appending; // Which part is currently being received const char *current_part; - // Which parts have already been received - bool_t received_author; - bool_t received_secret; - bool_t received_bundleid; - bool_t received_manifest; - bool_t received_payload; // For storing the "bundle-author" hex SID as we receive it char author_hex[SID_STRLEN]; size_t author_hex_len; @@ -132,12 +124,23 @@ typedef struct httpd_request // The "force-new" parameter char force_new_text[5]; // enough for "false" size_t force_new_text_len; - bool_t force_new; + // For storing the manifest text (malloc/realloc) as we receive it struct form_buf_malloc manifest; // For receiving the payload uint64_t payload_size; struct rhizome_write write; + + // If this is really a (journal) append request + bool_t appending:1; + bool_t importing:1; + // Which parts have already been received + bool_t received_author:1; + bool_t received_secret:1; + bool_t received_bundleid:1; + bool_t received_manifest:1; + bool_t received_payload:1; + bool_t force_new:1; } insert; diff --git a/rhizome.c b/rhizome.c index d5874581..8b988a15 100644 --- a/rhizome.c +++ b/rhizome.c @@ -475,6 +475,12 @@ enum rhizome_bundle_status rhizome_bundle_import_files(rhizome_manifest *m, rhiz if (ret != RHIZOME_BUNDLE_STATUS_NEW) goto end; + if (mout && *mout){ + if (m != *mout) + rhizome_manifest_free(*mout); + *mout = NULL; + } + enum rhizome_payload_status pstatus = RHIZOME_PAYLOAD_STATUS_EMPTY; if (m->filesize > 0){ diff --git a/rhizome_database.c b/rhizome_database.c index 75a30506..8386b62a 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -1428,6 +1428,12 @@ enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m, rh if (status != RHIZOME_BUNDLE_STATUS_NEW) return status; + if (mout && *mout){ + if (m != *mout) + rhizome_manifest_free(*mout); + *mout = NULL; + } + // manifest is complete, and not already stored /* Bind BAR to data field */ @@ -1509,6 +1515,8 @@ enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m, rh }else{ sync_rhizome(); } + if (mout) + *mout = m; return RHIZOME_BUNDLE_STATUS_NEW; } diff --git a/rhizome_restful.c b/rhizome_restful.c index 1b2665db..902f5b56 100644 --- a/rhizome_restful.c +++ b/rhizome_restful.c @@ -29,6 +29,7 @@ DEFINE_FEATURE(http_rest_rhizome); DECLARE_HANDLER("/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json); DECLARE_HANDLER("/restful/rhizome/newsince/", restful_rhizome_newsince); DECLARE_HANDLER("/restful/rhizome/insert", restful_rhizome_insert); +DECLARE_HANDLER("/restful/rhizome/import", restful_rhizome_import); DECLARE_HANDLER("/restful/rhizome/append", restful_rhizome_append); DECLARE_HANDLER("/restful/rhizome/", restful_rhizome_); @@ -446,6 +447,12 @@ static int restful_rhizome_insert(httpd_request *r, const char *remainder) return 1; } +static int restful_rhizome_import(httpd_request *r, const char *remainder) +{ + r->u.insert.importing = 1; + return restful_rhizome_insert(r, remainder); +} + static int restful_rhizome_append(httpd_request *r, const char *remainder) { r->u.insert.appending = 1; @@ -474,18 +481,36 @@ static int insert_make_manifest(httpd_request *r) assert(r->u.insert.manifest.length <= sizeof r->manifest->manifestdata); memcpy(r->manifest->manifestdata, r->u.insert.manifest.buffer, r->u.insert.manifest.length); r->manifest->manifest_all_bytes = r->u.insert.manifest.length; - rhizome_manifest *mout = NULL; int n = rhizome_manifest_parse(r->manifest); switch (n) { case 0: + if (r->u.insert.importing){ + const char *invalid_reason; + if ((invalid_reason = rhizome_manifest_validate_reason(r->manifest))){ + r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, invalid_reason); + break; + } + if (!rhizome_manifest_verify(r->manifest)){ + r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, "Invalid manifest: Invalid signature"); + break; + } + r->bundle_result.status = rhizome_manifest_check_stored(r->manifest, NULL); + break; + } + if (r->manifest->malformed) { r->bundle_result = rhizome_bundle_result_sprintf(RHIZOME_BUNDLE_STATUS_INVALID, "Malformed manifest: %s", r->manifest->malformed); } else { + rhizome_manifest *mout = NULL; r->bundle_result = rhizome_manifest_add_file(r->u.insert.appending, r->manifest, &mout, r->u.insert.received_bundleid ? &r->bid : NULL, r->u.insert.received_secret ? &r->u.insert.bundle_secret : NULL, r->u.insert.received_author ? &r->u.insert.author : NULL, NULL, 0, NULL); + if (mout && mout != r->manifest){ + rhizome_manifest_free(r->manifest); + r->manifest = mout; + } } break; case 1: @@ -498,13 +523,14 @@ static int insert_make_manifest(httpd_request *r) r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_ERROR, "Error while parsing manifest"); break; } + switch (r->bundle_result.status) { case RHIZOME_BUNDLE_STATUS_NEW: + break; case RHIZOME_BUNDLE_STATUS_OLD: case RHIZOME_BUNDLE_STATUS_SAME: case RHIZOME_BUNDLE_STATUS_DUPLICATE: case RHIZOME_BUNDLE_STATUS_NO_ROOM: - break; case RHIZOME_BUNDLE_STATUS_INCONSISTENT: case RHIZOME_BUNDLE_STATUS_INVALID: case RHIZOME_BUNDLE_STATUS_BUSY: @@ -514,11 +540,6 @@ static int insert_make_manifest(httpd_request *r) case RHIZOME_BUNDLE_STATUS_ERROR: return http_request_rhizome_response(r, 0, NULL); } - assert(mout != NULL); - if (mout != r->manifest) { - rhizome_manifest_free(r->manifest); - r->manifest = mout; - } assert(r->manifest != NULL); return 0; } @@ -534,7 +555,7 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa if (r->u.insert.received_author) return http_response_form_part(r, 400, "Duplicate", PART_AUTHOR, NULL, 0); // Reject a request if this parameter comes after the manifest part. - if (r->u.insert.received_manifest) + if (r->u.insert.received_manifest || r->u.insert.importing) return http_response_form_part(r, 400, "Spurious", PART_AUTHOR, NULL, 0); // TODO enforce correct content type r->u.insert.current_part = PART_AUTHOR; @@ -544,7 +565,7 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa if (r->u.insert.received_secret) return http_response_form_part(r, 400, "Duplicate", PART_SECRET, NULL, 0); // Reject a request if this parameter comes after the manifest part. - if (r->u.insert.received_manifest) + if (r->u.insert.received_manifest || r->u.insert.importing) return http_response_form_part(r, 400, "Spurious", PART_SECRET, NULL, 0); // TODO enforce correct content type r->u.insert.current_part = PART_SECRET; @@ -554,7 +575,7 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa if (r->u.insert.received_bundleid) return http_response_form_part(r, 400, "Duplicate", PART_BUNDLEID, NULL, 0); // Reject a request if this parameter comes after the manifest part. - if (r->u.insert.received_manifest) + if (r->u.insert.received_manifest || r->u.insert.importing) return http_response_form_part(r, 400, "Spurious", PART_BUNDLEID, NULL, 0); // TODO enforce correct content type r->u.insert.current_part = PART_BUNDLEID; @@ -582,37 +603,34 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa if (!r->u.insert.received_manifest) return http_response_form_part(r, 400, "Missing", PART_MANIFEST, NULL, 0); assert(r->manifest != NULL); + if (r->u.insert.importing && (r->manifest->filesize == 0 || !r->manifest->has_filehash)) + return http_response_form_part(r, 400, "Spurious", PART_PAYLOAD, NULL, 0); + // TODO enforce correct content type r->u.insert.current_part = PART_PAYLOAD; - // If the manifest does not contain a 'name' field, then assign it from the payload filename. - if ( strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0 - && r->manifest->name == NULL - && *h->content_disposition.filename - ) - rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename); - // Start writing the payload content into the Rhizome store. - if (r->u.insert.appending) { - r->payload_status = rhizome_write_open_journal(&r->u.insert.write, r->manifest, 0, RHIZOME_SIZE_UNSET); - if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) { - WHYF("rhizome_write_open_journal() returned %d %s", r->payload_status, rhizome_payload_status_message(r->payload_status)); - return http_request_rhizome_response(r, 500, "Error in payload open for write (journal)"); - } - } else { - // Note: r->manifest->filesize can be RHIZOME_SIZE_UNSET at this point, if the manifest did - // not contain a 'filesize' field. - r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest); - if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) { - WHYF("rhizome_write_open_manifest() returned %d %s", r->payload_status, rhizome_payload_status_message(r->payload_status)); - return http_request_rhizome_response(r, 500, "Error in payload open for write"); + + if (r->u.insert.importing){ + r->payload_status = rhizome_open_write(&r->u.insert.write, &r->manifest->filehash, r->manifest->filesize); + }else{ + // If the manifest does not contain a 'name' field, then assign it from the payload filename. + if (strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0 + && r->manifest->name == NULL + && *h->content_disposition.filename + ) + rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename); + // Start writing the payload content into the Rhizome store. + if (r->u.insert.appending) { + r->payload_status = rhizome_write_open_journal(&r->u.insert.write, r->manifest, 0, RHIZOME_SIZE_UNSET); + } else { + // Note: r->manifest->filesize can be RHIZOME_SIZE_UNSET at this point, if the manifest did + // not contain a 'filesize' field. + r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest); } } - switch (r->payload_status) { - case RHIZOME_PAYLOAD_STATUS_STORED: - // TODO: initialise payload hash so it can be compared with stored payload - break; - default: - break; // r->payload_status gets dealt with later - } + // complete early so the client doesn't have to send the payload + if (r->payload_status != RHIZOME_PAYLOAD_STATUS_NEW) + return http_request_rhizome_response(r, 0, NULL); + r->u.insert.payload_size = 0; } else @@ -772,23 +790,28 @@ static int restful_rhizome_insert_end(struct http_request *hr) } if (!status_valid) FATALF("rhizome_finish_store() returned status = %d", r->payload_status); - // Finalise the manifest and add it to the store. - const char *invalid_reason = rhizome_manifest_validate_reason(r->manifest); - if (invalid_reason) { - r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, invalid_reason); - return http_request_rhizome_response(r, 0, NULL); - } - if (r->manifest->malformed) { - r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, r->manifest->malformed); - return http_request_rhizome_response(r, 0, NULL); - } - if (!r->manifest->haveSecret) { - r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_READONLY, "Missing bundle secret"); - return http_request_rhizome_response(r, 0, NULL); - } + rhizome_manifest *mout = NULL; - rhizome_bundle_result_free(&r->bundle_result); - r->bundle_result = rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new); + if (r->u.insert.importing){ + r->bundle_result = rhizome_bundle_result(rhizome_add_manifest_to_store(r->manifest, &mout)); + }else{ + // Finalise the manifest and add it to the store. + const char *invalid_reason = rhizome_manifest_validate_reason(r->manifest); + if (invalid_reason) { + r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, invalid_reason); + return http_request_rhizome_response(r, 0, NULL); + } + if (r->manifest->malformed) { + r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, r->manifest->malformed); + return http_request_rhizome_response(r, 0, NULL); + } + if (!r->manifest->haveSecret) { + r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_READONLY, "Missing bundle secret"); + return http_request_rhizome_response(r, 0, NULL); + } + rhizome_bundle_result_free(&r->bundle_result); + r->bundle_result = rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new); + } int http_status = 500; switch (r->bundle_result.status) { case RHIZOME_BUNDLE_STATUS_NEW: @@ -796,19 +819,22 @@ static int restful_rhizome_insert_end(struct http_request *hr) rhizome_manifest_free(mout); http_status = 201; break; - case RHIZOME_BUNDLE_STATUS_SAME: case RHIZOME_BUNDLE_STATUS_OLD: + if (mout && mout != r->manifest) { + rhizome_manifest_free(r->manifest); + r->manifest = mout; + } + http_status = 202; + break; + case RHIZOME_BUNDLE_STATUS_SAME: case RHIZOME_BUNDLE_STATUS_DUPLICATE: if (mout && mout != r->manifest) { rhizome_manifest_free(r->manifest); r->manifest = mout; } - http_status = 201; + http_status = 200; break; case RHIZOME_BUNDLE_STATUS_ERROR: - rhizome_bundle_result_free(&r->bundle_result); - r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_ERROR, "Error in manifest finalise"); - // fall through case RHIZOME_BUNDLE_STATUS_INVALID: case RHIZOME_BUNDLE_STATUS_FAKE: case RHIZOME_BUNDLE_STATUS_INCONSISTENT: @@ -824,10 +850,14 @@ static int restful_rhizome_insert_end(struct http_request *hr) } if (http_status == 500) FATALF("rhizome_manifest_finalise() returned status=%d", r->bundle_result.status); - rhizome_authenticate_author(r->manifest); - http_request_response_static(&r->http, http_status, "rhizome-manifest/text", - (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes - ); + if (r->u.insert.importing){ + return http_request_rhizome_response(r, http_status, NULL); + }else{ + rhizome_authenticate_author(r->manifest); + http_request_response_static(&r->http, http_status, "rhizome-manifest/text", + (const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes + ); + } return 0; } diff --git a/rhizome_store.c b/rhizome_store.c index 8e9c33e1..7cf0e77b 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -787,6 +787,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write) if (write->id_known) { if (cmp_rhizome_filehash_t(&write->id, &hash_out) != 0) { WARNF("expected filehash=%s, got %s", alloca_tohex_rhizome_filehash_t(write->id), alloca_tohex_rhizome_filehash_t(hash_out)); + write->id = hash_out; status = RHIZOME_PAYLOAD_STATUS_WRONG_HASH; goto failure; } diff --git a/tests/rhizomerestful b/tests/rhizomerestful index 2c81a1b2..8720e6cf 100755 --- a/tests/rhizomerestful +++ b/tests/rhizomerestful @@ -1259,6 +1259,44 @@ test_RhizomeInsertIncorrectFilehash() { assert_rhizome_list } +doc_RhizomeImport="HTTP RESTful import Rhizome bundle" +setup_RhizomeImport() { + setup + set_instance +B + create_single_identity + rhizome_add_bundles $SIDB 1 1 + executeOk_servald rhizome export manifest "${BID[1]}" file1.manifest + set_instance +A +} +test_RhizomeImport() { + execute curl \ + -H "Expect:" \ + --silent --show-error --write-out '%{http_code}' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + --form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \ + --form "payload=@file1" \ + "http://$addr_localhost:$PORTA/restful/rhizome/import" + tfw_cat http.header http.body + assertStdoutIs 201 + assertJq http.body 'contains({"http_status_code": 201})' + assertJq http.body 'contains({"http_status_message": "Created"})' + execute curl \ + -H "Expect:" \ + --silent --show-error --write-out '%{http_code}' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + --form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \ + --form "payload=@file1" \ + "http://$addr_localhost:$PORTA/restful/rhizome/import" + tfw_cat http.header http.body + assertStdoutIs 200 + assertJq http.body 'contains({"http_status_code": 200})' + assertJq http.body 'contains({"http_status_message": "Bundle already in store"})' +} + doc_RhizomeJournalAppend="HTTP RESTful Rhizome journal create and append" setup_RhizomeJournalAppend() { setup