ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add restful import API

Browse Source
This commit is contained in:
Jeremy Lakeman 2017-05-17 10:16:46 +09:30
parent 9b7d8bfa23
commit a473304c06
6 changed files with 157 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
httpd.h
View File

@ -108,16 +108,8 @@ typedef struct httpd_request
/* For receiving RESTful Rhizome insert request
*/
struct {
// If this is really a (journal) append request
bool_t appending;
// Which part is currently being received
const char *current_part;
// Which parts have already been received
bool_t received_author;
bool_t received_secret;
bool_t received_bundleid;
bool_t received_manifest;
bool_t received_payload;
// For storing the "bundle-author" hex SID as we receive it
char author_hex[SID_STRLEN];
size_t author_hex_len;
@ -132,12 +124,23 @@ typedef struct httpd_request
// The "force-new" parameter
char force_new_text[5]; // enough for "false"
size_t force_new_text_len;
bool_t force_new;
// For storing the manifest text (malloc/realloc) as we receive it
struct form_buf_malloc manifest;
// For receiving the payload
uint64_t payload_size;
struct rhizome_write write;
// If this is really a (journal) append request
bool_t appending:1;
bool_t importing:1;
// Which parts have already been received
bool_t received_author:1;
bool_t received_secret:1;
bool_t received_bundleid:1;
bool_t received_manifest:1;
bool_t received_payload:1;
bool_t force_new:1;
}
insert;

6
rhizome.c
View File

@ -475,6 +475,12 @@ enum rhizome_bundle_status rhizome_bundle_import_files(rhizome_manifest *m, rhiz
if (ret != RHIZOME_BUNDLE_STATUS_NEW)
goto end;
if (mout && *mout){
if (m != *mout)
rhizome_manifest_free(*mout);
*mout = NULL;
}
enum rhizome_payload_status pstatus = RHIZOME_PAYLOAD_STATUS_EMPTY;
if (m->filesize > 0){

8
rhizome_database.c
View File

@ -1428,6 +1428,12 @@ enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m, rh
if (status != RHIZOME_BUNDLE_STATUS_NEW)
return status;
if (mout && *mout){
if (m != *mout)
rhizome_manifest_free(*mout);
*mout = NULL;
}
// manifest is complete, and not already stored
/* Bind BAR to data field */
@ -1509,6 +1515,8 @@ enum rhizome_bundle_status rhizome_add_manifest_to_store(rhizome_manifest *m, rh
}else{
sync_rhizome();
}
if (mout)
*mout = m;
return RHIZOME_BUNDLE_STATUS_NEW;
}

154
rhizome_restful.c
View File

@ -29,6 +29,7 @@ DEFINE_FEATURE(http_rest_rhizome);
DECLARE_HANDLER("/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json);
DECLARE_HANDLER("/restful/rhizome/newsince/", restful_rhizome_newsince);
DECLARE_HANDLER("/restful/rhizome/insert", restful_rhizome_insert);
DECLARE_HANDLER("/restful/rhizome/import", restful_rhizome_import);
DECLARE_HANDLER("/restful/rhizome/append", restful_rhizome_append);
DECLARE_HANDLER("/restful/rhizome/", restful_rhizome_);
@ -446,6 +447,12 @@ static int restful_rhizome_insert(httpd_request *r, const char *remainder)
return 1;
}
static int restful_rhizome_import(httpd_request *r, const char *remainder)
{
r->u.insert.importing = 1;
return restful_rhizome_insert(r, remainder);
}
static int restful_rhizome_append(httpd_request *r, const char *remainder)
{
r->u.insert.appending = 1;
@ -474,18 +481,36 @@ static int insert_make_manifest(httpd_request *r)
assert(r->u.insert.manifest.length <= sizeof r->manifest->manifestdata);
memcpy(r->manifest->manifestdata, r->u.insert.manifest.buffer, r->u.insert.manifest.length);
r->manifest->manifest_all_bytes = r->u.insert.manifest.length;
rhizome_manifest *mout = NULL;
int n = rhizome_manifest_parse(r->manifest);
switch (n) {
case 0:
if (r->u.insert.importing){
const char *invalid_reason;
if ((invalid_reason = rhizome_manifest_validate_reason(r->manifest))){
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, invalid_reason);
break;
}
if (!rhizome_manifest_verify(r->manifest)){
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, "Invalid manifest: Invalid signature");
break;
}
r->bundle_result.status = rhizome_manifest_check_stored(r->manifest, NULL);
break;
}
if (r->manifest->malformed) {
r->bundle_result = rhizome_bundle_result_sprintf(RHIZOME_BUNDLE_STATUS_INVALID, "Malformed manifest: %s", r->manifest->malformed);
} else {
rhizome_manifest *mout = NULL;
r->bundle_result = rhizome_manifest_add_file(r->u.insert.appending, r->manifest, &mout,
r->u.insert.received_bundleid ? &r->bid : NULL,
r->u.insert.received_secret ? &r->u.insert.bundle_secret : NULL,
r->u.insert.received_author ? &r->u.insert.author : NULL,
NULL, 0, NULL);
if (mout && mout != r->manifest){
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
}
break;
case 1:
@ -498,13 +523,14 @@ static int insert_make_manifest(httpd_request *r)
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_ERROR, "Error while parsing manifest");
break;
}
switch (r->bundle_result.status) {
case RHIZOME_BUNDLE_STATUS_NEW:
break;
case RHIZOME_BUNDLE_STATUS_OLD:
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_NO_ROOM:
break;
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_BUSY:
@ -514,11 +540,6 @@ static int insert_make_manifest(httpd_request *r)
case RHIZOME_BUNDLE_STATUS_ERROR:
return http_request_rhizome_response(r, 0, NULL);
}
assert(mout != NULL);
if (mout != r->manifest) {
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
assert(r->manifest != NULL);
return 0;
}
@ -534,7 +555,7 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
if (r->u.insert.received_author)
return http_response_form_part(r, 400, "Duplicate", PART_AUTHOR, NULL, 0);
// Reject a request if this parameter comes after the manifest part.
if (r->u.insert.received_manifest)
if (r->u.insert.received_manifest || r->u.insert.importing)
return http_response_form_part(r, 400, "Spurious", PART_AUTHOR, NULL, 0);
// TODO enforce correct content type
r->u.insert.current_part = PART_AUTHOR;
@ -544,7 +565,7 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
if (r->u.insert.received_secret)
return http_response_form_part(r, 400, "Duplicate", PART_SECRET, NULL, 0);
// Reject a request if this parameter comes after the manifest part.
if (r->u.insert.received_manifest)
if (r->u.insert.received_manifest || r->u.insert.importing)
return http_response_form_part(r, 400, "Spurious", PART_SECRET, NULL, 0);
// TODO enforce correct content type
r->u.insert.current_part = PART_SECRET;
@ -554,7 +575,7 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
if (r->u.insert.received_bundleid)
return http_response_form_part(r, 400, "Duplicate", PART_BUNDLEID, NULL, 0);
// Reject a request if this parameter comes after the manifest part.
if (r->u.insert.received_manifest)
if (r->u.insert.received_manifest || r->u.insert.importing)
return http_response_form_part(r, 400, "Spurious", PART_BUNDLEID, NULL, 0);
// TODO enforce correct content type
r->u.insert.current_part = PART_BUNDLEID;
@ -582,37 +603,34 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
if (!r->u.insert.received_manifest)
return http_response_form_part(r, 400, "Missing", PART_MANIFEST, NULL, 0);
assert(r->manifest != NULL);
if (r->u.insert.importing && (r->manifest->filesize == 0 || !r->manifest->has_filehash))
return http_response_form_part(r, 400, "Spurious", PART_PAYLOAD, NULL, 0);
// TODO enforce correct content type
r->u.insert.current_part = PART_PAYLOAD;
// If the manifest does not contain a 'name' field, then assign it from the payload filename.
if ( strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0
&& r->manifest->name == NULL
&& *h->content_disposition.filename
)
rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename);
// Start writing the payload content into the Rhizome store.
if (r->u.insert.appending) {
r->payload_status = rhizome_write_open_journal(&r->u.insert.write, r->manifest, 0, RHIZOME_SIZE_UNSET);
if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
WHYF("rhizome_write_open_journal() returned %d %s", r->payload_status, rhizome_payload_status_message(r->payload_status));
return http_request_rhizome_response(r, 500, "Error in payload open for write (journal)");
}
} else {
// Note: r->manifest->filesize can be RHIZOME_SIZE_UNSET at this point, if the manifest did
// not contain a 'filesize' field.
r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
WHYF("rhizome_write_open_manifest() returned %d %s", r->payload_status, rhizome_payload_status_message(r->payload_status));
return http_request_rhizome_response(r, 500, "Error in payload open for write");
if (r->u.insert.importing){
r->payload_status = rhizome_open_write(&r->u.insert.write, &r->manifest->filehash, r->manifest->filesize);
}else{
// If the manifest does not contain a 'name' field, then assign it from the payload filename.
if (strcasecmp(RHIZOME_SERVICE_FILE, r->manifest->service) == 0
&& r->manifest->name == NULL
&& *h->content_disposition.filename
)
rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename);
// Start writing the payload content into the Rhizome store.
if (r->u.insert.appending) {
r->payload_status = rhizome_write_open_journal(&r->u.insert.write, r->manifest, 0, RHIZOME_SIZE_UNSET);
} else {
// Note: r->manifest->filesize can be RHIZOME_SIZE_UNSET at this point, if the manifest did
// not contain a 'filesize' field.
r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
}
}
switch (r->payload_status) {
case RHIZOME_PAYLOAD_STATUS_STORED:
// TODO: initialise payload hash so it can be compared with stored payload
break;
default:
break; // r->payload_status gets dealt with later
}
// complete early so the client doesn't have to send the payload
if (r->payload_status != RHIZOME_PAYLOAD_STATUS_NEW)
return http_request_rhizome_response(r, 0, NULL);
r->u.insert.payload_size = 0;
}
else
@ -772,23 +790,28 @@ static int restful_rhizome_insert_end(struct http_request *hr)
}
if (!status_valid)
FATALF("rhizome_finish_store() returned status = %d", r->payload_status);
// Finalise the manifest and add it to the store.
const char *invalid_reason = rhizome_manifest_validate_reason(r->manifest);
if (invalid_reason) {
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, invalid_reason);
return http_request_rhizome_response(r, 0, NULL);
}
if (r->manifest->malformed) {
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, r->manifest->malformed);
return http_request_rhizome_response(r, 0, NULL);
}
if (!r->manifest->haveSecret) {
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_READONLY, "Missing bundle secret");
return http_request_rhizome_response(r, 0, NULL);
}
rhizome_manifest *mout = NULL;
rhizome_bundle_result_free(&r->bundle_result);
r->bundle_result = rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new);
if (r->u.insert.importing){
r->bundle_result = rhizome_bundle_result(rhizome_add_manifest_to_store(r->manifest, &mout));
}else{
// Finalise the manifest and add it to the store.
const char *invalid_reason = rhizome_manifest_validate_reason(r->manifest);
if (invalid_reason) {
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, invalid_reason);
return http_request_rhizome_response(r, 0, NULL);
}
if (r->manifest->malformed) {
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_INVALID, r->manifest->malformed);
return http_request_rhizome_response(r, 0, NULL);
}
if (!r->manifest->haveSecret) {
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_READONLY, "Missing bundle secret");
return http_request_rhizome_response(r, 0, NULL);
}
rhizome_bundle_result_free(&r->bundle_result);
r->bundle_result = rhizome_manifest_finalise(r->manifest, &mout, !r->u.insert.force_new);
}
int http_status = 500;
switch (r->bundle_result.status) {
case RHIZOME_BUNDLE_STATUS_NEW:
@ -796,19 +819,22 @@ static int restful_rhizome_insert_end(struct http_request *hr)
rhizome_manifest_free(mout);
http_status = 201;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_OLD:
if (mout && mout != r->manifest) {
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
http_status = 202;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
if (mout && mout != r->manifest) {
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
http_status = 201;
http_status = 200;
break;
case RHIZOME_BUNDLE_STATUS_ERROR:
rhizome_bundle_result_free(&r->bundle_result);
r->bundle_result = rhizome_bundle_result_static(RHIZOME_BUNDLE_STATUS_ERROR, "Error in manifest finalise");
// fall through
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_FAKE:
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
@ -824,10 +850,14 @@ static int restful_rhizome_insert_end(struct http_request *hr)
}
if (http_status == 500)
FATALF("rhizome_manifest_finalise() returned status=%d", r->bundle_result.status);
rhizome_authenticate_author(r->manifest);
http_request_response_static(&r->http, http_status, "rhizome-manifest/text",
(const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes
);
if (r->u.insert.importing){
return http_request_rhizome_response(r, http_status, NULL);
}else{
rhizome_authenticate_author(r->manifest);
http_request_response_static(&r->http, http_status, "rhizome-manifest/text",
(const char *)r->manifest->manifestdata, r->manifest->manifest_all_bytes
);
}
return 0;
}

1
rhizome_store.c
View File

@ -787,6 +787,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
if (write->id_known) {
if (cmp_rhizome_filehash_t(&write->id, &hash_out) != 0) {
WARNF("expected filehash=%s, got %s", alloca_tohex_rhizome_filehash_t(write->id), alloca_tohex_rhizome_filehash_t(hash_out));
write->id = hash_out;
status = RHIZOME_PAYLOAD_STATUS_WRONG_HASH;
goto failure;
}

38
tests/rhizomerestful
View File

@ -1259,6 +1259,44 @@ test_RhizomeInsertIncorrectFilehash() {
assert_rhizome_list
}
doc_RhizomeImport="HTTP RESTful import Rhizome bundle"
setup_RhizomeImport() {
setup
set_instance +B
create_single_identity
rhizome_add_bundles $SIDB 1 1
executeOk_servald rhizome export manifest "${BID[1]}" file1.manifest
set_instance +A
}
test_RhizomeImport() {
execute curl \
-H "Expect:" \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/import"
tfw_cat http.header http.body
assertStdoutIs 201
assertJq http.body 'contains({"http_status_code": 201})'
assertJq http.body 'contains({"http_status_message": "Created"})'
execute curl \
-H "Expect:" \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/import"
tfw_cat http.header http.body
assertStdoutIs 200
assertJq http.body 'contains({"http_status_code": 200})'
assertJq http.body 'contains({"http_status_message": "Bundle already in store"})'
}
doc_RhizomeJournalAppend="HTTP RESTful Rhizome journal create and append"
setup_RhizomeJournalAppend() {
setup