POST /restful/rhizome/append for journals

httpd.c

@@ -38,6 +38,7 @@ static HTTP_HANDLER static_page;
 HTTP_HANDLER restful_rhizome_bundlelist_json;
 HTTP_HANDLER restful_rhizome_newsince;
 HTTP_HANDLER restful_rhizome_insert;
+HTTP_HANDLER restful_rhizome_append;
 HTTP_HANDLER restful_rhizome_;
 HTTP_HANDLER restful_meshms_;
 HTTP_HANDLER restful_keyring_;
@@ -59,6 +60,7 @@ struct http_handler paths[]={
   {"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json},
   {"/restful/rhizome/newsince/", restful_rhizome_newsince},
   {"/restful/rhizome/insert", restful_rhizome_insert},
+  {"/restful/rhizome/append", restful_rhizome_append},
   {"/restful/rhizome/", restful_rhizome_},
   {"/restful/meshms/", restful_meshms_},
   {"/restful/keyring/", restful_keyring_},

httpd.h

@@ -104,6 +104,8 @@ typedef struct httpd_request
     /* For receiving RESTful Rhizome insert request
      */
     struct {
+      // If this is really a (journal) append request
+      bool_t is_append;
       // Which part is currently being received
       const char *current_part;
       // Which parts have already been received

rhizome.h

@@ -839,6 +839,7 @@ enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, cons
 int rhizome_write_buffer(struct rhizome_write *write_state, uint8_t *buffer, size_t data_size);
 int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, uint8_t *buffer, size_t data_size);
 enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
+enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t append_size);
 int rhizome_write_file(struct rhizome_write *write, const char *filename);
 void rhizome_fail_write(struct rhizome_write *write);
 enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write);

rhizome_restful.c

@@ -1,6 +1,6 @@
 /*
 Serval DNA Rhizome HTTP RESTful interface
-Copyright (C) 2013,2014 Serval Project Inc.
+Copyright (C) 2013-2015 Serval Project Inc.
  
 This program is free software; you can redistribute it and/or
 modify it under the terms of the GNU General Public License
@@ -355,6 +355,12 @@ int restful_rhizome_insert(httpd_request *r, const char *remainder)
   return 1;
 }
 
+int restful_rhizome_append(httpd_request *r, const char *remainder)
+{
+  r->u.insert.is_append = 1;
+  return restful_rhizome_insert(r, remainder);
+}
+
 static char PART_MANIFEST[] = "manifest";
 static char PART_PAYLOAD[] = "payload";
 static char PART_AUTHOR[] = "bundle-author";
@@ -445,20 +451,30 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
 	&& *h->content_disposition.filename
     )
       rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename);
-    // Start writing the payload content into the Rhizome store.  Note: r->manifest->filesize can be
-    // RHIZOME_SIZE_UNSET at this point, if the manifest did not contain a 'filesize' field.
+    // Start writing the payload content into the Rhizome store.
+    if (r->u.insert.is_append) {
+      r->payload_status = rhizome_write_open_journal(&r->u.insert.write, r->manifest, 0, RHIZOME_SIZE_UNSET);
+      if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
+	WHYF("rhizome_write_open_journal() returned %d", r->payload_status);
+	return 500;
+      }
+    } else {
+      // Note: r->manifest->filesize can be RHIZOME_SIZE_UNSET at this point, if the manifest did
+      // not contain a 'filesize' field.
       r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
-    r->u.insert.payload_size = 0;
-    switch (r->payload_status) {
-      case RHIZOME_PAYLOAD_STATUS_ERROR:
+      if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
 	WHYF("rhizome_write_open_manifest() returned %d", r->payload_status);
 	return 500;
+      }
+    }
+    switch (r->payload_status) {
       case RHIZOME_PAYLOAD_STATUS_STORED:
 	// TODO: initialise payload hash so it can be compared with stored payload
 	break;
       default:
 	break; // r->payload_status gets dealt with later
     }
+    r->u.insert.payload_size = 0;
   }
   else
     return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0);
@@ -548,9 +564,13 @@ static int insert_mime_part_end(struct http_request *hr)
       WHY("rhizome_fill_manifest() failed");
       return 500;
     }
-    if (r->manifest->is_journal)
-      return http_request_rhizome_response(r, 403, "Insert not supported for journals", NULL);
     assert(r->manifest != NULL);
+    if (r->u.insert.is_append) {
+      if (r->manifest->filesize == RHIZOME_SIZE_UNSET)
+	rhizome_manifest_set_filesize(r->manifest, 0);
+      if (!r->manifest->is_journal)
+	rhizome_manifest_set_tail(r->manifest, 0);
+    }
   }
   else if (r->u.insert.current_part == PART_PAYLOAD) {
     r->u.insert.received_payload = 1;
@@ -583,6 +603,10 @@ static int restful_rhizome_insert_end(struct http_request *hr)
     return http_response_form_part(r, "Missing", PART_PAYLOAD, NULL, 0);
   // Fill in the missing manifest fields and ensure payload and manifest are consistent.
   assert(r->manifest != NULL);
+  if (!r->u.insert.is_append && r->manifest->is_journal)
+    return http_request_rhizome_response(r, 403, "Insert not supported for journals", NULL);
+  else if (r->u.insert.is_append && !r->manifest->is_journal)
+    return http_request_rhizome_response(r, 403, "Append not supported for non-journals", NULL);
   assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET);
   int status_valid = 0;
   if (config.debug.rhizome)
@@ -629,6 +653,8 @@ static int restful_rhizome_insert_end(struct http_request *hr)
     return http_request_rhizome_response(r, 500, NULL, NULL);
   }
   // Finalise the manifest and add it to the store.
+  if (r->u.insert.is_append)
+    rhizome_manifest_set_version(r->manifest, r->manifest->filesize);
   if (r->manifest->filesize) {
     if (!r->manifest->has_filehash)
       rhizome_manifest_set_filehash(r->manifest, &r->u.insert.write.id);

rhizome_store.c

@@ -1588,18 +1588,20 @@ enum rhizome_payload_status rhizome_journal_pipe(struct rhizome_write *write, co
 }
 
 // open an existing journal bundle, advance the head pointer, duplicate the existing content and get ready to add more.
-enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t new_size)
+enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t append_size)
 {
-  assert(m->filesize != RHIZOME_SIZE_UNSET);
-  assert(m->filesize + new_size > 0);
   assert(m->is_journal);
+  assert(m->filesize != RHIZOME_SIZE_UNSET);
   assert(advance_by <= m->filesize);
   uint64_t copy_length = m->filesize - advance_by;
-  rhizome_manifest_set_filesize(m, m->filesize + new_size - advance_by);
+  uint64_t new_filesize = RHIZOME_SIZE_UNSET;
+  if (append_size != RHIZOME_SIZE_UNSET) {
+    assert(m->filesize + append_size > m->filesize); // no wraparound
+    new_filesize = m->filesize + append_size - advance_by;
+  }
   if (advance_by > 0)
     rhizome_manifest_set_tail(m, m->tail + advance_by);
-  rhizome_manifest_set_version(m, m->filesize);
-  enum rhizome_payload_status status = rhizome_open_write(write, NULL, m->filesize);
+  enum rhizome_payload_status status = rhizome_open_write(write, NULL, new_filesize);
   if (status == RHIZOME_PAYLOAD_STATUS_NEW && copy_length > 0) {
     // note that we don't need to bother decrypting the existing journal payload
     enum rhizome_payload_status rstatus = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length);
@@ -1613,6 +1615,15 @@ enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *wri
   return status;
 }
 
+// Call to finish any write started with rhizome_write_open_journal()
+static void rhizome_finish_journal(struct rhizome_write *write, rhizome_manifest *m)
+{
+  assert(m->is_journal);
+  rhizome_manifest_set_filesize(m, write->file_length);
+  rhizome_manifest_set_version(m, write->file_length);
+  rhizome_manifest_set_filehash(m, &write->id);
+}
+
 enum rhizome_payload_status rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, uint8_t *buffer, size_t len)
 {
   struct rhizome_write write;
@@ -1629,7 +1640,7 @@ enum rhizome_payload_status rhizome_append_journal_buffer(rhizome_manifest *m, u
     rhizome_fail_write(&write);
     return status;
   }
-  rhizome_manifest_set_filehash(m, &write.id);
+  rhizome_finish_journal(&write, m);
   return status;
 }
 
@@ -1652,6 +1663,6 @@ enum rhizome_payload_status rhizome_append_journal_file(rhizome_manifest *m, uin
     rhizome_fail_write(&write);
     return status;
   }
-  rhizome_manifest_set_filehash(m, &write.id);
+  rhizome_finish_journal(&write, m);
   return status;
 }

tests/rhizomeops

@@ -2,7 +2,7 @@
 
 # Tests for Serval rhizome command-line operations.
 #
-# Copyright 2012-2014 Serval Project, Inc.
+# Copyright 2012-2015 Serval Project, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License

tests/rhizomerestful

@@ -2,7 +2,7 @@
 
 # Tests for Serval DNA HTTP Rhizome RESTful interface
 #
-# Copyright 2013 Serval Project, Inc.
+# Copyright 2013-2015 Serval Project, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -27,7 +27,7 @@ shopt -s extglob
 
 setup() {
    CR='
'
-   VT='	'
+   HT='	'
    setup_curl 7
    setup_json
    setup_servald
@@ -539,7 +539,7 @@ extract_http_header() {
    local __headerfile="$2"
    local __header="$3"
    local __rexp="$4"
-   local __value=$($SED -n -e "/^$__header:[ $VT]*$__rexp$CR\$/s/^$__header:[ $VT]*\(.*\)$CR\$/\1/p" "$__headerfile")
+   local __value=$($SED -n -e "/^$__header:[ $HT]*$__rexp$CR\$/s/^$__header:[ $HT]*\(.*\)$CR\$/\1/p" "$__headerfile")
    assert --message="$__headerfile contains valid '$__header' header" \
           --dump-on-fail="$__headerfile" \
           [ -n "$__value" ]
@@ -1162,4 +1162,64 @@ test_RhizomeInsertIncorrectFilehash() {
    assert_rhizome_list
 }
 
+doc_RhizomeAppendJournalCreate="HTTP RESTful append creates new Rhizome journal "
+setup_RhizomeAppendJournalCreate() {
+   setup
+   echo 'File one' >file1
+   file1_size=$(cat file1 | wc -c)
+   >manifest1
+   echo "service=anything" >>manifest1
+   echo "filesize=$file1_size" >>manifest1
+   echo "name=hoopla" >>manifest1
+}
+test_RhizomeAppendJournalCreate() {
+   execute curl \
+         --silent --show-error --write-out '%{http_code}' \
+         --output file1.manifest \
+         --dump-header http.header \
+         --basic --user harry:potter \
+         --form "bundle-author=$SIDA" \
+         --form "manifest=@manifest1;type=rhizome/manifest;format=\"text+binarysig\"" \
+         --form "payload=@file1" \
+         "http://$addr_localhost:$PORTA/restful/rhizome/append"
+   tfw_cat http.header file1.manifest
+   assertExitStatus == 0
+   assertStdoutIs 201
+   extract_http_header H_BID http.header Serval-Rhizome-Bundle-Id "$rexp_manifestid"
+   extract_http_header H_VERSION http.header Serval-Rhizome-Bundle-Version "$rexp_version"
+   extract_http_header H_SIZE http.header Serval-Rhizome-Bundle-Filesize "$rexp_filesize"
+   extract_http_header H_HASH http.header Serval-Rhizome-Bundle-Filehash "$rexp_filehash"
+   extract_http_header H_DATE http.header Serval-Rhizome-Bundle-Date "$rexp_date"
+   extract_http_header H_ROWID http.header Serval-Rhizome-Bundle-Rowid "$rexp_rowid"
+   extract_http_header H_INSERTTIME http.header Serval-Rhizome-Bundle-Inserttime "$rexp_date"
+   extract_http_header H_SECRET http.header Serval-Rhizome-Bundle-Secret "$rexp_bundlesecret"
+   extract_http_header H_SERVICE http.header Serval-Rhizome-Bundle-Service ".*"
+   extract_http_header H_NAME http.header Serval-Rhizome-Bundle-Name ".*"
+   http_unquote_string H_NAME
+   extract_http_header H_BK http.header$n Serval-Rhizome-Bundle-BK "$rexp_bundlekey"
+   extract_http_header H_AUTHOR http.header$n Serval-Rhizome-Bundle-Author "$rexp_bundlekey"
+   assert [ $H_SIZE -eq $file1_size ]
+   assert [ "$H_SERVICE" = anything ]
+   assert [ "$H_NAME" = hoopla ]
+   assert [ "$H_AUTHOR" = $SIDA ]
+   extract_manifest_id BID file1.manifest
+   extract_manifest_version VERSION file1.manifest
+   extract_manifest_filesize SIZE file1.manifest
+   extract_manifest_filehash HASH file1.manifest
+   extract_manifest_date DATE file1.manifest
+   extract_manifest_service SERVICE file1.manifest
+   extract_manifest_name NAME file1.manifest
+   extract_manifest_BK BK file1.manifest
+   assert [ "$BID" = "$H_BID" ]
+   assert [ "$VERSION" = "$H_VERSION" ]
+   assert [ "$SIZE" = "$H_SIZE" ]
+   assert [ "$HASH" = "$H_HASH" ]
+   assert [ "$DATE" = "$H_DATE" ]
+   assert [ "$SERVICE" = "$H_SERVICE" ]
+   assert [ "$NAME" = "$H_NAME" ]
+   assert [ "$BK" = "$H_BK" ]
+   executeOk_servald rhizome list
+   assert_rhizome_list file1
+}
+
 runTests "$@"