Use chunked transfer encoding and expect header in Java API

http_server.c

@@ -1110,7 +1110,7 @@ static int http_request_parse_header(struct http_request *r)
   }
   _rewind(r);
   if (_skip_literal_nocase(r, "Transfer-Encoding:")) {
-    if (r->request_header.expect){
+    if (r->request_header.chunked){
       IDEBUGF(r->debug, "Skipping duplicate HTTP header Transfer-Encoding: %s", alloca_toprint(50, sol, r->end - sol));
       r->cursor = nextline;
       _commit(r);
@@ -1251,8 +1251,14 @@ static int http_request_decode_chunks(struct http_request *r){
 	len = r->chunk_size;
       r->chunk_size -= len;
       r->end += len;
-      if (r->chunk_size == 0)
+      if (r->chunk_size == 0){
 	r->chunk_state = CHUNK_NEWLINE;
+	if (r->end_received - r->end == 2 && r->end[0]=='\r' && r->end[1]=='\n'){
+	  // if we can cut the \r\n off the end, do it now
+	  r->chunk_state = CHUNK_SIZE;
+	  r->end_received = r->end;
+	}
+      }
       // give the parser a chance to deal with this chunk so we can avoid memmove
       return 0;
     }
@@ -1341,7 +1347,7 @@ static int http_request_start_body(struct http_request *r)
 		r->verb, r->request_header.content_type.type, r->request_header.content_type.subtype);
 	  return 400;
 	}
-	if (r->request_header.expect && _run_out(r)){
+	if (r->request_header.expect && _run_out(r) && r->end == r->end_received){
 	  r->parser = http_request_start_continue;
 	  return 0;
 	}else{
@@ -1828,8 +1834,8 @@ static void http_request_receive(struct http_request *r)
 	r->response.status_code = 500;
 	break;
       }
-      decode_more = 0;
     }
+    decode_more = 0;
     _rewind(r);
     if (_end_of_content(r)) {
       if (r->handle_content_end)

java-api/Makefile.in

@@ -33,7 +33,7 @@ classes: $(CLASSDIR)/dummy
 $(TESTCLASSDIR)/dummy: $(TEST_SOURCES) $(CLASSDIR)/dummy
 	@mkdir -p $(TESTCLASSDIR)
 	@echo "JAVAC $(TESTCLASSDIR)"
-	@$(JAVAC) -Xlint:unchecked -d $(TESTCLASSDIR) -classpath $(CLASSDIR) $(SOURCES)
+	@$(JAVAC) -Xlint:unchecked -d $(TESTCLASSDIR) -classpath $(CLASSDIR) $(TEST_SOURCES)
 	@touch $@
 
 testclasses: $(TESTCLASSDIR)/dummy

java-api/src/org/servalproject/servaldna/PostHelper.java

@@ -1,12 +1,15 @@
 package org.servalproject.servaldna;
 
 import org.servalproject.servaldna.rhizome.RhizomeIncompleteManifest;
+import org.servalproject.servaldna.rhizome.RhizomeManifest;
+import org.servalproject.servaldna.rhizome.RhizomeManifestSizeException;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.net.HttpURLConnection;
+import java.net.ProtocolException;
 
 /**
  * Created by jeremy on 5/10/16.
@@ -26,6 +29,8 @@ public class PostHelper {
         conn.setRequestMethod("POST");
         conn.setDoOutput(true);
         conn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
+        conn.setRequestProperty("Expect", "100-continue");
+        conn.setChunkedStreamingMode(0);
         conn.connect();
         output = conn.getOutputStream();
         writer = new PrintStream(output, false, "UTF-8");
@@ -84,12 +89,19 @@ public class PostHelper {
             output.write(buffer, 0, n);
     }
 
+    public void writeField(String name, RhizomeManifest manifest) throws IOException, RhizomeManifestSizeException {
+        writeHeading(name, null, "rhizome/manifest; format=\"text+binarysig\"", "binary");
+        manifest.toTextFormat(writer);
+    }
+
     public void writeField(String name, RhizomeIncompleteManifest manifest) throws IOException {
         writeHeading(name, null, "rhizome/manifest; format=\"text+binarysig\"", "binary");
         manifest.toTextFormat(writer);
     }
 
     public void close(){
+        if (writer==null)
+            return;
         writer.print("\r\n--" + boundary + "--\r\n");
         writer.flush();
         writer.close();

java-api/src/org/servalproject/servaldna/ServalDClient.java

@@ -37,12 +37,16 @@ import org.servalproject.servaldna.rhizome.RhizomeBundleList;
 import org.servalproject.servaldna.rhizome.RhizomeCommon;
 import org.servalproject.servaldna.rhizome.RhizomeDecryptionException;
 import org.servalproject.servaldna.rhizome.RhizomeEncryptionException;
+import org.servalproject.servaldna.rhizome.RhizomeException;
 import org.servalproject.servaldna.rhizome.RhizomeFakeManifestException;
+import org.servalproject.servaldna.rhizome.RhizomeImportStatus;
 import org.servalproject.servaldna.rhizome.RhizomeIncompleteManifest;
 import org.servalproject.servaldna.rhizome.RhizomeInconsistencyException;
 import org.servalproject.servaldna.rhizome.RhizomeInsertBundle;
 import org.servalproject.servaldna.rhizome.RhizomeInvalidManifestException;
+import org.servalproject.servaldna.rhizome.RhizomeManifest;
 import org.servalproject.servaldna.rhizome.RhizomeManifestBundle;
+import org.servalproject.servaldna.rhizome.RhizomeManifestSizeException;
 import org.servalproject.servaldna.rhizome.RhizomePayloadBundle;
 import org.servalproject.servaldna.rhizome.RhizomePayloadRawBundle;
 import org.servalproject.servaldna.rhizome.RhizomeReadOnlyException;
@@ -151,6 +155,10 @@ public class ServalDClient implements ServalDHttpConnectionFactory {
 		return RhizomeCommon.rhizomeInsert(this, author, manifest, secret, payloadStream, fileName);
 	}
 
+	public RhizomeImportStatus rhizomeImport(RhizomeManifest manifest, InputStream payloadStream) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException {
+		return RhizomeCommon.rhizomeImport(this, manifest, payloadStream);
+	}
+
 	public MeshMSConversationList meshmsListConversations(SubscriberId sid) throws ServalDInterfaceException, IOException, MeshMSException
 	{
 		MeshMSConversationList list = new MeshMSConversationList(this, sid);

java-api/src/org/servalproject/servaldna/ServalDCommand.java

@@ -22,6 +22,8 @@
 package org.servalproject.servaldna;
 
 import org.servalproject.servaldna.rhizome.RhizomeBundleStatus;
+import org.servalproject.servaldna.rhizome.RhizomeManifest;
+import org.servalproject.servaldna.rhizome.RhizomeManifestParseException;
 
 import java.io.File;
 import java.io.IOException;
@@ -343,7 +345,7 @@ public class ServalDCommand
 		public String service;
 		public String name;
 		public boolean readonly=true;
-		public byte[] manifest;
+		public byte[] manifestText;
 		public String secret;
 		public SubscriberId author;
 		public long rowId;
@@ -378,7 +380,13 @@ public class ServalDCommand
 		@Override
 		public void putBlob(byte[] value) {
 			if (columnName.equals("manifest"))
-				this.manifest = value;
+				this.manifestText = value;
+		}
+
+		public RhizomeManifest getManifest() throws RhizomeManifestParseException {
+			if (manifestText == null)
+				return null;
+			return RhizomeManifest.fromTextFormat(manifestText);
 		}
 
 		@Override

java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java

@@ -33,22 +33,25 @@ import org.servalproject.servaldna.ServalDInterfaceException;
 import org.servalproject.servaldna.ServalDNotImplementedException;
 import org.servalproject.servaldna.SubscriberId;
 
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.io.PrintStream;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.net.HttpURLConnection;
-import java.util.List;
+import java.net.ProtocolException;
-import java.util.Map;
+import java.net.URL;
 
 public class RhizomeCommon
 {
 
 	private static class Status {
+		URL url;
+		String contentType;
 		InputStream input_stream;
 		public int http_status_code;
 		public String http_status_message;
@@ -58,17 +61,6 @@ public class RhizomeCommon
 		String payload_status_message;
 	}
 
-	private static void dumpStatus(Status status, PrintStream out)
-	{
-		out.println("input_stream=" + status.input_stream);
-		out.println("http_status_code=" + status.http_status_code);
-		out.println("http_status_message=" + status.http_status_message);
-		out.println("bundle_status_code=" + status.bundle_status_code);
-		out.println("bundle_status_message=" + status.bundle_status_message);
-		out.println("payload_status_code=" + status.payload_status_code);
-		out.println("payload_status_message=" + status.payload_status_message);
-	}
-
 	protected static Status receiveResponse(HttpURLConnection conn, int expected_response_code) throws IOException, ServalDInterfaceException
 	{
 		int[] expected_response_codes = { expected_response_code };
@@ -78,6 +70,8 @@ public class RhizomeCommon
 	protected static Status receiveResponse(HttpURLConnection conn, int[] expected_response_codes) throws IOException, ServalDInterfaceException
 	{
 		Status status = new Status();
+		status.url = conn.getURL();
+		status.contentType = conn.getContentType();
 		status.http_status_code = conn.getResponseCode();
 		status.http_status_message = conn.getResponseMessage();
 		for (int code: expected_response_codes) {
@@ -86,8 +80,9 @@ public class RhizomeCommon
 				return status;
 			}
 		}
-		if (!conn.getContentType().equals("application/json"))
+		if (!status.contentType.equals("application/json"))
 			throw new ServalDInterfaceException("unexpected HTTP Content-Type: " + conn.getContentType());
+
 		if (status.http_status_code >= 300) {
 			JSONTokeniser json = new JSONTokeniser(conn.getErrorStream());
 			decodeRestfulStatus(status, json);
@@ -228,9 +223,7 @@ public class RhizomeCommon
 		conn.connect();
 		Status status = RhizomeCommon.receiveResponse(conn, HttpURLConnection.HTTP_OK);
 		try {
-			dumpHeaders(conn, System.err);
 			decodeHeaderBundleStatus(status, conn);
-			dumpStatus(status, System.err);
 			switch (status.bundle_status_code) {
 			case NEW:
 				return null;
@@ -261,9 +254,7 @@ public class RhizomeCommon
 		conn.connect();
 		Status status = RhizomeCommon.receiveResponse(conn, HttpURLConnection.HTTP_OK);
 		try {
-			dumpHeaders(conn, System.err);
 			decodeHeaderBundleStatus(status, conn);
-			dumpStatus(status, System.err);
 			switch (status.bundle_status_code) {
 			case ERROR:
 				throw new ServalDFailureException("received rhizome_bundle_status_code=ERROR(-1) from " + conn.getURL());
@@ -303,6 +294,33 @@ public class RhizomeCommon
 		throw unexpectedResponse(conn, status);
 	}
 
+	public static void WriteBundleZip(RhizomePayloadRawBundle payload, File output) throws IOException, RhizomeManifestSizeException {
+		try{
+			OutputStream out = new FileOutputStream(output);
+			try{
+				// read out all but the last two bytes
+				long toWrite = payload.manifest.filesize - 2;
+				byte[] buff = new byte[4096];
+				while(toWrite>0){
+					int len = toWrite > buff.length ? buff.length : (int) toWrite;
+					len = payload.rawPayloadInputStream.read(buff, 0, len);
+					out.write(buff, 0, len);
+					toWrite -= len;
+				}
+				// assume the apk is a zip file, write the manifest into a zip file comment at the end
+				byte[] manifestText = payload.manifest.toTextFormat();
+				buff[0] = (byte) (manifestText.length & 0xFF);
+				buff[1] = (byte) ((manifestText.length >> 8) & 0xFF);
+				out.write(buff, 0, 2);
+				out.write(manifestText);
+			}finally{
+				out.close();
+			}
+		}finally {
+			payload.rawPayloadInputStream.close();
+		}
+	}
+
 	public static RhizomePayloadBundle rhizomePayload(ServalDHttpConnectionFactory connector, BundleId bid)
 		throws IOException, ServalDInterfaceException, RhizomeDecryptionException
 	{
@@ -310,9 +328,7 @@ public class RhizomeCommon
 		conn.connect();
 		Status status = RhizomeCommon.receiveResponse(conn, HttpURLConnection.HTTP_OK);
 		try {
-			dumpHeaders(conn, System.err);
 			decodeHeaderBundleStatus(status, conn);
-			dumpStatus(status, System.err);
 			switch (status.bundle_status_code) {
 			case ERROR:
 				throw new ServalDFailureException("received rhizome_bundle_status_code=ERROR(-1) from " + conn.getURL());
@@ -369,6 +385,36 @@ public class RhizomeCommon
 		return rhizomeInsert(connector, author, manifest, secret, null, null);
 	}
 
+	protected static void checkBundleStatus(Status status) throws ServalDInterfaceException, IOException, RhizomeReadOnlyException, RhizomeInconsistencyException, RhizomeFakeManifestException, RhizomeInvalidManifestException {
+		switch (status.bundle_status_code) {
+			case ERROR:
+				throw new ServalDFailureException("received Rhizome bundle_status=ERROR " + quoteString(status.bundle_status_message) + " from " + status.url);
+			case INVALID:
+				throw new RhizomeInvalidManifestException(status.bundle_status_message, status.url);
+			case FAKE:
+				throw new RhizomeFakeManifestException(status.bundle_status_message, status.url);
+			case INCONSISTENT:
+				throw new RhizomeInconsistencyException(status.bundle_status_message, status.url);
+			case READONLY:
+				throw new RhizomeReadOnlyException(status.bundle_status_message, status.url);
+		}
+	}
+
+	protected static void checkPayloadStatus(Status status) throws ServalDFailureException, RhizomeInconsistencyException, RhizomeEncryptionException {
+		if (status.payload_status_code == null)
+			return;
+		switch (status.payload_status_code) {
+			case ERROR:
+				throw new ServalDFailureException("received Rhizome payload_status=ERROR " +
+						quoteString(status.payload_status_message) + " from " + status.url);
+			case WRONG_SIZE:
+			case WRONG_HASH:
+				throw new RhizomeInconsistencyException(status.payload_status_message, status.url);
+			case CRYPTO_FAIL:
+				throw new RhizomeEncryptionException(status.payload_status_message, status.url);
+		}
+	}
+
 	public static RhizomeInsertBundle rhizomeInsert(ServalDHttpConnectionFactory connector,
 													SubscriberId author,
 													RhizomeIncompleteManifest manifest,
@@ -398,55 +444,17 @@ public class RhizomeCommon
 		int[] expected_response_codes = { HttpURLConnection.HTTP_OK, HttpURLConnection.HTTP_CREATED };
 		Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes);
 		try {
-			dumpHeaders(conn, System.err);
 			decodeHeaderPayloadStatusOrNull(status, conn);
-			if (status.payload_status_code != null) {
+			checkPayloadStatus(status);
-				switch (status.payload_status_code) {
+
-				case ERROR:
-					dumpStatus(status, System.err);
-					throw new ServalDFailureException("received Rhizome payload_status=ERROR " + quoteString(status.payload_status_message) + " from " + conn.getURL());
-				case EMPTY:
-				case NEW:
-				case STORED:
-					break;
-				case TOO_BIG:
-				case EVICTED:
-					dumpStatus(status, System.err);
-					return null;
-				case WRONG_SIZE:
-				case WRONG_HASH:
-					dumpStatus(status, System.err);
-					throw new RhizomeInconsistencyException(status.payload_status_message, conn.getURL());
-				case CRYPTO_FAIL:
-					dumpStatus(status, System.err);
-					throw new RhizomeEncryptionException(status.payload_status_message, conn.getURL());
-				}
-			}
 			decodeHeaderBundleStatus(status, conn);
-			dumpStatus(status, System.err);
+			checkBundleStatus(status);
-			switch (status.bundle_status_code) {
+
-			case ERROR:
+			if (!status.contentType.equals("rhizome-manifest/text"))
-				throw new ServalDFailureException("received Rhizome bundle_status=ERROR " + quoteString(status.bundle_status_message) + " from " + conn.getURL());
+				throw new ServalDInterfaceException("unexpected HTTP Content-Type " + status.contentType + " from " + status.url);
-			case NEW:
+			RhizomeManifest returned_manifest = RhizomeManifest.fromTextFormat(status.input_stream);
-			case SAME:
+			BundleExtra extra = bundleExtraFromHeaders(conn);
-			case DUPLICATE:
+			return new RhizomeInsertBundle(status.bundle_status_code, status.payload_status_code, returned_manifest, extra.rowId, extra.insertTime, extra.author, extra.secret);
-			case OLD:
-			case NO_ROOM: {
-					if (!conn.getContentType().equals("rhizome-manifest/text"))
-						throw new ServalDInterfaceException("unexpected HTTP Content-Type " + conn.getContentType() + " from " + conn.getURL());
-					RhizomeManifest returned_manifest = RhizomeManifest.fromTextFormat(status.input_stream);
-					BundleExtra extra = bundleExtraFromHeaders(conn);
-					return new RhizomeInsertBundle(status.bundle_status_code, returned_manifest, extra.rowId, extra.insertTime, extra.author, extra.secret);
-				}
-			case INVALID:
-				throw new RhizomeInvalidManifestException(status.bundle_status_message, conn.getURL());
-			case FAKE:
-				throw new RhizomeFakeManifestException(status.bundle_status_message, conn.getURL());
-			case INCONSISTENT:
-				throw new RhizomeInconsistencyException(status.bundle_status_message, conn.getURL());
-			case READONLY:
-				throw new RhizomeReadOnlyException(status.bundle_status_message, conn.getURL());
-			}
 		}
 		catch (RhizomeManifestParseException e) {
 			throw new ServalDInterfaceException("malformed manifest from daemon", e);
@@ -455,15 +463,40 @@ public class RhizomeCommon
 			if (status.input_stream != null)
 				status.input_stream.close();
 		}
-		dumpStatus(status, System.err);
-		throw unexpectedResponse(conn, status);
 	}
 
-	private static void dumpHeaders(HttpURLConnection conn, PrintStream out)
+	public static RhizomeImportStatus rhizomeImport(ServalDHttpConnectionFactory connector, RhizomeManifest manifest, InputStream payloadStream) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException {
-	{
+		HttpURLConnection conn = connector.newServalDHttpConnection(
-		for (Map.Entry<String,List<String>> e: conn.getHeaderFields().entrySet())
+				"/restful/rhizome/import?id="+manifest.id.toHex()+"&version="+manifest.version);
-			for (String v: e.getValue())
+		PostHelper helper = new PostHelper(conn);
-				out.println("received header " + e.getKey() + ": " + v);
+		try {
+			helper.connect();
+			helper.writeField("manifest", manifest);
+			if (payloadStream != null)
+				helper.writeField("payload", null, payloadStream);
+			helper.close();
+		}catch (ProtocolException e){
+			// dodgy java implementation, only means that the server did not return 100-continue
+			// attempting to read the input stream will fail again
+			switch (conn.getResponseCode()){
+				case 200:
+					return new RhizomeImportStatus(RhizomeBundleStatus.SAME, null);
+				case 202:
+					return new RhizomeImportStatus(RhizomeBundleStatus.OLD, null);
+			}
+			throw e;
+		}
+
+		int[] expected_response_codes = { HttpURLConnection.HTTP_OK,
+				HttpURLConnection.HTTP_CREATED,
+				HttpURLConnection.HTTP_ACCEPTED};
+
+		Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes);
+		decodeHeaderPayloadStatusOrNull(status, conn);
+		checkPayloadStatus(status);
+		decodeHeaderBundleStatus(status, conn);
+		checkBundleStatus(status);
+		return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code);
 	}
 
 	private static RhizomeManifest manifestFromHeaders(HttpURLConnection conn) throws ServalDInterfaceException

java-api/src/org/servalproject/servaldna/rhizome/RhizomeImportStatus.java

@@ -0,0 +1,14 @@
+package org.servalproject.servaldna.rhizome;
+
+/**
+ * Created by jeremy on 23/05/17.
+ */
+public class RhizomeImportStatus {
+	public final RhizomeBundleStatus bundleStatus;
+	public final RhizomePayloadStatus payloadStatus;
+
+	RhizomeImportStatus(RhizomeBundleStatus bundleStatus, RhizomePayloadStatus payloadStatus) {
+		this.bundleStatus = bundleStatus;
+		this.payloadStatus = payloadStatus;
+	}
+}

java-api/src/org/servalproject/servaldna/rhizome/RhizomeIncompleteManifest.java

@@ -72,7 +72,7 @@ public class RhizomeIncompleteManifest {
 		this.date = m.date;
 		this.service = m.service;
 		this.name = m.name;
-		this.extraFields = (HashMap<String,String>) m.extraFields.clone(); // unchecked cast
+		this.extraFields = (HashMap<String,String>) (m.extraFields==null ? new HashMap<String,String>() : m.extraFields.clone()); // unchecked cast
 	}
 
 	/** Return the Rhizome manifest in its text format representation.

java-api/src/org/servalproject/servaldna/rhizome/RhizomeInsertBundle.java

@@ -22,13 +22,14 @@ package org.servalproject.servaldna.rhizome;
 
 import org.servalproject.servaldna.SubscriberId;
 import org.servalproject.servaldna.BundleSecret;
-import org.servalproject.servaldna.ServalDInterfaceException;
 
 public class RhizomeInsertBundle extends RhizomeManifestBundle {
 
 	public final RhizomeBundleStatus status;
+	public final RhizomePayloadStatus payloadStatus;
 
 	protected RhizomeInsertBundle(RhizomeBundleStatus status,
+								  RhizomePayloadStatus payloadStatus,
 								  RhizomeManifest manifest,
 								  Long rowId,
 								  Long insertTime,
@@ -37,6 +38,7 @@ public class RhizomeInsertBundle extends RhizomeManifestBundle {
 	{
 		super(manifest, rowId, insertTime, author, secret);
 		this.status = status;
+		this.payloadStatus = payloadStatus;
 	}
 
 }

java-api/src/org/servalproject/servaldna/rhizome/RhizomeManifest.java

@@ -20,6 +20,8 @@
 
 package org.servalproject.servaldna.rhizome;
 
+import java.io.File;
+import java.io.FileInputStream;
 import java.util.Map;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -108,39 +110,44 @@ public class RhizomeManifest {
 	 *
 	 * @author Andrew Bettison <andrew@servalproject.com>
 	 */
-	public byte[] toTextFormat() throws RhizomeManifestSizeException
+	public byte[] toTextFormat() throws RhizomeManifestSizeException {
-	{
+		buildTextformat();
-		if (this.textFormat == null) {
-			try {
-				ByteArrayOutputStream os = new ByteArrayOutputStream();
-				toTextFormat(os);
-				os.close();
-				if (os.size() > TEXT_FORMAT_MAX_SIZE)
-					throw new RhizomeManifestSizeException("manifest text format overflow", os.size(), TEXT_FORMAT_MAX_SIZE);
-				this.textFormat = os.toByteArray();
-			}
-			catch (IOException e) {
-				// should not happen with ByteArrayOutputStream
-				return new byte[0];
-			}
-		}
 		byte[] ret = new byte[this.textFormat.length];
 		System.arraycopy(this.textFormat, 0, ret, 0, ret.length);
 		return ret;
 	}
 
+
+	private void buildTextformat() throws RhizomeManifestSizeException {
+		if (textFormat!=null)
+			return;
+		try{
+			ByteArrayOutputStream os = new ByteArrayOutputStream();
+			try {
+				new RhizomeIncompleteManifest(this).toTextFormat(os);
+				if (signatureBlock!=null) {
+					os.write(0);
+					os.write(this.signatureBlock);
+				}
+				if (os.size() > TEXT_FORMAT_MAX_SIZE)
+					throw new RhizomeManifestSizeException("manifest text format overflow", os.size(), TEXT_FORMAT_MAX_SIZE);
+				textFormat = os.toByteArray();
+			} finally{
+				os.close();
+			}
+		} catch (IOException e) {
+			// Um....
+		}
+	}
+
 	/** Write the Rhizome manifest in its text format representation to the given output stream,
 	 * with the signature block at the end if present.
 	 *
 	 * @author Andrew Bettison <andrew@servalproject.com>
 	 */
-	public void toTextFormat(OutputStream os) throws IOException
+	public void toTextFormat(OutputStream os) throws IOException, RhizomeManifestSizeException {
-	{
+		buildTextformat();
-		new RhizomeIncompleteManifest(this).toTextFormat(os);
+		os.write(this.textFormat);
-		if (this.signatureBlock != null) {
-			os.write(0);
-			os.write(this.signatureBlock);
-		}
 	}
 
 	/** Construct a Rhizome manifest from its text format representation.
@@ -198,7 +205,7 @@ public class RhizomeManifest {
 	 *
 	 * @author Andrew Bettison <andrew@servalproject.com>
 	 */
-	static public RhizomeManifest fromTextFormat(InputStream in) throws IOException, RhizomeManifestParseException
+	public static RhizomeManifest fromTextFormat(InputStream in) throws IOException, RhizomeManifestParseException
 	{
 		byte[] bytes = new byte[TEXT_FORMAT_MAX_SIZE];
 		int n = 0;
@@ -213,6 +220,15 @@ public class RhizomeManifest {
 		return fromTextFormat(bytes, 0, offset);
 	}
 
+	public static RhizomeManifest fromTextFormat(File manifestFile) throws IOException, RhizomeManifestParseException {
+		InputStream in = new FileInputStream(manifestFile);
+		try {
+			return fromTextFormat(in);
+		}finally {
+			in.close();
+		}
+	}
+
 	private static boolean isFieldNameFirstChar(char c)
 	{
 		return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');

java-api/test/org/servalproject/test/Rhizome.java

@@ -35,6 +35,7 @@ import org.servalproject.servaldna.BundleSecret;
 import org.servalproject.servaldna.SubscriberId;
 import org.servalproject.servaldna.rhizome.RhizomeManifest;
 import org.servalproject.servaldna.rhizome.RhizomeIncompleteManifest;
+import org.servalproject.servaldna.rhizome.RhizomeImportStatus;
 import org.servalproject.servaldna.rhizome.RhizomeListBundle;
 import org.servalproject.servaldna.rhizome.RhizomeBundleList;
 import org.servalproject.servaldna.rhizome.RhizomeManifestBundle;
@@ -43,6 +44,7 @@ import org.servalproject.servaldna.rhizome.RhizomePayloadBundle;
 import org.servalproject.servaldna.rhizome.RhizomeInsertBundle;
 import org.servalproject.servaldna.rhizome.RhizomeException;
 import org.servalproject.servaldna.rhizome.RhizomeManifestParseException;
+import org.servalproject.servaldna.rhizome.RhizomeManifestSizeException;
 
 public class Rhizome {
 
@@ -211,6 +213,22 @@ public class Rhizome {
 		System.exit(0);
 	}
 
+	static void rhizome_import(String manifestPath, String payloadPath)
+		throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestParseException, RhizomeManifestSizeException{
+		ServalDClient client = new ServerControl().getRestfulClient();
+		RhizomeManifest manifest = RhizomeManifest.fromTextFormat(new File(manifestPath));
+		InputStream in = new FileInputStream(payloadPath);
+		try{
+			RhizomeImportStatus bundle = client.rhizomeImport(manifest, in);
+			System.out.println(
+				"_status=" + bundle.bundleStatus + "\n" +
+				"_payload_status=" + bundle.payloadStatus + "\n");
+		}finally{
+			in.close();
+		}
+		System.exit(0);
+	}
+
 	static void rhizome_insert(	String author,
 								String manifestPath,
 								String payloadPath,
@@ -238,6 +256,7 @@ public class Rhizome {
 				bundle = client.rhizomeInsert(authorSid, manifest, secret, new FileInputStream(payloadPath), payloadName);
 			System.out.println(
 					"_status=" + bundle.status + "\n" +
+					"_payload_status=" + bundle.payloadStatus + "\n" +
 					(bundle.rowId == null ? "" : "_rowId=" + bundle.rowId + "\n") +
 					(bundle.insertTime == null ? "" : "_insertTime=" + bundle.insertTime + "\n") +
 					(bundle.author == null ? "" : "_author=" + bundle.author + "\n") +
@@ -286,6 +305,8 @@ public class Rhizome {
 								args.length > 5 ? args[5] : null, // payload name
 								args.length > 6 ? args[6] : null  // bundle secret
 							  );
+			else if (methodName.equals("rhizome-import"))
+				rhizome_import(args[1], args[2]);
 		} catch (Exception e) {
 			e.printStackTrace();
 			System.exit(1);

java-api/test/org/servalproject/test/ServalDTests.java

@@ -38,7 +38,7 @@ import java.util.Date;
 import java.util.LinkedList;
 import java.util.List;
 
-class ServalDTests
+public class ServalDTests
 {
 	static void log(String msg) {
 		System.err.println(new Date().toString()+" "+msg);

tests/rhizomejava

@@ -353,18 +353,33 @@ test_RhizomeInsert() {
       tfw_cat --stdout --stderr -v nfile$n.manifest
       if [ -n "${author[$n]}" ]; then
          assertStdoutGrep '^_status=NEW$'
+         assertStdoutGrep '^_payload_status=NEW$'
          assertStdoutGrep "^id=${BID[$n]}\$"
-         assertStderrGrep --matches=1 "^bundle_status_code=NEW\$"
-         assertStderrGrep --matches=1 --ignore-case "^bundle_status_message=.*bundle new to store.*\$"
-         assertStderrGrep --matches=1 "^payload_status_code=NEW\$"
-         assertStderrGrep --matches=1 --ignore-case "^payload_status_message=.*payload new to store.*\$"
       else
          assertStdoutGrep RhizomeReadOnlyException
-         assertStderrGrep --ignore-case "missing bundle secret"
       fi
    done
 }
 
+doc_RhizomeImport="Java API import existing bundle"
+setup_RhizomeImport() {
+   setup
+   set_instance +B
+   create_single_identity
+   rhizome_add_bundles $SIDB 1 1
+   executeOk_servald rhizome export manifest "${BID[1]}" file1.manifest
+   set_instance +A
+}
+test_RhizomeImport() {
+   executeJavaOk org.servalproject.test.Rhizome rhizome-import 'file1.manifest' 'file1'
+   tfw_cat --stdout --stderr
+   assertStdoutGrep '^_status=NEW$'
+   assertStdoutGrep '^_payload_status=NEW$'
+   executeJavaOk org.servalproject.test.Rhizome rhizome-import 'file1.manifest' 'file1'
+   tfw_cat --stdout --stderr
+   assertStdoutGrep '^_status=SAME$'
+}
+
 doc_RhizomeInsertAnon="Java API update anonymous Rhizome bundle"
 setup_RhizomeInsertAnon() {
    setup
@@ -381,11 +396,8 @@ test_RhizomeInsertAnon() {
    executeJavaOk org.servalproject.test.Rhizome rhizome-insert '' file2.manifest file2 ifile2.manifest "file2" "$SECRET"
    tfw_cat --stdout --stderr -v ifile2.manifest
    assertStdoutGrep '^_status=NEW$'
+   assertStdoutGrep '^_payload_status=NEW$'
    assertStdoutGrep "^id=$BID\$"
-   assertStderrGrep --matches=1 "^bundle_status_code=NEW\$"
-   assertStderrGrep --matches=1 --ignore-case "^bundle_status_message=.*bundle new to store.*\$"
-   assertStderrGrep --matches=1 "^payload_status_code=NEW\$"
-   assertStderrGrep --matches=1 --ignore-case "^payload_status_message=.*payload new to store.*\$"
    executeOk_servald rhizome list
    assert_rhizome_list --fromhere=0 --manifest=ifile2.manifest file2
 }
@@ -401,13 +413,10 @@ test_RhizomeInsertEmptyNew() {
    tfw_cat --stdout --stderr -v empty.manifest
    extract_manifest_id BID empty.manifest
    assertStdoutGrep '^_status=NEW$'
+   assertStdoutGrep '^_payload_status=EMPTY$'
    assertStdoutGrep "^id=$BID\$"
    assertStdoutGrep "^filesize=0\$"
    assertStdoutGrep --matches=0 "^filehash="
-   assertStderrGrep --matches=1 "^bundle_status_code=NEW\$"
-   assertStderrGrep --matches=1 --ignore-case "^bundle_status_message=.*bundle new to store.*\$"
-   assertStderrGrep --matches=1 "^payload_status_code=EMPTY\$"
-   assertStderrGrep --matches=1 --ignore-case "^payload_status_message=.*payload empty.*\$"
    executeOk_servald rhizome list
    assert_rhizome_list empty
    executeOk_servald rhizome extract bundle "$BID" xempty.manifest xempty
@@ -431,13 +440,10 @@ test_RhizomeInsertEmptyUpdate() {
    executeJavaOk org.servalproject.test.Rhizome rhizome-insert '' iempty.manifest empty empty.manifest
    tfw_cat --stdout --stderr -v empty.manifest
    assertStdoutGrep '^_status=NEW$'
+   assertStdoutGrep '^_payload_status=EMPTY$'
    assertStdoutGrep "^id=$BID\$"
    assertStdoutGrep "^filesize=0\$"
    assertStdoutGrep --matches=0 "^filehash="
-   assertStderrGrep --matches=1 "^bundle_status_code=NEW\$"
-   assertStderrGrep --matches=1 --ignore-case "^bundle_status_message=.*bundle new to store.*\$"
-   assertStderrGrep --matches=1 "^payload_status_code=EMPTY\$"
-   assertStderrGrep --matches=1 --ignore-case "^payload_status_message=.*payload empty.*\$"
    executeOk_servald rhizome list
    assert_rhizome_list empty
    executeOk_servald rhizome extract bundle "$BID" xempty.manifest xempty
@@ -456,7 +462,6 @@ test_RhizomeInsertJournal() {
    tfw_cat --stdout --stderr
    # TODO: need special exception for this case, not RhizomeInvalidManifestException
    assertStdoutGrep RhizomeInvalidManifestException
-   assertStderrGrep --ignore-case "cannot add.*journal"
    executeOk_servald rhizome list
    assert_rhizome_list
 }