diff --git a/http_server.c b/http_server.c index 411218dd..f02ab847 100644 --- a/http_server.c +++ b/http_server.c @@ -99,7 +99,6 @@ static int http_request_parse_http_version(struct http_request *r); static int http_request_start_parsing_headers(struct http_request *r); static int http_request_parse_header(struct http_request *r); static int http_request_start_body(struct http_request *r); -static int http_request_reject_content(struct http_request *r); static int http_request_parse_body_form_data(struct http_request *r); static void http_request_start_response(struct http_request *r); @@ -1006,18 +1005,7 @@ static int http_request_parse_header(struct http_request *r) _skip_eol(r); if (eol == r->parsed) { // if EOL is at start of line (ie, blank line)... _commit(r); - if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { - size_t unparsed = r->end - r->parsed; - if (unparsed > r->request_header.content_length) { - WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length)); - r->request_content_remaining = 0; - } - else - r->request_content_remaining = r->request_header.content_length - unparsed; - } r->parser = http_request_start_body; - if (r->handle_headers) - return r->handle_headers(r); return 0; } char *const nextline = r->cursor; @@ -1193,6 +1181,8 @@ static int http_request_decode_chunks(struct http_request *r){ r->chunk_state = CHUNK_SIZE; if (r->request_content_remaining == 0){ r->decoder = NULL; + if (r->end_received>r->end) + return WHY("Unexpected data"); return 0; } // fall through @@ -1206,13 +1196,13 @@ static int http_request_decode_chunks(struct http_request *r){ return 100; } if (ret!=1 || p[0]!='\r' || p[1]!='\n') - return WHY("Expected [size]\r\n"); + return WHY("Expected [size]\\r\\n"); r->decode_ptr = (char*)p+2; r->chunk_state = CHUNK_DATA; - IDEBUGF(r->debug, "Chunk size %zu (parsed %d, unparsed %d, heading %d, data %d) %s", - r->chunk_size, + IDEBUGF(r->debug, "Chunk size %u (parsed %d, unparsed %d, heading %d, data %d) %s", + (int)r->chunk_size, (int)(r->parsed - r->received), (int)(r->end - r->parsed), (int)(r->decode_ptr - r->end), @@ -1254,8 +1244,11 @@ static int http_request_decode_chunks(struct http_request *r){ // if we can cut the \r\n off the end, do it now r->chunk_state = CHUNK_SIZE; r->end_received = r->end; - if (r->request_content_remaining == 0) + if (r->request_content_remaining == 0){ r->decoder = NULL; + if (r->end_received>r->end) + return WHY("Unexpected data"); + } } } // give the parser a chance to deal with this chunk so we can avoid memmove @@ -1288,7 +1281,8 @@ static int http_request_start_continue(struct http_request *r){ return 0; } - r->response_sent=0; + r->response_sent = 0; + r->request_header.expect = 0; r->parser = http_request_parse_body_form_data; r->form_data_state = START; if (_run_out(r)) @@ -1310,9 +1304,12 @@ static int http_request_start_body(struct http_request *r) assert(r->path != NULL); assert(r->version_major != 0); assert(r->parsed <= r->end); + if (r->verb == HTTP_VERB_GET) { // TODO: Implement HEAD requests (only send response header, not body) - if (r->request_header.content_length != 0 && r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { + if (r->request_header.content_length == CONTENT_LENGTH_UNKNOWN) + r->request_header.content_length = 0; + if (r->request_header.content_length != 0) { IDEBUGF(r->debug, "Malformed HTTP %s request: non-zero Content-Length not allowed", r->verb); return 400; } @@ -1332,7 +1329,7 @@ static int http_request_start_body(struct http_request *r) return 411; // Length Required } if (r->request_header.content_length == 0) { - r->parser = http_request_reject_content; + r->parser = NULL; } else { if (r->request_header.content_type.type[0] == '\0') { IDEBUGF(r->debug, "Malformed HTTP %s request: missing Content-Type header", r->verb); @@ -1346,9 +1343,8 @@ static int http_request_start_body(struct http_request *r) r->verb, r->request_header.content_type.type, r->request_header.content_type.subtype); return 400; } - if (r->request_header.expect && _run_out(r) && r->end == r->end_received){ + if (r->request_header.expect){ r->parser = http_request_start_continue; - return 0; }else{ r->parser = http_request_parse_body_form_data; r->form_data_state = START; @@ -1360,30 +1356,31 @@ static int http_request_start_body(struct http_request *r) } } } - else { - IDEBUGF(r->debug, "Unsupported HTTP %s request", r->verb); - r->parser = NULL; - return 405; // Method Not Allowed + + if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) { + size_t unparsed = r->end - r->parsed; + if (unparsed > r->request_header.content_length) { + IDEBUGF(r->debug, "Malformed request: already read %zu bytes past end of content", + (size_t)(unparsed - r->request_header.content_length)); + return 431; // Request Header Fields Too Large + } + else + r->request_content_remaining = r->request_header.content_length - unparsed; } + + if (r->handle_headers){ + int ret = r->handle_headers(r); + if (ret!=0){ + r->parser = NULL; + return ret; + } + } + if (_run_out(r)) return 100; return 0; } -/* A special content parser that rejects any content, used when a Content-Type: 0 header was - * received. - * - * @author Andrew Bettison - */ -static int http_request_reject_content(struct http_request *r) -{ - if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) - IDEBUGF(r->debug, "Malformed HTTP %s request (Content-Length %"PRIhttp_size_t"): spurious content", r->verb, r->request_header.content_length); - else - IDEBUGF(r->debug, "Malformed HTTP %s request: spurious content", r->verb); - return 400; -} - /* Returns 1 if a MIME delimiter is skipped, 2 if a MIME close-delimiter is skipped. */ static int _skip_mime_boundary(struct http_request *r) @@ -1760,7 +1757,22 @@ static ssize_t http_request_read(struct http_request *r, char *buf, size_t len) static void http_request_receive(struct http_request *r) { IN(); - assert(r->phase == RECEIVE); + if (r->phase != RECEIVE){ + // just read & throw away any data + char buff[1024]; + ssize_t len = http_request_read(r, buff, sizeof buff); + if (len <0) + RETURNVOID; + if (r->request_content_remaining!=CONTENT_LENGTH_UNKNOWN){ + if ((size_t)len > r->request_content_remaining){ + IDEBUG(r->debug, "Buffer size reached, reporting overflow"); + http_request_simple_response(r, 431, NULL); // Request Header Fields Too Large + RETURNVOID; + } + r->request_content_remaining -= len; + } + RETURNVOID; + } const char *const bufend = r->buffer + sizeof r->buffer; assert(r->end_received <= bufend); assert(r->decode_ptr <= r->end_received); @@ -1832,7 +1844,7 @@ static void http_request_receive(struct http_request *r) RETURNVOID; // poll again } if (result != 0){ - r->response.status_code = 500; + r->response.status_code = 400; break; } } @@ -2042,7 +2054,7 @@ static void _http_request_start_transmitting(struct http_request *r) { assert(r->phase == RECEIVE || r->phase == PAUSE); r->phase = TRANSMIT; - r->alarm.poll.events = POLLOUT; + r->alarm.poll.events = POLLIN|POLLOUT; watch(&r->alarm); http_request_set_idle_timeout(r); } diff --git a/java-api/src/org/servalproject/servaldna/PostHelper.java b/java-api/src/org/servalproject/servaldna/PostHelper.java index 786f2f39..6d6d1fcb 100644 --- a/java-api/src/org/servalproject/servaldna/PostHelper.java +++ b/java-api/src/org/servalproject/servaldna/PostHelper.java @@ -29,10 +29,18 @@ public class PostHelper { conn.setRequestMethod("POST"); conn.setDoOutput(true); conn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); + // we try to set an expect header so we can gracefully deal with the server aborting early + // however implementations don't seem to support it well and will throw a ProtocolException + // if the server doesn't return a 100-Continue + // Other implementations (like android), just strip the header. + // Then if the server closes the connection early, throw some form of IOException conn.setRequestProperty("Expect", "100-continue"); + // If we don't set this, Java might try to re-use a connection that the server closed + conn.setRequestProperty("Connection", "close"); conn.setChunkedStreamingMode(0); conn.connect(); output = conn.getOutputStream(); + output.flush(); writer = new PrintStream(output, false, "UTF-8"); } @@ -52,7 +60,7 @@ public class PostHelper { sb.append('"'); } - public void writeHeading(String name, String filename, String type, String encoding) + protected void writeHeading(String name, String filename, String type, String encoding) { StringBuilder sb = new StringBuilder(); sb.append("\r\n--").append(boundary).append("\r\n"); @@ -80,30 +88,52 @@ public class PostHelper { writer.print(value.toHex()); } - public void writeField(String name, String filename, InputStream stream) throws IOException { + public OutputStream beginFileField(String name, String filename){ writeHeading(name, filename, "application/octet-stream", "binary"); writer.flush(); + return output; + } + + public void writeField(String name, String filename, InputStream stream) throws IOException { + beginFileField(name, filename); byte[] buffer = new byte[4096]; int n; while ((n = stream.read(buffer)) > 0) output.write(buffer, 0, n); } + public void writeField(String name, String type, byte value[]) throws IOException { + writeHeading(name, null, type, "binary"); + writer.flush(); + output.write(value); + } + + public void writeField(String name, String type, byte value[], int offset, int length) throws IOException { + writeHeading(name, null, type, "binary"); + writer.flush(); + output.write(value, offset, length); + } + public void writeField(String name, RhizomeManifest manifest) throws IOException, RhizomeManifestSizeException { - writeHeading(name, null, "rhizome/manifest; format=\"text+binarysig\"", "binary"); + writeHeading(name, null, RhizomeManifest.MIME_TYPE, "binary"); manifest.toTextFormat(writer); } public void writeField(String name, RhizomeIncompleteManifest manifest) throws IOException { - writeHeading(name, null, "rhizome/manifest; format=\"text+binarysig\"", "binary"); + writeHeading(name, null, RhizomeManifest.MIME_TYPE, "binary"); manifest.toTextFormat(writer); } - public void close(){ - if (writer==null) - return; - writer.print("\r\n--" + boundary + "--\r\n"); - writer.flush(); - writer.close(); + public void close() throws IOException { + if (writer!=null) { + writer.print("\r\n--" + boundary + "--\r\n"); + writer.flush(); + writer.close(); + writer=null; + } + if (output!=null) { + output.close(); + output = null; + } } } diff --git a/java-api/src/org/servalproject/servaldna/ServalDClient.java b/java-api/src/org/servalproject/servaldna/ServalDClient.java index a6efca51..1ace6e6d 100644 --- a/java-api/src/org/servalproject/servaldna/ServalDClient.java +++ b/java-api/src/org/servalproject/servaldna/ServalDClient.java @@ -46,11 +46,13 @@ import org.servalproject.servaldna.rhizome.RhizomeInsertBundle; import org.servalproject.servaldna.rhizome.RhizomeInvalidManifestException; import org.servalproject.servaldna.rhizome.RhizomeManifest; import org.servalproject.servaldna.rhizome.RhizomeManifestBundle; +import org.servalproject.servaldna.rhizome.RhizomeManifestParseException; import org.servalproject.servaldna.rhizome.RhizomeManifestSizeException; import org.servalproject.servaldna.rhizome.RhizomePayloadBundle; import org.servalproject.servaldna.rhizome.RhizomePayloadRawBundle; import org.servalproject.servaldna.rhizome.RhizomeReadOnlyException; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; @@ -159,6 +161,10 @@ public class ServalDClient implements ServalDHttpConnectionFactory { return RhizomeCommon.rhizomeImport(this, manifest, payloadStream); } + public RhizomeImportStatus rhizomeImportZip(File zipFile) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException, RhizomeManifestParseException { + return RhizomeCommon.rhizomeImportZip(this, zipFile); + } + public MeshMSConversationList meshmsListConversations(SubscriberId sid) throws ServalDInterfaceException, IOException, MeshMSException { MeshMSConversationList list = new MeshMSConversationList(this, sid); diff --git a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java index 13cfb060..32f8502e 100644 --- a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java +++ b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java @@ -34,10 +34,12 @@ import org.servalproject.servaldna.ServalDNotImplementedException; import org.servalproject.servaldna.SubscriberId; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.RandomAccessFile; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -465,16 +467,45 @@ public class RhizomeCommon } } - public static RhizomeImportStatus rhizomeImport(ServalDHttpConnectionFactory connector, RhizomeManifest manifest, InputStream payloadStream) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException { + public static RhizomeImportStatus rhizomeImportZip(ServalDHttpConnectionFactory connector, File zipFile) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException, RhizomeManifestParseException { + RandomAccessFile file = new RandomAccessFile(zipFile, "r"); + RhizomeManifest manifest = RhizomeManifest.fromZipComment(file); + HttpURLConnection conn = connector.newServalDHttpConnection( "/restful/rhizome/import?id="+manifest.id.toHex()+"&version="+manifest.version); PostHelper helper = new PostHelper(conn); try { helper.connect(); helper.writeField("manifest", manifest); - if (payloadStream != null) - helper.writeField("payload", null, payloadStream); + OutputStream out = helper.beginFileField("payload", null); + + file.seek(0); + + long readLength = manifest.filesize-2; + byte buff[] = new byte[4096]; + while (readLength>0){ + int len = readLength > buff.length ? buff.length : (int)readLength; + int read = file.read(buff, 0, len); + out.write(buff, 0, read); + readLength -= read; + } + buff[0]=0; + buff[1]=0; + out.write(buff, 0, 2); + helper.close(); + + int[] expected_response_codes = { HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_ACCEPTED}; + + Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes); + decodeHeaderPayloadStatusOrNull(status, conn); + checkPayloadStatus(status); + decodeHeaderBundleStatus(status, conn); + checkBundleStatus(status); + return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code); + }catch (ProtocolException e){ // dodgy java implementation, only means that the server did not return 100-continue // attempting to read the input stream will fail again @@ -486,17 +517,40 @@ public class RhizomeCommon } throw e; } + } - int[] expected_response_codes = { HttpURLConnection.HTTP_OK, - HttpURLConnection.HTTP_CREATED, - HttpURLConnection.HTTP_ACCEPTED}; + public static RhizomeImportStatus rhizomeImport(ServalDHttpConnectionFactory connector, RhizomeManifest manifest, InputStream payloadStream) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException { + HttpURLConnection conn = connector.newServalDHttpConnection( + "/restful/rhizome/import?id="+manifest.id.toHex()+"&version="+manifest.version); + PostHelper helper = new PostHelper(conn); + try { + helper.connect(); + helper.writeField("manifest", manifest); + if (manifest.filesize>0 && payloadStream != null) + helper.writeField("payload", null, payloadStream); + helper.close(); - Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes); - decodeHeaderPayloadStatusOrNull(status, conn); - checkPayloadStatus(status); - decodeHeaderBundleStatus(status, conn); - checkBundleStatus(status); - return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code); + int[] expected_response_codes = { HttpURLConnection.HTTP_OK, + HttpURLConnection.HTTP_CREATED, + HttpURLConnection.HTTP_ACCEPTED}; + + Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes); + decodeHeaderPayloadStatusOrNull(status, conn); + checkPayloadStatus(status); + decodeHeaderBundleStatus(status, conn); + checkBundleStatus(status); + return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code); + }catch (ProtocolException e){ + // dodgy java implementation, only means that the server did not return 100-continue + // attempting to read the input stream will fail again + switch (conn.getResponseCode()){ + case 200: + return new RhizomeImportStatus(RhizomeBundleStatus.SAME, null); + case 202: + return new RhizomeImportStatus(RhizomeBundleStatus.OLD, null); + } + throw e; + } } private static RhizomeManifest manifestFromHeaders(HttpURLConnection conn) throws ServalDInterfaceException diff --git a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeManifest.java b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeManifest.java index 126469b5..0088d5c3 100644 --- a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeManifest.java +++ b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeManifest.java @@ -22,6 +22,7 @@ package org.servalproject.servaldna.rhizome; import java.io.File; import java.io.FileInputStream; +import java.io.RandomAccessFile; import java.util.Map; import java.util.HashMap; import java.util.HashSet; @@ -41,6 +42,7 @@ import org.servalproject.servaldna.BundleKey; public class RhizomeManifest { public final static int TEXT_FORMAT_MAX_SIZE = 8192; + public static final String MIME_TYPE = "rhizome/manifest; format=\"text+binarysig\""; // Core fields used for routing and expiry (cannot be null) public final BundleId id; @@ -229,6 +231,39 @@ public class RhizomeManifest { } } + public static RhizomeManifest fromZipComment(RandomAccessFile file) throws IOException, RhizomeManifestParseException { + int readLen = RhizomeManifest.TEXT_FORMAT_MAX_SIZE + 22; + file.seek(file.length() - readLen); + byte buff[] = new byte[readLen]; + file.readFully(buff); + int offset = buff.length - 21; + while(offset>0) { + if (buff[--offset] != 0x06) + continue; + if (buff[--offset] != 0x05) + continue; + if (buff[--offset] != 0x4b) + continue; + if (buff[--offset] != 0x50) + continue; + + // located zip EOCD record marker 0x504b0506 + offset += 20; + int manifestLen = (buff[offset++]&0xFF) | ((buff[offset++] & 0xFF) << 8); + if (manifestLen != readLen - offset) + throw new RhizomeManifestParseException("Zip Comment length ("+manifestLen+") doesn't align with end of file ("+readLen+", "+offset+")"); + if (manifestLen == 0) + throw new RhizomeManifestParseException("No Zip Comment"); + + RhizomeManifest manifest = RhizomeManifest.fromTextFormat(buff, offset, manifestLen); + long expectedFileSize = file.length() - readLen + offset; + if (manifest.filesize != expectedFileSize) + throw new RhizomeManifestParseException("Manifest filesize doesn't match zip file length"); + return manifest; + } + throw new RhizomeManifestParseException("Zip EOCD record not found"); + } + private static boolean isFieldNameFirstChar(char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); diff --git a/tests/rhizomerestful b/tests/rhizomerestful index b59fba7c..3582d723 100755 --- a/tests/rhizomerestful +++ b/tests/rhizomerestful @@ -1240,8 +1240,11 @@ setup_RhizomeImport() { setup set_instance +B create_single_identity - rhizome_add_bundles $SIDB 1 1 - executeOk_servald rhizome export manifest "${BID[1]}" file1.manifest + create_file file1 100 + executeOk_servald rhizome add file $SIDB file1 file1.manifest + extract_manifest_id manifestid file1.manifest + extract_manifest_version version file1.manifest + executeOk_servald rhizome export manifest "${manifestid}" file1.manifest set_instance +A } test_RhizomeImport() { @@ -1253,7 +1256,7 @@ test_RhizomeImport() { --basic --user harry:potter \ --form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \ --form "payload=@file1" \ - "http://$addr_localhost:$PORTA/restful/rhizome/import?id=${BID[1]}&version=${VERSION[1]}" + "http://$addr_localhost:$PORTA/restful/rhizome/import?id=${manifestid}&version=${version}" tfw_cat http.header http.body assertStdoutIs 201 assertGrep http.header '100 Continue' @@ -1267,7 +1270,7 @@ test_RhizomeImport() { --basic --user harry:potter \ --form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \ --form "payload=@file1" \ - "http://$addr_localhost:$PORTA/restful/rhizome/import?id=${BID[1]}&version=${VERSION[1]}" + "http://$addr_localhost:$PORTA/restful/rhizome/import?id=${manifestid}&version=${version}" tfw_cat http.header http.body assertStdoutIs 200 assertGrep --matches=0 http.header '100 Continue' @@ -1275,6 +1278,35 @@ test_RhizomeImport() { assertJq http.body 'contains({"http_status_message": "Bundle already in store"})' } +doc_RhizomeImportLarge="HTTP RESTful import 50 MiB Rhizome bundle" +setup_RhizomeImportLarge() { + setup + set_instance +B + create_single_identity + create_file file1 50m + executeOk_servald rhizome add file $SIDB file1 file1.manifest + extract_manifest_id manifestid file1.manifest + extract_manifest_version version file1.manifest + executeOk_servald rhizome export manifest "${manifestid}" file1.manifest + set_instance +A +} +test_RhizomeImportLarge() { + execute curl \ + --silent --show-error --write-out '%{http_code}' \ + --header 'Transfer-Encoding: chunked' \ + --output http.body \ + --dump-header http.header \ + --basic --user harry:potter \ + --form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \ + --form "payload=@file1" \ + "http://$addr_localhost:$PORTA/restful/rhizome/import?id=${manifestid}&version=${version}" + tfw_cat http.header http.body + assertStdoutIs 201 + assertGrep http.header '100 Continue' + assertJq http.body 'contains({"http_status_code": 201})' + assertJq http.body 'contains({"http_status_message": "Created"})' +} + doc_RhizomeJournalAppend="HTTP RESTful Rhizome journal create and append" setup_RhizomeJournalAppend() { setup