Add Java API for importing bundles with manifests in zip comments

This commit is contained in:
Jeremy Lakeman 2017-05-24 13:33:03 +09:30
parent c7de17b552
commit af2d32c25b
6 changed files with 237 additions and 68 deletions

View File

@ -99,7 +99,6 @@ static int http_request_parse_http_version(struct http_request *r);
static int http_request_start_parsing_headers(struct http_request *r);
static int http_request_parse_header(struct http_request *r);
static int http_request_start_body(struct http_request *r);
static int http_request_reject_content(struct http_request *r);
static int http_request_parse_body_form_data(struct http_request *r);
static void http_request_start_response(struct http_request *r);
@ -1006,18 +1005,7 @@ static int http_request_parse_header(struct http_request *r)
_skip_eol(r);
if (eol == r->parsed) { // if EOL is at start of line (ie, blank line)...
_commit(r);
if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) {
size_t unparsed = r->end - r->parsed;
if (unparsed > r->request_header.content_length) {
WARNF("HTTP parsing: already read %zu bytes past end of content", (size_t)(unparsed - r->request_header.content_length));
r->request_content_remaining = 0;
}
else
r->request_content_remaining = r->request_header.content_length - unparsed;
}
r->parser = http_request_start_body;
if (r->handle_headers)
return r->handle_headers(r);
return 0;
}
char *const nextline = r->cursor;
@ -1193,6 +1181,8 @@ static int http_request_decode_chunks(struct http_request *r){
r->chunk_state = CHUNK_SIZE;
if (r->request_content_remaining == 0){
r->decoder = NULL;
if (r->end_received>r->end)
return WHY("Unexpected data");
return 0;
}
// fall through
@ -1206,13 +1196,13 @@ static int http_request_decode_chunks(struct http_request *r){
return 100;
}
if (ret!=1 || p[0]!='\r' || p[1]!='\n')
return WHY("Expected [size]\r\n");
return WHY("Expected [size]\\r\\n");
r->decode_ptr = (char*)p+2;
r->chunk_state = CHUNK_DATA;
IDEBUGF(r->debug, "Chunk size %zu (parsed %d, unparsed %d, heading %d, data %d) %s",
r->chunk_size,
IDEBUGF(r->debug, "Chunk size %u (parsed %d, unparsed %d, heading %d, data %d) %s",
(int)r->chunk_size,
(int)(r->parsed - r->received),
(int)(r->end - r->parsed),
(int)(r->decode_ptr - r->end),
@ -1254,8 +1244,11 @@ static int http_request_decode_chunks(struct http_request *r){
// if we can cut the \r\n off the end, do it now
r->chunk_state = CHUNK_SIZE;
r->end_received = r->end;
if (r->request_content_remaining == 0)
if (r->request_content_remaining == 0){
r->decoder = NULL;
if (r->end_received>r->end)
return WHY("Unexpected data");
}
}
}
// give the parser a chance to deal with this chunk so we can avoid memmove
@ -1288,7 +1281,8 @@ static int http_request_start_continue(struct http_request *r){
return 0;
}
r->response_sent=0;
r->response_sent = 0;
r->request_header.expect = 0;
r->parser = http_request_parse_body_form_data;
r->form_data_state = START;
if (_run_out(r))
@ -1310,9 +1304,12 @@ static int http_request_start_body(struct http_request *r)
assert(r->path != NULL);
assert(r->version_major != 0);
assert(r->parsed <= r->end);
if (r->verb == HTTP_VERB_GET) {
// TODO: Implement HEAD requests (only send response header, not body)
if (r->request_header.content_length != 0 && r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) {
if (r->request_header.content_length == CONTENT_LENGTH_UNKNOWN)
r->request_header.content_length = 0;
if (r->request_header.content_length != 0) {
IDEBUGF(r->debug, "Malformed HTTP %s request: non-zero Content-Length not allowed", r->verb);
return 400;
}
@ -1332,7 +1329,7 @@ static int http_request_start_body(struct http_request *r)
return 411; // Length Required
}
if (r->request_header.content_length == 0) {
r->parser = http_request_reject_content;
r->parser = NULL;
} else {
if (r->request_header.content_type.type[0] == '\0') {
IDEBUGF(r->debug, "Malformed HTTP %s request: missing Content-Type header", r->verb);
@ -1346,9 +1343,8 @@ static int http_request_start_body(struct http_request *r)
r->verb, r->request_header.content_type.type, r->request_header.content_type.subtype);
return 400;
}
if (r->request_header.expect && _run_out(r) && r->end == r->end_received){
if (r->request_header.expect){
r->parser = http_request_start_continue;
return 0;
}else{
r->parser = http_request_parse_body_form_data;
r->form_data_state = START;
@ -1360,30 +1356,31 @@ static int http_request_start_body(struct http_request *r)
}
}
}
else {
IDEBUGF(r->debug, "Unsupported HTTP %s request", r->verb);
r->parser = NULL;
return 405; // Method Not Allowed
if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN) {
size_t unparsed = r->end - r->parsed;
if (unparsed > r->request_header.content_length) {
IDEBUGF(r->debug, "Malformed request: already read %zu bytes past end of content",
(size_t)(unparsed - r->request_header.content_length));
return 431; // Request Header Fields Too Large
}
else
r->request_content_remaining = r->request_header.content_length - unparsed;
}
if (r->handle_headers){
int ret = r->handle_headers(r);
if (ret!=0){
r->parser = NULL;
return ret;
}
}
if (_run_out(r))
return 100;
return 0;
}
/* A special content parser that rejects any content, used when a Content-Type: 0 header was
* received.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static int http_request_reject_content(struct http_request *r)
{
if (r->request_header.content_length != CONTENT_LENGTH_UNKNOWN)
IDEBUGF(r->debug, "Malformed HTTP %s request (Content-Length %"PRIhttp_size_t"): spurious content", r->verb, r->request_header.content_length);
else
IDEBUGF(r->debug, "Malformed HTTP %s request: spurious content", r->verb);
return 400;
}
/* Returns 1 if a MIME delimiter is skipped, 2 if a MIME close-delimiter is skipped.
*/
static int _skip_mime_boundary(struct http_request *r)
@ -1760,7 +1757,22 @@ static ssize_t http_request_read(struct http_request *r, char *buf, size_t len)
static void http_request_receive(struct http_request *r)
{
IN();
assert(r->phase == RECEIVE);
if (r->phase != RECEIVE){
// just read & throw away any data
char buff[1024];
ssize_t len = http_request_read(r, buff, sizeof buff);
if (len <0)
RETURNVOID;
if (r->request_content_remaining!=CONTENT_LENGTH_UNKNOWN){
if ((size_t)len > r->request_content_remaining){
IDEBUG(r->debug, "Buffer size reached, reporting overflow");
http_request_simple_response(r, 431, NULL); // Request Header Fields Too Large
RETURNVOID;
}
r->request_content_remaining -= len;
}
RETURNVOID;
}
const char *const bufend = r->buffer + sizeof r->buffer;
assert(r->end_received <= bufend);
assert(r->decode_ptr <= r->end_received);
@ -1832,7 +1844,7 @@ static void http_request_receive(struct http_request *r)
RETURNVOID; // poll again
}
if (result != 0){
r->response.status_code = 500;
r->response.status_code = 400;
break;
}
}
@ -2042,7 +2054,7 @@ static void _http_request_start_transmitting(struct http_request *r)
{
assert(r->phase == RECEIVE || r->phase == PAUSE);
r->phase = TRANSMIT;
r->alarm.poll.events = POLLOUT;
r->alarm.poll.events = POLLIN|POLLOUT;
watch(&r->alarm);
http_request_set_idle_timeout(r);
}

View File

@ -29,10 +29,18 @@ public class PostHelper {
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
// we try to set an expect header so we can gracefully deal with the server aborting early
// however implementations don't seem to support it well and will throw a ProtocolException
// if the server doesn't return a 100-Continue
// Other implementations (like android), just strip the header.
// Then if the server closes the connection early, throw some form of IOException
conn.setRequestProperty("Expect", "100-continue");
// If we don't set this, Java might try to re-use a connection that the server closed
conn.setRequestProperty("Connection", "close");
conn.setChunkedStreamingMode(0);
conn.connect();
output = conn.getOutputStream();
output.flush();
writer = new PrintStream(output, false, "UTF-8");
}
@ -52,7 +60,7 @@ public class PostHelper {
sb.append('"');
}
public void writeHeading(String name, String filename, String type, String encoding)
protected void writeHeading(String name, String filename, String type, String encoding)
{
StringBuilder sb = new StringBuilder();
sb.append("\r\n--").append(boundary).append("\r\n");
@ -80,30 +88,52 @@ public class PostHelper {
writer.print(value.toHex());
}
public void writeField(String name, String filename, InputStream stream) throws IOException {
public OutputStream beginFileField(String name, String filename){
writeHeading(name, filename, "application/octet-stream", "binary");
writer.flush();
return output;
}
public void writeField(String name, String filename, InputStream stream) throws IOException {
beginFileField(name, filename);
byte[] buffer = new byte[4096];
int n;
while ((n = stream.read(buffer)) > 0)
output.write(buffer, 0, n);
}
public void writeField(String name, String type, byte value[]) throws IOException {
writeHeading(name, null, type, "binary");
writer.flush();
output.write(value);
}
public void writeField(String name, String type, byte value[], int offset, int length) throws IOException {
writeHeading(name, null, type, "binary");
writer.flush();
output.write(value, offset, length);
}
public void writeField(String name, RhizomeManifest manifest) throws IOException, RhizomeManifestSizeException {
writeHeading(name, null, "rhizome/manifest; format=\"text+binarysig\"", "binary");
writeHeading(name, null, RhizomeManifest.MIME_TYPE, "binary");
manifest.toTextFormat(writer);
}
public void writeField(String name, RhizomeIncompleteManifest manifest) throws IOException {
writeHeading(name, null, "rhizome/manifest; format=\"text+binarysig\"", "binary");
writeHeading(name, null, RhizomeManifest.MIME_TYPE, "binary");
manifest.toTextFormat(writer);
}
public void close(){
if (writer==null)
return;
writer.print("\r\n--" + boundary + "--\r\n");
writer.flush();
writer.close();
public void close() throws IOException {
if (writer!=null) {
writer.print("\r\n--" + boundary + "--\r\n");
writer.flush();
writer.close();
writer=null;
}
if (output!=null) {
output.close();
output = null;
}
}
}

View File

@ -46,11 +46,13 @@ import org.servalproject.servaldna.rhizome.RhizomeInsertBundle;
import org.servalproject.servaldna.rhizome.RhizomeInvalidManifestException;
import org.servalproject.servaldna.rhizome.RhizomeManifest;
import org.servalproject.servaldna.rhizome.RhizomeManifestBundle;
import org.servalproject.servaldna.rhizome.RhizomeManifestParseException;
import org.servalproject.servaldna.rhizome.RhizomeManifestSizeException;
import org.servalproject.servaldna.rhizome.RhizomePayloadBundle;
import org.servalproject.servaldna.rhizome.RhizomePayloadRawBundle;
import org.servalproject.servaldna.rhizome.RhizomeReadOnlyException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
@ -159,6 +161,10 @@ public class ServalDClient implements ServalDHttpConnectionFactory {
return RhizomeCommon.rhizomeImport(this, manifest, payloadStream);
}
public RhizomeImportStatus rhizomeImportZip(File zipFile) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException, RhizomeManifestParseException {
return RhizomeCommon.rhizomeImportZip(this, zipFile);
}
public MeshMSConversationList meshmsListConversations(SubscriberId sid) throws ServalDInterfaceException, IOException, MeshMSException
{
MeshMSConversationList list = new MeshMSConversationList(this, sid);

View File

@ -34,10 +34,12 @@ import org.servalproject.servaldna.ServalDNotImplementedException;
import org.servalproject.servaldna.SubscriberId;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@ -465,16 +467,45 @@ public class RhizomeCommon
}
}
public static RhizomeImportStatus rhizomeImport(ServalDHttpConnectionFactory connector, RhizomeManifest manifest, InputStream payloadStream) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException {
public static RhizomeImportStatus rhizomeImportZip(ServalDHttpConnectionFactory connector, File zipFile) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException, RhizomeManifestParseException {
RandomAccessFile file = new RandomAccessFile(zipFile, "r");
RhizomeManifest manifest = RhizomeManifest.fromZipComment(file);
HttpURLConnection conn = connector.newServalDHttpConnection(
"/restful/rhizome/import?id="+manifest.id.toHex()+"&version="+manifest.version);
PostHelper helper = new PostHelper(conn);
try {
helper.connect();
helper.writeField("manifest", manifest);
if (payloadStream != null)
helper.writeField("payload", null, payloadStream);
OutputStream out = helper.beginFileField("payload", null);
file.seek(0);
long readLength = manifest.filesize-2;
byte buff[] = new byte[4096];
while (readLength>0){
int len = readLength > buff.length ? buff.length : (int)readLength;
int read = file.read(buff, 0, len);
out.write(buff, 0, read);
readLength -= read;
}
buff[0]=0;
buff[1]=0;
out.write(buff, 0, 2);
helper.close();
int[] expected_response_codes = { HttpURLConnection.HTTP_OK,
HttpURLConnection.HTTP_CREATED,
HttpURLConnection.HTTP_ACCEPTED};
Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes);
decodeHeaderPayloadStatusOrNull(status, conn);
checkPayloadStatus(status);
decodeHeaderBundleStatus(status, conn);
checkBundleStatus(status);
return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code);
}catch (ProtocolException e){
// dodgy java implementation, only means that the server did not return 100-continue
// attempting to read the input stream will fail again
@ -486,17 +517,40 @@ public class RhizomeCommon
}
throw e;
}
}
int[] expected_response_codes = { HttpURLConnection.HTTP_OK,
HttpURLConnection.HTTP_CREATED,
HttpURLConnection.HTTP_ACCEPTED};
public static RhizomeImportStatus rhizomeImport(ServalDHttpConnectionFactory connector, RhizomeManifest manifest, InputStream payloadStream) throws ServalDInterfaceException, IOException, RhizomeException, RhizomeManifestSizeException {
HttpURLConnection conn = connector.newServalDHttpConnection(
"/restful/rhizome/import?id="+manifest.id.toHex()+"&version="+manifest.version);
PostHelper helper = new PostHelper(conn);
try {
helper.connect();
helper.writeField("manifest", manifest);
if (manifest.filesize>0 && payloadStream != null)
helper.writeField("payload", null, payloadStream);
helper.close();
Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes);
decodeHeaderPayloadStatusOrNull(status, conn);
checkPayloadStatus(status);
decodeHeaderBundleStatus(status, conn);
checkBundleStatus(status);
return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code);
int[] expected_response_codes = { HttpURLConnection.HTTP_OK,
HttpURLConnection.HTTP_CREATED,
HttpURLConnection.HTTP_ACCEPTED};
Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes);
decodeHeaderPayloadStatusOrNull(status, conn);
checkPayloadStatus(status);
decodeHeaderBundleStatus(status, conn);
checkBundleStatus(status);
return new RhizomeImportStatus(status.bundle_status_code, status.payload_status_code);
}catch (ProtocolException e){
// dodgy java implementation, only means that the server did not return 100-continue
// attempting to read the input stream will fail again
switch (conn.getResponseCode()){
case 200:
return new RhizomeImportStatus(RhizomeBundleStatus.SAME, null);
case 202:
return new RhizomeImportStatus(RhizomeBundleStatus.OLD, null);
}
throw e;
}
}
private static RhizomeManifest manifestFromHeaders(HttpURLConnection conn) throws ServalDInterfaceException

View File

@ -22,6 +22,7 @@ package org.servalproject.servaldna.rhizome;
import java.io.File;
import java.io.FileInputStream;
import java.io.RandomAccessFile;
import java.util.Map;
import java.util.HashMap;
import java.util.HashSet;
@ -41,6 +42,7 @@ import org.servalproject.servaldna.BundleKey;
public class RhizomeManifest {
public final static int TEXT_FORMAT_MAX_SIZE = 8192;
public static final String MIME_TYPE = "rhizome/manifest; format=\"text+binarysig\"";
// Core fields used for routing and expiry (cannot be null)
public final BundleId id;
@ -229,6 +231,39 @@ public class RhizomeManifest {
}
}
public static RhizomeManifest fromZipComment(RandomAccessFile file) throws IOException, RhizomeManifestParseException {
int readLen = RhizomeManifest.TEXT_FORMAT_MAX_SIZE + 22;
file.seek(file.length() - readLen);
byte buff[] = new byte[readLen];
file.readFully(buff);
int offset = buff.length - 21;
while(offset>0) {
if (buff[--offset] != 0x06)
continue;
if (buff[--offset] != 0x05)
continue;
if (buff[--offset] != 0x4b)
continue;
if (buff[--offset] != 0x50)
continue;
// located zip EOCD record marker 0x504b0506
offset += 20;
int manifestLen = (buff[offset++]&0xFF) | ((buff[offset++] & 0xFF) << 8);
if (manifestLen != readLen - offset)
throw new RhizomeManifestParseException("Zip Comment length ("+manifestLen+") doesn't align with end of file ("+readLen+", "+offset+")");
if (manifestLen == 0)
throw new RhizomeManifestParseException("No Zip Comment");
RhizomeManifest manifest = RhizomeManifest.fromTextFormat(buff, offset, manifestLen);
long expectedFileSize = file.length() - readLen + offset;
if (manifest.filesize != expectedFileSize)
throw new RhizomeManifestParseException("Manifest filesize doesn't match zip file length");
return manifest;
}
throw new RhizomeManifestParseException("Zip EOCD record not found");
}
private static boolean isFieldNameFirstChar(char c)
{
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z');

View File

@ -1240,8 +1240,11 @@ setup_RhizomeImport() {
setup
set_instance +B
create_single_identity
rhizome_add_bundles $SIDB 1 1
executeOk_servald rhizome export manifest "${BID[1]}" file1.manifest
create_file file1 100
executeOk_servald rhizome add file $SIDB file1 file1.manifest
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
executeOk_servald rhizome export manifest "${manifestid}" file1.manifest
set_instance +A
}
test_RhizomeImport() {
@ -1253,7 +1256,7 @@ test_RhizomeImport() {
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/import?id=${BID[1]}&version=${VERSION[1]}"
"http://$addr_localhost:$PORTA/restful/rhizome/import?id=${manifestid}&version=${version}"
tfw_cat http.header http.body
assertStdoutIs 201
assertGrep http.header '100 Continue'
@ -1267,7 +1270,7 @@ test_RhizomeImport() {
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/import?id=${BID[1]}&version=${VERSION[1]}"
"http://$addr_localhost:$PORTA/restful/rhizome/import?id=${manifestid}&version=${version}"
tfw_cat http.header http.body
assertStdoutIs 200
assertGrep --matches=0 http.header '100 Continue'
@ -1275,6 +1278,35 @@ test_RhizomeImport() {
assertJq http.body 'contains({"http_status_message": "Bundle already in store"})'
}
doc_RhizomeImportLarge="HTTP RESTful import 50 MiB Rhizome bundle"
setup_RhizomeImportLarge() {
setup
set_instance +B
create_single_identity
create_file file1 50m
executeOk_servald rhizome add file $SIDB file1 file1.manifest
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
executeOk_servald rhizome export manifest "${manifestid}" file1.manifest
set_instance +A
}
test_RhizomeImportLarge() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--header 'Transfer-Encoding: chunked' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "manifest=@file1.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/import?id=${manifestid}&version=${version}"
tfw_cat http.header http.body
assertStdoutIs 201
assertGrep http.header '100 Continue'
assertJq http.body 'contains({"http_status_code": 201})'
assertJq http.body 'contains({"http_status_message": "Created"})'
}
doc_RhizomeJournalAppend="HTTP RESTful Rhizome journal create and append"
setup_RhizomeJournalAppend() {
setup