ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-02-20 17:33:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge RESTful Rhizome journal append into development

Browse Source 
Support for appending to Rhizome journals using the RESTful API
This commit is contained in:
Andrew Bettison 2015-04-13 16:59:06 +09:30
commit 30f4a398ea
30 changed files with 1497 additions and 597 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cli.c
View File

@ -379,11 +379,16 @@ int cli_optional_bundle_secret_key(const char *arg)
return !arg[0] || str_to_rhizome_bsk_t(NULL, arg) != -1;
}
int cli_manifestid(const char *arg)
int cli_bid(const char *arg)
{
return str_to_rhizome_bid_t(NULL, arg) != -1;
}
int cli_optional_bid(const char *arg)
{
return !arg[0] || str_to_rhizome_bid_t(NULL, arg) != -1;
}
int cli_fileid(const char *arg)
{
return str_to_rhizome_filehash_t(NULL, arg) != -1;

18
cli.h
View File

@ -80,16 +80,30 @@ int cli_usage_args(const int argc, const char *const *args, const struct cli_sch
int cli_usage_parsed(const struct cli_parsed *parsed, XPRINTF xpf);
int cli_parse(const int argc, const char *const *args, const struct cli_schema *commands, const struct cli_schema *end_commands, struct cli_parsed *parsed);
int cli_invoke(const struct cli_parsed *parsed, struct cli_context *context);
int _cli_arg(struct __sourceloc __whence, const struct cli_parsed *parsed, char *label, const char **dst, int (*validator)(const char *arg), char *defaultvalue);
/* First, assign 'defaultvalue' to '*dst', to guarantee that '*dst' is in a
* known state regardless of the return value and provide the caller with an
* alternative way to check if an argument was found.
*
* Then, if there is an argument labelled 'label' present on the 'parsed'
* command line:
* - if a validator function was supplied (not NULL) and it returns false (0)
* when invoked on '*dst', return -1, otherwise
* - assign the argument's value as a NUL-terminated string to '*dst' and
* return 0.
*
* Otherwise, there is no argument labelled 'label', so return 1.
*/
#define cli_arg(parsed, label, dst, validator, defaultvalue) _cli_arg(__WHENCE__, parsed, label, dst, validator, defaultvalue)
int _cli_arg(struct __sourceloc __whence, const struct cli_parsed *parsed, char *label, const char **dst, int (*validator)(const char *arg), char *defaultvalue);
int cli_lookup_did(const char *text);
int cli_path_regular(const char *arg);
int cli_absolute_path(const char *arg);
int cli_optional_sid(const char *arg);
int cli_optional_bundle_secret_key(const char *arg);
int cli_manifestid(const char *arg);
int cli_bid(const char *arg);
int cli_optional_bid(const char *arg);
int cli_fileid(const char *arg);
int cli_optional_bundle_crypt_key(const char *arg);
int cli_interval_ms(const char *arg);

1
conf_schema.h
View File

@ -263,7 +263,6 @@ ATOM(bool_t, rhizome_ads, 0, boolean,, "")
ATOM(bool_t, rhizome_mdp_rx, 0, boolean,, "")
ATOM(bool_t, subscriber, 0, boolean,, "")
ATOM(bool_t, meshms, 0, boolean,, "")
ATOM(bool_t, manifests, 0, boolean,, "")
ATOM(bool_t, vomp, 0, boolean,, "")
ATOM(bool_t, profiling, 0, boolean,, "")
ATOM(bool_t, linkstate, 0, boolean,, "")

3
crypto.c
View File

@ -121,7 +121,7 @@ int crypto_sign_message(struct keyring_identity *identity, unsigned char *conten
return ret;
}
int crypto_sign_compute_public_key(const unsigned char *skin, unsigned char *pk)
void crypto_sign_compute_public_key(const unsigned char *skin, unsigned char *pk)
{
IN();
unsigned char h[64];
@ -135,6 +135,5 @@ int crypto_sign_compute_public_key(const unsigned char *skin, unsigned char *pk)
ge_scalarmult_base(&A,h);
ge_p3_tobytes(pk,&A);
RETURN(0);
OUT();
}

2
crypto.h
View File

@ -32,6 +32,6 @@ int crypto_create_signature(unsigned char *key,
unsigned char *content, int content_len,
unsigned char *signature, int *sig_length);
int crypto_sign_message(struct keyring_identity *identity, unsigned char *content, size_t buffer_len, size_t *content_len);
int crypto_sign_compute_public_key(const unsigned char *skin, unsigned char *pk);
void crypto_sign_compute_public_key(const unsigned char *skin, unsigned char *pk);
#endif

80
dataformats.c
View File

@ -32,27 +32,38 @@ int cmp_sid_t(const sid_t *a, const sid_t *b)
int str_to_sid_t(sid_t *sidp, const char *hex)
{
const char *end;
return strn_to_sid_t(sidp, hex, SIZE_MAX, &end) != -1 && *end == '\0' ? 0 : -1;
return parse_sid_t(sidp, hex, -1, NULL); // checks for nul terminator
}
int strn_to_sid_t(sid_t *sidp, const char *hex, size_t hexlen, const char **endp)
int strn_to_sid_t(sid_t *sidp, const char *hex, size_t hexlen)
{
if (strn_startswith(hex, hexlen, "broadcast", endp)) {
return parse_sid_t(sidp, hex, hexlen, NULL); // does not check for nul terminator
}
int parse_sid_t(sid_t *sidp, const char *hex, ssize_t hexlen, const char **endp)
{
const char *end = NULL;
if (strn_startswith(hex, hexlen, "broadcast", &end)) {
if (endp)
*endp = end;
else if (hexlen == -1 && *end != '\0')
return -1;
if (sidp)
*sidp = SID_BROADCAST;
return 0;
}
sid_t tmp;
if (hexlen < sizeof tmp.binary * 2)
if (hexlen != -1 && hexlen != SID_STRLEN)
return -1;
sid_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
if (endp)
*endp = hex + SID_STRLEN;
else if (hexlen == -1 && hex[SID_STRLEN] != '\0')
return -1;
if (sidp)
*sidp = tmp;
if (endp)
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
@ -84,19 +95,28 @@ int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b)
int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex)
{
return bid ? fromhexstr(bid->binary, hex, sizeof bid->binary) : is_xstring(hex, RHIZOME_BUNDLE_ID_STRLEN) ? 0 : -1;
return parse_rhizome_bid_t(bid, hex, -1, NULL); // checks for nul terminator
}
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp)
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, size_t hexlen)
{
return parse_rhizome_bid_t(bid, hex, hexlen, NULL); // does not check for nul terminator
}
int parse_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, ssize_t hexlen, const char **endp)
{
if (hexlen != -1 && hexlen != RHIZOME_BUNDLE_ID_STRLEN)
return -1;
rhizome_bid_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
if (endp)
*endp = hex + RHIZOME_BUNDLE_ID_STRLEN;
else if (hexlen == -1 && hex[RHIZOME_BUNDLE_ID_STRLEN] != '\0')
return -1;
if (bid)
*bid = tmp;
if (endp)
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
@ -107,37 +127,55 @@ int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t
int str_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex)
{
return hashp ? fromhexstr(hashp->binary, hex, sizeof hashp->binary) : is_xstring(hex, RHIZOME_FILEHASH_STRLEN) ? 0 : -1;
return parse_rhizome_filehash_t(hashp, hex, -1, NULL); // checks for nul terminator
}
int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const char **endp)
int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, size_t hexlen)
{
return parse_rhizome_filehash_t(hashp, hex, hexlen, NULL); // does not check for nul terminator
}
int parse_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, ssize_t hexlen, const char **endp)
{
if (hexlen != -1 && hexlen != RHIZOME_FILEHASH_STRLEN)
return -1;
rhizome_filehash_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
if (endp)
*endp = hex + RHIZOME_FILEHASH_STRLEN;
else if (hexlen == -1 && hex[RHIZOME_FILEHASH_STRLEN] != '\0')
return -1;
if (hashp)
*hashp = tmp;
if (endp)
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
int str_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex)
{
return bkp ? fromhexstr(bkp->binary, hex, sizeof bkp->binary) : is_xstring(hex, RHIZOME_BUNDLE_KEY_STRLEN) ? 0 : -1;
return parse_rhizome_bk_t(bkp, hex, -1, NULL); // checks for nul terminator
}
int strn_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex, const char **endp)
int strn_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex, size_t hexlen)
{
return parse_rhizome_bk_t(bkp, hex, hexlen, NULL); // does not check for nul terminator
}
int parse_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex, ssize_t hexlen, const char **endp)
{
if (hexlen != -1 && hexlen != RHIZOME_BUNDLE_KEY_STRLEN)
return -1;
rhizome_bk_t tmp;
int n = fromhex(tmp.binary, hex, sizeof tmp.binary);
if (n != sizeof tmp.binary)
return -1;
if (endp)
*endp = hex + RHIZOME_BUNDLE_KEY_STRLEN;
else if (hexlen == -1 && hex[RHIZOME_BUNDLE_KEY_STRLEN] != '\0')
return -1;
if (bkp)
*bkp = tmp;
if (endp)
*endp = hex + sizeof tmp.binary * 2;
return 0;
}
@ -157,7 +195,7 @@ int strn_to_rhizome_bsk_t(rhizome_bk_t *bskp, const char *text, size_t textlen)
strn_digest_passphrase(bskp->binary, sizeof bskp->binary, text, textlen);
return 0;
}
return strn_to_rhizome_bk_t(bskp, text, NULL);
return strn_to_rhizome_bk_t(bskp, text, textlen);
}
int rhizome_strn_is_bundle_crypt_key(const char *key)

2
httpd.c
View File

@ -38,6 +38,7 @@ static HTTP_HANDLER static_page;
HTTP_HANDLER restful_rhizome_bundlelist_json;
HTTP_HANDLER restful_rhizome_newsince;
HTTP_HANDLER restful_rhizome_insert;
HTTP_HANDLER restful_rhizome_append;
HTTP_HANDLER restful_rhizome_;
HTTP_HANDLER restful_meshms_;
HTTP_HANDLER restful_keyring_;
@ -59,6 +60,7 @@ struct http_handler paths[]={
{"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json},
{"/restful/rhizome/newsince/", restful_rhizome_newsince},
{"/restful/rhizome/insert", restful_rhizome_insert},
{"/restful/rhizome/append", restful_rhizome_append},
{"/restful/rhizome/", restful_rhizome_},
{"/restful/meshms/", restful_meshms_},
{"/restful/keyring/", restful_keyring_},

6
httpd.h
View File

@ -104,11 +104,14 @@ typedef struct httpd_request
/* For receiving RESTful Rhizome insert request
*/
struct {
// If this is really a (journal) append request
bool_t appending;
// Which part is currently being received
const char *current_part;
// Which parts have already been received
bool_t received_author;
bool_t received_secret;
bool_t received_bundleid;
bool_t received_manifest;
bool_t received_payload;
// For storing the "bundle-author" hex SID as we receive it
@ -119,6 +122,9 @@ typedef struct httpd_request
char secret_text[RHIZOME_BUNDLE_SECRET_MAX_STRLEN];
size_t secret_text_len;
rhizome_bk_t bundle_secret;
// For storing the "bundle-id" hex as we receive it
char bid_text[RHIZOME_BUNDLE_ID_STRLEN];
size_t bid_text_len;
// The "force-new" parameter
char force_new_text[5]; // enough for "false"
size_t force_new_text_len;

92
java/org/servalproject/servaldna/rhizome/RhizomeCommon.java
View File

@ -138,6 +138,12 @@ public class RhizomeCommon
status.payload_status_message = headerString(conn, "Serval-Rhizome-Result-Payload-Status-Message");
}
protected static void decodeHeaderPayloadStatusOrNull(Status status, HttpURLConnection conn) throws ServalDInterfaceException
{
status.payload_status_code = headerOrNull(conn, "Serval-Rhizome-Result-Payload-Status-Code", RhizomePayloadStatus.class);
status.payload_status_message = headerStringOrNull(conn, "Serval-Rhizome-Result-Payload-Status-Message");
}
protected static void decodeRestfulStatus(Status status, JSONTokeniser json) throws IOException, ServalDInterfaceException
{
try {
@ -424,51 +430,53 @@ public class RhizomeCommon
Status status = RhizomeCommon.receiveResponse(conn, expected_response_codes);
try {
dumpHeaders(conn, System.err);
decodeHeaderPayloadStatus(status, conn);
switch (status.payload_status_code) {
case ERROR:
dumpStatus(status, System.err);
throw new ServalDFailureException("received Rhizome payload_status=ERROR " + quoteString(status.payload_status_message) + " from " + conn.getURL());
case EMPTY:
case NEW:
case STORED:
decodeHeaderBundleStatus(status, conn);
dumpStatus(status, System.err);
switch (status.bundle_status_code) {
decodeHeaderPayloadStatusOrNull(status, conn);
if (status.payload_status_code != null) {
switch (status.payload_status_code) {
case ERROR:
throw new ServalDFailureException("received Rhizome bundle_status=ERROR " + quoteString(status.bundle_status_message) + " from " + conn.getURL());
dumpStatus(status, System.err);
throw new ServalDFailureException("received Rhizome payload_status=ERROR " + quoteString(status.payload_status_message) + " from " + conn.getURL());
case EMPTY:
case NEW:
case SAME:
case DUPLICATE:
case OLD:
case NO_ROOM: {
if (!conn.getContentType().equals("rhizome-manifest/text"))
throw new ServalDInterfaceException("unexpected HTTP Content-Type " + conn.getContentType() + " from " + conn.getURL());
RhizomeManifest returned_manifest = RhizomeManifest.fromTextFormat(status.input_stream);
BundleExtra extra = bundleExtraFromHeaders(conn);
return new RhizomeInsertBundle(status.bundle_status_code, returned_manifest, extra.rowId, extra.insertTime, extra.author, extra.secret);
}
case INVALID:
throw new RhizomeInvalidManifestException(status.bundle_status_message, conn.getURL());
case FAKE:
throw new RhizomeFakeManifestException(status.bundle_status_message, conn.getURL());
case INCONSISTENT:
throw new RhizomeInconsistencyException(status.bundle_status_message, conn.getURL());
case READONLY:
throw new RhizomeReadOnlyException(status.bundle_status_message, conn.getURL());
case STORED:
break;
case TOO_BIG:
case EVICTED:
dumpStatus(status, System.err);
return null;
case WRONG_SIZE:
case WRONG_HASH:
dumpStatus(status, System.err);
throw new RhizomeInconsistencyException(status.payload_status_message, conn.getURL());
case CRYPTO_FAIL:
dumpStatus(status, System.err);
throw new RhizomeEncryptionException(status.payload_status_message, conn.getURL());
}
break;
case TOO_BIG:
case EVICTED:
dumpStatus(status, System.err);
return null;
case WRONG_SIZE:
case WRONG_HASH:
dumpStatus(status, System.err);
throw new RhizomeInconsistencyException(status.payload_status_message, conn.getURL());
case CRYPTO_FAIL:
dumpStatus(status, System.err);
throw new RhizomeEncryptionException(status.payload_status_message, conn.getURL());
}
decodeHeaderBundleStatus(status, conn);
dumpStatus(status, System.err);
switch (status.bundle_status_code) {
case ERROR:
throw new ServalDFailureException("received Rhizome bundle_status=ERROR " + quoteString(status.bundle_status_message) + " from " + conn.getURL());
case NEW:
case SAME:
case DUPLICATE:
case OLD:
case NO_ROOM: {
if (!conn.getContentType().equals("rhizome-manifest/text"))
throw new ServalDInterfaceException("unexpected HTTP Content-Type " + conn.getContentType() + " from " + conn.getURL());
RhizomeManifest returned_manifest = RhizomeManifest.fromTextFormat(status.input_stream);
BundleExtra extra = bundleExtraFromHeaders(conn);
return new RhizomeInsertBundle(status.bundle_status_code, returned_manifest, extra.rowId, extra.insertTime, extra.author, extra.secret);
}
case INVALID:
throw new RhizomeInvalidManifestException(status.bundle_status_message, conn.getURL());
case FAKE:
throw new RhizomeFakeManifestException(status.bundle_status_message, conn.getURL());
case INCONSISTENT:
throw new RhizomeInconsistencyException(status.bundle_status_message, conn.getURL());
case READONLY:
throw new RhizomeReadOnlyException(status.bundle_status_message, conn.getURL());
}
}
catch (RhizomeManifestParseException e) {

2
mdp_filter.c
View File

@ -452,7 +452,7 @@ static int _endpoint(Cursor c, uint8_t *flagsp, uint8_t port_flag, struct subscr
preload(c, SID_STRLEN);
if (skip(c, "*")) {
*subscr = NULL;
} else if (strn_to_sid_t(&sid, preloaded(c), available(c), &end) == 0) {
} else if (parse_sid_t(&sid, preloaded(c), available(c), &end) == 0) {
if ((*subscr = find_subscriber(sid.binary, sizeof sid.binary, 1)) == NULL)
return 0;
advance_to(c, end);

6
meshms.c
View File

@ -69,8 +69,8 @@ static enum meshms_status get_my_conversation_bundle(const sid_t *my_sidp, rhizo
if (m->haveSecret == NEW_BUNDLE_ID) {
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
rhizome_manifest_set_name(m, "");
if (rhizome_fill_manifest(m, NULL, my_sidp) == -1)
return WHY("Invalid manifest");
if (rhizome_fill_manifest(m, NULL, my_sidp) != NULL)
return WHY("Invalid conversation manifest");
if (config.debug.meshms) {
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
@ -205,7 +205,7 @@ static int create_ply(const sid_t *my_sid, struct meshms_conversations *conv, rh
rhizome_manifest_set_recipient(m, &conv->them);
rhizome_manifest_set_filesize(m, 0);
rhizome_manifest_set_tail(m, 0);
if (rhizome_fill_manifest(m, NULL, my_sid))
if (rhizome_fill_manifest(m, NULL, my_sid) != NULL)
return -1;
assert(m->haveSecret);
assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);

4
meshms_restful.c
View File

@ -124,7 +124,7 @@ int restful_meshms_(httpd_request *r, const char *remainder)
http_size_t content_length = CONTENT_LENGTH_UNKNOWN;
HTTP_HANDLER *handler = NULL;
const char *end;
if (strn_to_sid_t(&r->sid1, remainder, SIZE_MAX, &end) != -1) {
if (parse_sid_t(&r->sid1, remainder, -1, &end) != -1) {
remainder = end;
if (strcmp(remainder, "/conversationlist.json") == 0) {
handler = restful_meshms_conversationlist_json;
@ -136,7 +136,7 @@ int restful_meshms_(httpd_request *r, const char *remainder)
content_length = 0;
remainder = "";
}
else if (*remainder == '/' && strn_to_sid_t(&r->sid2, remainder + 1, SIZE_MAX, &end) != -1) {
else if (*remainder == '/' && parse_sid_t(&r->sid2, remainder + 1, -1, &end) != -1) {
remainder = end;
if (strcmp(remainder, "/messagelist.json") == 0) {
handler = restful_meshms_messagelist_json;

284
rhizome.c
View File

@ -93,6 +93,255 @@ int rhizome_fetch_delay_ms()
return config.rhizome.fetch_delay_ms;
}
/* Create a manifest structure to accompany adding a file to Rhizome or appending to a journal.
* This function is used by all application-facing APIs (eg, CLI, RESTful HTTP). It differs from
* the import operation in that if the caller does not supply a complete, signed manifest then this
* operation will create it using the fields supplied. Also, the caller can supply a clear-text
* payload with the 'crypt=1' field to cause it to be stored encrypted.
*
* - if 'appending' is true then the new bundle will be a journal bundle, otherwise it will be a
* normal bundle. Any existing manifest must be consistent; eg, an append will fail if a bundle
* with the same Bundle Id already exists in the store and is not a journal.
*
* - 'm' must point to a manifest structure into which any supplied partial manifest has already
* been parsed. If the caller supplied no (partial) manifest at all, then the manifest 'm' will
* be blank.
*
* - 'mout' must point to a manifest pointer which is updated to hold the constructed manifest.
*
* - 'bid' must point to a supplied bundle id parameter, or NULL if none was supplied.
*
* - 'bsk' must point to a supplied bundle secret parameter, or NULL if none was supplied.
*
* - 'author' must point to a supplied author parameter, or NULL if none was supplied.
*
* - 'file_path' can point to a supplied payload file name (eg, if the payload was read from a named
* file), or can be NULL. If not NULL, then the file's name will be used to fill in the 'name'
* field of the manifest if it was not explicitly supplied in 'm' or in the existing manifest.
*
* - 'nassignments' and 'assignments' describe an array of field assignments that override the
* fields supplied in 'm' and also the fields in any existing manifest with the same Bundle Id.
*
* - 'reason' may either be NULL or points to a strbuf to which descriptive text is appended if the
* manifest creation fails.
*
* If the add is successful, modifies '*mout' to point to the constructed Manifest, which might be
* 'm' or might be another manifest, and returns 0. It is the caller's responsibility to free
* '*mout'.
*
* If the add fails because of invalid field settings that violate Rhizome semantics (eg, a missing
* mandatory field, a malformed field name or value), then if 'reason' is not NULL, appends a text
* string to the 'reason' strbuf that describes the cause of the failure, does not alter '*mout',
* and returns 1.
*
* If the add fails because of a recoverable error (eg, database locking) then if 'reason' is not
* NULL, appends a text string to the 'reason' strbuf that describes the cause of the failure, does
* not alter '*mout', and returns 2.
*
* If the add fails because of an unrecoverable error (eg, out of memory, i/o failure)
* then if 'reason' is not NULL, appends a text string to the 'reason' strbuf that describes the
* cause of the failure, does not alter '*mout', and returns -1.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
enum rhizome_add_file_result rhizome_manifest_add_file(int appending,
rhizome_manifest *m,
rhizome_manifest **mout,
const rhizome_bid_t *bid,
const rhizome_bk_t *bsk,
const sid_t *author,
const char *file_path,
unsigned nassignments,
const struct rhizome_manifest_field_assignment *assignments,
strbuf reason
)
{
const char *cause = NULL;
enum rhizome_add_file_result result = RHIZOME_ADD_FILE_ERROR;
rhizome_manifest *existing_manifest = NULL;
rhizome_manifest *new_manifest = NULL;
assert(m != NULL);
// Caller must not supply a malformed manifest (but an invalid one is okay because missing
// fields will be filled in, so we don't check validity here).
assert(!m->malformed);
// If appending to a journal, caller must not supply 'version', 'filesize' or 'filehash' fields,
// because these will be calculated by the journal append logic.
if (appending) {
if (m->version)
DEBUG(cause = "Cannot set 'version' field in journal append");
else if (m->filesize != RHIZOME_SIZE_UNSET)
DEBUG(cause = "Cannot set 'filesize' field in journal append");
else if (m->has_filehash)
DEBUG(cause = "Cannot set 'filehash' field in journal append");
if (cause) {
result = RHIZOME_ADD_FILE_INVALID_FOR_JOURNAL;
goto error;
}
}
if (bid) {
if (config.debug.rhizome)
DEBUGF("Reading manifest from database: id=%s", alloca_tohex_rhizome_bid_t(*bid));
if ((existing_manifest = rhizome_new_manifest()) == NULL) {
WHY(cause = "Manifest struct could not be allocated");
goto error;
}
enum rhizome_bundle_status status = rhizome_retrieve_manifest(bid, existing_manifest);
switch (status) {
case RHIZOME_BUNDLE_STATUS_NEW:
// No manifest with that bundle ID exists in the store, so we are building a bundle from
// scratch.
rhizome_manifest_free(existing_manifest);
existing_manifest = NULL;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
// Found a manifest with the same bundle ID. If appending to a journal, then keep the
// existing 'version', 'filesize' and 'filehash' (so they can be verified when the existing
// payload is copied) and don't allow the supplied manifest to overwrite them. If not a
// journal, then unset the 'version', 'filesize' and 'filehash' fields, then overwrite the
// existing manifest with the supplied manifest.
if (!appending) {
rhizome_manifest_del_version(existing_manifest);
rhizome_manifest_del_filesize(existing_manifest);
rhizome_manifest_del_filehash(existing_manifest);
}
if (rhizome_manifest_overwrite(existing_manifest, m) == -1) {
WHY(cause = "Existing manifest could not be overwritten");
goto error;
}
new_manifest = existing_manifest;
existing_manifest = NULL;
break;
case RHIZOME_BUNDLE_STATUS_BUSY:
WARN(cause = "Existing manifest not retrieved due to Rhizome store locking");
result = RHIZOME_ADD_FILE_BUSY;
goto error;
case RHIZOME_BUNDLE_STATUS_ERROR:
WHY(cause = "Error retrieving existing manifest from Rhizome store");
goto error;
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
case RHIZOME_BUNDLE_STATUS_OLD:
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_FAKE:
case RHIZOME_BUNDLE_STATUS_INCONSISTENT:
case RHIZOME_BUNDLE_STATUS_NO_ROOM:
case RHIZOME_BUNDLE_STATUS_READONLY:
FATALF("rhizome_retrieve_manifest() returned %s", rhizome_bundle_status_message(status));
}
}
// If no existing bundle has been identified, we are building a bundle from scratch.
if (!new_manifest) {
new_manifest = m;
// A new journal manifest needs the 'filesize' and 'tail' fields set so that the first append can
// succeed.
if (appending) {
if (new_manifest->filesize == RHIZOME_SIZE_UNSET)
rhizome_manifest_set_filesize(new_manifest, 0);
if (new_manifest->tail == RHIZOME_SIZE_UNSET)
rhizome_manifest_set_tail(new_manifest, 0);
}
}
// Apply the field assignments, overriding the existing manifest fields.
if (nassignments) {
unsigned i;
for (i = 0; i != nassignments; ++i) {
const struct rhizome_manifest_field_assignment *asg = &assignments[i];
rhizome_manifest_remove_field(new_manifest, asg->label, asg->labellen);
if (asg->value) {
const char *label = alloca_strndup(asg->label, asg->labellen);
enum rhizome_manifest_parse_status status = rhizome_manifest_parse_field(new_manifest, asg->label, asg->labellen, asg->value, asg->valuelen);
int status_ok = 0;
switch (status) {
case RHIZOME_MANIFEST_ERROR:
WHYF("Fatal error updating manifest field");
if (reason)
strbuf_sprintf(reason, "Fatal error updating manifest field: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
goto error;
case RHIZOME_MANIFEST_OK:
status_ok = 1;
break;
case RHIZOME_MANIFEST_SYNTAX_ERROR:
if (config.debug.rhizome)
DEBUGF("Manifest syntax error: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
if (reason)
strbuf_sprintf(reason, "Manifest syntax error: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
result = RHIZOME_ADD_FILE_INVALID;
goto error;
case RHIZOME_MANIFEST_DUPLICATE_FIELD:
// We already deleted the field, so if this happens, its a nasty bug
FATALF("Duplicate field should not occur: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
case RHIZOME_MANIFEST_INVALID:
if (config.debug.rhizome)
DEBUGF("Manifest invalid field: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
if (reason)
strbuf_sprintf(reason, "Manifest invalid field: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
result = RHIZOME_ADD_FILE_INVALID;
goto error;
case RHIZOME_MANIFEST_MALFORMED:
if (config.debug.rhizome)
DEBUGF("Manifest malformed field: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
if (reason)
strbuf_sprintf(reason, "Manifest malformed field: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
result = RHIZOME_ADD_FILE_INVALID;
goto error;
case RHIZOME_MANIFEST_OVERFLOW:
if (config.debug.rhizome)
DEBUGF("Too many fields in manifest at: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
if (reason)
strbuf_sprintf(reason, "Too many fields in manifest at: %s=%s", label, alloca_toprint(-1, asg->value, asg->valuelen));
result = RHIZOME_ADD_FILE_INVALID;
goto error;
}
if (!status_ok)
FATALF("status = %d", status);
}
}
}
if (appending && !new_manifest->is_journal) {
cause = "Cannot append to a non-journal";
if (config.debug.rhizome)
DEBUG(cause);
result = RHIZOME_ADD_FILE_REQUIRES_JOURNAL;
goto error;
}
if (!appending && new_manifest->is_journal) {
cause = "Cannot add a journal bundle (use append instead)";
if (config.debug.rhizome)
DEBUG(cause);
result = RHIZOME_ADD_FILE_INVALID_FOR_JOURNAL;
goto error;
}
if (bsk) {
if (new_manifest->has_id) {
if (!rhizome_apply_bundle_secret(new_manifest, bsk)) {
WHY(cause = "Supplied bundle secret does not match Bundle Id");
result = RHIZOME_ADD_FILE_WRONG_SECRET;
goto error;
}
} else {
rhizome_new_bundle_from_secret(new_manifest, bsk);
}
}
// TODO: one day there will be no default service, but for now if no service
// is specified, it defaults to 'file' (file sharing).
if (new_manifest->service == NULL) {
WARNF("Manifest 'service' field not supplied - setting to '%s'", RHIZOME_SERVICE_FILE);
rhizome_manifest_set_service(new_manifest, RHIZOME_SERVICE_FILE);
}
if ((cause = rhizome_fill_manifest(new_manifest, file_path, author ? author : NULL)) != NULL)
goto error;
*mout = new_manifest;
return RHIZOME_ADD_FILE_OK;
error:
assert(result != RHIZOME_ADD_FILE_OK);
if (cause && reason)
strbuf_puts(reason, cause);
if (new_manifest && new_manifest != m && new_manifest != existing_manifest)
rhizome_manifest_free(new_manifest);
if (existing_manifest)
rhizome_manifest_free(existing_manifest);
return result;
}
/* Import a bundle from a pair of files, one containing the manifest and the optional other
* containing the payload. The work is all done by rhizome_bundle_import() and
* rhizome_store_manifest().
@ -310,6 +559,29 @@ enum rhizome_bundle_status rhizome_manifest_check_stored(rhizome_manifest *m, rh
return result;
}
/* Insert the manifest 'm' into the Rhizome store. This function encapsulates all the invariants
* that a manifest must satisfy before it is allowed into the store, so it is used by both the sync
* protocol and the application layer.
*
* - If the manifest is not valid then returns RHIZOME_BUNDLE_STATUS_INVALID. A valid manifest is
* one with all the core (transport) fields present and consistent ('id', 'version', 'filesize',
* 'filehash', 'tail'), all mandatory application fields present and consistent ('service',
* 'date') and any other service-dependent mandatory fields present (eg, 'sender', 'recipient').
*
* - If the manifest's signature does not verify, then returns RHIZOME_BUNDLE_STATUS_FAKE.
*
* - If the manifest has a payload (filesize != 0) but the payload is not present in the store
* (filehash), then returns an internal error RHIZOME_BUNDLE_STATUS_ERROR (-1).
*
* - If the store will not accept the manifest because there is already the same or a newer
* manifest in the store, then returns RHIZOME_BUNDLE_STATUS_SAME or RHIZOME_BUNDLE_STATUS_OLD.
*
* This function then attempts to store the manifest. If this fails due to an internal error,
* then returns RHIZOME_BUNDLE_STATUS_ERROR (-1), otherwise returns RHIZOME_BUNDLE_STATUS_NEW to
* indicate that the manifest was successfully stored.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
enum rhizome_bundle_status rhizome_add_manifest(rhizome_manifest *m, rhizome_manifest **mout)
{
if (config.debug.rhizome) {
@ -361,6 +633,12 @@ const char *rhizome_bundle_status_message(enum rhizome_bundle_status status)
return NULL;
}
const char *rhizome_bundle_status_message_nonnull(enum rhizome_bundle_status status)
{
const char *message = rhizome_bundle_status_message(status);
return message ? message : "Invalid";
}
const char *rhizome_payload_status_message(enum rhizome_payload_status status)
{
switch (status) {
@ -376,3 +654,9 @@ const char *rhizome_payload_status_message(enum rhizome_payload_status status)
}
return NULL;
}
const char *rhizome_payload_status_message_nonnull(enum rhizome_payload_status status)
{
const char *message = rhizome_payload_status_message(status);
return message ? message : "Invalid";
}

54
rhizome.h
View File

@ -251,8 +251,11 @@ typedef struct rhizome_manifest
*/
#define rhizome_manifest_set_id(m,v) _rhizome_manifest_set_id(__WHENCE__,(m),(v))
#define rhizome_manifest_set_version(m,v) _rhizome_manifest_set_version(__WHENCE__,(m),(v))
#define rhizome_manifest_del_version(m) _rhizome_manifest_del_version(__WHENCE__,(m))
#define rhizome_manifest_set_filesize(m,v) _rhizome_manifest_set_filesize(__WHENCE__,(m),(v))
#define rhizome_manifest_del_filesize(m) _rhizome_manifest_del_filesize(__WHENCE__,(m))
#define rhizome_manifest_set_filehash(m,v) _rhizome_manifest_set_filehash(__WHENCE__,(m),(v))
#define rhizome_manifest_del_filehash(m) _rhizome_manifest_del_filehash(__WHENCE__,(m))
#define rhizome_manifest_set_tail(m,v) _rhizome_manifest_set_tail(__WHENCE__,(m),(v))
#define rhizome_manifest_set_bundle_key(m,v) _rhizome_manifest_set_bundle_key(__WHENCE__,(m),(v))
#define rhizome_manifest_del_bundle_key(m) _rhizome_manifest_del_bundle_key(__WHENCE__,(m))
@ -274,8 +277,11 @@ typedef struct rhizome_manifest
void _rhizome_manifest_set_id(struct __sourceloc, rhizome_manifest *, const rhizome_bid_t *);
void _rhizome_manifest_set_version(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_del_version(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_filesize(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_del_filesize(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_filehash(struct __sourceloc, rhizome_manifest *, const rhizome_filehash_t *);
void _rhizome_manifest_del_filehash(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_tail(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_bundle_key(struct __sourceloc, rhizome_manifest *, const rhizome_bk_t *);
void _rhizome_manifest_del_bundle_key(struct __sourceloc, rhizome_manifest *);
@ -295,6 +301,10 @@ void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, ti
void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);
#define rhizome_manifest_overwrite(dstm,srcm) _rhizome_manifest_overwrite(__WHENCE__,(dstm),(srcm))
int _rhizome_manifest_overwrite(struct __sourceloc, rhizome_manifest *m, const rhizome_manifest *srcm);
enum rhizome_manifest_parse_status {
RHIZOME_MANIFEST_ERROR = -1, // unrecoverable error while constructing manifest
RHIZOME_MANIFEST_OK = 0, // field parsed ok; manifest updated
@ -305,6 +315,21 @@ enum rhizome_manifest_parse_status {
RHIZOME_MANIFEST_OVERFLOW = 5, // maximum field count exceeded
};
/* This structure represents a manifest field assignment as received by the API
* operations "add file" or "journal append" or any other operation that takes an
* existing manifest and modifies it to produce a new one.
*
* The 'label' and 'value' strings are pointer-length instead of NUL terminated,
* to allow them to refer directly to fragments of an existing, larger text
* without requiring the caller to allocate new strings to hold them.
*/
struct rhizome_manifest_field_assignment {
const char *label;
size_t labellen;
const char *value;
size_t valuelen;
};
int rhizome_manifest_field_label_is_valid(const char *field_label, size_t field_label_len);
int rhizome_manifest_field_value_is_valid(const char *field_value, size_t field_value_len);
enum rhizome_manifest_parse_status
@ -364,7 +389,7 @@ void rhizome_vacuum_db(sqlite_retry_state *retry);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
int rhizome_get_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk);
int rhizome_new_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk);
void rhizome_new_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk);
struct rhizome_manifest_summary {
rhizome_bid_t bid;
@ -391,6 +416,7 @@ enum rhizome_bundle_status {
#define INVALID_RHIZOME_BUNDLE_STATUS ((enum rhizome_bundle_status)-2)
const char *rhizome_bundle_status_message(enum rhizome_bundle_status);
const char *rhizome_bundle_status_message_nonnull(enum rhizome_bundle_status);
enum rhizome_payload_status {
RHIZOME_PAYLOAD_STATUS_ERROR = -1,
@ -407,6 +433,7 @@ enum rhizome_payload_status {
#define INVALID_RHIZOME_PAYLOAD_STATUS ((enum rhizome_payload_status)-2)
const char *rhizome_payload_status_message(enum rhizome_payload_status);
const char *rhizome_payload_status_message_nonnull(enum rhizome_payload_status);
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_manifest_selfsign(rhizome_manifest *m);
@ -425,10 +452,31 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc);
int rhizome_store_manifest(rhizome_manifest *m);
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
enum rhizome_add_file_result {
RHIZOME_ADD_FILE_ERROR = -1,
RHIZOME_ADD_FILE_OK = 0, // manifest created successfully
RHIZOME_ADD_FILE_INVALID, // manifest not created due to invalid input
RHIZOME_ADD_FILE_BUSY, // manifest not created because database busy
RHIZOME_ADD_FILE_REQUIRES_JOURNAL, // operation is only valid for a journal
RHIZOME_ADD_FILE_INVALID_FOR_JOURNAL, // operation is not valid for a journal
RHIZOME_ADD_FILE_WRONG_SECRET, // incorrect bundle secret supplied
};
enum rhizome_add_file_result rhizome_manifest_add_file(int appending,
rhizome_manifest *m,
rhizome_manifest **mout,
const rhizome_bid_t *bid,
const rhizome_bk_t *bsk,
const sid_t *author,
const char *file_path,
unsigned nassignments,
const struct rhizome_manifest_field_assignment *assignments,
strbuf reason
);
int rhizome_bundle_import_files(rhizome_manifest *m, rhizome_manifest **m_out, const char *manifest_path, const char *filepath);
int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepath);
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp);
const char * rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp);
int rhizome_apply_bundle_secret(rhizome_manifest *, const rhizome_bk_t *);
int rhizome_manifest_add_bundle_key(rhizome_manifest *);
@ -839,9 +887,11 @@ enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, cons
int rhizome_write_buffer(struct rhizome_write *write_state, uint8_t *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, uint8_t *buffer, size_t data_size);
enum rhizome_payload_status rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t append_size);
int rhizome_write_file(struct rhizome_write *write, const char *filename);
void rhizome_fail_write(struct rhizome_write *write);
enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write);
enum rhizome_payload_status rhizome_finish_store(struct rhizome_write *write, rhizome_manifest *m, enum rhizome_payload_status status);
enum rhizome_payload_status rhizome_import_payload_from_file(rhizome_manifest *m, const char *filepath);
enum rhizome_payload_status rhizome_import_buffer(rhizome_manifest *m, uint8_t *buffer, size_t length);
enum rhizome_payload_status rhizome_stat_payload_file(rhizome_manifest *m, const char *filepath);

251
rhizome_bundle.c
View File

@ -156,6 +156,11 @@ void _rhizome_manifest_set_version(struct __sourceloc __whence, rhizome_manifest
m->finalised = 0;
}
void _rhizome_manifest_del_version(struct __sourceloc __whence, rhizome_manifest *m)
{
_rhizome_manifest_set_version(__whence, m, 0);
}
void _rhizome_manifest_set_filesize(struct __sourceloc __whence, rhizome_manifest *m, uint64_t size)
{
if (size == RHIZOME_SIZE_UNSET) {
@ -168,6 +173,11 @@ void _rhizome_manifest_set_filesize(struct __sourceloc __whence, rhizome_manifes
m->finalised = 0;
}
void _rhizome_manifest_del_filesize(struct __sourceloc __whence, rhizome_manifest *m)
{
_rhizome_manifest_set_filesize(__whence, m, RHIZOME_SIZE_UNSET);
}
/* Must always set file size before setting the file hash, to avoid assertion failures.
*/
void _rhizome_manifest_set_filehash(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_filehash_t *hash)
@ -185,6 +195,11 @@ void _rhizome_manifest_set_filehash(struct __sourceloc __whence, rhizome_manifes
m->finalised = 0;
}
void _rhizome_manifest_del_filehash(struct __sourceloc __whence, rhizome_manifest *m)
{
_rhizome_manifest_set_filehash(__whence, m, NULL);
}
void _rhizome_manifest_set_tail(struct __sourceloc __whence, rhizome_manifest *m, uint64_t tail)
{
if (tail == RHIZOME_SIZE_UNSET) {
@ -193,9 +208,9 @@ void _rhizome_manifest_set_tail(struct __sourceloc __whence, rhizome_manifest *m
} else {
const char *v = rhizome_manifest_set_ui64(m, "tail", tail);
assert(v); // TODO: remove known manifest fields from vars[]
m->tail = tail;
m->is_journal = 1;
}
m->tail = tail;
m->finalised = 0;
}
@ -509,7 +524,7 @@ int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifes
eol = p;
if (has_bid == 1) {
const char *e;
if (strn_to_rhizome_bid_t(&summ->bid, begin, &e) == 0 && e == eol)
if (parse_rhizome_bid_t(&summ->bid, begin, eol - begin, &e) == 0 && e == eol)
has_bid = 2;
else
state = Error; // invalid "id" field
@ -664,18 +679,23 @@ int rhizome_manifest_parse(rhizome_manifest *m)
OUT();
}
typedef int MANIFEST_FIELD_TESTER(rhizome_manifest *);
typedef void MANIFEST_FIELD_UNSETTER(rhizome_manifest *);
typedef int MANIFEST_FIELD_TESTER(const rhizome_manifest *);
typedef void MANIFEST_FIELD_UNSETTER(struct __sourceloc, rhizome_manifest *);
typedef void MANIFEST_FIELD_COPIER(struct __sourceloc, rhizome_manifest *, const rhizome_manifest *);
typedef int MANIFEST_FIELD_PARSER(rhizome_manifest *, const char *);
static int _rhizome_manifest_test_id(rhizome_manifest *m)
static int _rhizome_manifest_test_id(const rhizome_manifest *m)
{
return m->has_id;
}
static void _rhizome_manifest_unset_id(rhizome_manifest *m)
static void _rhizome_manifest_unset_id(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_id(m, NULL);
}
static void _rhizome_manifest_copy_id(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_id(m, srcm->has_id ? &srcm->cryptoSignPublic : NULL);
}
static int _rhizome_manifest_parse_id(rhizome_manifest *m, const char *text)
{
rhizome_bid_t bid;
@ -685,13 +705,17 @@ static int _rhizome_manifest_parse_id(rhizome_manifest *m, const char *text)
return 1;
}
static int _rhizome_manifest_test_version(rhizome_manifest *m)
static int _rhizome_manifest_test_version(const rhizome_manifest *m)
{
return m->version != 0;
}
static void _rhizome_manifest_unset_version(rhizome_manifest *m)
static void _rhizome_manifest_unset_version(struct __sourceloc __whence, rhizome_manifest *m)
{
m->version = 0;
rhizome_manifest_del_version(m);
}
static void _rhizome_manifest_copy_version(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_version(m, srcm->version);
}
static int _rhizome_manifest_parse_version(rhizome_manifest *m, const char *text)
{
@ -702,14 +726,18 @@ static int _rhizome_manifest_parse_version(rhizome_manifest *m, const char *text
return 1;
}
static int _rhizome_manifest_test_filehash(rhizome_manifest *m)
static int _rhizome_manifest_test_filehash(const rhizome_manifest *m)
{
return m->has_filehash;
}
static void _rhizome_manifest_unset_filehash(rhizome_manifest *m)
static void _rhizome_manifest_unset_filehash(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_filehash(m, NULL);
}
static void _rhizome_manifest_copy_filehash(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_filehash(m, srcm->has_filehash ? &srcm->filehash : NULL);
}
static int _rhizome_manifest_parse_filehash(rhizome_manifest *m, const char *text)
{
rhizome_filehash_t hash;
@ -719,14 +747,18 @@ static int _rhizome_manifest_parse_filehash(rhizome_manifest *m, const char *tex
return 1;
}
static int _rhizome_manifest_test_filesize(rhizome_manifest *m)
static int _rhizome_manifest_test_filesize(const rhizome_manifest *m)
{
return m->filesize != RHIZOME_SIZE_UNSET;
}
static void _rhizome_manifest_unset_filesize(rhizome_manifest *m)
static void _rhizome_manifest_unset_filesize(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_filesize(m, RHIZOME_SIZE_UNSET);
}
static void _rhizome_manifest_copy_filesize(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_filesize(m, srcm->filesize);
}
static int _rhizome_manifest_parse_filesize(rhizome_manifest *m, const char *text)
{
uint64_t size;
@ -736,14 +768,18 @@ static int _rhizome_manifest_parse_filesize(rhizome_manifest *m, const char *tex
return 1;
}
static int _rhizome_manifest_test_tail(rhizome_manifest *m)
static int _rhizome_manifest_test_tail(const rhizome_manifest *m)
{
return m->is_journal;
}
static void _rhizome_manifest_unset_tail(rhizome_manifest *m)
static void _rhizome_manifest_unset_tail(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_tail(m, RHIZOME_SIZE_UNSET);
}
static void _rhizome_manifest_copy_tail(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_tail(m, srcm->tail);
}
static int _rhizome_manifest_parse_tail(rhizome_manifest *m, const char *text)
{
uint64_t tail;
@ -753,13 +789,17 @@ static int _rhizome_manifest_parse_tail(rhizome_manifest *m, const char *text)
return 1;
}
static int _rhizome_manifest_test_BK(rhizome_manifest *m)
static int _rhizome_manifest_test_BK(const rhizome_manifest *m)
{
return m->has_bundle_key;
}
static void _rhizome_manifest_unset_BK(rhizome_manifest *m)
static void _rhizome_manifest_unset_BK(struct __sourceloc __whence, rhizome_manifest *m)
{
m->has_bundle_key = 0;
rhizome_manifest_del_bundle_key(m);
}
static void _rhizome_manifest_copy_BK(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_bundle_key(m, srcm->has_bundle_key ? &srcm->bundle_key : NULL);
}
static int _rhizome_manifest_parse_BK(rhizome_manifest *m, const char *text)
{
@ -770,14 +810,18 @@ static int _rhizome_manifest_parse_BK(rhizome_manifest *m, const char *text)
return 1;
}
static int _rhizome_manifest_test_service(rhizome_manifest *m)
static int _rhizome_manifest_test_service(const rhizome_manifest *m)
{
return m->service != NULL;
}
static void _rhizome_manifest_unset_service(rhizome_manifest *m)
static void _rhizome_manifest_unset_service(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_del_service(m);
}
static void _rhizome_manifest_copy_service(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_service(m, srcm->service);
}
static int _rhizome_manifest_parse_service(rhizome_manifest *m, const char *text)
{
if (!rhizome_str_is_manifest_service(text))
@ -786,14 +830,21 @@ static int _rhizome_manifest_parse_service(rhizome_manifest *m, const char *text
return 1;
}
static int _rhizome_manifest_test_date(rhizome_manifest *m)
static int _rhizome_manifest_test_date(const rhizome_manifest *m)
{
return m->has_date;
}
static void _rhizome_manifest_unset_date(rhizome_manifest *m)
static void _rhizome_manifest_unset_date(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_del_date(m);
}
static void _rhizome_manifest_copy_date(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
if (srcm->has_date)
rhizome_manifest_set_date(m, srcm->date);
else
rhizome_manifest_del_date(m);
}
static int _rhizome_manifest_parse_date(rhizome_manifest *m, const char *text)
{
int64_t date;
@ -803,14 +854,18 @@ static int _rhizome_manifest_parse_date(rhizome_manifest *m, const char *text)
return 1;
}
static int _rhizome_manifest_test_sender(rhizome_manifest *m)
static int _rhizome_manifest_test_sender(const rhizome_manifest *m)
{
return m->has_sender;
}
static void _rhizome_manifest_unset_sender(rhizome_manifest *m)
static void _rhizome_manifest_unset_sender(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_sender(m, NULL);
}
static void _rhizome_manifest_copy_sender(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_sender(m, srcm->has_sender ? &srcm->sender : NULL);
}
static int _rhizome_manifest_parse_sender(rhizome_manifest *m, const char *text)
{
sid_t sid;
@ -820,14 +875,18 @@ static int _rhizome_manifest_parse_sender(rhizome_manifest *m, const char *text)
return 1;
}
static int _rhizome_manifest_test_recipient(rhizome_manifest *m)
static int _rhizome_manifest_test_recipient(const rhizome_manifest *m)
{
return m->has_recipient;
}
static void _rhizome_manifest_unset_recipient(rhizome_manifest *m)
static void _rhizome_manifest_unset_recipient(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_recipient(m, NULL);
}
static void _rhizome_manifest_copy_recipient(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_recipient(m, srcm->has_recipient ? &srcm->recipient : NULL);
}
static int _rhizome_manifest_parse_recipient(rhizome_manifest *m, const char *text)
{
sid_t sid;
@ -837,28 +896,36 @@ static int _rhizome_manifest_parse_recipient(rhizome_manifest *m, const char *te
return 1;
}
static int _rhizome_manifest_test_name(rhizome_manifest *m)
static int _rhizome_manifest_test_name(const rhizome_manifest *m)
{
return m->name != NULL;
}
static void _rhizome_manifest_unset_name(rhizome_manifest *m)
static void _rhizome_manifest_unset_name(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_del_name(m);
}
static void _rhizome_manifest_copy_name(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_name(m, srcm->name);
}
static int _rhizome_manifest_parse_name(rhizome_manifest *m, const char *text)
{
rhizome_manifest_set_name(m, text);
return 1;
}
static int _rhizome_manifest_test_crypt(rhizome_manifest *m)
static int _rhizome_manifest_test_crypt(const rhizome_manifest *m)
{
return m->payloadEncryption != PAYLOAD_CRYPT_UNKNOWN;
}
static void _rhizome_manifest_unset_crypt(rhizome_manifest *m)
static void _rhizome_manifest_unset_crypt(struct __sourceloc __whence, rhizome_manifest *m)
{
rhizome_manifest_set_crypt(m, PAYLOAD_CRYPT_UNKNOWN);
}
static void _rhizome_manifest_copy_crypt(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
rhizome_manifest_set_crypt(m, srcm->payloadEncryption);
}
static int _rhizome_manifest_parse_crypt(rhizome_manifest *m, const char *text)
{
if (!(strcmp(text, "0") == 0 || strcmp(text, "1") == 0))
@ -872,23 +939,65 @@ static struct rhizome_manifest_field_descriptor {
int core;
MANIFEST_FIELD_TESTER *test;
MANIFEST_FIELD_UNSETTER *unset;
MANIFEST_FIELD_COPIER *copy;
MANIFEST_FIELD_PARSER *parse;
}
rhizome_manifest_fields[] = {
{ "id", 1, _rhizome_manifest_test_id, _rhizome_manifest_unset_id, _rhizome_manifest_parse_id },
{ "version", 1, _rhizome_manifest_test_version, _rhizome_manifest_unset_version, _rhizome_manifest_parse_version },
{ "filehash", 1, _rhizome_manifest_test_filehash, _rhizome_manifest_unset_filehash, _rhizome_manifest_parse_filehash },
{ "filesize", 1, _rhizome_manifest_test_filesize, _rhizome_manifest_unset_filesize, _rhizome_manifest_parse_filesize },
{ "tail", 1, _rhizome_manifest_test_tail, _rhizome_manifest_unset_tail, _rhizome_manifest_parse_tail },
{ "BK", 0, _rhizome_manifest_test_BK, _rhizome_manifest_unset_BK, _rhizome_manifest_parse_BK },
{ "service", 0, _rhizome_manifest_test_service, _rhizome_manifest_unset_service, _rhizome_manifest_parse_service },
{ "date", 0, _rhizome_manifest_test_date, _rhizome_manifest_unset_date, _rhizome_manifest_parse_date },
{ "sender", 0, _rhizome_manifest_test_sender, _rhizome_manifest_unset_sender, _rhizome_manifest_parse_sender },
{ "recipient", 0, _rhizome_manifest_test_recipient, _rhizome_manifest_unset_recipient, _rhizome_manifest_parse_recipient },
{ "name", 0, _rhizome_manifest_test_name, _rhizome_manifest_unset_name, _rhizome_manifest_parse_name },
{ "crypt", 0, _rhizome_manifest_test_crypt, _rhizome_manifest_unset_crypt, _rhizome_manifest_parse_crypt },
#define FIELD(CORE, NAME) \
{ #NAME, CORE, _rhizome_manifest_test_ ## NAME, _rhizome_manifest_unset_ ## NAME, _rhizome_manifest_copy_ ## NAME, _rhizome_manifest_parse_ ## NAME }
FIELD(1, id),
FIELD(1, version),
FIELD(1, filehash),
FIELD(1, filesize),
FIELD(1, tail),
FIELD(0, BK),
FIELD(0, service),
FIELD(0, date),
FIELD(0, sender),
FIELD(0, recipient),
FIELD(0, name),
FIELD(0, crypt),
#undef FIELD
};
static struct rhizome_manifest_field_descriptor *get_rhizome_manifest_field_descriptor(const char *label)
{
unsigned i;
for (i = 0; i < NELS(rhizome_manifest_fields); ++i)
if (strcasecmp(label, rhizome_manifest_fields[i].label) == 0)
return &rhizome_manifest_fields[i];
return NULL;
}
/* Overwrite a Rhizome manifest with fields from another. Used in the "add bundle" application API
* when the application supplies a partial manifest to override or add to existing manifest fields.
*
* Returns -1 if a field in the destination manifest cannot be overwritten for an unrecoverable
* reason, eg, out of memory or too many variables, leaving the destination manifest in an undefined
* state.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int _rhizome_manifest_overwrite(struct __sourceloc __whence, rhizome_manifest *m, const rhizome_manifest *srcm)
{
unsigned i;
for (i = 0; i < NELS(rhizome_manifest_fields); ++i) {
struct rhizome_manifest_field_descriptor *desc = &rhizome_manifest_fields[i];
if (desc->test(srcm)) {
if (config.debug.rhizome_manifest)
DEBUGF("COPY manifest[%d].%s to:", srcm->manifest_record_number, desc->label);
desc->copy(__whence, m, srcm);
}
}
for (i = 0; i < srcm->var_count; ++i) {
struct rhizome_manifest_field_descriptor *desc = get_rhizome_manifest_field_descriptor(srcm->vars[i]);
if (!desc)
if (_rhizome_manifest_set(__whence, m, srcm->vars[i], srcm->values[i]) == NULL)
return -1;
}
return 0;
}
int rhizome_manifest_field_label_is_valid(const char *field_label, size_t field_label_len)
{
if (field_label_len == 0 || field_label_len > MAX_MANIFEST_FIELD_LABEL_LEN)
@ -959,11 +1068,7 @@ rhizome_manifest_parse_field(rhizome_manifest *m, const char *field_label, size_
return RHIZOME_MANIFEST_SYNTAX_ERROR;
}
const char *value = alloca_strndup(field_value, field_value_len);
struct rhizome_manifest_field_descriptor *desc = NULL;
unsigned i;
for (i = 0; desc == NULL && i < NELS(rhizome_manifest_fields); ++i)
if (strcasecmp(label, rhizome_manifest_fields[i].label) == 0)
desc = &rhizome_manifest_fields[i];
struct rhizome_manifest_field_descriptor *desc = get_rhizome_manifest_field_descriptor(label);
enum rhizome_manifest_parse_status status = RHIZOME_MANIFEST_OK;
assert(m->var_count <= NELS(m->vars));
if (desc ? desc->test(m) : rhizome_manifest_get(m, label) != NULL) {
@ -980,7 +1085,7 @@ rhizome_manifest_parse_field(rhizome_manifest *m, const char *field_label, size_
DEBUGF("Manifest field parse failed at %s=%s", label, alloca_toprint(100, field_value, field_value_len));
status = desc->core ? RHIZOME_MANIFEST_INVALID : RHIZOME_MANIFEST_MALFORMED;
}
} else if ((rhizome_manifest_set(m, label, value)) == NULL)
} else if (rhizome_manifest_set(m, label, value) == NULL)
status = RHIZOME_MANIFEST_ERROR;
if (status != RHIZOME_MANIFEST_OK) {
if (config.debug.rhizome_manifest)
@ -1010,7 +1115,7 @@ int rhizome_manifest_remove_field(rhizome_manifest *m, const char *field_label,
return rhizome_manifest_del(m, label);
if (!desc->test(m))
return 0;
desc->unset(m);
desc->unset(__WHENCE__, m);
return 1;
}
@ -1133,14 +1238,14 @@ int manifest_first_free=-1;
struct __sourceloc manifest_alloc_whence[MAX_RHIZOME_MANIFESTS];
struct __sourceloc manifest_free_whence[MAX_RHIZOME_MANIFESTS];
static void _log_manifest_trace(struct __sourceloc __whence, const char *operation)
static unsigned _count_free_manifests()
{
int count_free = 0;
unsigned count_free = 0;
unsigned i;
for (i = 0; i != MAX_RHIZOME_MANIFESTS; ++i)
if (manifest_free[i])
++count_free;
DEBUGF("%s(): count_free = %d", operation, count_free);
return count_free;
}
rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
@ -1187,7 +1292,8 @@ rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence)
for (; manifest_first_free < MAX_RHIZOME_MANIFESTS && !manifest_free[manifest_first_free]; ++manifest_first_free)
;
if (config.debug.manifests) _log_manifest_trace(__whence, __FUNCTION__);
if (config.debug.rhizome_manifest)
DEBUGF("NEW manifest[%d], count_free=%u", m->manifest_record_number, _count_free_manifests());
// Set global defaults for a manifest (which are not zero)
rhizome_manifest_clear(m);
@ -1201,12 +1307,12 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
int mid=m->manifest_record_number;
if (m!=&manifests[mid])
FATALF("%s(): asked to free manifest %p, which claims to be manifest slot #%d (%p), but isn't",
FATALF("%s(): manifest at %p claims to be manifest[%d] (%p) but isn't",
__FUNCTION__, m, mid, &manifests[mid]
);
if (manifest_free[mid])
FATALF("%s(): asked to free manifest slot #%d (%p), which was already freed at %s:%d:%s()",
FATALF("%s(): manifest[%d] (%p) was already freed at %s:%d:%s()",
__FUNCTION__, mid, m,
manifest_free_whence[mid].file,
manifest_free_whence[mid].line,
@ -1226,7 +1332,8 @@ void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m)
manifest_free_whence[mid]=__whence;
if (mid<manifest_first_free) manifest_first_free=mid;
if (config.debug.manifests) _log_manifest_trace(__whence, __FUNCTION__);
if (config.debug.rhizome_manifest)
DEBUGF("FREE manifest[%d], count_free=%u", m->manifest_record_number, _count_free_manifests());
return;
}
@ -1327,9 +1434,12 @@ enum rhizome_bundle_status rhizome_manifest_finalise(rhizome_manifest *m, rhizom
assert(*mout == NULL);
if (!m->finalised && !rhizome_manifest_validate(m))
RETURN(RHIZOME_BUNDLE_STATUS_INVALID);
// if a manifest was supplied with an ID, don't bother to check for a duplicate.
// we only want to filter out added files with no existing manifest.
if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID) {
// The duplicate detection logic exists to filter out files repeatedly added with no existing
// manifest (ie, "de-bounce" for the "Add File" user interface action).
// 1. If a manifest was supplied with a bundle ID, don't check for a duplicate.
// 2. Never perform duplicate detection on journals (the first append does not supply a bundle ID,
// but all subsequent appends supply a bundle ID, so are caught by case (1)).
if (deduplicate && m->haveSecret != EXISTING_BUNDLE_ID && !m->is_journal) {
enum rhizome_bundle_status status = rhizome_find_duplicate(m, mout);
switch (status) {
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
@ -1389,9 +1499,14 @@ int rhizome_manifest_set_name_from_path(rhizome_manifest *m, const char *filepat
* - use the given author SID, or the 'sender' if present, as the author
* - create an ID if there is none, otherwise authenticate the existing one
* - if service is file, then use the payload file's basename for "name"
*
* Return NULL if successful, otherwise a pointer to a static text string describing the reason for
* the failure (always an internal/unrecoverable error).
*/
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp)
const char * rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp)
{
const char *reason = NULL;
/* Set version of manifest from current time if not already set. */
if (m->version == 0)
rhizome_manifest_set_version(m, gettime_ms());
@ -1420,8 +1535,10 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
}
if (config.debug.rhizome)
DEBUG("creating new bundle");
if (rhizome_manifest_createid(m) == -1)
return WHY("Could not bind manifest to an ID");
if (rhizome_manifest_createid(m) == -1) {
WHY(reason = "Could not bind manifest to an ID");
return reason;
}
// fall through to set the BK field...
case NEW_BUNDLE_ID:
valid_haveSecret = 1;
@ -1446,10 +1563,12 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
/* Service field must already be set.
*/
if (m->service == NULL)
return WHYF("missing 'service'");
if (m->service == NULL) {
WHYF(reason = "Missing 'service' field");
return reason;
}
if (config.debug.rhizome)
DEBUGF("manifest service=%s", m->service);
DEBUGF("manifest contains service=%s", m->service);
/* Fill in 'date' field to current time unless already set.
*/
@ -1484,7 +1603,7 @@ int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t
rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED);
}
return 0;
return NULL;
}
/* Work out the authorship status of the bundle without performing any cryptographic checks.

182
rhizome_cli.c
View File

@ -104,49 +104,50 @@ static int app_rhizome_hash_file(const struct cli_parsed *parsed, struct cli_con
DEFINE_CMD(app_rhizome_add_file, 0,
"Add a file to Rhizome and optionally write its manifest to the given path",
"rhizome","add","file" KEYRING_PIN_OPTIONS,"[--force-new]","<author_sid>","<filepath>","[<manifestpath>]","[<bsk>]","...");
"rhizome","add","file" KEYRING_PIN_OPTIONS,"[--bundle=<bundleid>]","[--force-new]","<author_sid>","<filepath>","[<manifestpath>]","[<bsk>]","...");
DEFINE_CMD(app_rhizome_add_file, 0,
"Append content to a journal bundle",
"rhizome", "journal", "append" KEYRING_PIN_OPTIONS, "<author_sid>", "<manifestid>", "<filepath>", "[<bsk>]");
"rhizome", "journal", "append" KEYRING_PIN_OPTIONS, "<author_sid>", "<bundleid>", "<filepath>", "[<bsk>]");
static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *context)
{
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *filepath, *manifestpath, *manifestid, *authorSidHex, *bsktext;
const char *filepath, *manifestpath, *bundleIdHex, *authorSidHex, *bsktext;
int force_new = 0 == cli_arg(parsed, "--force-new", NULL, NULL, NULL);
cli_arg(parsed, "filepath", &filepath, NULL, "");
if (cli_arg(parsed, "author_sid", &authorSidHex, cli_optional_sid, "") == -1)
return -1;
cli_arg(parsed, "manifestpath", &manifestpath, NULL, "");
cli_arg(parsed, "manifestid", &manifestid, NULL, "");
cli_arg(parsed, "--bundle", &bundleIdHex, cli_bid, "") == 0 || cli_arg(parsed, "bundleid", &bundleIdHex, cli_optional_bid, "");
if (cli_arg(parsed, "bsk", &bsktext, cli_optional_bundle_secret_key, NULL) == -1)
return -1;
sid_t authorSid;
if (authorSidHex[0] && str_to_sid_t(&authorSid, authorSidHex) == -1)
if (!authorSidHex || !*authorSidHex)
authorSidHex = NULL;
else if (str_to_sid_t(&authorSid, authorSidHex) == -1)
return WHYF("invalid author_sid: %s", authorSidHex);
// treat empty string the same as null
if (bsktext && !*bsktext)
bsktext = NULL;
rhizome_bid_t bid;
if (!bundleIdHex || !*bundleIdHex)
bundleIdHex = NULL;
else if (str_to_rhizome_bid_t(&bid, bundleIdHex) == -1)
return WHYF("Invalid bundle ID: %s", alloca_str_toprint(bundleIdHex));
rhizome_bk_t bsk;
if (bsktext && str_to_rhizome_bsk_t(&bsk, bsktext) == -1)
return WHYF("invalid bsk: \"%s\"", bsktext);
if (!bsktext || !*bsktext)
bsktext = NULL;
else if (str_to_rhizome_bsk_t(&bsk, bsktext) == -1)
return WHYF("invalid BSK: \"%s\"", bsktext);
unsigned nfields = (parsed->varargi == -1) ? 0 : parsed->argc - (unsigned)parsed->varargi;
struct field {
const char *label;
size_t labellen;
const char *value;
size_t valuelen;
}
fields[nfields];
struct rhizome_manifest_field_assignment fields[nfields];
if (nfields) {
assert(parsed->varargi >= 0);
unsigned i;
for (i = 0; i < nfields; ++i) {
struct field *field = &fields[i];
struct rhizome_manifest_field_assignment *field = &fields[i];
unsigned n = (unsigned)parsed->varargi + i;
assert(n < parsed->argc);
const char *arg = parsed->args[n];
@ -170,7 +171,7 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
}
}
int journal = strcasecmp(parsed->args[1], "journal")==0;
int appending = strcasecmp(parsed->args[1], "journal")==0;
if (create_serval_instance_dir() == -1)
return -1;
@ -183,112 +184,72 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
if (rhizome_opendb() == -1)
goto finish;
/* Create a new manifest that will represent the file. If a manifest file was supplied, then read
* it, otherwise create a blank manifest. */
/* Create a manifest in memory that to describe the added file. Initially the manifest is blank.
* If a manifest file is supplied, then read and parse it, barfing if it contains any duplicate
* fields or invalid values. If it successfully parses, then overwrite it with any command-line
* manifest field settings, overriding the values parsed from the file. Barf if any of these new
* values are malformed. We don't validate the resulting manifest, it order to allow the user to
* supply an incomplete manifest. Any missing fields will be filled in later.
*/
if ((m = rhizome_new_manifest()) == NULL){
ret = WHY("Manifest struct could not be allocated -- not added to rhizome");
ret = WHY("Manifest struct could not be allocated -- not added");
goto finish;
}
if (manifestpath && *manifestpath && access(manifestpath, R_OK) == 0) {
if (config.debug.rhizome)
DEBUGF("reading manifest from %s", manifestpath);
/* Don't verify the manifest, because it will fail if it is incomplete.
This is okay, because we fill in any missing bits and sanity check before
trying to write it out. However, we do insist that whatever we load is
parsed okay and not malformed. */
if (rhizome_read_manifest_from_file(m, manifestpath) || m->malformed) {
ret = WHY("Manifest file could not be loaded -- not added to rhizome");
goto finish;
}
} else if (manifestid && *manifestid) {
if (config.debug.rhizome)
DEBUGF("Reading manifest from database");
rhizome_bid_t bid;
if (str_to_rhizome_bid_t(&bid, manifestid) == -1) {
ret = WHYF("Invalid bundle ID: %s", alloca_str_toprint(manifestid));
goto finish;
}
if (rhizome_retrieve_manifest(&bid, m) != RHIZOME_BUNDLE_STATUS_SAME) {
ret = WHY("Existing manifest could not be loaded -- not added to rhizome");
goto finish;
}
} else {
if (config.debug.rhizome)
DEBUGF("Creating new manifest");
if (journal) {
rhizome_manifest_set_filesize(m, 0);
rhizome_manifest_set_tail(m, 0);
}
}
if (journal && !m->is_journal){
// TODO: return a special status code for this case
ret = WHY("Existing manifest is not a journal");
/* Create an in-memory manifest for the file being added.
*/
rhizome_manifest *mout = NULL;
enum rhizome_add_file_result result = rhizome_manifest_add_file(appending, m, &mout,
bundleIdHex ? &bid : NULL,
bsktext ? &bsk : NULL,
authorSidHex ? &authorSid : NULL,
filepath,
nfields, fields,
NULL);
int result_valid = 0;
switch (result) {
case RHIZOME_ADD_FILE_ERROR:
ret = -1;
goto finish;
case RHIZOME_ADD_FILE_OK:
result_valid = 1;
break;
case RHIZOME_ADD_FILE_INVALID:
ret = RHIZOME_BUNDLE_STATUS_INVALID; // TODO separate enum for CLI return codes
goto finish;
case RHIZOME_ADD_FILE_BUSY:
ret = RHIZOME_BUNDLE_STATUS_BUSY; // TODO separate enum for CLI return codes
goto finish;
case RHIZOME_ADD_FILE_REQUIRES_JOURNAL:
ret = RHIZOME_BUNDLE_STATUS_INVALID; // TODO separate enum for CLI return codes
goto finish;
case RHIZOME_ADD_FILE_INVALID_FOR_JOURNAL:
ret = RHIZOME_BUNDLE_STATUS_INVALID; // TODO separate enum for CLI return codes
goto finish;
case RHIZOME_ADD_FILE_WRONG_SECRET:
ret = RHIZOME_BUNDLE_STATUS_READONLY; // TODO separate enum for CLI return codes
goto finish;
}
if (!journal && m->is_journal) {
// TODO: return a special status code for this case
ret = WHY("Existing manifest is a journal");
goto finish;
if (!result_valid)
FATALF("result = %d", result);
assert(mout != NULL);
if (mout != m) {
rhizome_manifest_free(m);
m = mout;
}
mout = NULL;
if (nfields) {
unsigned i;
for (i = 0; i != nfields; ++i) {
struct field *field = &fields[i];
rhizome_manifest_remove_field(m, field->label, field->labellen);
if (field->value) {
const char *label = alloca_strndup(field->label, field->labellen);
enum rhizome_manifest_parse_status status = rhizome_manifest_parse_field(m, field->label, field->labellen, field->value, field->valuelen);
int status_ok = 0;
switch (status) {
case RHIZOME_MANIFEST_ERROR:
ret = WHY("Fatal error while updating manifest field");
goto finish;
case RHIZOME_MANIFEST_OK:
status_ok = 1;
break;
case RHIZOME_MANIFEST_SYNTAX_ERROR:
ret = WHYF("Manifest syntax error: %s=%s", label, alloca_toprint(-1, field->value, field->valuelen));
goto finish;
case RHIZOME_MANIFEST_DUPLICATE_FIELD:
abort(); // should not happen, field was removed first
case RHIZOME_MANIFEST_INVALID:
ret = WHYF("Manifest invalid field: %s=%s", label, alloca_toprint(-1, field->value, field->valuelen));
goto finish;
case RHIZOME_MANIFEST_MALFORMED:
ret = WHYF("Manifest malformed field: %s=%s", label, alloca_toprint(-1, field->value, field->valuelen));
goto finish;
case RHIZOME_MANIFEST_OVERFLOW:
ret = WHYF("Too many fields in manifest at: %s=%s", label, alloca_toprint(-1, field->value, field->valuelen));
goto finish;
}
if (!status_ok)
FATALF("status = %d", status);
}
}
}
if (bsktext) {
if (m->has_id) {
if (!rhizome_apply_bundle_secret(m, &bsk)) {
ret = WHY("Supplied bundle secret does not match Bundle Id");
goto finish;
}
} else {
if (rhizome_new_bundle_from_secret(m, &bsk) == -1) {
ret = WHY("Failed to create bundle from given secret");
goto finish;
}
}
}
if (m->service == NULL)
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
if (rhizome_fill_manifest(m, filepath, *authorSidHex ? &authorSid : NULL))
goto finish;
// Insert the payload into the Rhizome store.
enum rhizome_payload_status pstatus;
if (journal){
if (appending) {
pstatus = rhizome_append_journal_file(m, 0, filepath);
if (config.debug.rhizome)
DEBUGF("rhizome_append_journal_file() returned %d %s", pstatus, rhizome_payload_status_message(pstatus));
@ -335,7 +296,6 @@ static int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_cont
}
if (!pstatus_valid)
FATALF("pstatus = %d", pstatus);
rhizome_manifest *mout = NULL;
if (status == RHIZOME_BUNDLE_STATUS_NEW) {
if (!rhizome_manifest_validate(m) || m->malformed)
status = RHIZOME_BUNDLE_STATUS_INVALID;
@ -466,7 +426,7 @@ static int app_rhizome_delete(const struct cli_parsed *parsed, struct cli_contex
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *manifestid, *fileid;
if (cli_arg(parsed, "manifestid", &manifestid, cli_manifestid, NULL) == -1)
if (cli_arg(parsed, "manifestid", &manifestid, cli_bid, NULL) == -1)
return -1;
if (cli_arg(parsed, "fileid", &fileid, cli_fileid, NULL) == -1)
return -1;
@ -572,7 +532,7 @@ static int app_rhizome_extract(const struct cli_parsed *parsed, struct cli_conte
if (config.debug.verbose)
DEBUG_cli_parsed(parsed);
const char *manifestpath, *filepath, *manifestid, *bsktext;
if ( cli_arg(parsed, "manifestid", &manifestid, cli_manifestid, "") == -1
if ( cli_arg(parsed, "manifestid", &manifestid, cli_bid, "") == -1
|| cli_arg(parsed, "manifestpath", &manifestpath, NULL, "") == -1
|| cli_arg(parsed, "filepath", &filepath, NULL, "") == -1
|| cli_arg(parsed, "bsk", &bsktext, cli_optional_bundle_secret_key, NULL) == -1)

21
rhizome_crypto.c
View File

@ -47,16 +47,14 @@ struct signing_key {
};
/* generate a keypair from a given secret key */
static int generate_keypair_from_secret(const rhizome_bk_t *bsk, struct signing_key *key)
static void generate_keypair_from_secret(const rhizome_bk_t *bsk, struct signing_key *key)
{
bcopy(bsk->binary, key->Private, sizeof bsk->binary); // first 32 bytes
if (crypto_sign_compute_public_key(key->Private, key->Public.binary) == -1)
return WHY("Could not generate public key");
crypto_sign_compute_public_key(key->Private, key->Public.binary);
// The last 32 bytes of the private key should be identical to the public key. This is what
// crypto_sign_edwards25519sha512batch_keypair() returns, and there is code that depends on it.
// TODO: Refactor the Rhizome private/public keypair to eliminate this duplication.
bcopy(key->Public.binary, key->Private + RHIZOME_BUNDLE_KEY_BYTES, sizeof key->Public.binary);
return 0;
}
/* Generate a new empty manifest from the given keypair.
@ -90,8 +88,7 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
int rhizome_get_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
{
struct signing_key key;
if (generate_keypair_from_secret(bsk, &key))
return -1;
generate_keypair_from_secret(bsk, &key);
switch (rhizome_retrieve_manifest(&key.Public, m)) {
case RHIZOME_BUNDLE_STATUS_NEW:
rhizome_new_bundle_from_keypair(m, &key);
@ -109,13 +106,11 @@ int rhizome_get_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
/* Generate a bundle id deterministically from the given bundle secret key.
* Then initialise a new empty manifest.
*/
int rhizome_new_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
void rhizome_new_bundle_from_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
{
struct signing_key key;
if (generate_keypair_from_secret(bsk, &key))
return -1;
generate_keypair_from_secret(bsk, &key);
rhizome_new_bundle_from_keypair(m, &key);
return 0;
}
/* Given a Rhizome Secret (RS) and bundle ID (BID), XOR a bundle key 'bkin' (private or public) with
@ -446,10 +441,8 @@ int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned cha
{
IN();
rhizome_bid_t pk;
if (crypto_sign_compute_public_key(sk, pk.binary) == -1)
RETURN(0);
int ret = bcmp(pkin, pk.binary, sizeof pk.binary) == 0;
RETURN(ret);
crypto_sign_compute_public_key(sk, pk.binary);
RETURN(bcmp(pkin, pk.binary, sizeof pk.binary) == 0);
}
int rhizome_sign_hash(rhizome_manifest *m, rhizome_signature *out)

5
rhizome_direct_http.c
View File

@ -249,10 +249,11 @@ static int rhizome_direct_addfile_end(struct http_request *hr)
if (m->service == NULL)
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
const sid_t *author = is_sid_t_any(config.rhizome.api.addfile.default_author) ? NULL : &config.rhizome.api.addfile.default_author;
if (rhizome_fill_manifest(m, r->u.direct_import.data_file_name, author)) {
const char *reason = rhizome_fill_manifest(m, r->u.direct_import.data_file_name, author);
if (reason) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest");
http_request_simple_response(&r->http, 500, alloca_sprintf(-1, "Internal Error: %s", reason));
return 0;
}
rhizome_manifest_set_crypt(m, PAYLOAD_CLEAR);

269
rhizome_restful.c
View File

@ -1,6 +1,6 @@
/*
Serval DNA Rhizome HTTP RESTful interface
Copyright (C) 2013,2014 Serval Project Inc.
Copyright (C) 2013-2015 Serval Project Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -68,10 +68,10 @@ static int http_request_rhizome_response(struct httpd_request *r, uint16_t resul
uint16_t rhizome_result = 0;
switch (r->bundle_status) {
case RHIZOME_BUNDLE_STATUS_NEW:
rhizome_result = 201;
break;
case RHIZOME_BUNDLE_STATUS_SAME:
case RHIZOME_BUNDLE_STATUS_DUPLICATE:
rhizome_result = 201;
break;
case RHIZOME_BUNDLE_STATUS_OLD:
case RHIZOME_BUNDLE_STATUS_NO_ROOM:
rhizome_result = 200;
@ -105,10 +105,10 @@ static int http_request_rhizome_response(struct httpd_request *r, uint16_t resul
rhizome_result = 0;
switch (r->payload_status) {
case RHIZOME_PAYLOAD_STATUS_NEW:
rhizome_result = 201;
break;
case RHIZOME_PAYLOAD_STATUS_STORED:
case RHIZOME_PAYLOAD_STATUS_EMPTY:
rhizome_result = 201;
break;
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_EVICTED:
rhizome_result = 200;
@ -341,6 +341,7 @@ int restful_rhizome_insert(httpd_request *r, const char *remainder)
assert(r->u.insert.current_part == NULL);
assert(!r->u.insert.received_author);
assert(!r->u.insert.received_secret);
assert(!r->u.insert.received_bundleid);
assert(!r->u.insert.received_manifest);
assert(!r->u.insert.received_payload);
bzero(&r->u.insert.write, sizeof r->u.insert.write);
@ -355,8 +356,15 @@ int restful_rhizome_insert(httpd_request *r, const char *remainder)
return 1;
}
int restful_rhizome_append(httpd_request *r, const char *remainder)
{
r->u.insert.appending = 1;
return restful_rhizome_insert(r, remainder);
}
static char PART_MANIFEST[] = "manifest";
static char PART_PAYLOAD[] = "payload";
static char PART_BUNDLEID[] = "bundle-id";
static char PART_AUTHOR[] = "bundle-author";
static char PART_SECRET[] = "bundle-secret";
@ -371,32 +379,68 @@ static int insert_make_manifest(httpd_request *r)
{
if (!r->u.insert.received_manifest)
return http_response_form_part(r, "Missing", PART_MANIFEST, NULL, 0);
if ((r->manifest = rhizome_new_manifest())) {
if (r->u.insert.manifest.length == 0)
return 0;
assert(r->u.insert.manifest.length <= sizeof r->manifest->manifestdata);
memcpy(r->manifest->manifestdata, r->u.insert.manifest.buffer, r->u.insert.manifest.length);
r->manifest->manifest_all_bytes = r->u.insert.manifest.length;
int n = rhizome_manifest_parse(r->manifest);
switch (n) {
case 0:
if (!r->manifest->malformed)
return 0;
// fall through
case 1:
rhizome_manifest_free(r->manifest);
r->manifest = NULL;
r->bundle_status = RHIZOME_BUNDLE_STATUS_INVALID;
return http_request_rhizome_response(r, 403, "Malformed manifest", NULL);
default:
WHYF("rhizome_manifest_parse() returned %d", n);
// fall through
case -1:
r->bundle_status = RHIZOME_BUNDLE_STATUS_ERROR;
if ((r->manifest = rhizome_new_manifest()) == NULL)
return http_request_rhizome_response(r, 500, "Internal Error: Out of manifests", NULL);
assert(r->u.insert.manifest.length <= sizeof r->manifest->manifestdata);
memcpy(r->manifest->manifestdata, r->u.insert.manifest.buffer, r->u.insert.manifest.length);
r->manifest->manifest_all_bytes = r->u.insert.manifest.length;
int n = rhizome_manifest_parse(r->manifest);
switch (n) {
case 0:
if (!r->manifest->malformed)
break;
}
// fall through
case 1:
rhizome_manifest_free(r->manifest);
r->manifest = NULL;
r->bundle_status = RHIZOME_BUNDLE_STATUS_INVALID;
return http_request_rhizome_response(r, 403, "Malformed manifest", NULL);
default:
WHYF("rhizome_manifest_parse() returned %d", n);
// fall through
case -1:
r->bundle_status = RHIZOME_BUNDLE_STATUS_ERROR;
break;
}
return 500;
rhizome_manifest *mout = NULL;
char message[150];
enum rhizome_add_file_result result = rhizome_manifest_add_file(r->u.insert.appending, r->manifest, &mout,
r->u.insert.received_bundleid ? &r->bid: NULL,
r->u.insert.received_secret ? &r->u.insert.bundle_secret : NULL,
r->u.insert.received_author ? &r->u.insert.author: NULL,
NULL, 0, NULL, strbuf_local(message, sizeof message));
int result_valid = 0;
switch (result) {
case RHIZOME_ADD_FILE_ERROR:
return http_request_rhizome_response(r, 500, message, NULL);
case RHIZOME_ADD_FILE_OK:
result_valid = 1;
break;
case RHIZOME_ADD_FILE_INVALID:
r->bundle_status = RHIZOME_BUNDLE_STATUS_INVALID; // TODO separate enum for CLI return codes
return http_request_rhizome_response(r, 403, message, NULL);
case RHIZOME_ADD_FILE_BUSY:
r->bundle_status = RHIZOME_BUNDLE_STATUS_BUSY; // TODO separate enum for CLI return codes
return http_request_rhizome_response(r, 403, message, NULL);
case RHIZOME_ADD_FILE_REQUIRES_JOURNAL:
r->bundle_status = RHIZOME_BUNDLE_STATUS_INVALID; // TODO separate enum for CLI return codes
return http_request_rhizome_response(r, 403, message, NULL);
case RHIZOME_ADD_FILE_INVALID_FOR_JOURNAL:
r->bundle_status = RHIZOME_BUNDLE_STATUS_INVALID; // TODO separate enum for CLI return codes
return http_request_rhizome_response(r, 403, message, NULL);
case RHIZOME_ADD_FILE_WRONG_SECRET:
r->bundle_status = RHIZOME_BUNDLE_STATUS_READONLY; // TODO separate enum for CLI return codes
return http_request_rhizome_response(r, 403, message, NULL);
}
if (!result_valid)
FATALF("result = %d", result);
assert(mout != NULL);
if (mout != r->manifest) {
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
assert(r->manifest != NULL);
return 0;
}
static int insert_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
@ -407,15 +451,30 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
if (strcmp(h->content_disposition.name, PART_AUTHOR) == 0) {
if (r->u.insert.received_author)
return http_response_form_part(r, "Duplicate", PART_AUTHOR, NULL, 0);
// Reject a request if this parameter comes after the manifest part.
if (r->u.insert.received_manifest)
return http_response_form_part(r, "Spurious", PART_AUTHOR, NULL, 0);
r->u.insert.current_part = PART_AUTHOR;
assert(r->u.insert.author_hex_len == 0);
}
else if (strcmp(h->content_disposition.name, PART_SECRET) == 0) {
if (r->u.insert.received_secret)
return http_response_form_part(r, "Duplicate", PART_SECRET, NULL, 0);
// Reject a request if this parameter comes after the manifest part.
if (r->u.insert.received_manifest)
return http_response_form_part(r, "Spurious", PART_SECRET, NULL, 0);
r->u.insert.current_part = PART_SECRET;
assert(r->u.insert.secret_text_len == 0);
}
else if (strcmp(h->content_disposition.name, PART_BUNDLEID) == 0) {
if (r->u.insert.received_bundleid)
return http_response_form_part(r, "Duplicate", PART_BUNDLEID, NULL, 0);
// Reject a request if this parameter comes after the manifest part.
if (r->u.insert.received_manifest)
return http_response_form_part(r, "Spurious", PART_BUNDLEID, NULL, 0);
r->u.insert.current_part = PART_BUNDLEID;
assert(r->u.insert.bid_text_len == 0);
}
else if (strcmp(h->content_disposition.name, PART_MANIFEST) == 0) {
// Reject a request if it has a repeated manifest part.
if (r->u.insert.received_manifest)
@ -445,20 +504,30 @@ static int insert_mime_part_header(struct http_request *hr, const struct mime_pa
&& *h->content_disposition.filename
)
rhizome_manifest_set_name_from_path(r->manifest, h->content_disposition.filename);
// Start writing the payload content into the Rhizome store. Note: r->manifest->filesize can be
// RHIZOME_SIZE_UNSET at this point, if the manifest did not contain a 'filesize' field.
r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
r->u.insert.payload_size = 0;
switch (r->payload_status) {
case RHIZOME_PAYLOAD_STATUS_ERROR:
WHYF("rhizome_write_open_manifest() returned %d", r->payload_status);
// Start writing the payload content into the Rhizome store.
if (r->u.insert.appending) {
r->payload_status = rhizome_write_open_journal(&r->u.insert.write, r->manifest, 0, RHIZOME_SIZE_UNSET);
if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
WHYF("rhizome_write_open_journal() returned %d %s", r->payload_status, rhizome_payload_status_message(r->payload_status));
return 500;
}
} else {
// Note: r->manifest->filesize can be RHIZOME_SIZE_UNSET at this point, if the manifest did
// not contain a 'filesize' field.
r->payload_status = rhizome_write_open_manifest(&r->u.insert.write, r->manifest);
if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
WHYF("rhizome_write_open_manifest() returned %d %s", r->payload_status, rhizome_payload_status_message(r->payload_status));
return 500;
}
}
switch (r->payload_status) {
case RHIZOME_PAYLOAD_STATUS_STORED:
// TODO: initialise payload hash so it can be compared with stored payload
break;
default:
break; // r->payload_status gets dealt with later
}
r->u.insert.payload_size = 0;
}
else
return http_response_form_part(r, "Unsupported", h->content_disposition.name, NULL, 0);
@ -482,6 +551,13 @@ static int insert_mime_part_body(struct http_request *hr, char *buf, size_t len)
&r->u.insert.secret_text_len,
buf, len);
}
else if (r->u.insert.current_part == PART_BUNDLEID) {
accumulate_text(r, PART_BUNDLEID,
r->u.insert.bid_text,
sizeof r->u.insert.bid_text,
&r->u.insert.bid_text_len,
buf, len);
}
else if (r->u.insert.current_part == PART_MANIFEST) {
form_buf_malloc_accumulate(r, PART_MANIFEST, &r->u.insert.manifest, buf, len);
}
@ -508,7 +584,7 @@ static int insert_mime_part_end(struct http_request *hr)
httpd_request *r = (httpd_request *) hr;
if (r->u.insert.current_part == PART_AUTHOR) {
if ( r->u.insert.author_hex_len != sizeof r->u.insert.author_hex
|| strn_to_sid_t(&r->u.insert.author, r->u.insert.author_hex, sizeof r->u.insert.author_hex, NULL) == -1
|| strn_to_sid_t(&r->u.insert.author, r->u.insert.author_hex, sizeof r->u.insert.author_hex) == -1
)
return http_response_form_part(r, "Invalid", PART_AUTHOR, r->u.insert.author_hex, r->u.insert.author_hex_len);
r->u.insert.received_author = 1;
@ -522,6 +598,13 @@ static int insert_mime_part_end(struct http_request *hr)
if (config.debug.rhizome)
DEBUGF("received %s = %s", PART_SECRET, alloca_tohex_rhizome_bk_t(r->u.insert.bundle_secret));
}
else if (r->u.insert.current_part == PART_BUNDLEID) {
if (strn_to_rhizome_bid_t(&r->bid, r->u.insert.bid_text, r->u.insert.bid_text_len) == -1)
return http_response_form_part(r, "Invalid", PART_BUNDLEID, r->u.insert.secret_text, r->u.insert.secret_text_len);
r->u.insert.received_bundleid = 1;
if (config.debug.rhizome)
DEBUGF("received %s = %s", PART_BUNDLEID, alloca_tohex_rhizome_bid_t(r->bid));
}
else if (r->u.insert.current_part == PART_MANIFEST) {
r->u.insert.received_manifest = 1;
if (config.debug.rhizome)
@ -529,45 +612,12 @@ static int insert_mime_part_end(struct http_request *hr)
int result = insert_make_manifest(r);
if (result)
return result;
if (r->u.insert.received_secret) {
if (r->manifest->has_id) {
if (!rhizome_apply_bundle_secret(r->manifest, &r->u.insert.bundle_secret)) {
http_request_simple_response(&r->http, 403, "Secret does not match Bundle Id");
return 403;
}
} else {
if (rhizome_new_bundle_from_secret(r->manifest, &r->u.insert.bundle_secret) == -1) {
WHY("Failed to create bundle from secret");
return 500;
}
}
}
if (r->manifest->service == NULL)
rhizome_manifest_set_service(r->manifest, RHIZOME_SERVICE_FILE);
if (rhizome_fill_manifest(r->manifest, NULL, r->u.insert.received_author ? &r->u.insert.author: NULL) == -1) {
WHY("rhizome_fill_manifest() failed");
return 500;
}
if (r->manifest->is_journal)
return http_request_rhizome_response(r, 403, "Insert not supported for journals", NULL);
assert(r->manifest != NULL);
}
else if (r->u.insert.current_part == PART_PAYLOAD) {
r->u.insert.received_payload = 1;
switch (r->payload_status) {
case RHIZOME_PAYLOAD_STATUS_NEW:
r->payload_status = rhizome_finish_write(&r->u.insert.write);
if (r->payload_status == RHIZOME_PAYLOAD_STATUS_ERROR) {
WHYF("rhizome_finish_write() returned status = %d", r->payload_status);
return 500;
}
break;
case RHIZOME_PAYLOAD_STATUS_STORED:
// TODO: finish calculating payload hash and compare it with stored payload
break;
default:
break;
}
if (config.debug.rhizome)
DEBUGF("received %s, %zd bytes", PART_PAYLOAD, r->u.insert.payload_size);
r->payload_status = rhizome_finish_write(&r->u.insert.write);
} else
FATALF("current_part = %s", alloca_str_toprint(r->u.insert.current_part));
r->u.insert.current_part = NULL;
@ -583,58 +633,69 @@ static int restful_rhizome_insert_end(struct http_request *hr)
return http_response_form_part(r, "Missing", PART_PAYLOAD, NULL, 0);
// Fill in the missing manifest fields and ensure payload and manifest are consistent.
assert(r->manifest != NULL);
assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET);
int status_valid = 0;
if (config.debug.rhizome)
DEBUGF("r->payload_status=%d", r->payload_status);
DEBUGF("r->payload_status=%d %s", r->payload_status, rhizome_payload_status_message(r->payload_status));
assert(r->u.insert.write.file_length != RHIZOME_SIZE_UNSET);
if (r->u.insert.appending) {
// For journal appends, the user cannot supply a 'filesize' field. This will have been caught
// by previous logic. The existing manifest should have a 'filesize' field. The new payload
// size should be the sum of 'filesize' and the appended portion.
assert(r->manifest->is_journal);
assert(r->manifest->filesize != RHIZOME_SIZE_UNSET);
if (config.debug.rhizome)
DEBUGF("file_length=%"PRIu64" filesize=%"PRIu64" payload_size=%"PRIu64,
r->u.insert.write.file_length,
r->manifest->filesize,
r->u.insert.payload_size);
if (r->u.insert.write.file_length != r->manifest->filesize + r->u.insert.payload_size)
r->payload_status = RHIZOME_PAYLOAD_STATUS_WRONG_SIZE;
} else {
// The Rhizome CLI 'add file' operation allows the user to supply a 'filesize' field which is
// smaller than the supplied file, for convenience, to allow only the first part of a file to be
// added as a payload. But the RESTful interface doesn't allow that.
assert(!r->manifest->is_journal);
if (r->manifest->filesize != RHIZOME_SIZE_UNSET && r->u.insert.payload_size != r->manifest->filesize)
r->payload_status = RHIZOME_PAYLOAD_STATUS_WRONG_SIZE;
}
r->payload_status = rhizome_finish_store(&r->u.insert.write, r->manifest, r->payload_status);
int status_valid = 0;
switch (r->payload_status) {
case RHIZOME_PAYLOAD_STATUS_NEW:
if (r->manifest->filesize == RHIZOME_SIZE_UNSET)
rhizome_manifest_set_filesize(r->manifest, r->u.insert.write.file_length);
// fall through
case RHIZOME_PAYLOAD_STATUS_STORED:
assert(r->manifest->filesize != RHIZOME_SIZE_UNSET);
// TODO: check that stored hash matches received payload's hash
// fall through
case RHIZOME_PAYLOAD_STATUS_EMPTY:
status_valid = 1;
if (r->manifest->filesize == RHIZOME_SIZE_UNSET)
rhizome_manifest_set_filesize(r->manifest, 0);
if (r->u.insert.payload_size == r->manifest->filesize)
break;
// fall through
break;
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
r->payload_status = RHIZOME_PAYLOAD_STATUS_WRONG_SIZE;
r->bundle_status = RHIZOME_BUNDLE_STATUS_INCONSISTENT;
{
strbuf msg = strbuf_alloca(200);
strbuf_sprintf(msg, "Payload size (%"PRIu64") contradicts manifest (filesize=%"PRIu64")", r->u.insert.payload_size, r->manifest->filesize);
return http_request_rhizome_response(r, 403, NULL, strbuf_str(msg));
return http_request_rhizome_response(r, 403, "Inconsistent filesize", strbuf_str(msg));
}
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
r->bundle_status = RHIZOME_BUNDLE_STATUS_INCONSISTENT;
return http_request_rhizome_response(r, 403, NULL, NULL);
{
strbuf msg = strbuf_alloca(200);
strbuf_sprintf(msg, "Payload hash (%s) contradicts manifest (filehash=%s)",
alloca_tohex_rhizome_filehash_t(r->u.insert.write.id),
alloca_tohex_rhizome_filehash_t(r->manifest->filehash));
return http_request_rhizome_response(r, 403, "Inconsistent filehash", strbuf_str(msg));
}
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
r->bundle_status = RHIZOME_BUNDLE_STATUS_READONLY;
return http_request_rhizome_response(r, 403, "Missing bundle secret", NULL);
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
r->bundle_status = RHIZOME_BUNDLE_STATUS_NO_ROOM;
return http_request_rhizome_response(r, 403, "Bundle too big", NULL);
case RHIZOME_PAYLOAD_STATUS_EVICTED:
r->bundle_status = RHIZOME_BUNDLE_STATUS_NO_ROOM;
// fall through
return http_request_rhizome_response(r, 403, "Bundle evicted", NULL);
case RHIZOME_PAYLOAD_STATUS_ERROR:
return http_request_rhizome_response(r, 403, NULL, NULL);
}
if (!status_valid) {
WHYF("r->payload_status = %d", r->payload_status);
return http_request_rhizome_response(r, 500, NULL, NULL);
return http_request_rhizome_response(r, 500, NULL, NULL);
}
if (!status_valid)
FATALF("rhizome_finish_store() returned status = %d", r->payload_status);
// Finalise the manifest and add it to the store.
if (r->manifest->filesize) {
if (!r->manifest->has_filehash)
rhizome_manifest_set_filehash(r->manifest, &r->u.insert.write.id);
else
assert(cmp_rhizome_filehash_t(&r->u.insert.write.id, &r->manifest->filehash) == 0);
}
const char *invalid_reason = rhizome_manifest_validate_reason(r->manifest);
if (invalid_reason) {
r->bundle_status = RHIZOME_BUNDLE_STATUS_INVALID;
@ -666,7 +727,7 @@ static int restful_rhizome_insert_end(struct http_request *hr)
rhizome_manifest_free(r->manifest);
r->manifest = mout;
}
result = 200;
result = 201;
break;
case RHIZOME_BUNDLE_STATUS_INVALID:
case RHIZOME_BUNDLE_STATUS_FAKE:
@ -706,7 +767,7 @@ int restful_rhizome_(httpd_request *r, const char *remainder)
HTTP_HANDLER *handler = NULL;
rhizome_bid_t bid;
const char *end;
if (strn_to_rhizome_bid_t(&bid, remainder, &end) != -1) {
if (parse_rhizome_bid_t(&bid, remainder, -1, &end) != -1) {
if (strcmp(end, ".rhm") == 0) {
handler = restful_rhizome_bid_rhm;
remainder = "";

185
rhizome_store.c
View File

@ -310,6 +310,9 @@ int rhizome_store_cleanup(struct rhizome_cleanup_report *report)
enum rhizome_payload_status rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length)
{
if (config.debug.rhizome_store)
DEBUGF("file_length=%"PRIu64, file_length);
if (file_length == 0)
return RHIZOME_PAYLOAD_STATUS_EMPTY;
@ -681,6 +684,9 @@ void rhizome_fail_write(struct rhizome_write *write)
enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
{
if (config.debug.rhizome_store)
DEBUGF("blob_fd=%d file_offset=%"PRIu64"", write->blob_fd, write->file_offset);
enum rhizome_payload_status status = RHIZOME_PAYLOAD_STATUS_NEW;
// Once the whole file has been processed, we should finally know its length
@ -789,6 +795,7 @@ enum rhizome_payload_status rhizome_finish_write(struct rhizome_write *write)
}
if (config.debug.rhizome_store)
DEBUGF("Payload id=%s already present, removed id='%"PRIu64"'", alloca_tohex_rhizome_filehash_t(write->id), write->temp_id);
status = RHIZOME_PAYLOAD_STATUS_STORED;
}else{
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
goto dbfailure;
@ -991,33 +998,11 @@ enum rhizome_payload_status rhizome_store_payload_file(rhizome_manifest *m, cons
}
if (!status_ok)
FATALF("rhizome_write_open_manifest() returned status = %d", status);
if (rhizome_write_file(&write, filepath) == -1) {
rhizome_fail_write(&write);
return RHIZOME_PAYLOAD_STATUS_ERROR;
}
status = rhizome_finish_write(&write);
switch (status) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
assert(write.file_length == 0);
assert(m->filesize == 0);
return status;
case RHIZOME_PAYLOAD_STATUS_NEW:
assert(m->filesize == write.file_length);
if (m->has_filehash)
assert(cmp_rhizome_filehash_t(&m->filehash, &write.id) == 0);
else
rhizome_manifest_set_filehash(m, &write.id);
return status;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_STORED:
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_EVICTED:
return status;
}
FATALF("rhizome_finish_write() returned status = %d", status);
if (rhizome_write_file(&write, filepath) == -1)
status = RHIZOME_PAYLOAD_STATUS_ERROR;
else
status = rhizome_finish_write(&write);
return rhizome_finish_store(&write, m, status);
}
/* Return RHIZOME_PAYLOAD_STATUS_STORED if file blob found, RHIZOME_PAYLOAD_STATUS_NEW if not found.
@ -1588,28 +1573,120 @@ enum rhizome_payload_status rhizome_journal_pipe(struct rhizome_write *write, co
}
// open an existing journal bundle, advance the head pointer, duplicate the existing content and get ready to add more.
enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t new_size)
enum rhizome_payload_status rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t append_size)
{
assert(m->filesize != RHIZOME_SIZE_UNSET);
assert(m->filesize + new_size > 0);
assert(m->is_journal);
assert(m->filesize != RHIZOME_SIZE_UNSET);
assert(advance_by <= m->filesize);
uint64_t copy_length = m->filesize - advance_by;
rhizome_manifest_set_filesize(m, m->filesize + new_size - advance_by);
uint64_t new_filesize = RHIZOME_SIZE_UNSET;
if (append_size != RHIZOME_SIZE_UNSET) {
assert(m->filesize + append_size > m->filesize); // no wraparound
new_filesize = m->filesize + append_size - advance_by;
}
if (advance_by > 0)
rhizome_manifest_set_tail(m, m->tail + advance_by);
rhizome_manifest_set_version(m, m->filesize);
enum rhizome_payload_status status = rhizome_open_write(write, NULL, m->filesize);
enum rhizome_payload_status status = rhizome_open_write(write, NULL, new_filesize);
if (config.debug.rhizome)
DEBUGF("rhizome_open_write() returned %d %s", status, rhizome_payload_status_message(status));
if (status == RHIZOME_PAYLOAD_STATUS_NEW && copy_length > 0) {
// note that we don't need to bother decrypting the existing journal payload
// we don't need to bother decrypting the existing journal payload
enum rhizome_payload_status rstatus = rhizome_journal_pipe(write, &m->filehash, advance_by, copy_length);
if (rstatus != RHIZOME_PAYLOAD_STATUS_STORED)
status = RHIZOME_PAYLOAD_STATUS_ERROR;
if (config.debug.rhizome)
DEBUGF("rhizome_journal_pipe() returned %d %s", rstatus, rhizome_payload_status_message(rstatus));
int rstatus_valid = 0;
switch (rstatus) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
case RHIZOME_PAYLOAD_STATUS_NEW:
case RHIZOME_PAYLOAD_STATUS_STORED:
rstatus_valid = 1;
break;
case RHIZOME_PAYLOAD_STATUS_ERROR:
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
rstatus_valid = 1;
status = rstatus;
break;
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
case RHIZOME_PAYLOAD_STATUS_EVICTED:
// rhizome_journal_pipe() should not return any of these codes
FATALF("rhizome_journal_pipe() returned %d %s", rstatus, rhizome_payload_status_message(rstatus));
}
if (!rstatus_valid)
FATALF("rstatus = %d", rstatus);
}
if (status == RHIZOME_PAYLOAD_STATUS_NEW)
if (status == RHIZOME_PAYLOAD_STATUS_NEW) {
status = rhizome_write_derive_key(m, write);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
if (config.debug.rhizome)
DEBUGF("rhizome_write_derive_key() returned %d %s", status, rhizome_payload_status_message(status));
}
if (status != RHIZOME_PAYLOAD_STATUS_NEW) {
rhizome_fail_write(write);
}
return status;
}
// Call to finish any payload store operation
enum rhizome_payload_status rhizome_finish_store(struct rhizome_write *write, rhizome_manifest *m, enum rhizome_payload_status status)
{
if (config.debug.rhizome)
DEBUGF("write=%p m=manifest[%d], status=%d %s", write, m->manifest_record_number, status, rhizome_payload_status_message_nonnull(status));
switch (status) {
case RHIZOME_PAYLOAD_STATUS_NEW:
break;
default:
break;
}
int status_valid = 0;
switch (status) {
case RHIZOME_PAYLOAD_STATUS_EMPTY:
status_valid = 1;
assert(write->file_length == 0);
break;
case RHIZOME_PAYLOAD_STATUS_NEW:
assert(write->file_length != RHIZOME_SIZE_UNSET);
status_valid = 1;
break;
case RHIZOME_PAYLOAD_STATUS_STORED:
assert(write->file_length != RHIZOME_SIZE_UNSET);
status_valid = 1;
// TODO: check that stored hash matches received payload's hash
break;
case RHIZOME_PAYLOAD_STATUS_WRONG_SIZE:
case RHIZOME_PAYLOAD_STATUS_WRONG_HASH:
case RHIZOME_PAYLOAD_STATUS_TOO_BIG:
case RHIZOME_PAYLOAD_STATUS_CRYPTO_FAIL:
case RHIZOME_PAYLOAD_STATUS_EVICTED:
case RHIZOME_PAYLOAD_STATUS_ERROR:
status_valid = 1;
rhizome_fail_write(write);
return status;
}
if (!status_valid)
FATALF("status = %d", status);
// Fill in missing manifest fields and check consistency with existing fields.
if (m->is_journal || m->filesize == RHIZOME_SIZE_UNSET)
rhizome_manifest_set_filesize(m, write->file_length);
else if (m->filesize != write->file_length) {
if (config.debug.rhizome)
DEBUGF("m->filesize=%"PRIu64", write->file_length=%"PRIu64, m->filesize, write->file_length);
return RHIZOME_PAYLOAD_STATUS_WRONG_SIZE;
}
if (m->is_journal)
rhizome_manifest_set_version(m, m->filesize);
if (m->filesize) {
if (m->is_journal || !m->has_filehash)
rhizome_manifest_set_filehash(m, &write->id);
else if (cmp_rhizome_filehash_t(&write->id, &m->filehash) != 0) {
if (config.debug.rhizome)
DEBUGF("m->filehash=%s, write->id=%s", alloca_tohex_rhizome_filehash_t(m->filehash), alloca_tohex_rhizome_filehash_t(write->id));
return RHIZOME_PAYLOAD_STATUS_WRONG_HASH;
}
} else if (m->is_journal)
rhizome_manifest_del_filehash(m);
else if (m->has_filehash)
return RHIZOME_PAYLOAD_STATUS_WRONG_HASH;
return status;
}
@ -1620,17 +1697,11 @@ enum rhizome_payload_status rhizome_append_journal_buffer(rhizome_manifest *m, u
enum rhizome_payload_status status = rhizome_write_open_journal(&write, m, advance_by, (uint64_t) len);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
return status;
if (buffer && len && rhizome_write_buffer(&write, buffer, len) == -1) {
rhizome_fail_write(&write);
return RHIZOME_PAYLOAD_STATUS_ERROR;
}
status = rhizome_finish_write(&write);
if (status != RHIZOME_PAYLOAD_STATUS_NEW) {
rhizome_fail_write(&write);
return status;
}
rhizome_manifest_set_filehash(m, &write.id);
return status;
if (buffer && len && rhizome_write_buffer(&write, buffer, len) == -1)
status = RHIZOME_PAYLOAD_STATUS_ERROR;
else
status = rhizome_finish_write(&write);
return rhizome_finish_store(&write, m, status);
}
enum rhizome_payload_status rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename)
@ -1643,15 +1714,9 @@ enum rhizome_payload_status rhizome_append_journal_file(rhizome_manifest *m, uin
enum rhizome_payload_status status = rhizome_write_open_journal(&write, m, advance_by, stat.st_size);
if (status != RHIZOME_PAYLOAD_STATUS_NEW)
return status;
if (stat.st_size != 0 && rhizome_write_file(&write, filename) == -1) {
rhizome_fail_write(&write);
return RHIZOME_PAYLOAD_STATUS_ERROR;
}
status = rhizome_finish_write(&write);
if (status != RHIZOME_PAYLOAD_STATUS_NEW) {
rhizome_fail_write(&write);
return status;
}
rhizome_manifest_set_filehash(m, &write.id);
return status;
if (stat.st_size != 0 && rhizome_write_file(&write, filename) == -1)
status = RHIZOME_PAYLOAD_STATUS_ERROR;
else
status = rhizome_finish_write(&write);
return rhizome_finish_store(&write, m, status);
}

9
rhizome_types.h
View File

@ -80,7 +80,8 @@ typedef struct rhizome_bid_binary {
#define alloca_tohex_rhizome_bid_t(bid) alloca_tohex((bid).binary, sizeof (*(rhizome_bid_t*)0).binary)
int cmp_rhizome_bid_t(const rhizome_bid_t *a, const rhizome_bid_t *b);
int str_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex);
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, const char **endp);
int strn_to_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, size_t hexlen);
int parse_rhizome_bid_t(rhizome_bid_t *bid, const char *hex, ssize_t hexlen, const char **endp);
/* Fundamental data type: Rhizome File Hash
*
@ -97,7 +98,8 @@ typedef struct rhizome_filehash_binary {
#define alloca_tohex_rhizome_filehash_t(fh) alloca_tohex((fh).binary, sizeof (*(rhizome_filehash_t*)0).binary)
int cmp_rhizome_filehash_t(const rhizome_filehash_t *a, const rhizome_filehash_t *b);
int str_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex);
int strn_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex, const char **endp);
int strn_to_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex, size_t hexlen);
int parse_rhizome_filehash_t(rhizome_filehash_t *fh, const char *hex, ssize_t hexlen, const char **endp);
/* Fundamental data type: Rhizome Bundle Key (BK)
*
@ -119,7 +121,8 @@ int cmp_rhizome_bk_t(const rhizome_bk_t *a, const rhizome_bk_t *b);
// The BK field can only be in hex format
int str_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex);
int strn_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex, const char **endp);
int strn_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex, size_t hexlen);
int parse_rhizome_bk_t(rhizome_bk_t *bk, const char *hex, ssize_t hexlen, const char **endp);
// The Bundle Secret can be given as hex or as a passphrase
int str_to_rhizome_bsk_t(rhizome_bk_t *bsk, const char *text);

3
serval_types.h
View File

@ -54,7 +54,8 @@ typedef struct sid_binary {
int cmp_sid_t(const sid_t *a, const sid_t *b);
int str_to_sid_t(sid_t *sid, const char *hex);
int strn_to_sid_t(sid_t *sid, const char *hex, size_t hexlen, const char **endp);
int strn_to_sid_t(sid_t *sid, const char *hex, size_t hexlen);
int parse_sid_t(sid_t *sid, const char *hex, ssize_t hexlen, const char **endp);
#define alloca_tohex_sas(sas) alloca_tohex((sas), SAS_SIZE)

20
str.c
View File

@ -563,8 +563,9 @@ int str_startswith(const char *str, const char *substring, const char **afterp)
return 1;
}
int strn_startswith(const char *str, size_t len, const char *substring, const char **afterp)
int strn_startswith(const char *str, ssize_t len, const char *substring, const char **afterp)
{
// if len == -1 then str must be nul terminated
while (len && *substring && *substring == *str)
--len, ++substring, ++str;
if (*substring)
@ -893,6 +894,19 @@ int str_to_uint64_interval_ms(const char *str, int64_t *result, const char **aft
return 1;
}
/* Compute the length of the string produced by sprintf(fmt, ...).
@author Andrew Bettison <andrew@servalproject.com>
*/
size_t sprintf_len(const char *fmt, ...)
{
strbuf b = strbuf_local(NULL, 0);
va_list ap;
va_start(ap, fmt);
strbuf_vsprintf(b, fmt, ap);
va_end(ap);
return strbuf_count(b);
}
/* Format a buffer of data as a printable representation, eg: "Abc\x0b\n\0", for display
in log messages.
@author Andrew Bettison <andrew@servalproject.com>
@ -904,9 +918,7 @@ char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcByt
return dstStr;
}
/* Compute the length of the string produced by toprint(). If dstStrLen == -1 then returns the
exact number of characters in the printable representation (excluding the terminating nul),
otherwise returns dstStrLen.
/* Compute the length of the string produced by toprint().
@author Andrew Bettison <andrew@servalproject.com>
*/
size_t toprint_len(const char *srcBuf, size_t srcBytes, const char quotes[2])

9
str.h
View File

@ -283,6 +283,11 @@ __SERVAL_DNA__STR_INLINE int hexvalue(char c) {
return isxdigit(c) ? _serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_HEX_MASK : -1;
}
/* -------------------- In-line string formatting -------------------- */
size_t sprintf_len(const char *fmt, ...);
#define alloca_sprintf(dstlen, fmt,...) strbuf_str(strbuf_sprintf(strbuf_alloca((dstlen) == -1 ? sprintf_len((fmt), ##__VA_ARGS__) + 1 : (size_t)(dstlen)), (fmt), ##__VA_ARGS__))
/* -------------------- Printable string representation -------------------- */
char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcBytes, const char quotes[2]);
@ -366,9 +371,11 @@ int str_startswith(const char *str, const char *substring, const char **afterp);
* sub-string. If so, return 1 and, if afterp is not NULL, set *afterp to point to the character
* immediately following the substring. Otherwise return 0.
*
* If len == -1 then is equivalent to str_startswith().
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int strn_startswith(const char *str, size_t len, const char *substring, const char **afterp);
int strn_startswith(const char *str, ssize_t len, const char *substring, const char **afterp);
/* Case-insensitive form of str_startswith().
* @author Andrew Bettison <andrew@servalproject.com>

10
strbuf.c
View File

@ -101,16 +101,16 @@ strbuf strbuf_putc(strbuf sb, char ch)
return sb;
}
int strbuf_sprintf(strbuf sb, const char *fmt, ...)
strbuf strbuf_sprintf(strbuf sb, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
int n = strbuf_vsprintf(sb, fmt, ap);
strbuf_vsprintf(sb, fmt, ap);
va_end(ap);
return n;
return sb;
}
int strbuf_vsprintf(strbuf sb, const char *fmt, va_list ap)
strbuf strbuf_vsprintf(strbuf sb, const char *fmt, va_list ap)
{
int n;
if (sb->start && !sb->end) {
@ -126,7 +126,7 @@ int strbuf_vsprintf(strbuf sb, const char *fmt, va_list ap)
}
if (n != -1)
sb->current += n;
return n;
return sb;
}
char *strbuf_substr(const_strbuf sb, int offset)

6
strbuf.h
View File

@ -353,7 +353,7 @@ strbuf strbuf_putc(strbuf sb, char ch);
/** Append the results of sprintf(fmt,...) to the string buffer, truncating if
* necessary to avoid buffer overrun. Return sprintf()'s return value.
* necessary to avoid buffer overrun. Return a pointer to the strbuf.
*
* This is equivalent to char tmp[...]; sprintf(tmp, fmt, ...); strbuf_puts(tmp);
* assuming that tmp[] is large enough to contain the entire string produced by
@ -361,8 +361,8 @@ strbuf strbuf_putc(strbuf sb, char ch);
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int strbuf_sprintf(strbuf sb, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
int strbuf_vsprintf(strbuf sb, const char *fmt, va_list ap);
strbuf strbuf_sprintf(strbuf sb, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
strbuf strbuf_vsprintf(strbuf sb, const char *fmt, va_list ap);
/** Return a pointer to the current nul-terminated string in the strbuf.

11
testdefs_rhizome.sh
View File

@ -23,6 +23,7 @@ rexp_bundlekey='[0-9a-fA-F]\{64\}'
rexp_bundlesecret="$rexp_bundlekey"
rexp_filehash='[0-9a-fA-F]\{128\}'
rexp_filesize='[0-9]\{1,\}'
rexp_tail='[0-9]\{1,\}'
rexp_version='[0-9]\{1,\}'
rexp_crypt='[01]'
rexp_date='[0-9]\{1,\}'
@ -193,6 +194,7 @@ unpack_manifest_for_grep() {
re_recipient="\($rexp_sid\)\{0,1\}"
re_filesize="$rexp_filesize"
re_filehash="\($rexp_filehash\)\{0,1\}"
re_tail="$rexp_tail"
re_name=$(escape_grep_basic "${filename##*/}")
if [ -e "$manifestname" ]; then
re_filesize=$($SED -n -e '/^filesize=/s///p' "$manifestname")
@ -206,6 +208,7 @@ unpack_manifest_for_grep() {
re_service=$(escape_grep_basic "$re_service")
re_manifestid=$($SED -n -e '/^id=/s///p' "$manifestname")
re_version=$($SED -n -e '/^version=/s///p' "$manifestname")
re_tail=$($SED -n -e '/^tail=/s///p' "$manifestname")
re_date=$($SED -n -e '/^date=/s///p' "$manifestname")
re_crypt=$($SED -n -e '/^crypt=/s///p' "$manifestname")
re_name=$($SED -n -e '/^name=/s///p' "$manifestname")
@ -283,6 +286,10 @@ extract_stdout_filehash() {
extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
}
extract_stdout_tail() {
extract_stdout_keyvalue "$1" tail "$rexp_tail"
}
extract_stdout_crypt() {
extract_stdout_keyvalue "$1" crypt "$rexp_crypt"
}
@ -347,6 +354,10 @@ extract_manifest_filesize() {
extract_manifest "$1" "$2" filesize "$rexp_filesize"
}
extract_manifest_tail() {
extract_manifest "$1" "$2" tail "$rexp_tail"
}
extract_manifest_filehash() {
extract_manifest "$1" "$2" filehash "$rexp_filehash"
}

5
tests/rhizomejava
View File

@ -512,8 +512,9 @@ setup_RhizomeInsertJournal() {
test_RhizomeInsertJournal() {
executeJavaOk org.servalproject.test.Rhizome rhizome-insert '' file1.manifest file1 ifile1.manifest
tfw_cat --stdout --stderr
assertStdoutGrep ServalDNotImplementedException
assertStdoutGrep --ignore-case "not supported.*journal"
# TODO: need special exception for this case, not RhizomeInvalidManifestException
assertStdoutGrep RhizomeInvalidManifestException
assertStderrGrep --ignore-case "cannot add.*journal"
executeOk_servald rhizome list
assert_rhizome_list
}

354
tests/rhizomeops
View File

@ -2,7 +2,7 @@
# Tests for Serval rhizome command-line operations.
#
# Copyright 2012-2014 Serval Project, Inc.
# Copyright 2012-2015 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@ -25,18 +25,32 @@ source "${0%/*}/../testdefs_rhizome.sh"
shopt -s extglob
setup_rhizome() {
[ -z "$A_IDENTITY_COUNT" ] && A_IDENTITY_COUNT=1
[ -z "$B_IDENTITY_COUNT" ] && B_IDENTITY_COUNT=0
>sids
if [ $A_IDENTITY_COUNT -ne 0 ]; then
set_instance +A
set_rhizome_config
if [ $A_IDENTITY_COUNT -eq 1 ]; then
create_single_identity
echo $SIDA >>sids
else
create_identities $A_IDENTITY_COUNT
for ((i=0; i!=A_IDENTITY_COUNT; ++i)); do
eval echo \"\$SIDA$i\" >>sids
done
fi
fi
if [ $B_IDENTITY_COUNT -ne 0 ]; then
set_instance +B
set_rhizome_config
create_identities $B_IDENTITY_COUNT
for ((i=0; i!=B_IDENTITY_COUNT; ++i)); do
eval echo \"\$SIDB$i\" >>sids
done
fi
assert [ $(sort sids | uniq | wc -l) -eq $((A_IDENTITY_COUNT + B_IDENTITY_COUNT)) ]
set_instance +A
set_rhizome_config
create_single_identity
echo "$SIDA1" >sids
set_instance +B
set_rhizome_config
create_identities 4
echo "$SIDB1" >>sids
echo "$SIDB2" >>sids
echo "$SIDB3" >>sids
echo "$SIDB4" >>sids
assert [ $(sort sids | uniq | wc -l) -eq 5 ]
}
set_rhizome_config() {
@ -83,9 +97,9 @@ setup_AddNoManifest() {
echo "Another test file" >file2
}
test_AddNoManifest() {
executeOk_servald rhizome add file $SIDB1 file1
executeOk_servald rhizome add file $SIDA file1
assert_stdout_add_file file1
executeOk_servald rhizome add file $SIDB1 "$PWD/file2"
executeOk_servald rhizome add file $SIDA "$PWD/file2"
assert_stdout_add_file file2
}
@ -100,7 +114,7 @@ setup_AddManifestFieldUnsupported() {
echo "bogus=one" >file1.manifest
}
test_AddManifestFieldUnsupported() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
assert_stdout_add_file file1
tfw_cat -v file1.manifest
assert_manifest_complete file1.manifest
@ -117,7 +131,7 @@ setup_AddManifestFieldUnsupportedArgs() {
echo "Another test file" >file2
}
test_AddManifestFieldUnsupportedArgs() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest '' bogus=two
executeOk_servald rhizome add file $SIDA file1 file1.manifest '' bogus=two
assert_stdout_add_file file1
tfw_cat -v file1.manifest
assert_manifest_complete file1.manifest
@ -177,13 +191,13 @@ setup_AddNonExistManifest() {
}
test_AddNonExistManifest() {
assert --error-on-fail [ ! -e file1.manifest ]
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
assert_stdout_add_file file1
assert [ -r file1.manifest ]
assert_manifest_complete file1.manifest
assert_manifest_fields file1.manifest service=file name=file1
assert --error-on-fail [ ! -e file2.manifest ]
executeOk_servald rhizome add file $SIDB1 "$PWD/file2" file2.manifest
executeOk_servald rhizome add file $SIDA "$PWD/file2" file2.manifest
assert_stdout_add_file file2
assert [ -r file2.manifest ]
assert_manifest_complete file2.manifest
@ -200,7 +214,7 @@ setup_AddManifest() {
echo -e 'name=wah\ndate=12345' >file1.manifest
}
test_AddManifest() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr -v file1.manifest
assert_stdout_add_file file1 name=wah
assert_manifest_complete file1.manifest
@ -216,7 +230,7 @@ setup_AddManifestArgs() {
echo "A test file" >file1
}
test_AddManifestArgs() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest '' name=wah date=12345
executeOk_servald rhizome add file $SIDA file1 file1.manifest '' name=wah date=12345
tfw_cat --stdout --stderr -v file1.manifest
assert_stdout_add_file file1 name=wah
assert_manifest_complete file1.manifest
@ -231,13 +245,13 @@ setup_AddEmpty() {
assert_rhizome_list
}
test_AddEmpty() {
executeOk_servald rhizome add file $SIDB1 '' empty.manifest
executeOk_servald rhizome add file $SIDA '' empty.manifest
tfw_cat --stdout --stderr -v empty.manifest
assert_stdout_add_file --manifest=empty.manifest ''
assert_manifest_complete empty.manifest
assert_manifest_fields empty.manifest service=file name= filesize=0
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 --manifest=empty.manifest ''
assert_rhizome_list --fromhere=1 --author=$SIDA --manifest=empty.manifest ''
}
doc_AddThenList="List contains one file after one add"
@ -251,13 +265,13 @@ setup_AddThenList() {
}
test_AddThenList() {
# Add first file
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
assert_rhizome_list --fromhere=1 --author=$SIDA file1
# Add second file
executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
executeOk_servald rhizome add file $SIDA file2 file2.manifest
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}
doc_ExtractManifestAfterAdd="Export manifest after one add"
@ -265,10 +279,10 @@ setup_ExtractManifestAfterAdd() {
setup_servald
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
assert_rhizome_list --fromhere=1 --author=$SIDA file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
@ -290,7 +304,7 @@ test_ExtractManifestAfterAdd() {
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assert [ -e file1x.manifest ]
@ -302,10 +316,10 @@ setup_ExtractManifestFileAfterAdd() {
setup_servald
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
assert_rhizome_list --fromhere=1 --author=$SIDA file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
@ -327,7 +341,7 @@ test_ExtractManifestFileAfterAdd() {
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assert [ -e file1x.manifest ]
@ -342,14 +356,14 @@ setup_ExtractManifestFileFromExtBlob() {
setup_rhizome
executeOk_servald config set rhizome.max_blob_size 0
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_stdout_rowid rowid1
executeOk_servald config set rhizome.max_blob_size 1000
echo "Another test file" >file2
executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
executeOk_servald rhizome add file $SIDA file2 file2.manifest
extract_stdout_rowid rowid2
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
extract_manifest_id manifestid1 file1.manifest
extract_manifest_version version1 file1.manifest
extract_manifest_filehash filehash1 file1.manifest
@ -376,7 +390,7 @@ test_ExtractManifestFileFromExtBlob() {
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid1\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assert [ -e file1x.manifest ]
@ -397,7 +411,7 @@ test_ExtractManifestFileFromExtBlob() {
assertStdoutGrep --matches=1 "^name:file2\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid2\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assert [ -e file2x.manifest ]
@ -410,7 +424,6 @@ doc_LargePayload="Export huge bundle after one add"
setup_LargePayload() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set debug.rhizome_store on
}
test_LargePayload() {
@ -428,7 +441,7 @@ setup_CorruptExternalBlob() {
setup_rhizome
executeOk_servald config set rhizome.max_blob_size 0
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_manifest_id manifestid file1.manifest
extract_manifest_filehash filehash file1.manifest
assert cmp file1 "$SERVALINSTANCE_PATH/blob/$filehash"
@ -444,7 +457,7 @@ setup_ExtractManifestToStdout() {
setup_servald
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_stdout_rowid rowid
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
@ -466,7 +479,7 @@ test_ExtractManifestToStdout() {
assertStdoutGrep --line=..13 --matches=1 "^name:file1\$"
assertStdoutGrep --line=..13 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=..13 --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --line=..13 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..13 --matches=1 "^\.author:$SIDA\$"
assertStdoutGrep --line=..13 --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --line=..13 --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --line=14 --matches=1 "^manifest:"
@ -543,11 +556,11 @@ setup_ExtractFileAfterAdd() {
setup_servald
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stderr
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
assert_rhizome_list --fromhere=1 --author=$SIDA file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
@ -570,7 +583,7 @@ test_ExtractFileAfterAdd() {
assertStdoutGrep --matches=1 "^name:file1\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDA\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
}
@ -629,34 +642,34 @@ setup_AddDeDuplicate() {
echo "Another test file" >file2
echo "A test file, second version" >file1_2
# Add first file
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
extract_stdout_secret file1_secret
# Add second file
executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
executeOk_servald rhizome add file $SIDA file2 file2.manifest
extract_stdout_secret file2_secret
# Make sure they are both in the list.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}
test_AddDeDuplicate() {
# Add first file again - should return a "duplicate" status code and nothing
# should change in its manifests.
execute --exit-status=2 --stderr --core-backtrace $servald rhizome add file $SIDB1 file1 file1.manifestA
execute --exit-status=2 --stderr --core-backtrace $servald rhizome add file $SIDA file1 file1.manifestA
assert [ -s file1.manifestA ]
assert_stdout_add_file file1
extract_stdout_secret file1_dup_secret
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
strip_signatures file1.manifest file1.manifestA
assert diff file1.manifest file1.manifestA
assert [ $file1_secret = $file1_dup_secret ]
# Repeat for second file.
execute --exit-status=2 --stderr $servald rhizome add file $SIDB1 file2 file2.manifestA
execute --exit-status=2 --stderr $servald rhizome add file $SIDA file2 file2.manifestA
assert [ -s file2.manifestA ]
assert_stdout_add_file file2
extract_stdout_secret file2_dup_secret
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
strip_signatures file2.manifest file2.manifestA
assert diff file2.manifest file2.manifestA
assert [ $file2_secret = $file2_dup_secret ]
@ -669,22 +682,22 @@ setup_AddForceDuplicate() {
test_AddForceDuplicate() {
# Add first file again with the --force-new option. A new manifest
# should be created with a new ID.
executeOk_servald rhizome add file --force-new $SIDB1 file1 file1.manifestA
executeOk_servald rhizome add file --force-new $SIDA file1 file1.manifestA
assert [ -s file1.manifestA ]
assert_stdout_add_file --manifest=file1.manifestA file1
extract_stdout_secret file1_dup_secret
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2 --manifest=file1.manifestA file1
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2 --manifest=file1.manifestA file1
strip_signatures file1.manifest file1.manifestA
assert ! diff file1.manifest file1.manifestA
assert [ $file1_secret != $file1_dup_secret ]
# Repeat for second file.
executeOk_servald rhizome add file --force-new $SIDB1 file2 file2.manifestA
executeOk_servald rhizome add file --force-new $SIDA file2 file2.manifestA
assert [ -s file2.manifestA ]
assert_stdout_add_file --manifest=file2.manifestA file2
extract_stdout_secret file2_dup_secret
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2 --manifest=file1.manifestA file1 --manifest=file2.manifestA file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2 --manifest=file1.manifestA file1 --manifest=file2.manifestA file2
strip_signatures file2.manifest file2.manifestA
assert ! diff file2.manifest file2.manifestA
assert [ $file2_secret != $file2_dup_secret ]
@ -699,13 +712,13 @@ test_AddMismatched() {
# code indicating inconsistency.
cp file1.manifest file1_2.manifest
# Exit status 6 means manifest and payload do not match (filesize/filehash).
execute --exit-status=6 --stderr --core-backtrace $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
execute --exit-status=6 --stderr --core-backtrace $servald rhizome add file $SIDA file1_2 file1_2.manifest
tfw_cat file1.manifest file1_2.manifest
# Output manifest should be the same as the re-used manigfest
assert --stderr cmp file1.manifest file1_2.manifest
# And rhizome store should be unchanged.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}
doc_AddUpdateSameVersion="Add new payload to existing manifest with same version fails"
@ -722,13 +735,13 @@ test_AddUpdateSameVersion() {
# Try to add another file using an existing manifest Id and Version, should
# fail and update the manifest file to show existing bundle's manifest.
tfw_cat -v file1_2.manifest
execute $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
execute $servald rhizome add file $SIDA file1_2 file1_2.manifest
assertExitStatus --stderr '==' 1
tfw_cat -v file1_2.manifest file1.manifest
assert cmp file1_2.manifest file1.manifest
# And rhizome store should be unchanged.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1 file2
}
doc_AddUpdateNewVersion="Add new payload to existing manifest with new version"
@ -741,13 +754,13 @@ setup_AddUpdateNewVersion() {
}
test_AddUpdateNewVersion() {
tfw_cat -v file1_2.manifest
executeOk_servald rhizome add file $SIDB1 file1_2 file1_2.manifest
executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest
tfw_cat --stderr
assert_stdout_add_file file1_2 name=file1
assert_manifest_newer file1.manifest file1_2.manifest
# Rhizome store contents reflect new payload.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1_2 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1_2 file2
}
doc_AddUpdateDiscoverAuthor="Add new payload to manifest with author discovery"
@ -760,22 +773,34 @@ test_AddUpdateDiscoverAuthor() {
tfw_cat --stderr
# Rhizome store contents have new payload.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1_2 file2
assert_rhizome_list --fromhere=1 --author=$SIDA file1_2 file2
}
doc_AddUpdateNoAuthor="Cannot add new payload to authorless manifest"
setup_AddUpdateNoAuthor() {
setup_AddUpdateNewVersion
$SED -i -e '/^BK=/d' file1_2.manifest
setup_servald
setup_rhizome
executeOk_servald rhizome list
assert_rhizome_list
echo "A test file" >file1
echo "A test file, second version" >file1_2
# Add first file
executeOk_servald rhizome add file '' file1 file1.manifest
extract_stdout_secret file1_secret
assert_manifest_fields file1.manifest !BK
# Create second manifest
cp file1.manifest file1_2.manifest
strip_signatures file1_2.manifest
$SED -i -e '/^date=/d;/^filehash=/d;/^filesize=/d;/^version=/d' file1_2.manifest
tfw_cat -v file1_2.manifest
}
test_AddUpdateNoAuthor() {
tfw_cat -v file1_2.manifest
execute $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
execute $servald rhizome add file $SIDA file1_2 file1_2.manifest
tfw_cat --stderr
assertExitStatus '!=' 0
# Rhizome store contents have old payload, with the original author.
# Rhizome store contents still have old payload
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
assert_rhizome_list --fromhere=0 file1
}
doc_AddUpdateNoAuthorWithSecret="Add new payload to authorless manifest with bundle secret"
@ -783,13 +808,12 @@ setup_AddUpdateNoAuthorWithSecret() {
setup_AddUpdateNoAuthor
}
test_AddUpdateNoAuthorWithSecret() {
tfw_cat -v file1_2.manifest
executeOk_servald rhizome add file $SIDB1 file1_2 file1_2.manifest "$file1_secret"
executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest "$file1_secret"
tfw_cat --stderr
# Rhizome store contents have new payload, but it has lost its author (no BK
# field any more).
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1_2 --fromhere=1 --author=$SIDB1 file2
assert_rhizome_list --fromhere=0 file1_2
}
doc_AddUpdateAutoVersion="Add new payload to existing manifest with automatic version"
@ -801,7 +825,7 @@ setup_AddUpdateAutoVersion() {
test_AddUpdateAutoVersion() {
tfw_cat -v file1_2.manifest
sleep 0.001 # Ensure that at least one millisecond has elapsed
executeOk_servald rhizome add file $SIDB1 file1_2 file1_2.manifest
executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest
assert_manifest_newer file1.manifest file1_2.manifest
# Rhizome store contents reflect new payload.
executeOk_servald rhizome list
@ -816,7 +840,7 @@ setup_AddUpdateArgs() {
}
test_AddUpdateArgs() {
sleep 0.001 # Ensure that at least one millisecond has elapsed
executeOk_servald rhizome add file $SIDB1 file1_2 file1_2.manifest '' !version !filesize !filehash !date
executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest '' !version !filesize !filehash !date
assert_stdout_add_file file1_2 name=file1
assert_manifest_fields file1_2.manifest name=file1
extract_manifest_id BID2 file1_2.manifest
@ -835,7 +859,7 @@ setup_AddServiceInvalid() {
echo 'service=Fubar!' >file1.manifest
}
test_AddServiceInvalid() {
execute $servald rhizome add file $SIDB1 file1 file1.manifest
execute $servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr
assertExitStatus '!=' 0
}
@ -848,7 +872,7 @@ setup_AddServiceUnsupported() {
echo 'service=Fubar' >file1.manifest
}
test_AddServiceUnsupported() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr
}
@ -888,16 +912,16 @@ setup_AddUpdateWithPassphrase() {
pass='On the Ning Nang Nong'
}
test_AddUpdateWithPassphrase() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest "#$pass"
executeOk_servald rhizome add file $SIDA file1 file1.manifest "#$pass"
tfw_cat --stdout --stderr
assert_stdout_add_file file1 .author=$SIDB1
assert_stdout_add_file file1 .author=$SIDA
assert_manifest_complete file1.manifest
extract_manifest_id BID file1.manifest
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 file1
executeOk_servald rhizome add file $SIDB1 file1_2 file1_2.manifest "#$pass"
executeOk_servald rhizome add file $SIDA file1_2 file1_2.manifest "#$pass"
tfw_cat --stdout --stderr
assert_stdout_add_file file1_2 .author=$SIDB1
assert_stdout_add_file file1_2 .author=$SIDA
assert_manifest_complete file1_2.manifest
extract_manifest_id BID2 file1.manifest
assert [ "$BID" = "$BID2" ]
@ -916,7 +940,7 @@ setup_EncryptedPayload() {
echo -e "service=file\nname=private\ncrypt=1" >file1.manifest
}
test_EncryptedPayload() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr
assert_stdout_add_file file1
assert_manifest_complete file1.manifest
@ -933,13 +957,14 @@ test_EncryptedPayload() {
doc_RecipientIsEncrypted="Sender & recipient triggers encryption by default"
setup_RecipientIsEncrypted() {
A_IDENTITY_COUNT=2
setup_servald
setup_rhizome
echo "Clear Text" >file1
echo -e "service=file\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
echo -e "service=file\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
}
test_RecipientIsEncrypted() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA1 file1 file1.manifest
tfw_cat --stdout --stderr
assert_stdout_add_file file1
assert_manifest_complete file1.manifest
@ -960,10 +985,10 @@ setup_BroadcastNotEncrypted() {
setup_servald
setup_rhizome
echo "Clear Text" >file1
echo -e "service=file\nsender=$SIDB1\nrecipient=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" >file1.manifest
echo -e "service=file\nsender=$SIDA\nrecipient=FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF" >file1.manifest
}
test_BroadcastNotEncrypted() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr
assert_stdout_add_file file1
assert_manifest_complete file1.manifest
@ -974,6 +999,26 @@ test_BroadcastNotEncrypted() {
assert diff file1 file1y
}
doc_AddReuseManifest="Add --bundle copies fields from existing manifest"
setup_AddReuseManifest() {
B_IDENTITY_COUNT=1
setup_servald
setup_rhizome
echo "First content" >file1
echo "Second content" >file1a
echo -e "service=wazoo\nunexpected=true\nrecipient=$SIDB1\nnothing=here" >file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stderr
extract_stdout_manifestid BID
}
test_AddReuseManifest() {
executeOk_servald rhizome add file --bundle=$BID $SIDA file1a file1a.manifest '' !nothing something=there
tfw_cat --stderr
assert_manifest_fields file1a.manifest service=wazoo unexpected=true recipient=$SIDB1 !nothing something=there
executeOk_servald rhizome export bundle $BID file1ax.manifest file1ax
assert diff file1a file1ax
}
doc_JournalAppend="Create and append to a journal"
setup_JournalAppend() {
setup_servald
@ -983,17 +1028,51 @@ setup_JournalAppend() {
cat file1 file2 > file
}
test_JournalAppend() {
executeOk_servald rhizome journal append $SIDB1 "" file1
executeOk_servald rhizome journal append $SIDA "" file1
tfw_cat --stdout --stderr
assert_stdout_add_file file1
extract_stdout_keyvalue BID 'manifestid' '[0-9A-F]\+'
executeOk_servald rhizome journal append $SIDB1 $BID file2
extract_stdout_manifestid BID
executeOk_servald rhizome journal append $SIDA $BID file2
tfw_cat --stdout --stderr
executeOk_servald rhizome extract file $BID filex
tfw_cat --stdout --stderr
assert diff file filex
}
doc_JournalAppendSharedPayload="Journal append produces a shared payload"
setup_JournalAppendSharedPayload() {
setup_servald
setup_rhizome
executeOk_servald config set rhizome.max_blob_size 0
create_file file1 101
>manifest1
create_file file2 102
>manifest2
cat file1 file2 >file12
executeOk_servald rhizome add file '' file1
extract_stdout_filehash HASH1
assert cmp file1 "$SERVALINSTANCE_PATH/blob/$HASH1"
executeOk_servald rhizome add file '' file12
extract_stdout_filehash HASH12
assert cmp file12 "$SERVALINSTANCE_PATH/blob/$HASH12"
assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
}
test_JournalAppendSharedPayload() {
executeOk_servald rhizome journal append $SIDA "" file1
tfw_cat --stdout --stderr
assert_stdout_add_file file1
extract_stdout_filehash addedhash
assert [ $addedhash = $HASH1 ]
assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
extract_stdout_manifestid BID
executeOk_servald rhizome journal append $SIDA $BID file2
tfw_cat --stdout --stderr
assert_stdout_add_file file12 !name
extract_stdout_filehash addedhash
assert [ $addedhash = $HASH12 ]
assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
}
doc_JournalAddCreate="Cannot create a journal using file add"
setup_JournalAddCreate() {
setup_servald
@ -1003,8 +1082,8 @@ setup_JournalAddCreate() {
}
test_JournalAddCreate() {
# TODO: servald should return a status code reserved for this case, instead
# of the generic 255 error
execute --exit-status=255 $servald rhizome add file $SIDB1 file1 file1.manifest
# of the 4 error (which means "invalid manifest")
execute --exit-status=4 $servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr
}
@ -1013,9 +1092,9 @@ setup_JournalAddUpdate() {
setup_servald
setup_rhizome
echo "Part One" > file1
executeOk_servald rhizome journal append $SIDB1 "" file1
executeOk_servald rhizome journal append $SIDA "" file1
assert_stdout_add_file file1
extract_stdout_keyvalue BID 'manifestid' '[0-9A-F]\+'
extract_stdout_manifestid BID
executeOk_servald rhizome extract bundle $BID file1x.manifest file1x
assert diff file1 file1x
extract_manifest_version version file1x.manifest
@ -1025,8 +1104,8 @@ setup_JournalAddUpdate() {
}
test_JournalAddUpdate() {
# TODO: servald should return a status code reserved for this case, instead
# of the generic 255 error
execute --exit-status=255 $servald rhizome add file $SIDB1 file1x file1x.manifest
# of the 4 error (which means "invalid manifest")
execute --exit-status=4 $servald rhizome add file $SIDA file1x file1x.manifest
tfw_cat --stdout --stderr
}
@ -1036,27 +1115,28 @@ setup_AppendFile() {
setup_rhizome
echo "Part One" > file1
echo "Part Two" > file2
executeOk_servald rhizome add file $SIDB1 file1
executeOk_servald rhizome add file $SIDA file1
tfw_cat --stdout --stderr
assert_stdout_add_file file1
extract_stdout_keyvalue BID 'manifestid' '[0-9A-F]\+'
extract_stdout_manifestid BID
}
test_AppendFile() {
# TODO: servald should return a status code reserved for this case, instead
# of the generic 255 error
execute --exit-status=255 $servald rhizome journal append $SIDB1 $BID file2
# of the 4 error (which means "invalid manifest")
execute --exit-status=4 $servald rhizome journal append $SIDA $BID file2
tfw_cat --stdout --stderr
}
doc_MeshMSAddCreate="First add MeshMS creates manifest"
setup_MeshMSAddCreate() {
B_IDENTITY_COUNT=1
setup_servald
setup_rhizome
echo "Message1" >file1
echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB1" >file1.manifest
}
test_MeshMSAddCreate() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
assert_stdout_add_file file1
assert_manifest_complete file1.manifest
extract_manifest_crypt crypt file1.manifest
@ -1070,15 +1150,16 @@ test_MeshMSAddCreate() {
doc_MeshMSAddGrow="Subsequent add MeshMS updates manifest and removes old payload"
setup_MeshMSAddGrow() {
B_IDENTITY_COUNT=1
setup_servald
setup_rhizome
executeOk_servald config set rhizome.clean_on_open on
export SERVALD_ORPHAN_PAYLOAD_PERSIST_MS=0
echo "Message1" >file1
echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
echo -e "service=MeshMS1\nsender=$SIDA\nrecipient=$SIDB1" >file1.manifest
}
test_MeshMSAddGrow() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
assert_stdout_add_file file1
assert_manifest_complete file1.manifest
executeOk_servald rhizome list
@ -1089,9 +1170,9 @@ test_MeshMSAddGrow() {
local -a ofilehashes=()
for m in 2 3 4 5; do
ofilehashes+=("$filehash")
echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
echo -e "id=$id\nBK=$bk\nservice=MeshMS1\nsender=$SIDA\nrecipient=$SIDB1" >file1.manifest
echo "Message$m" >>file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 file1
extract_manifest_id idx file1.manifest
@ -1111,13 +1192,14 @@ test_MeshMSAddGrow() {
doc_MeshMSAddMissingSender="Add MeshMS without sender fails"
setup_MeshMSAddMissingSender() {
B_IDENTITY_COUNT=1
setup_servald
setup_rhizome
echo "Message1" >file1
echo -e "service=MeshMS1\nrecipient=$SIDB2" >file1.manifest
echo -e "service=MeshMS1\nrecipient=$SIDB1" >file1.manifest
}
test_MeshMSAddMissingSender() {
execute $servald rhizome add file $SIDB1 file1 file1.manifest
execute $servald rhizome add file $SIDA file1 file1.manifest
assertExitStatus --stdout --stderr '!=' 0
}
@ -1128,19 +1210,20 @@ setup_MeshMSAddMissingRecipient() {
executeOk_servald rhizome list
assert_rhizome_list
echo "Message1" >file1
echo -e "service=MeshMS1\nsender=$SIDB1" >file1.manifest
echo -e "service=MeshMS1\nsender=$SIDA" >file1.manifest
}
test_MeshMSAddMissingRecipient() {
execute $servald rhizome add file $SIDB1 file1 file1.manifest
execute $servald rhizome add file $SIDA file1 file1.manifest
assertExitStatus '!=' 0
}
doc_MeshMSAddMissingAuthor="Add MeshMS without author uses sender"
setup_MeshMSAddMissingAuthor() {
A_IDENTITY_COUNT=2
setup_servald
setup_rhizome
echo "Message1" >file1
echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
}
test_MeshMSAddMissingAuthor() {
executeOk_servald rhizome add file '' file1 file1.manifest
@ -1178,16 +1261,17 @@ test_ListFilter() {
doc_MeshMSListFilter="List MeshMS manifests by filter"
setup_MeshMSListFilter() {
A_IDENTITY_COUNT=4
setup_servald
setup_rhizome
echo "Message1" >file1
echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB2" >file1.manifest
echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA2" >file1.manifest
echo "Message2" >file2
echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB3" >file2.manifest
echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA3" >file2.manifest
echo "Message3" >file3
echo -e "service=MeshMS1\nsender=$SIDB1\nrecipient=$SIDB4" >file3.manifest
echo -e "service=MeshMS1\nsender=$SIDA1\nrecipient=$SIDA4" >file3.manifest
echo "Message3" >file4
echo -e "service=MeshMS1\nsender=$SIDB2\nrecipient=$SIDB3" >file4.manifest
echo -e "service=MeshMS1\nsender=$SIDA2\nrecipient=$SIDA3" >file4.manifest
executeOk_servald rhizome add file '' file1 file1.manifest
assert_stdout_add_file file1 !.author !BK
assert_manifest_complete file1.manifest
@ -1208,39 +1292,40 @@ test_MeshMSListFilter() {
assert_rhizome_list
executeOk_servald rhizome list MeshMS1
assert_rhizome_list --fromhere=1 file1 file2 file3 file4
executeOk_servald rhizome list '' '' $SIDB1
executeOk_servald rhizome list '' '' $SIDA1
assert_rhizome_list --fromhere=1 file1 file2 file3
executeOk_servald rhizome list '' '' $SIDB2
executeOk_servald rhizome list '' '' $SIDA2
assert_rhizome_list --fromhere=1 file4
executeOk_servald rhizome list '' '' $SIDB3
executeOk_servald rhizome list '' '' $SIDA3
assert_rhizome_list
executeOk_servald rhizome list '' '' $SIDB4
executeOk_servald rhizome list '' '' $SIDA4
assert_rhizome_list
executeOk_servald rhizome list '' '' '' $SIDB1
executeOk_servald rhizome list '' '' '' $SIDA1
assert_rhizome_list
executeOk_servald rhizome list '' '' '' $SIDB2
executeOk_servald rhizome list '' '' '' $SIDA2
assert_rhizome_list --fromhere=1 file1
executeOk_servald rhizome list '' '' '' $SIDB3
executeOk_servald rhizome list '' '' '' $SIDA3
assert_rhizome_list --fromhere=1 file2 file4
executeOk_servald rhizome list file '' '' $SIDB3
executeOk_servald rhizome list file '' '' $SIDA3
assert_rhizome_list
executeOk_servald rhizome list '' '' '' $SIDB4
executeOk_servald rhizome list '' '' '' $SIDA4
assert_rhizome_list --fromhere=1 file3
executeOk_servald rhizome list '' '' $SIDB1 $SIDB4
executeOk_servald rhizome list '' '' $SIDA1 $SIDA4
assert_rhizome_list --fromhere=1 file3
executeOk_servald rhizome list '' '' $SIDB2 $SIDB4
executeOk_servald rhizome list '' '' $SIDA2 $SIDA4
assert_rhizome_list
executeOk_servald rhizome list '' '' $SIDB2 $SIDB3
executeOk_servald rhizome list '' '' $SIDA2 $SIDA3
assert_rhizome_list --fromhere=1 file4
}
doc_ImportForeignBundle="Import a bundle created by another instance"
setup_ImportForeignBundle() {
B_IDENTITY_COUNT=1
setup_servald
setup_rhizome
set_instance +A
echo "Hello from A" >fileA
executeOk_servald rhizome add file $SIDA1 fileA fileA.manifest
executeOk_servald rhizome add file $SIDA fileA fileA.manifest
assert_stdout_add_file fileA
set_instance +B
}
@ -1256,8 +1341,11 @@ test_ImportForeignBundle() {
doc_ImportOwnBundle="Import a bundle created by same instance"
setup_ImportOwnBundle() {
A_IDENTITY_COUNT=0
B_IDENTITY_COUNT=2
setup_servald
setup_rhizome
set_instance +B
echo "Hello from B" >fileB
executeOk_servald rhizome add file $SIDB2 fileB fileB.manifest
assert_stdout_add_file fileB
@ -1309,9 +1397,8 @@ setup_ImportCombinedBundle() {
# manifest appended to the end.
setup_servald
setup_rhizome
set_instance +A
echo "Hello from A" >fileA
executeOk_servald rhizome add file $SIDA1 fileA fileA.manifest
executeOk_servald rhizome add file $SIDA fileA fileA.manifest
assert_stdout_add_file fileA
extract_manifest_id manifestid fileA.manifest
extract_manifest_filehash filehash fileA.manifest
@ -1335,15 +1422,15 @@ test_ImportCombinedBundle() {
doc_ImportJournal="Import a journal bundle"
setup_ImportJournal() {
B_IDENTITY_COUNT=1
setup_servald
setup_rhizome
echo "Part One" > file1
echo "Part Two2" > file2
cat file1 file2 >file3
set_instance +A
executeOk_servald rhizome journal append $SIDA1 "" file1
executeOk_servald rhizome journal append $SIDA "" file1
assert_stdout_add_file file1
extract_stdout_keyvalue BID 'manifestid' '[0-9A-F]\+'
extract_stdout_manifestid BID
executeOk_servald rhizome extract bundle $BID file1x.manifest file1x
assert diff file1 file1x
}
@ -1363,7 +1450,7 @@ test_ImportJournal() {
# Grow the journal and import it again.
begin_fixture
set_instance +A
executeOk_servald rhizome journal append $SIDA1 $BID file2
executeOk_servald rhizome journal append $SIDA $BID file2
assert_stdout_add_file file3 manifestid=$BID name=file1
executeOk_servald rhizome extract bundle $BID file3x.manifest file3x
assert diff file3 file3x
@ -1378,7 +1465,6 @@ test_ImportJournal() {
setup_delete() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.clean_on_open off
rhizome_add_files file{1..4}
for i in {1..4}; do
@ -1470,7 +1556,6 @@ doc_payloadTooBig="Fail to insert a payload that is larger than the database"
setup_payloadTooBig() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.database_size 32K
}
test_payloadTooBig(){
@ -1483,7 +1568,6 @@ doc_payloadUninteresting="Fail to insert a payload that is uninteresting"
setup_payloadUninteresting() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.database_size 64K
}
test_payloadUninteresting(){
@ -1498,7 +1582,6 @@ doc_evictUninteresting="Evict a large payload to make room for smaller payloads"
setup_evictUninteresting() {
setup_servald
setup_rhizome
set_instance +A
executeOk_servald config set rhizome.database_size 1M
create_file file1 512K
create_file file2 256K
@ -1523,7 +1606,6 @@ doc_evictFreeSpace="Reduce database size due to insufficient free space"
setup_evictFreeSpace() {
setup_servald
setup_rhizome
set_instance +A
create_file file1 512K
create_file file2 256K
create_file file3 128K

193
tests/rhizomerestful
View File

@ -2,7 +2,7 @@
# Tests for Serval DNA HTTP Rhizome RESTful interface
#
# Copyright 2013 Serval Project, Inc.
# Copyright 2013-2015 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@ -27,7 +27,7 @@ shopt -s extglob
setup() {
CR=' '
VT=' '
HT=' '
setup_curl 7
setup_json
setup_servald
@ -539,7 +539,7 @@ extract_http_header() {
local __headerfile="$2"
local __header="$3"
local __rexp="$4"
local __value=$($SED -n -e "/^$__header:[ $VT]*$__rexp$CR\$/s/^$__header:[ $VT]*\(.*\)$CR\$/\1/p" "$__headerfile")
local __value=$($SED -n -e "/^$__header:[ $HT]*$__rexp$CR\$/s/^$__header:[ $HT]*\(.*\)$CR\$/\1/p" "$__headerfile")
assert --message="$__headerfile contains valid '$__header' header" \
--dump-on-fail="$__headerfile" \
[ -n "$__value" ]
@ -973,13 +973,13 @@ test_RhizomeInsertDuplicateManifest() {
assert_rhizome_list
}
doc_RhizomeInsertJournal="HTTP RESTful insert Rhizome bundle does not accept journals"
setup_RhizomeInsertJournal() {
doc_RhizomeInsertJournalForbidden="HTTP RESTful insert Rhizome bundle does not accept journals"
setup_RhizomeInsertJournalForbidden() {
setup
echo 'File one' >file1
echo 'tail=0' >file1.manifest
}
test_RhizomeInsertJournal() {
test_RhizomeInsertJournalForbidden() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
@ -992,7 +992,7 @@ test_RhizomeInsertJournal() {
assertExitStatus == 0
assertStdoutIs 403
assertJq http.body 'contains({"http_status_code": 403})'
assertJqGrep --ignore-case http.body '.http_status_message' 'not supported.*journal'
assertJqGrep --ignore-case http.body '.http_status_message' 'cannot add.*journal'
executeOk_servald rhizome list
assert_rhizome_list
}
@ -1162,4 +1162,183 @@ test_RhizomeInsertIncorrectFilehash() {
assert_rhizome_list
}
doc_RhizomeJournalAppend="HTTP RESTful Rhizome journal create and append"
setup_RhizomeJournalAppend() {
setup
echo 'File one' >file1
file1_size=$(cat file1 | wc -c)
>manifest1
echo "service=anything" >>manifest1
echo "name=hoopla" >>manifest1
echo "random=rubbish" >>manifest1
echo 'File two two two' >file2
>manifest2
file2_size=$(cat file2 | wc -c)
}
test_RhizomeJournalAppend() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output file1.manifest \
--dump-header http.header \
--basic --user harry:potter \
--form "bundle-author=$SIDA" \
--form "manifest=@manifest1;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/append"
tfw_cat http.header file1.manifest
assertExitStatus == 0
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Code: 0$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Message: .*bundle new to store.*$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Code: 1$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Message: .*payload new to store.*$CR\$"
assertStdoutIs 201
extract_http_header H_BID http.header Serval-Rhizome-Bundle-Id "$rexp_manifestid"
extract_http_header H_VERSION http.header Serval-Rhizome-Bundle-Version "$rexp_version"
extract_http_header H_SIZE http.header Serval-Rhizome-Bundle-Filesize "$rexp_filesize"
extract_http_header H_HASH http.header Serval-Rhizome-Bundle-Filehash "$rexp_filehash"
extract_http_header H_TAIL http.header Serval-Rhizome-Bundle-Tail "$rexp_tail"
extract_http_header H_DATE http.header Serval-Rhizome-Bundle-Date "$rexp_date"
extract_http_header H_ROWID http.header Serval-Rhizome-Bundle-Rowid "$rexp_rowid"
extract_http_header H_INSERTTIME http.header Serval-Rhizome-Bundle-Inserttime "$rexp_date"
extract_http_header H_SECRET http.header Serval-Rhizome-Bundle-Secret "$rexp_bundlesecret"
extract_http_header H_SERVICE http.header Serval-Rhizome-Bundle-Service ".*"
extract_http_header H_NAME http.header Serval-Rhizome-Bundle-Name ".*"
http_unquote_string H_NAME
extract_http_header H_BK http.header$n Serval-Rhizome-Bundle-BK "$rexp_bundlekey"
extract_http_header H_AUTHOR http.header$n Serval-Rhizome-Bundle-Author "$rexp_bundlekey"
assert [ $H_SIZE -eq $file1_size ]
assert [ "$H_SERVICE" = anything ]
assert [ "$H_NAME" = hoopla ]
assert [ "$H_AUTHOR" = $SIDA ]
extract_manifest_id BID file1.manifest
extract_manifest_version VERSION file1.manifest
extract_manifest_filesize SIZE file1.manifest
extract_manifest_filehash HASH file1.manifest
extract_manifest_tail TAIL file1.manifest
extract_manifest_date DATE file1.manifest
extract_manifest_service SERVICE file1.manifest
extract_manifest_name NAME file1.manifest
extract_manifest_BK BK file1.manifest
assert [ "$BID" = "$H_BID" ]
assert [ "$VERSION" = "$H_VERSION" ]
assert [ "$SIZE" = "$H_SIZE" ]
assert [ "$HASH" = "$H_HASH" ]
assert [ "$TAIL" = "$H_TAIL" ]
assert [ "$DATE" = "$H_DATE" ]
assert [ "$SERVICE" = "$H_SERVICE" ]
assert [ "$NAME" = "$H_NAME" ]
assert [ "$BK" = "$H_BK" ]
executeOk_servald rhizome list
assert_rhizome_list file1
executeOk_servald rhizome extract file "$BID" file1x
assert --message="extracted payload is correct" diff file1 file1x
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output file2.manifest \
--dump-header http.headers \
--basic --user harry:potter \
--form "bundle-id=$BID" \
--form "bundle-author=$SIDA" \
--form "manifest=@manifest2;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file2" \
"http://$addr_localhost:$PORTA/restful/rhizome/append"
tfw_cat http.header file2.manifest
assertExitStatus == 0
assertStdoutIs 201
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Code: 0$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Message: .*bundle new to store.*$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Code: 1$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Message: .*payload new to store.*$CR\$"
}
doc_RhizomeJournalAppendSharedPayload="HTTP RESTful Rhizome journal append with shared payload"
setup_RhizomeJournalAppendSharedPayload() {
set_extra_config() {
executeOk_servald config set rhizome.max_blob_size 0
}
setup
echo 'File one' >file1
>manifest1
echo 'File two two' >file2
>manifest2
cat file1 file2 >file12
executeOk_servald rhizome add file '' file1
extract_stdout_filehash HASH1
assert cmp file1 "$SERVALINSTANCE_PATH/blob/$HASH1"
executeOk_servald rhizome add file '' file12
extract_stdout_filehash HASH12
assert cmp file12 "$SERVALINSTANCE_PATH/blob/$HASH12"
assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
}
test_RhizomeJournalAppendSharedPayload() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output file1.manifest \
--dump-header http.header \
--basic --user harry:potter \
--form "bundle-author=$SIDA" \
--form "manifest=@manifest1;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file1" \
"http://$addr_localhost:$PORTA/restful/rhizome/append"
tfw_cat http.header file1.manifest
assertExitStatus == 0
assertStdoutIs 201
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Bundle-Filehash: $HASH1$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Code: 0$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Message: .*bundle new to store.*$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Code: 2$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Message: .*payload already in store.*$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Message: .*payload already in store.*$CR\$"
assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
extract_http_header BID http.header Serval-Rhizome-Bundle-Id "$rexp_manifestid"
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output file2.manifest \
--dump-header http.header \
--basic --user harry:potter \
--form "bundle-id=$BID" \
--form "manifest=@manifest2;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file2" \
"http://$addr_localhost:$PORTA/restful/rhizome/append"
tfw_cat http.header file2.manifest
assertExitStatus == 0
assertStdoutIs 201
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Bundle-Filehash: $HASH12$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Code: 0$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Bundle-Status-Message: .*bundle new to store.*$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Code: 2$CR\$"
assertGrep --matches=1 --ignore-case http.header "^Serval-Rhizome-Result-Payload-Status-Message: .*payload already in store.*$CR\$"
assert [ $(ls "$SERVALINSTANCE_PATH/blob" | wc -l) -eq 2 ]
}
doc_RhizomeAppendNonJournalForbidden="HTTP RESTful Rhizome cannot append to non-journal"
setup_RhizomeAppendNonJournalForbidden() {
setup
echo "File One" > file1
echo "File Two" > file2
>file2.manifest
executeOk_servald rhizome add file $SIDA file1 file1.manifest
tfw_cat --stdout --stderr
assert_stdout_add_file file1
extract_stdout_manifestid BID
}
test_RhizomeAppendNonJournalForbidden() {
execute curl \
--silent --show-error --write-out '%{http_code}' \
--output http.body \
--dump-header http.header \
--basic --user harry:potter \
--form "bundle-id=$BID" \
--form "manifest=@file2.manifest;type=rhizome/manifest;format=\"text+binarysig\"" \
--form "payload=@file2" \
"http://$addr_localhost:$PORTA/restful/rhizome/append"
tfw_cat http.header http.body
assertExitStatus == 0
assertStdoutIs 403
assertJq http.body 'contains({"http_status_code": 403})'
assertJqGrep --ignore-case http.body '.http_status_message' 'cannot append.*non.*journal'
executeOk_servald rhizome list
assert_rhizome_list file1
}
runTests "$@"