ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor add via HTTP to be equivalent to command line add

Browse Source
This commit is contained in:
Jeremy Lakeman 2012-12-20 15:18:59 +10:30
parent 181d1363f1
commit f64de66b34
8 changed files with 314 additions and 639 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

237
commandline.c
View File

@ -1059,23 +1059,28 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
cli_arg(argc, argv, o, "manifestpath", &manifestpath, NULL, "");
if (cli_arg(argc, argv, o, "bsk", &bskhex, cli_optional_bundle_key, "") == -1)
return -1;
unsigned char authorSid[SID_SIZE];
if (authorSidHex[0] && fromhexstr(authorSid, authorSidHex, SID_SIZE) == -1)
sid_t authorSid;
if (authorSidHex[0] && fromhexstr(authorSid.binary, authorSidHex, SID_SIZE) == -1)
return WHYF("invalid author_sid: %s", authorSidHex);
unsigned char bsk[RHIZOME_BUNDLE_KEY_BYTES];
if (bskhex[0] && fromhexstr(bsk, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1)
rhizome_bk_t bsk;
if (bskhex[0] && fromhexstr(bsk.binary, bskhex, RHIZOME_BUNDLE_KEY_BYTES) == -1)
return WHYF("invalid bsk: %s", bskhex);
if (create_serval_instance_dir() == -1)
return -1;
if (!(keyring = keyring_open_with_pins((char *)pin)))
return -1;
if (rhizome_opendb() == -1)
return -1;
/* Create a new manifest that will represent the file. If a manifest file was supplied, then read
* it, otherwise create a blank manifest. */
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
return WHY("Manifest struct could not be allocated -- not added to rhizome");
if (manifestpath[0] && access(manifestpath, R_OK) == 0) {
if (config.debug.rhizome) DEBUGF("reading manifest from %s", manifestpath);
/* Don't verify the manifest, because it will fail if it is incomplete.
@ -1088,181 +1093,32 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
} else {
if (config.debug.rhizome) DEBUGF("manifest file %s does not exist -- creating new manifest", manifestpath);
}
/* Fill in a few missing manifest fields, to make it easier to use when adding new files:
- the default service is FILE
- use the current time for "date"
- if service is file, then use the payload file's basename for "name"
*/
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (service == NULL) {
rhizome_manifest_set(m, "service", (service = RHIZOME_SERVICE_FILE));
if (config.debug.rhizome) DEBUGF("missing 'service', set default service=%s", service);
} else {
if (config.debug.rhizome) DEBUGF("manifest contains service=%s", service);
}
if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) {
rhizome_manifest_set_ll(m, "date", (long long) gettime_ms());
if (config.debug.rhizome) DEBUGF("missing 'date', set default date=%s", rhizome_manifest_get(m, "date", NULL, 0));
}
if (strcasecmp(RHIZOME_SERVICE_FILE, service) == 0) {
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
if (name == NULL) {
name = strrchr(filepath, '/');
name = name ? name + 1 : filepath;
rhizome_manifest_set(m, "name", name);
if (config.debug.rhizome) DEBUGF("missing 'name', set default name=\"%s\"", name);
} else {
if (config.debug.rhizome) DEBUGF("manifest contains name=\"%s\"", name);
}
}
/* If the author was not specified on the command-line, then the manifest's "sender"
field is used, if present. */
const char *sender = NULL;
if (!authorSidHex[0] && (sender = rhizome_manifest_get(m, "sender", NULL, 0)) != NULL) {
if (fromhexstr(authorSid, sender, SID_SIZE) == -1)
return WHYF("invalid sender: %s", sender);
authorSidHex = sender;
}
/* Bind an ID to the manifest, and also bind the file. Then finalise the manifest.
But if the manifest already contains an ID, don't override it. */
if (authorSidHex[0]) {
if (config.debug.rhizome) DEBUGF("author=%s", authorSidHex);
memcpy(m->author, authorSid, SID_SIZE);
}
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
if (id == NULL) {
if (config.debug.rhizome) DEBUG("creating new bundle");
if (rhizome_manifest_bind_id(m) == -1) {
rhizome_manifest_free(m);
return WHY("Could not bind manifest to an ID");
}
} else {
if (config.debug.rhizome) DEBUGF("modifying existing bundle bid=%s", id);
// Modifying an existing bundle. If an author SID is supplied, we must ensure that it is valid,
// ie, that identity has permission to alter the bundle. If no author SID is supplied but a BSK
// is supplied, then use that to alter the bundle. Otherwise, search the keyring for an
// identity with permission to alter the bundle.
if (!is_sid_any(m->author)) {
// Check that the given author has permission to alter the bundle, and extract the secret
// bundle key if so.
int result = rhizome_extract_privatekey(m);
switch (result) {
case -1:
rhizome_manifest_free(m);
return WHY("error in rhizome_extract_privatekey()");
case 0:
break;
case 1:
if (bskhex[0])
break;
rhizome_manifest_free(m);
return WHY("Manifest does not have BK field");
case 2:
rhizome_manifest_free(m);
return WHY("Author unknown");
case 3:
rhizome_manifest_free(m);
return WHY("Author does not have a Rhizome Secret");
case 4:
rhizome_manifest_free(m);
return WHY("Author does not have permission to modify manifest");
default:
rhizome_manifest_free(m);
return WHYF("Unknown result from rhizome_extract_privatekey(): %d", result);
}
}
if (bskhex[0]) {
if (config.debug.rhizome) DEBUGF("bskhex=%s", bskhex);
if (m->haveSecret) {
// If a bundle secret key was supplied that does not match the secret key derived from the
// author, then warn but carry on using the author's.
if (memcmp(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES) != 0)
WARNF("Supplied bundle secret key is invalid -- ignoring");
} else {
// The caller provided the bundle secret key, so ensure that it corresponds to the bundle's
// public key (its bundle ID), otherwise it won't work.
memcpy(m->cryptoSignSecret, bsk, RHIZOME_BUNDLE_KEY_BYTES);
if (rhizome_verify_bundle_privatekey(m,m->cryptoSignSecret,
m->cryptoSignPublic) == -1) {
rhizome_manifest_free(m);
return WHY("Incorrect BID secret key.");
}
}
}
// If we still don't know the bundle secret or the author, then search for an author.
if (!m->haveSecret && is_sid_any(m->author)) {
if (config.debug.rhizome) DEBUG("bundle author not specified, searching keyring");
int result = rhizome_find_bundle_author(m);
if (result != 0) {
rhizome_manifest_free(m);
switch (result) {
case -1:
return WHY("error in rhizome_find_bundle_author()");
case 4:
return WHY("Manifest does not have BK field");
case 1:
return WHY("No author found");
default:
return WHYF("Unknown result from rhizome_find_bundle_author(): %d", result);
}
}
}
}
if (rhizome_stat_file(m, filepath))
return -1;
if (rhizome_fill_manifest(m, filepath, *authorSidHex?&authorSid:NULL, &bsk))
return -1;
/* Keep note as to whether we are supposed to be encrypting this file or not */
// TODO should we encrypt??
m->payloadEncryption=0;
rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0);
if (rhizome_add_file(m, filepath))
if (m->fileLength){
if (rhizome_add_file(m, filepath))
return -1;
}
rhizome_manifest *mout = NULL;
int ret=rhizome_manifest_finalise(m,&mout);
if (ret<0)
return -1;
/* Add the manifest and its associated file to the Rhizome database,
generating an "id" in the process.
PGS @20121003 - Hang on, didn't we create the ID above? Presumably the
following does NOT in fact generate a bundle ID.
*/
int ret=0;
rhizome_manifest *mout = NULL;
if (rhizome_manifest_check_duplicate(m, &mout) == 2) {
/* duplicate found -- verify it so that we can write it out later */
rhizome_manifest_verify(mout);
ret=2;
} else {
/* set version of manifest, either from version variable, or using current time */
if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
{
/* No version set */
m->version = gettime_ms();
rhizome_manifest_set_ll(m,"version",m->version);
}
else
m->version = rhizome_manifest_get_ll(m,"version");
/* Convert to final form for signing and writing to disk */
if (rhizome_manifest_pack_variables(m))
return WHY("Could not convert manifest to wire format");
/* Sign it */
if (rhizome_manifest_selfsign(m))
return WHY("Could not sign manifest");
/* mark manifest as finalised */
m->finalised=1;
if (rhizome_add_manifest(m, 255 /* TTL */)) {
rhizome_manifest_free(m);
return WHY("Manifest not added to Rhizome database");
}
}
/* If successfully added, overwrite the manifest file so that the Java component that is
invoking this command can read it to obtain feedback on the result. */
rhizome_manifest *mwritten = mout ? mout : m;
if (manifestpath[0]
&& rhizome_write_manifest_file(mwritten, manifestpath) == -1)
&& rhizome_write_manifest_file(mout, manifestpath) == -1)
ret = WHY("Could not overwrite manifest file.");
service = rhizome_manifest_get(mwritten, "service", NULL, 0);
const char *service = rhizome_manifest_get(mout, "service", NULL, 0);
if (service) {
cli_puts("service");
cli_delim(":");
@ -1271,7 +1127,7 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
}
{
char bid[RHIZOME_MANIFEST_ID_STRLEN + 1];
rhizome_bytes_to_hex_upper(mwritten->cryptoSignPublic, bid, RHIZOME_MANIFEST_ID_BYTES);
rhizome_bytes_to_hex_upper(mout->cryptoSignPublic, bid, RHIZOME_MANIFEST_ID_BYTES);
cli_puts("manifestid");
cli_delim(":");
cli_puts(bid);
@ -1279,7 +1135,7 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
}
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(mwritten->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
rhizome_bytes_to_hex_upper(mout->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
cli_puts("secret");
cli_delim(":");
cli_puts(secret);
@ -1287,24 +1143,24 @@ int app_rhizome_add_file(int argc, const char *const *argv, const struct command
}
cli_puts("filesize");
cli_delim(":");
cli_printf("%lld", mwritten->fileLength);
cli_printf("%lld", mout->fileLength);
cli_delim("\n");
if (mwritten->fileLength != 0) {
if (mout->fileLength != 0) {
cli_puts("filehash");
cli_delim(":");
cli_puts(mwritten->fileHexHash);
cli_puts(mout->fileHexHash);
cli_delim("\n");
}
const char *name = rhizome_manifest_get(mwritten, "name", NULL, 0);
const char *name = rhizome_manifest_get(mout, "name", NULL, 0);
if (name) {
cli_puts("name");
cli_delim(":");
cli_puts(name);
cli_delim("\n");
}
rhizome_manifest_free(m);
if (mout != m)
rhizome_manifest_free(mout);
rhizome_manifest_free(m);
return ret;
}
@ -1320,38 +1176,11 @@ int app_rhizome_import_bundle(int argc, const char *const *argv, const struct co
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
return WHY("Out of manifests.");
int status=0;
if (rhizome_read_manifest_file(m, manifestpath, 0) == -1) {
status = WHY("could not read manifest file");
goto cleanup;
}
if (rhizome_manifest_verify(m)){
status = WHY("could not verify manifest");
goto cleanup;
}
/* Make sure we store signatures */
// TODO, why do we need this? Why isn't the state correct from rhizome_read_manifest_file?
// This feels like a hack...
m->manifest_bytes=m->manifest_all_bytes;
status = rhizome_import_file(m, filepath);
int status=rhizome_bundle_import_files(m, manifestpath, filepath);
if (status<0)
goto cleanup;
status = rhizome_manifest_check_duplicate(m, NULL);
if (status<0)
goto cleanup;
if (status==0){
if (rhizome_add_manifest(m, 1) == -1) { // ttl = 1
status = WHY("rhizome_add_manifest() failed");
goto cleanup;
}
}else
INFO("Duplicate found in store");
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (service) {
cli_puts("service");

161
rhizome.c
View File

@ -65,38 +65,39 @@ int rhizome_fetch_delay_ms()
that function and manages file and object buffers and lifetimes.
*/
int rhizome_bundle_import_files(const char *manifest_path, const char *payload_path, int ttl)
int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, const char *filepath)
{
if (config.debug.rhizome)
DEBUGF("(manifest_path=%s, payload_path=%s, ttl=%d)",
DEBUGF("(manifest_path=%s, filepath=%s)",
manifest_path ? alloca_str_toprint(manifest_path) : "NULL",
payload_path ? alloca_str_toprint(payload_path) : "NULL",
ttl
);
/* Read manifest file if no manifest was given */
if (!manifest_path)
return WHY("No manifest supplied");
int ret = 0;
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
ret = WHY("Out of manifests");
else if (rhizome_read_manifest_file(m, manifest_path, 0 /* file not buffer */) == -1)
ret = WHY("Could not read manifest file");
else if (rhizome_manifest_verify(m))
ret = WHY("Verification of manifest file failed");
else {
/* Make sure we store signatures */
m->manifest_bytes=m->manifest_all_bytes;
m->dataFileName = strdup(payload_path);
if (rhizome_manifest_check_file(m))
ret = WHY("Payload does not belong to manifest");
else
ret = rhizome_bundle_import(m, ttl);
}
if (m)
rhizome_manifest_free(m);
return ret;
filepath ? alloca_str_toprint(filepath) : "NULL");
if (rhizome_read_manifest_file(m, manifest_path, 0) == -1)
return WHY("could not read manifest file");
if (rhizome_manifest_verify(m))
return WHY("could not verify manifest");
/* Make sure we store signatures */
// TODO, why do we need this? Why isn't the state correct from rhizome_read_manifest_file?
// This feels like a hack...
m->manifest_bytes=m->manifest_all_bytes;
int status = rhizome_import_file(m, filepath);
if (status<0)
return status;
status = rhizome_manifest_check_duplicate(m, NULL);
if (status<0)
return status;
if (status==0){
if (rhizome_add_manifest(m, 1) == -1) { // ttl = 1
return WHY("rhizome_add_manifest() failed");
}
}else
INFO("Duplicate found in store");
return status;
}
/* Import a bundle from a finalised manifest struct. The dataFileName element must give the path
@ -109,9 +110,6 @@ int rhizome_bundle_import(rhizome_manifest *m, int ttl)
{
if (config.debug.rhizome)
DEBUGF("(m=%p, ttl=%d)", m, ttl);
/* Add the manifest and its payload to the Rhizome database. */
if (rhizome_manifest_check_file(m))
return WHY("File does not belong to manifest");
int ret = rhizome_manifest_check_duplicate(m, NULL);
if (ret == 0) {
ret = rhizome_add_manifest(m, ttl);
@ -203,99 +201,6 @@ int rhizome_manifest_bind_id(rhizome_manifest *m_in)
return 0;
}
int rhizome_manifest_bind_file(rhizome_manifest *m_in,const char *filename,int encryptP)
{
/* Keep payload file name handy for later */
m_in->dataFileName = strdup(filename);
/* Keep note as to whether we are supposed to be encrypting this file or not */
m_in->payloadEncryption=encryptP;
if (encryptP) rhizome_manifest_set_ll(m_in,"crypt",1);
else rhizome_manifest_set_ll(m_in,"crypt",0);
/* Get length of payload. An empty filename means empty payload. */
if (filename[0]) {
struct stat stat;
if (lstat(filename,&stat))
return WHYF("Could not stat() payload file '%s'",filename);
m_in->fileLength = stat.st_size;
} else
m_in->fileLength = 0;
if (config.debug.rhizome)
DEBUGF("filename=%s, fileLength=%lld", filename, m_in->fileLength);
rhizome_manifest_set_ll(m_in,"filesize",m_in->fileLength);
/* Compute hash of non-empty payload */
if (m_in->fileLength != 0) {
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,filename, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
rhizome_manifest_set(m_in, "filehash", m_in->fileHexHash);
m_in->fileHashedP = 1;
} else {
m_in->fileHexHash[0] = '\0';
rhizome_manifest_del(m_in, "filehash");
m_in->fileHashedP = 0;
}
return 0;
}
int rhizome_manifest_check_file(rhizome_manifest *m_in)
{
// Don't bother to check the file if we already have it
if (rhizome_exists(m_in->fileHexHash))
return 0;
/* Find out whether the payload is expected to be encrypted or not */
m_in->payloadEncryption=rhizome_manifest_get_ll(m_in, "crypt");
/* Check payload file is accessible and discover its length, then check that it
matches the file size stored in the manifest */
long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize");
m_in->fileLength = 0;
if (m_in->dataFileName && m_in->dataFileName[0]) {
struct stat stat;
if (lstat(m_in->dataFileName,&stat) == -1) {
if (errno != ENOENT || mfilesize != 0)
return WHYF_perror("stat(%s)", m_in->dataFileName);
} else {
m_in->fileLength = stat.st_size;
}
}
if (config.debug.rhizome)
DEBUGF("filename=%s, fileLength=%lld", m_in->dataFileName ? alloca_str_toprint(m_in->dataFileName) : "NULL", m_in->fileLength);
if (mfilesize != -1 && mfilesize != m_in->fileLength) {
WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m_in->fileLength);
return -1;
}
/* If payload is empty, ensure manifest has not file hash, otherwis compute the hash of the
payload and check that it matches manifest. */
const char *mhexhash = rhizome_manifest_get(m_in, "filehash", NULL, 0);
if (m_in->fileLength != 0) {
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,m_in->dataFileName, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
m_in->fileHashedP = 1;
if (!mhexhash) return WHY("manifest contains no file hash");
if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) {
WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash);
return -1;
}
} else {
if (mhexhash != NULL) {
WHYF("Manifest.filehash (%s) should be absent for empty payload", mhexhash);
return -1;
}
}
return 0;
}
/* Check if a manifest is already stored for the same payload with the same details.
This catches the case of "dna rhizome add file <filename>" on the same file more than once.
(Debounce!) */
@ -334,8 +239,10 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
if (rhizome_manifest_check_sanity(m_in))
return WHY("Sanity checks on manifest failed");
if (rhizome_manifest_check_file(m_in))
return WHY("File does not belong to this manifest");
if (m_in->fileLength){
if (!rhizome_exists(m_in->fileHexHash))
return WHY("File has not been imported");
}
/* Get manifest version number. */
m_in->version = rhizome_manifest_get_ll(m_in, "version");

8
rhizome.h
View File

@ -238,17 +238,16 @@ int rhizome_store_bundle(rhizome_manifest *m);
int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
int rhizome_clean_payload(const char *fileidhex);
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
int rhizome_bundle_import_files(const char *manifest_path, const char *payload_path, int ttl);
int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, const char *filepath);
int rhizome_bundle_import(rhizome_manifest *m, int ttl);
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSid, rhizome_bk_t *bsk);
int rhizome_manifest_verify(rhizome_manifest *m);
int rhizome_manifest_check_sanity(rhizome_manifest *m_in);
int rhizome_manifest_check_file(rhizome_manifest *m_in);
int rhizome_manifest_check_duplicate(rhizome_manifest *m_in,rhizome_manifest **m_out);
int rhizome_manifest_bind_id(rhizome_manifest *m_in);
int rhizome_manifest_bind_file(rhizome_manifest *m_in,const char *filename,int encryptP);
int rhizome_manifest_finalise(rhizome_manifest *m);
int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout);
int rhizome_add_manifest(rhizome_manifest *m_in,int ttl);
void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount);
@ -618,6 +617,7 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename);
int rhizome_fail_write(struct rhizome_write *write);
int rhizome_finish_write(struct rhizome_write *write);
int rhizome_import_file(rhizome_manifest *m, const char *filepath);
int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
int rhizome_add_file(rhizome_manifest *m, const char *filepath);
#endif //__SERVALDNA__RHIZOME_H

209
rhizome_bundle.c
View File

@ -626,57 +626,174 @@ int rhizome_manifest_dump(rhizome_manifest *m, const char *msg)
return 0;
}
int rhizome_manifest_finalise(rhizome_manifest *m)
int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout)
{
/* set fileLength and "filesize" var */
if (m->dataFileName[0]) {
struct stat stat;
if (lstat(m->dataFileName, &stat)) {
WHY_perror("lstat");
return WHY("Could not stat() associated file");
}
m->fileLength = stat.st_size;
} else
m->fileLength = 0;
rhizome_manifest_set_ll(m, "filesize", m->fileLength);
/* set fileHexHash and "filehash" var */
if (m->fileLength != 0) {
if (!m->fileHashedP) {
if (rhizome_hash_file(m, m->dataFileName, m->fileHexHash))
return WHY("rhizome_hash_file() failed during finalisation of manifest.");
m->fileHashedP = 1;
}
rhizome_manifest_set(m, "filehash", m->fileHexHash);
/* Add the manifest and its associated file to the Rhizome database,
generating an "id" in the process.
PGS @20121003 - Hang on, didn't we create the ID above? Presumably the
following does NOT in fact generate a bundle ID.
*/
int ret=0;
if (rhizome_manifest_check_duplicate(m, mout) == 2) {
/* duplicate found -- verify it so that we can write it out later */
rhizome_manifest_verify(*mout);
ret=2;
} else {
m->fileHexHash[0] = '\0';
m->fileHashedP = 0;
rhizome_manifest_del(m, "filehash");
}
/* set fileHighestPriority based on group associations.
XXX - Should probably be set as groups are added */
/* set version of manifest, either from version variable, or using current time */
if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
*mout=m;
/* set version of manifest, either from version variable, or using current time */
if (rhizome_manifest_get(m,"version",NULL,0)==NULL)
{
/* No version set */
m->version = gettime_ms();
rhizome_manifest_set_ll(m,"version",m->version);
}
else
m->version = rhizome_manifest_get_ll(m,"version");
/* Convert to final form for signing and writing to disk */
if (rhizome_manifest_pack_variables(m))
return WHY("Could not convert manifest to wire format");
/* Sign it */
if (rhizome_manifest_selfsign(m))
return WHY("Could not sign manifest");
/* mark manifest as finalised */
m->finalised=1;
return 0;
else
m->version = rhizome_manifest_get_ll(m,"version");
/* Convert to final form for signing and writing to disk */
if (rhizome_manifest_pack_variables(m))
return WHY("Could not convert manifest to wire format");
/* Sign it */
if (rhizome_manifest_selfsign(m))
return WHY("Could not sign manifest");
/* mark manifest as finalised */
m->finalised=1;
if (rhizome_add_manifest(m, 255 /* TTL */)) {
rhizome_manifest_free(m);
return WHY("Manifest not added to Rhizome database");
}
}
return ret;
}
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSid, rhizome_bk_t *bsk){
/* Fill in a few missing manifest fields, to make it easier to use when adding new files:
- the default service is FILE
- use the current time for "date"
- if service is file, then use the payload file's basename for "name"
*/
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (service == NULL) {
rhizome_manifest_set(m, "service", (service = RHIZOME_SERVICE_FILE));
if (config.debug.rhizome) DEBUGF("missing 'service', set default service=%s", service);
} else {
if (config.debug.rhizome) DEBUGF("manifest contains service=%s", service);
}
if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) {
rhizome_manifest_set_ll(m, "date", (long long) gettime_ms());
if (config.debug.rhizome) DEBUGF("missing 'date', set default date=%s", rhizome_manifest_get(m, "date", NULL, 0));
}
if (strcasecmp(RHIZOME_SERVICE_FILE, service) == 0) {
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
if (name == NULL) {
if (filepath && *filepath){
name = strrchr(filepath, '/');
name = name ? name + 1 : filepath;
}else
name="";
rhizome_manifest_set(m, "name", name);
if (config.debug.rhizome) DEBUGF("missing 'name', set default name=\"%s\"", name);
} else {
if (config.debug.rhizome) DEBUGF("manifest contains name=\"%s\"", name);
}
}
/* If the author was not specified, then the manifest's "sender"
field is used, if present. */
if (authorSid){
memcpy(m->author, authorSid, SID_SIZE);
}else{
const char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
if (sender){
if (fromhexstr(m->author, sender, SID_SIZE) == -1)
return WHYF("invalid sender: %s", sender);
}
}
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
if (id == NULL) {
if (config.debug.rhizome) DEBUG("creating new bundle");
if (rhizome_manifest_bind_id(m) == -1) {
rhizome_manifest_free(m);
return WHY("Could not bind manifest to an ID");
}
} else {
if (config.debug.rhizome) DEBUGF("modifying existing bundle bid=%s", id);
// Modifying an existing bundle. If an author SID is supplied, we must ensure that it is valid,
// ie, that identity has permission to alter the bundle. If no author SID is supplied but a BSK
// is supplied, then use that to alter the bundle. Otherwise, search the keyring for an
// identity with permission to alter the bundle.
if (!is_sid_any(m->author)) {
// Check that the given author has permission to alter the bundle, and extract the secret
// bundle key if so.
int result = rhizome_extract_privatekey(m);
switch (result) {
case -1:
rhizome_manifest_free(m);
return WHY("error in rhizome_extract_privatekey()");
case 0:
break;
case 1:
if (!rhizome_is_bk_none(bsk))
break;
rhizome_manifest_free(m);
return WHY("Manifest does not have BK field");
case 2:
rhizome_manifest_free(m);
return WHY("Author unknown");
case 3:
rhizome_manifest_free(m);
return WHY("Author does not have a Rhizome Secret");
case 4:
rhizome_manifest_free(m);
return WHY("Author does not have permission to modify manifest");
default:
rhizome_manifest_free(m);
return WHYF("Unknown result from rhizome_extract_privatekey(): %d", result);
}
}
if (!rhizome_is_bk_none(bsk)){
if (config.debug.rhizome) DEBUGF("bskhex=%s", alloca_tohex(bsk->binary, RHIZOME_BUNDLE_KEY_BYTES));
if (m->haveSecret) {
// If a bundle secret key was supplied that does not match the secret key derived from the
// author, then warn but carry on using the author's.
if (memcmp(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES) != 0)
WARNF("Supplied bundle secret key is invalid -- ignoring");
} else {
// The caller provided the bundle secret key, so ensure that it corresponds to the bundle's
// public key (its bundle ID), otherwise it won't work.
memcpy(m->cryptoSignSecret, bsk, RHIZOME_BUNDLE_KEY_BYTES);
if (rhizome_verify_bundle_privatekey(m,m->cryptoSignSecret,
m->cryptoSignPublic) == -1) {
rhizome_manifest_free(m);
return WHY("Incorrect BID secret key.");
}
}
}
// If we still don't know the bundle secret or the author, then search for an author.
if (!m->haveSecret && is_sid_any(m->author)) {
if (config.debug.rhizome) DEBUG("bundle author not specified, searching keyring");
int result = rhizome_find_bundle_author(m);
if (result != 0) {
rhizome_manifest_free(m);
switch (result) {
case -1:
return WHY("error in rhizome_find_bundle_author()");
case 4:
return WHY("Manifest does not have BK field");
case 1:
return WHY("No author found");
default:
return WHYF("Unknown result from rhizome_find_bundle_author(): %d", result);
}
}
}
}
return 0;
}

181
rhizome_database.c
View File

@ -739,10 +739,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
/* rhizome_store_file() checks if it is already in the database, so we just
call it normally. */
if (rhizome_store_file(m, NULL))
return WHY("Could not store file");
if (!rhizome_exists(filehash))
return WHY("File should already be stored by now");
} else {
filehash[0] = '\0';
}
@ -999,181 +997,6 @@ insert_row_fail:
return rowid;
}
/* The following function just stores the file (or silently returns if it already exists).
The relationships of manifests to this file are the responsibility of the caller. */
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key)
{
const char *file=m->dataFileName;
const char *hash=m->fileHexHash;
int priority=m->fileHighestPriority;
if (m->payloadEncryption)
return WHY("Writing encrypted payloads not implemented");
if (!m->fileHashedP)
return WHY("Cannot store bundle file until it has been hashed");
int fd = -1;
/* See if the file is already stored, and if so, don't bother storing it again.
Do this check BEFORE trying to open the associated file, because if the caller
has received a manifest and checked that it exists in the database, it may
(sensibly) elect not supply the file. Rhizome Direct does this. */
long long count = 0;
if (sqlite_exec_int64(&count, "SELECT COUNT(*) FROM FILES WHERE id='%s' AND datavalid<>0;", hash) < 1) {
WHY("Failed to count stored files");
goto error;
}
if (count >= 1) {
/* File is already stored, so just update the highestPriority field if required. */
long long storedPriority = -1;
if (sqlite_exec_int64(&storedPriority, "SELECT highestPriority FROM FILES WHERE id='%s' AND datavalid!=0", hash) == -1) {
WHY("Failed to select highest priority");
goto error;
}
if (storedPriority<priority) {
if (sqlite_exec_void("UPDATE FILES SET highestPriority=%d WHERE id='%s';", priority, hash) == -1) {
WHY("SQLite failed to update highestPriority field for stored file.");
goto error;
}
}
return 0;
}
fd = open(file, O_RDONLY);
if (fd == -1) {
WHYF_perror("open(%s)", alloca_str_toprint(file));
WHY("Could not open associated file");
goto error;
}
struct stat stat;
if (fstat(fd, &stat)) {
WHYF_perror("fstat(%d)", fd);
WHY("Could not stat() associated file");
goto error;
}
if (stat.st_size < m->fileLength) {
WHYF("File has shrunk by %lld bytes from %lld to %lld, not stored",
(long long)(m->fileLength - stat.st_size), (long long) m->fileLength, (long long) stat.st_size
);
goto error;
} else if (stat.st_size > m->fileLength) {
// If the file has grown, store the original , in the hope that it will match the hash.
WARNF("File has grown by +%lld bytes to %lld, only storing %lld",
(long long)(stat.st_size - m->fileLength), (long long) stat.st_size, (long long) m->fileLength
);
}
unsigned char *addr = mmap(NULL, m->fileLength, PROT_READ, MAP_SHARED, fd, 0);
if (addr==MAP_FAILED) {
WHYF_perror("mmap(NULL, %lld, PROT_READ, MAP_SHARED, %d, 0)", (long long) m->fileLength, fd);
WHY("mmap() of associated file failed.");
goto error;
}
int64_t rowid=rhizome_database_create_blob_for(hash,m->fileLength,priority);
if (rowid<1) {
WHYF("query failed, %s", sqlite3_errmsg(rhizome_db));
WHYF("Failed to get row ID of newly inserted row for fileid=%s", hash);
goto error;
}
// We write the blob inside a transaction so that we can't get SQLITE_BUSY from
// sqlite3_blob_close(), which cannot be retried. Using an explicit transaction, defers BUSY
// detection to the COMMIT, which can be retried.
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;") == -1)
goto error;
sqlite3_blob *blob;
int ret;
do ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", rowid, 1 /* read/write */, &blob);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_open"));
if (ret != SQLITE_OK) {
WHYF("sqlite3_blob_open() failed, %s", sqlite3_errmsg(rhizome_db));
goto rollback_blob;
}
sqlite_retry_done(&retry, "sqlite3_blob_open");
/* Calculate hash of file as we go, so that we can report if
the contents have changed during import. This is also why we
use the m->fileLength instead of size returned by stat, in case
the file has been appended, e.g., if a journal is being appended to
by a separate process. This has already been shown to happen with
Serval Maps, and it is also quite possible with MeshMS and other
services. */
char hash_out[crypto_hash_sha512_BYTES*2+1];
{
unsigned char nonce[crypto_stream_xsalsa20_NONCEBYTES];
unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
bzero(nonce, sizeof nonce);
SHA512_CTX context;
SHA512_Init(&context);
long long i;
for (i = 0; i < m->fileLength; i += RHIZOME_CRYPT_PAGE_SIZE) {
int n = RHIZOME_CRYPT_PAGE_SIZE;
if (i + n > m->fileLength)
n = m->fileLength - i;
const unsigned char *writeable = &addr[i];
SHA512_Update(&context, writeable, n);
if (key) {
/* calculate block nonce */
int j;
for (j=0;j<8;j++)
nonce[i]=(i>>(j*8))&0xff;
crypto_stream_xsalsa20_xor(buffer, writeable, n, nonce, key);
writeable = buffer;
}
do ret = sqlite3_blob_write(blob, writeable, n, i);
while (sqlite_code_busy(ret) && sqlite_retry(&retry, "sqlite3_blob_write"));
if (ret != SQLITE_OK) {
WHYF("sqlite3_blob_write() failed, %s", sqlite3_errmsg(rhizome_db));
goto rollback_blob;
}
sqlite_retry_done(&retry, "sqlite3_blob_write");
}
SHA512_End(&context, (char *)hash_out);
str_toupper_inplace(hash_out);
}
if (strcasecmp(hash_out, hash) != 0) {
WHYF("File hash %s does not match computed hash %s -- has file been modified while being stored?",
hash_out, hash
);
goto rollback_blob;
}
// sqlite3_blob_close() always closes the blob, regardless of return value, so it cannot be
// retried on returning SQLITE_BUSY.
ret = sqlite3_blob_close(blob);
blob = NULL;
if (!sqlite_code_ok(ret)) {
WHYF("sqlite3_blob_close() failed, %s", sqlite3_errmsg(rhizome_db));
goto rollback_blob;
}
if (sqlite_exec_void_retry(&retry, "COMMIT;") == -1)
goto rollback;
/* Mark file as up-to-date */
if (sqlite_exec_void_retry(&retry, "UPDATE FILES SET datavalid=1 WHERE id='%s';", hash) != 0) {
WHY("Failed to set datavalid");
goto error;
}
close(fd);
return 0;
rollback_blob:
WHYF("Failed to write blob in newly inserted row for fileid=%s", hash);
if (blob)
sqlite3_blob_close(blob);
rollback:
sqlite_exec_void_retry(&retry, "ROLLBACK;");
error:
if (fd != -1)
close(fd);
return -1;
}
void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
{
(void) tohex(out, in, byteCount);

115
rhizome_direct_http.c
View File

@ -55,10 +55,17 @@ int rhizome_direct_form_received(rhizome_http_request *r)
strbuf payload_path = strbuf_alloca(50);
strbuf_sprintf(manifest_path, "rhizomedirect.%d.manifest", r->alarm.poll.fd);
strbuf_sprintf(payload_path, "rhizomedirect.%d.data", r->alarm.poll.fd);
int ret = rhizome_bundle_import_files(strbuf_str(manifest_path), strbuf_str(payload_path), 1); // ttl = 1
DEBUGF("Import returned %d",ret);
int ret=0;
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
ret=WHY("Out of manifests.");
else{
ret=rhizome_bundle_import_files(m, strbuf_str(manifest_path), strbuf_str(payload_path));
rhizome_manifest_free(m);
}
rhizome_direct_clear_temporary_files(r);
/* report back to caller.
200 = ok, which is probably appropriate for when we already had the bundle.
@ -221,87 +228,49 @@ int rhizome_direct_form_received(rhizome_http_request *r)
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.manifesttemplate can't be read as a manifest.");
}
/* Fill in a few missing manifest fields, to make it easier to use when adding new files:
- the default service is FILE
- use the current time for "date"
- if service is file, then use the payload file's basename for "name"
*/
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (service == NULL) {
rhizome_manifest_set(m, "service", (service = RHIZOME_SERVICE_FILE));
if (config.debug.rhizome) DEBUGF("missing 'service', set default service=%s", service);
} else {
if (config.debug.rhizome) DEBUGF("manifest contains service=%s", service);
if (rhizome_stat_file(m, filepath)){
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"Could not store file");
}
if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) {
rhizome_manifest_set_ll(m, "date", (long long) gettime_ms());
if (config.debug.rhizome) DEBUGF("missing 'date', set default date=%s", rhizome_manifest_get(m, "date", NULL, 0));
}
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
if (name == NULL) {
name=r->data_file_name;
rhizome_manifest_set(m, "name", r->data_file_name);
if (config.debug.rhizome) DEBUGF("missing 'name', set name=\"%s\" from HTTP post field filename specification", name);
} else {
if (config.debug.rhizome) DEBUGF("manifest contains name=\"%s\"", name);
}
const char *senderhex = rhizome_manifest_get(m, "sender", NULL, 0);
if (senderhex)
fromhexstr(m->author, senderhex, SID_SIZE);
else if (!is_sid_any(config.rhizome.api.addfile.default_author.binary))
memcpy(m->author, config.rhizome.api.addfile.default_author.binary, sizeof m->author); // TODO replace with sid_t struct assignment
/* Bind an ID to the manifest, and also bind the file. Then finalise the
manifest. But if the manifest already contains an ID, don't override it. */
if (rhizome_manifest_get(m, "id", NULL, 0) == NULL) {
if (rhizome_manifest_bind_id(m)) {
rhizome_manifest_free(m);
m = NULL;
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"Could not bind manifest to an ID");
}
} else if (!rhizome_is_bk_none(&config.rhizome.api.addfile.bundle_secret_key)) {
/* Allow user to specify a bundle secret key so that the same bundle can
be updated, rather than creating a new bundle each time. */
memcpy(m->cryptoSignSecret, config.rhizome.api.addfile.bundle_secret_key.binary, RHIZOME_BUNDLE_KEY_BYTES);
if (rhizome_verify_bundle_privatekey(m,m->cryptoSignSecret,m->cryptoSignPublic) == -1) {
rhizome_manifest_free(m);
m = NULL;
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.bundlesecretkey did not verify. Using the right key for the right bundle?");
}
} else {
/* Bundle ID specified, but without a BSK or sender SID specified.
Therefore we cannot work out the bundle key, and cannot update the
bundle. */
sid_t *author=NULL;
if (!is_sid_any(config.rhizome.api.addfile.default_author.binary))
author = &config.rhizome.api.addfile.default_author;
rhizome_bk_t bsk;
memcpy(bsk.binary, config.rhizome.api.addfile.bundle_secret_key.binary, RHIZOME_BUNDLE_KEY_BYTES);
if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)){
rhizome_manifest_free(m);
m = NULL;
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"rhizome.api.addfile.bundlesecretkey not set, and manifest template contains no sender, but template contains a hard-wired bundle ID. You must specify at least one, or not supply id= in the manifest template.");
return rhizome_server_simple_http_response(r,500,"Could not fill manifest default values");
}
int encryptP = 0; // TODO Determine here whether payload is to be encrypted.
if (rhizome_manifest_bind_file(m, filepath, encryptP)) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"Could not bind manifest to file");
m->payloadEncryption=0;
rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0);
// import file contents
// TODO, stream file into database
if (m->fileLength){
if (rhizome_add_file(m, filepath)){
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,"Could not store file");
}
}
if (rhizome_manifest_finalise(m)) {
rhizome_manifest *mout = NULL;
if (rhizome_manifest_finalise(m, &mout)) {
if (mout && mout!=m)
rhizome_manifest_free(mout);
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,
"Could not finalise manifest");
}
if (rhizome_add_manifest(m,255 /* TTL */)) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
return rhizome_server_simple_http_response(r,500,
"Add manifest operation failed");
}
DEBUGF("Import sans-manifest appeared to succeed");
@ -309,6 +278,8 @@ int rhizome_direct_form_received(rhizome_http_request *r)
rhizome_server_simple_http_response(r, 200, (char *)m->manifestdata);
/* clean up after ourselves */
if (mout && mout!=m)
rhizome_manifest_free(mout);
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);

38
rhizome_store.c
View File

@ -263,9 +263,7 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
return 0;
}
// import a file for a new bundle with an unknown file hash
// update the manifest with the details of the file
int rhizome_add_file(rhizome_manifest *m, const char *filepath)
int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
{
m->fileLength = 0;
if (filepath[0]) {
@ -281,9 +279,14 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
m->fileHexHash[0] = '\0';
rhizome_manifest_del(m, "filehash");
m->fileHashedP = 0;
return 0;
}
return 0;
}
// import a file for a new bundle with an unknown file hash
// update the manifest with the details of the file
int rhizome_add_file(rhizome_manifest *m, const char *filepath)
{
// Stream the file directly into the database, encrypting & hashing as we go.
struct rhizome_write write;
bzero(&write, sizeof(write));
@ -304,3 +307,28 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
rhizome_manifest_set(m, "filehash", m->fileHexHash);
return 0;
}
/*
int rhizome_open_append(struct rhizome_write *write, int64_t size, const char *expectedFileHash, const char *existingFileHash){
}
struct rhizome_read{
};
int rhizome_open_read(struct rhizome_read *read, ){
}
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length){
}
int rhizome_seek(struct rhizome_read *read, int64_t offset){
}
*/

4
tests/rhizomeops
View File

@ -350,7 +350,6 @@ test_AddDuplicate() {
assert [ -s file1.manifestA ]
assert_stdout_add_file file1
extract_stdout_secret file1_dup_secret
assert [ $file1_secret = $file1_dup_secret ]
executeOk_servald rhizome list ''
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
strip_signatures file1.manifest file1.manifestA
@ -360,11 +359,12 @@ test_AddDuplicate() {
assert [ -s file2.manifestA ]
assert_stdout_add_file file2
extract_stdout_secret file2_dup_secret
assert [ $file2_secret = $file2_dup_secret ]
executeOk_servald rhizome list ''
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
strip_signatures file2.manifest file2.manifestA
assert diff file2.manifest file2.manifestA
assert [ $file1_secret = $file1_dup_secret ]
assert [ $file2_secret = $file2_dup_secret ]
}
doc_AddMismatched="Add mismatched manifest/payload fails"