mirror of
https://github.com/servalproject/serval-dna.git
synced 2025-04-07 11:08:36 +00:00
Make "dna rhizome add" work for more test cases
- adding a bundle using an existing manifest with an incorrect payload should fail with an error - adding a bundle using an existing manifest to update the payload to a new version should succeed - improve format of "dna rhizome list" output to one bundle per line
This commit is contained in:
Andrew Bettison
parent
45369b0891
commit
de88d3db21
@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include "serval.h"
|
||||
#include "rhizome.h"
|
||||
@ -627,27 +628,36 @@ int app_rhizome_add_file(int argc, char **argv, struct command_line_option *o)
|
||||
/* Create a new manifest that will represent the file. If a manifest file was supplied, then read
|
||||
* it, otherwise create a blank manifest. */
|
||||
rhizome_manifest *m = NULL;
|
||||
if (manifestpath[0]) {
|
||||
int manifest_file_supplied = 0;
|
||||
if (manifestpath[0] && access(manifestpath, R_OK) == 0) {
|
||||
m = rhizome_read_manifest_file(manifestpath, 0, 0); // no verify
|
||||
if (!m)
|
||||
return WHY("Manifest file could not be loaded -- not added to rhizome");
|
||||
manifest_file_supplied = 1;
|
||||
} else {
|
||||
m = rhizome_new_manifest();
|
||||
if (!m)
|
||||
return WHY("Manifest struct could not be allocated -- not added to rhizome");
|
||||
}
|
||||
/* Use the file's basename to fill in a missing "name". */
|
||||
/* Fill in a few missing manifest fields, to make it easier to use when adding new files:
|
||||
- the payload file's basename for "name"
|
||||
- current time for "date"
|
||||
*/
|
||||
if (rhizome_manifest_get(m, "name", NULL, 0) == NULL) {
|
||||
const char *name = strrchr(filepath, '/');
|
||||
name = name ? name + 1 : filepath;
|
||||
rhizome_manifest_set(m, "name", name);
|
||||
}
|
||||
/* Use current time to fill in a missing "date". */
|
||||
if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) {
|
||||
rhizome_manifest_set_ll(m, "date", overlay_gettime_ms());
|
||||
}
|
||||
/* Add the manifest and its associated file to the Rhizome database, generating an "id" in the
|
||||
* process */
|
||||
int ret = rhizome_add_manifest(m, filepath,
|
||||
rhizome_manifest *mout = NULL;
|
||||
int ret = rhizome_add_manifest(m, &mout, filepath,
|
||||
NULL, // no groups - XXX should allow them
|
||||
255, // ttl - XXX should read from somewhere
|
||||
manifestpath[0] != 0, // int verifyP
|
||||
manifest_file_supplied, // int verifyP
|
||||
1, // int checkFileP
|
||||
1 // int signP
|
||||
);
|
||||
@ -656,11 +666,13 @@ int app_rhizome_add_file(int argc, char **argv, struct command_line_option *o)
|
||||
} else {
|
||||
/* If successfully added, overwrite the manifest file so that the Java component that is
|
||||
* invoking this command can read it to obtain feedback on the result. */
|
||||
if (manifestpath[0] && rhizome_write_manifest_file(m, manifestpath) == -1) {
|
||||
if (manifestpath[0] && rhizome_write_manifest_file(mout, manifestpath) == -1) {
|
||||
ret = WHY("Could not overwrite manifest file.");
|
||||
}
|
||||
}
|
||||
rhizome_manifest_free(m);
|
||||
if (mout != m)
|
||||
rhizome_manifest_free(mout);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
2
dna.c
2
dna.c
@ -581,7 +581,7 @@ int main(int argc,char **argv)
|
||||
manifest.
|
||||
A different calling would be required to import an existing pre-signed
|
||||
manifest */
|
||||
return rhizome_bundle_import(NULL,optarg,
|
||||
return rhizome_bundle_import(NULL, NULL, optarg,
|
||||
NULL /* no groups - XXX should allow them */,
|
||||
255 /* ttl - XXX should read from somewhere,
|
||||
e.g., bar if being imported */,
|
||||
|
||||
132
rhizome.c
132
rhizome.c
@ -28,12 +28,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
file and object buffers and lifetimes.
|
||||
*/
|
||||
|
||||
int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
|
||||
int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out, char *bundle,
|
||||
char *groups[], int ttl,
|
||||
int verifyP, int checkFileP, int signP)
|
||||
{
|
||||
if (m_out) *m_out = NULL;
|
||||
|
||||
char filename[1024];
|
||||
char manifestname[1024];
|
||||
|
||||
if (snprintf(filename, sizeof(filename), "%s/import/file.%s", rhizome_datastore_path, bundle) >= sizeof(filename)
|
||||
|| snprintf(manifestname, sizeof(manifestname), "%s/import/manifest.%s", rhizome_datastore_path, bundle) >= sizeof(manifestname)) {
|
||||
return WHY("Manifest bundle name too long");
|
||||
@ -51,7 +53,8 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in
|
||||
}
|
||||
|
||||
/* Add the manifest and its associated file to the Rhizome database. */
|
||||
int ret = rhizome_add_manifest(m, filename, groups, ttl, verifyP, checkFileP, signP);
|
||||
rhizome_manifest *dupm;
|
||||
int ret = rhizome_add_manifest(m, &dupm, filename, groups, ttl, verifyP, checkFileP, signP);
|
||||
unlink(filename);
|
||||
if (ret == -1) {
|
||||
unlink(manifestname);
|
||||
@ -62,9 +65,11 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in
|
||||
ret = WHY("Could not write manifest file.");
|
||||
}
|
||||
|
||||
/* If the manifest structure was allocated in this function, then this function is responsible for
|
||||
freeing it */
|
||||
if (!m_in)
|
||||
/* If the manifest structure was allocated in this function, and it is not being returned to the
|
||||
caller, then this function is responsible for freeing it */
|
||||
if (m_out)
|
||||
*m_out = m;
|
||||
else if (!m_in)
|
||||
rhizome_manifest_free(m);
|
||||
|
||||
return ret;
|
||||
@ -121,7 +126,8 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in
|
||||
|
||||
*/
|
||||
|
||||
int rhizome_add_manifest(rhizome_manifest *m,
|
||||
int rhizome_add_manifest(rhizome_manifest *m_in,
|
||||
rhizome_manifest **m_out,
|
||||
const char *filename,
|
||||
char *groups[],
|
||||
int ttl,
|
||||
@ -130,22 +136,20 @@ int rhizome_add_manifest(rhizome_manifest *m,
|
||||
int signP
|
||||
)
|
||||
{
|
||||
char *id = NULL;
|
||||
char hexhash[SHA512_DIGEST_STRING_LENGTH];
|
||||
int verifyErrors = 0;
|
||||
if (m_out) *m_out = NULL;
|
||||
|
||||
/* Ensure manifest meets basic sanity checks. */
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
const char *name = rhizome_manifest_get(m_in, "name", NULL, 0);
|
||||
if (name == NULL || !name[0])
|
||||
return WHY("Manifest missing 'name' field");
|
||||
if (rhizome_manifest_get_ll(m, "date") == -1)
|
||||
if (rhizome_manifest_get_ll(m_in, "date") == -1)
|
||||
return WHY("Manifest missing 'date' field");
|
||||
|
||||
/* Keep payload file name handy for later */
|
||||
m->dataFileName = strdup(filename);
|
||||
m_in->dataFileName = strdup(filename);
|
||||
|
||||
/* Store time to live, clamped to within legal range */
|
||||
m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
|
||||
m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
|
||||
|
||||
/* Check payload file is accessible and discover its length, then check that it matches
|
||||
the file size stored in the manifest */
|
||||
@ -153,115 +157,121 @@ int rhizome_add_manifest(rhizome_manifest *m,
|
||||
struct stat stat;
|
||||
if (lstat(filename,&stat))
|
||||
return WHY("Could not stat() payload file");
|
||||
m->fileLength = stat.st_size;
|
||||
long long mfilesize = rhizome_manifest_get_ll(m, "filesize");
|
||||
if (mfilesize != -1 && mfilesize != m->fileLength) {
|
||||
WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m->fileLength);
|
||||
++verifyErrors;
|
||||
m_in->fileLength = stat.st_size;
|
||||
long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize");
|
||||
if (mfilesize != -1 && mfilesize != m_in->fileLength) {
|
||||
WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m_in->fileLength);
|
||||
if (verifyP)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Bail out now if errors occurred loading the manifest file, eg signature failed to validate */
|
||||
if (verifyP && m_in->errors)
|
||||
return WHYF("Manifest.errors (%d) is non-zero", m_in->errors);
|
||||
|
||||
/* Compute hash of payload unless we know verification has already failed */
|
||||
if (checkFileP ? !(verifyP && (m->errors || verifyErrors)) : signP) {
|
||||
if (checkFileP || signP) {
|
||||
char hexhash[SHA512_DIGEST_STRING_LENGTH];
|
||||
if (rhizome_hash_file(filename, hexhash))
|
||||
return WHY("Could not hash file.");
|
||||
memcpy(&m->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH);
|
||||
m->fileHashedP = 1;
|
||||
memcpy(&m_in->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH);
|
||||
m_in->fileHashedP = 1;
|
||||
}
|
||||
|
||||
/* Check that paylod hash matches manifest */
|
||||
/* Check that payload hash matches manifest */
|
||||
if (checkFileP) {
|
||||
const char *mhexhash = rhizome_manifest_get(m, "filehash", NULL, 0);
|
||||
if (mhexhash && strcmp(hexhash, mhexhash)) {
|
||||
WHYF("Manifest.filehash (%s) != actual file hash (%s)", mhexhash, hexhash);
|
||||
++verifyErrors;
|
||||
const char *mhexhash = rhizome_manifest_get(m_in, "filehash", NULL, 0);
|
||||
if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) {
|
||||
WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash);
|
||||
if (verifyP)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* If any signature errors were encountered on loading, or manifest is inconsistent with payload,
|
||||
then bail out now. */
|
||||
if (verifyP) {
|
||||
if (m->errors)
|
||||
WHYF("Manifest.errors (%d) is non-zero", m->errors);
|
||||
if (verifyErrors || m->errors)
|
||||
return WHY("Errors encountered verifying bundle manifest");
|
||||
}
|
||||
|
||||
/* Fill in the manifest so that duplicate detection can be performed, and to avoid redundant work
|
||||
by rhizome_manifest_finalise() below. */
|
||||
if (checkFileP) {
|
||||
rhizome_manifest_set(m, "filehash", hexhash);
|
||||
rhizome_manifest_set_ll(m, "first_byte", 0);
|
||||
rhizome_manifest_set_ll(m, "last_byte", m->fileLength);
|
||||
rhizome_manifest_set(m_in, "filehash", m_in->fileHexHash);
|
||||
rhizome_manifest_set_ll(m_in, "first_byte", 0);
|
||||
rhizome_manifest_set_ll(m_in, "last_byte", m_in->fileLength);
|
||||
}
|
||||
|
||||
/* Check if a manifest is already stored for the same payload with the same details.
|
||||
This catches the case of "dna rhizome add file <filename>" on the same file more than once.
|
||||
(Debounce!) */
|
||||
rhizome_manifest *dupm = NULL;
|
||||
if (rhizome_find_duplicate(m, &dupm) == -1)
|
||||
if (rhizome_find_duplicate(m_in, &dupm) == -1)
|
||||
return WHY("Errors encountered searching for duplicate manifest");
|
||||
if (dupm) {
|
||||
if (debug & DEBUG_RHIZOME) fprintf(stderr, "Not adding manifest for payload name=\"%s\" hexhash=%s - duplicate found in rhizome store\n", name, hexhash);
|
||||
if (debug & DEBUG_RHIZOME)
|
||||
fprintf(stderr, "Not adding manifest for payload name=\"%s\" hexhash=%s - duplicate found in rhizome store\n", name, m_in->fileHexHash);
|
||||
#if 0
|
||||
/* TODO Upgrade the version of the duplicate? */
|
||||
long long version = rhizome_manifest_get_ll(m, "version");
|
||||
long long version = rhizome_manifest_get_ll(m_in, "version");
|
||||
long long dupversion = rhizome_manifest_get_ll(dupm, "version");
|
||||
if (version > dupversion) {
|
||||
rhizome_manifest_set_ll(dupm, "version", version);
|
||||
...
|
||||
}
|
||||
#endif
|
||||
rhizome_manifest_free(dupm);
|
||||
if (m_out) {
|
||||
/* Finish completing the manifest */
|
||||
if (rhizome_manifest_finalise(dupm, 0))
|
||||
return WHY("Failed to finalise manifest.\n");
|
||||
*m_out = dupm;
|
||||
}
|
||||
else
|
||||
rhizome_manifest_free(dupm);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Supply manifest version number if missing, so we can do the version check below */
|
||||
if (rhizome_manifest_get(m, "version", NULL, 0) == NULL) {
|
||||
rhizome_manifest_set_ll(m, "version", overlay_gettime_ms());
|
||||
if (rhizome_manifest_get(m_in, "version", NULL, 0) == NULL) {
|
||||
rhizome_manifest_set_ll(m_in, "version", overlay_gettime_ms());
|
||||
}
|
||||
|
||||
/* If the manifest already has an ID, look to see if we possess its private key */
|
||||
if ((id = rhizome_manifest_get(m, "id", NULL, 0))) {
|
||||
rhizome_hex_to_bytes(id, m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2);
|
||||
if (!rhizome_find_keypair_bytes(m->cryptoSignPublic, m->cryptoSignSecret))
|
||||
m->haveSecret=1;
|
||||
}
|
||||
|
||||
/* Discard the new manifest it is older than the most recent known version with the same ID */
|
||||
if (id) {
|
||||
/* If the manifest already has an ID */
|
||||
char *id = NULL;
|
||||
if ((id = rhizome_manifest_get(m_in, "id", NULL, 0))) {
|
||||
/* Discard the new manifest it is older than the most recent known version with the same ID */
|
||||
long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';", id);
|
||||
if (storedversion > rhizome_manifest_get_ll(m, "version")) {
|
||||
if (storedversion > rhizome_manifest_get_ll(m_in, "version")) {
|
||||
return WHY("Newer version exists");
|
||||
}
|
||||
/* Check if we know its private key */
|
||||
rhizome_hex_to_bytes(id, m_in->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2);
|
||||
if (!rhizome_find_keypair_bytes(m_in->cryptoSignPublic, m_in->cryptoSignSecret))
|
||||
m_in->haveSecret=1;
|
||||
} else {
|
||||
/* The manifest had no ID (256 bit random string being a public key in the NaCl CryptoSign
|
||||
crypto system), so create one. */
|
||||
printf("manifest does not have an id\n");
|
||||
rhizome_manifest_createid(m);
|
||||
rhizome_manifest_createid(m_in);
|
||||
/* The ID is implicit in transit, but we need to store it in the file, so that reimporting
|
||||
manifests on receiver nodes works easily. We might implement something that strips the id
|
||||
variable out of the manifest when sending it, or some other scheme to avoid sending all the
|
||||
extra bytes. */
|
||||
id = rhizome_bytes_to_hex(m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
|
||||
rhizome_manifest_set(m, "id", id);
|
||||
id = rhizome_bytes_to_hex(m_in->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
|
||||
rhizome_manifest_set(m_in, "id", id);
|
||||
}
|
||||
|
||||
/* Add group memberships */
|
||||
if (groups) {
|
||||
int i;
|
||||
for(i = 0; groups[i]; i++)
|
||||
rhizome_manifest_add_group(m, groups[i]);
|
||||
rhizome_manifest_add_group(m_in, groups[i]);
|
||||
}
|
||||
|
||||
/* Finish completing the manifest */
|
||||
if (rhizome_manifest_finalise(m,signP))
|
||||
if (rhizome_manifest_finalise(m_in, signP))
|
||||
return WHY("Failed to finalise manifest.\n");
|
||||
|
||||
/* Okay, it is written, and can be put directly into the rhizome database now */
|
||||
if (rhizome_store_bundle(m, filename) == -1)
|
||||
if (rhizome_store_bundle(m_in, filename) == -1)
|
||||
return WHY("rhizome_store_bundle() failed.");
|
||||
|
||||
if (m_out) *m_out = m_in;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -182,9 +182,11 @@ int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
|
||||
int rhizome_store_file(const char *file,char *hash,int priortity);
|
||||
char *rhizome_safe_encode(unsigned char *in,int len);
|
||||
int rhizome_finish_sqlstatement(sqlite3_stmt *statement);
|
||||
int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl,
|
||||
int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out, char *bundle,
|
||||
char *groups[], int ttl,
|
||||
int verifyP, int checkFileP, int signP);
|
||||
int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl,
|
||||
int rhizome_add_manifest(rhizome_manifest *m_in, rhizome_manifest **m_out, const char *filename,
|
||||
char *groups[], int ttl,
|
||||
int verifyP, int checkFileP, int signP);
|
||||
int rhizome_manifest_finalise(rhizome_manifest *m,int signP);
|
||||
char *rhizome_bytes_to_hex(unsigned char *in,int byteCount);
|
||||
|
||||
@ -504,7 +504,11 @@ char *rhizome_safe_encode(unsigned char *in,int len)
|
||||
int rhizome_list_manifests(int limit, int offset)
|
||||
{
|
||||
char sqlcmd[1024];
|
||||
int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id");
|
||||
int n = snprintf(sqlcmd, sizeof(sqlcmd),
|
||||
"SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime"
|
||||
" FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id"
|
||||
" ORDER BY files.id ASC"
|
||||
);
|
||||
if (n >= sizeof(sqlcmd))
|
||||
return WHY("SQL command too long");
|
||||
if (limit) {
|
||||
@ -539,21 +543,23 @@ int rhizome_list_manifests(int limit, int offset)
|
||||
ret = WHY("Incorrect statement column");
|
||||
break;
|
||||
}
|
||||
size_t filesize = sqlite3_column_int(statement, 1);
|
||||
const char *manifestblob = (char *) sqlite3_column_blob(statement, 4);
|
||||
size_t manifestblobsize = sqlite3_column_bytes(statement, 4); // must call after sqlite3_column_blob()
|
||||
//printf("manifest blob = %s\n", manifestblob);
|
||||
rhizome_manifest *m = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0);
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
printf("file id = %s\nfile length = %u\nfile datavalid = %u\nfile name = \"%s\"\n\n",
|
||||
long long date = rhizome_manifest_get_ll(m, "date");
|
||||
printf("fileid=%s:manifestid=%s:version=%d:inserttime=%lld:length=%u:datavalid=%u:date=%lld:name=%s\n",
|
||||
sqlite3_column_text(statement, 0),
|
||||
filesize,
|
||||
sqlite3_column_text(statement, 3),
|
||||
sqlite3_column_int(statement, 5),
|
||||
(long long) sqlite3_column_int64(statement, 6),
|
||||
sqlite3_column_int(statement, 1),
|
||||
sqlite3_column_int(statement, 2),
|
||||
date,
|
||||
name
|
||||
);
|
||||
rhizome_manifest_free(m);
|
||||
}
|
||||
printf("Found %lu rows\n", (unsigned long) rows);
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
return ret;
|
||||
@ -758,6 +764,9 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
||||
{
|
||||
if (!m->fileHashedP)
|
||||
return WHY("Manifest payload is not hashed");
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
if (!name)
|
||||
return WHY("Manifest has no name");
|
||||
char sqlcmd[1024];
|
||||
int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT manifests.id, manifests.manifest FROM filemanifests, manifests WHERE filemanifests.fileid = ? AND filemanifests.manifestid = manifests.id");
|
||||
if (n >= sizeof(sqlcmd))
|
||||
@ -766,12 +775,10 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
||||
sqlite3_stmt *statement;
|
||||
const char *cmdtail;
|
||||
if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) {
|
||||
sqlite3_finalize(statement);
|
||||
ret = WHY(sqlite3_errmsg(rhizome_db));
|
||||
} else {
|
||||
if (debug & DEBUG_RHIZOME) fprintf(stderr, "fileHaxHash = \"%s\"\n", m->fileHexHash);
|
||||
if (debug & DEBUG_RHIZOME) fprintf(stderr, "fileHexHash = \"%s\"\n", m->fileHexHash);
|
||||
sqlite3_bind_text(statement, 1, m->fileHexHash, -1, SQLITE_STATIC);
|
||||
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
|
||||
size_t rows = 0;
|
||||
while (sqlite3_step(statement) == SQLITE_ROW) {
|
||||
++rows;
|
||||
@ -793,12 +800,20 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
||||
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
|
||||
rhizome_manifest *mq = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0);
|
||||
const char *nameq = rhizome_manifest_get(mq, "name", NULL, 0);
|
||||
/* No need to compare "filehash" here, but we do so as a precaution. */
|
||||
const char *filehashq = rhizome_manifest_get(mq, "filehash", NULL, 0);
|
||||
long long lengthq = rhizome_manifest_get_ll(mq, "filesize");
|
||||
if (debug & DEBUG_RHIZOME) fprintf(stderr, "Consider manifest.id=%s manifest.name=\"%s\"\n", manifestid, nameq);
|
||||
if ( !strcmp(nameq, name)
|
||||
&& !strncmp(rhizome_manifest_get(mq, "filehash", NULL, 0), m->fileHexHash, SHA512_DIGEST_STRING_LENGTH)) {
|
||||
/* No need to compare "filehash" or "filesize" here, but we do so as a precaution if present */
|
||||
if ( nameq && !strcmp(nameq, name)
|
||||
&& (!filehashq || strncmp(filehashq, m->fileHexHash, SHA512_DIGEST_STRING_LENGTH) == 0)
|
||||
&& (lengthq == -1 || lengthq == m->fileLength)
|
||||
) {
|
||||
memcpy(mq->fileHexHash, m->fileHexHash, SHA512_DIGEST_STRING_LENGTH);
|
||||
mq->fileHashedP = 1;
|
||||
mq->fileLength = m->fileLength;
|
||||
*found = mq;
|
||||
ret = 1;
|
||||
if (debug & DEBUG_RHIZOME) fprintf(stderr, "found\n");
|
||||
break;
|
||||
}
|
||||
rhizome_manifest_free(mq);
|
||||
|
||||
@ -393,7 +393,7 @@ int rhizome_queue_manifest_import(rhizome_manifest *m,
|
||||
rhizome_datastore_path,
|
||||
rhizome_manifest_get(m,"id",NULL,0));
|
||||
if (!rhizome_write_manifest_file(m,filename)) {
|
||||
rhizome_bundle_import(m,rhizome_manifest_get(m,"id",NULL,0),
|
||||
rhizome_bundle_import(m, NULL, rhizome_manifest_get(m,"id",NULL,0),
|
||||
NULL /* no additional groups */,
|
||||
m->ttl-1 /* TTL */,
|
||||
1 /* do verify */,
|
||||
@ -522,17 +522,15 @@ int rhizome_fetch_poll()
|
||||
q->manifest->finalised=1;
|
||||
q->manifest->manifest_bytes=q->manifest->manifest_all_bytes;
|
||||
if (!rhizome_write_manifest_file(q->manifest,filename)) {
|
||||
rhizome_bundle_import(q->manifest,
|
||||
rhizome_manifest_get(q->manifest,
|
||||
"id",NULL,0),
|
||||
NULL /* no additional groups */,
|
||||
q->manifest->ttl-1 /* TTL */,
|
||||
1 /* do verify */,
|
||||
1 /* do check hash of file */,
|
||||
0 /* do not sign it, just keep existing
|
||||
signatures */);
|
||||
rhizome_bundle_import(q->manifest, NULL,
|
||||
rhizome_manifest_get(q->manifest, "id", NULL, 0),
|
||||
NULL /* no additional groups */,
|
||||
q->manifest->ttl - 1 /* TTL */,
|
||||
1 /* do verify */,
|
||||
1 /* do check hash of file */,
|
||||
0 /* do not sign it, just keep existing signatures */);
|
||||
q->manifest=NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
@ -26,9 +26,29 @@ setup_dna_rhizome() {
|
||||
executeOk $dna config set debug rhizome
|
||||
}
|
||||
|
||||
assert_rhizome_list_empty() {
|
||||
assert_rhizome_list() {
|
||||
executeOk $dna rhizome list
|
||||
assertStdoutIs -e "Found 0 rows\n"
|
||||
assertStdoutLineCount '==' $#
|
||||
assertStdoutGrep --matches=$# '^fileid=.*:name='
|
||||
local filename
|
||||
for filename; do
|
||||
local filehash='[^:]\+'
|
||||
# If there is a manifest file that looks like it matches this payload
|
||||
# file, then use its file hash to check the rhizome list output.
|
||||
if [ -r "$filename.manifest" ]; then
|
||||
local name=$(sed -n -e '/^name=/s///p' "$filename.manifest")
|
||||
if [ "$name" == "$filename" ]; then
|
||||
filehash=$(sed -n -e '/^filehash=/s///p' "$filename.manifest")
|
||||
fi
|
||||
fi
|
||||
assertStdoutGrep --matches=1 "^\(.*:\)\?fileid=$filehash:.*:name=$filename\$"
|
||||
done
|
||||
}
|
||||
|
||||
strip_signatures() {
|
||||
for file; do
|
||||
cat -v "$file" | sed -e '/^^@/,$d' >"tmp.$file" && mv -f "tmp.$file" "$file"
|
||||
done
|
||||
}
|
||||
|
||||
doc_InitialEmptyList="Initial rhizome list is empty"
|
||||
@ -36,15 +56,47 @@ setup_InitialEmptyList() {
|
||||
setup_dna_rhizome
|
||||
}
|
||||
test_InitialEmptyList() {
|
||||
assert_rhizome_list_empty
|
||||
assert_rhizome_list
|
||||
}
|
||||
|
||||
doc_AddNoManifest="Rhizome add with no manifest file"
|
||||
setup_AddNoManifest() {
|
||||
setup_dna_rhizome
|
||||
assert_rhizome_list
|
||||
echo "A test file" >file1
|
||||
echo "Another test file" >file2
|
||||
}
|
||||
test_AddNoManifest() {
|
||||
executeOk $dna rhizome add file file1
|
||||
}
|
||||
|
||||
doc_AddNonExistManifest="Rhizome add with non-existent manifest file"
|
||||
setup_AddNonExistManifest() {
|
||||
setup_dna_rhizome
|
||||
assert_rhizome_list
|
||||
echo "A test file" >file1
|
||||
echo "Another test file" >file2
|
||||
}
|
||||
test_AddNonExistManifest() {
|
||||
assert --error-on-fail [ ! -e file1.manifest ]
|
||||
executeOk $dna rhizome add file file1 file1.manifest
|
||||
assert [ -r file1.manifest ]
|
||||
tfw_cat -v file1.manifest
|
||||
assertGrep file1.manifest '^name=file1$'
|
||||
assertGrep file1.manifest '^date=[0-9]\+$'
|
||||
assertGrep file1.manifest '^version=[0-9]\+$'
|
||||
assertGrep file1.manifest '^id=[0-9a-fA-F]\+$'
|
||||
assertGrep file1.manifest "^filesize=$(cat file1 | wc --bytes)\$"
|
||||
assertGrep file1.manifest "^first_byte=0$"
|
||||
assertGrep file1.manifest "^last_byte=$(cat file1 | wc --bytes)\$"
|
||||
}
|
||||
|
||||
doc_AddManifest="Rhizome add with manifest"
|
||||
setup_AddManifest() {
|
||||
setup_dna_rhizome
|
||||
assert_rhizome_list_empty
|
||||
assert_rhizome_list
|
||||
echo "A test file" >file1
|
||||
echo 'name=wah' >file1.manifest
|
||||
echo -e 'name=wah\ndate=12345' >file1.manifest
|
||||
echo "Another test file" >file2
|
||||
}
|
||||
test_AddManifest() {
|
||||
@ -53,7 +105,7 @@ test_AddManifest() {
|
||||
tfw_cat --stderr
|
||||
tfw_cat -v file1.manifest
|
||||
assertGrep file1.manifest '^name=wah$'
|
||||
assertGrep file1.manifest '^date=[0-9]\+$'
|
||||
assertGrep file1.manifest '^date=12345$'
|
||||
assertGrep file1.manifest '^version=[0-9]\+$'
|
||||
assertGrep file1.manifest '^id=[0-9a-fA-F]\+$'
|
||||
assertGrep file1.manifest "^filesize=$(cat file1 | wc --bytes)\$"
|
||||
@ -64,55 +116,86 @@ test_AddManifest() {
|
||||
doc_AddThenList="Rhizome list contains one file after one add"
|
||||
setup_AddThenList() {
|
||||
setup_dna_rhizome
|
||||
assert_rhizome_list_empty
|
||||
assert_rhizome_list
|
||||
echo "A test file" >file1
|
||||
echo "Another test file" >file2
|
||||
}
|
||||
test_AddThenList() {
|
||||
# Add first file
|
||||
executeOk $dna rhizome add file file1
|
||||
executeOk $dna rhizome list
|
||||
assertStdoutGrep "^Found 1 rows$"
|
||||
assertStdoutLineCount '==' 6
|
||||
assertStdoutGrep --matches=1 '^file name = "file1"$'
|
||||
assertStdoutGrep --matches=0 '^file name = "file2"$'
|
||||
replayStdout >add1.stdout
|
||||
executeOk $dna rhizome add file file1 file1.manifest
|
||||
assert_rhizome_list file1
|
||||
# Add second file
|
||||
executeOk $dna rhizome add file file2
|
||||
executeOk $dna rhizome list
|
||||
assertStdoutGrep "^Found 2 rows$"
|
||||
assertStdoutLineCount '==' 11
|
||||
assertStdoutGrep --matches=1 '^file name = "file1"$'
|
||||
assertStdoutGrep --matches=1 '^file name = "file2"$'
|
||||
executeOk $dna rhizome add file file2 file2.manifest
|
||||
assert_rhizome_list file1 file2
|
||||
}
|
||||
|
||||
doc_AddDuplicate="Rhizome add of same file"
|
||||
setup_AddDuplicate() {
|
||||
setup_dna_rhizome
|
||||
assert_rhizome_list_empty
|
||||
assert_rhizome_list
|
||||
echo "A test file" >file1
|
||||
echo "Another test file" >file2
|
||||
echo "A test file, second version" >file1_2
|
||||
# Add first file
|
||||
executeOk $dna rhizome add file file1
|
||||
executeOk $dna rhizome add file file1 file1.manifest
|
||||
# Add second file
|
||||
executeOk $dna rhizome add file file2
|
||||
executeOk $dna rhizome add file file2 file2.manifest
|
||||
# Make sure they are both in the list.
|
||||
executeOk $dna rhizome list
|
||||
assertStdoutGrep "^Found 2 rows$"
|
||||
assertStdoutGrep --matches=1 '^file name = "file1"$'
|
||||
assertStdoutGrep --matches=1 '^file name = "file2"$'
|
||||
assert_rhizome_list file1 file2
|
||||
}
|
||||
test_AddDuplicate() {
|
||||
# Add first file again (nothing should change except its date).
|
||||
executeOk $dna config get debug
|
||||
tfw_cat --stdout
|
||||
executeOk $dna rhizome add file file1
|
||||
tfw_cat --stderr
|
||||
executeOk $dna rhizome list
|
||||
assertStdoutGrep "^Found 2 rows$"
|
||||
assertStdoutLineCount '==' 11
|
||||
assertStdoutGrep --matches=1 '^file name = "file1"$'
|
||||
assertStdoutGrep --matches=1 '^file name = "file2"$'
|
||||
# Add first file again - nothing should change in its manifests, and it
|
||||
# should appear that the add command succeeded (with perhaps some grumbling
|
||||
# on stderr).
|
||||
executeOk $dna rhizome add file file1 file1.manifestA
|
||||
assert [ -s file1.manifestA ]
|
||||
assert_rhizome_list file1 file2
|
||||
strip_signatures file1.manifest file1.manifestA
|
||||
assert diff file1.manifest file1.manifestA
|
||||
# Repeat for second file.
|
||||
executeOk $dna rhizome add file file2 file2.manifestA
|
||||
assert [ -s file2.manifestA ]
|
||||
assert_rhizome_list file1 file2
|
||||
strip_signatures file2.manifest file2.manifestA
|
||||
assert diff file2.manifest file2.manifestA
|
||||
}
|
||||
|
||||
doc_AddMismatched="Rhizome add fails on mismatched manifest and payload"
|
||||
setup_AddMismatched() {
|
||||
setup_AddDuplicate
|
||||
}
|
||||
test_AddMismatched() {
|
||||
# Try to add another file using an existing manifest, should fail and leave
|
||||
# the manifest file unchanged.
|
||||
cp file1.manifest file1_2.manifest
|
||||
execute $dna rhizome add file file1_2 file1_2.manifest
|
||||
assertExitStatus '!=' 0
|
||||
assert cmp file1.manifest file1_2.manifest
|
||||
# And rhizome store should be unchanged.
|
||||
assert_rhizome_list file1 file2
|
||||
}
|
||||
|
||||
doc_AddUpdate="Rhizome add new version of existing file"
|
||||
setup_AddUpdate() {
|
||||
setup_AddDuplicate
|
||||
}
|
||||
test_AddUpdate() {
|
||||
# Change payload associated with first manifest.
|
||||
cp file1.manifest file1_2.manifest
|
||||
strip_signatures file1_2.manifest
|
||||
sed -i -e '/^date=/d' -e '/^filehash=/d' -e '/^filesize=/d' file1_2.manifest
|
||||
tfw_cat file1_2.manifest
|
||||
assertGrep --matches=0 --error-on-fail file1_2.manifest '^filehash='
|
||||
executeOk $dna rhizome add file file1_2 file1_2.manifest
|
||||
# The new manifest must now have a different filehash from the original.
|
||||
filehash1=$(sed -n -e '/^filehash=/s///p' file1.manifest)
|
||||
filehash1_2=$(sed -n -e '/^filehash=/s///p' file1_2.manifest)
|
||||
assert [ -n "$filehash1" ]
|
||||
assert [ -n "$filehash1_2" ]
|
||||
assert [ "$filehash1" != "$filehash1_2" ]
|
||||
# Rhizome store contents reflect new payload.
|
||||
mv -f file1_2.manifest file1.manifest
|
||||
assert_rhizome_list file1 file2
|
||||
}
|
||||
|
||||
runTests "$@"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user