From de88d3db2177eb93c55ee3ecc5de803e4638d2b9 Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Fri, 13 Apr 2012 17:47:20 +0930 Subject: [PATCH] Make "dna rhizome add" work for more test cases - adding a bundle using an existing manifest with an incorrect payload should fail with an error - adding a bundle using an existing manifest to update the payload to a new version should succeed - improve format of "dna rhizome list" output to one bundle per line --- commandline.c | 24 +++++-- dna.c | 2 +- rhizome.c | 132 +++++++++++++++++++------------------ rhizome.h | 6 +- rhizome_database.c | 39 +++++++---- rhizome_fetch.c | 20 +++--- tests/dna_rhizome | 157 ++++++++++++++++++++++++++++++++++----------- 7 files changed, 250 insertions(+), 130 deletions(-) diff --git a/commandline.c b/commandline.c index a493db20..04609463 100644 --- a/commandline.c +++ b/commandline.c @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include #include +#include #include #include "serval.h" #include "rhizome.h" @@ -627,27 +628,36 @@ int app_rhizome_add_file(int argc, char **argv, struct command_line_option *o) /* Create a new manifest that will represent the file. If a manifest file was supplied, then read * it, otherwise create a blank manifest. */ rhizome_manifest *m = NULL; - if (manifestpath[0]) { + int manifest_file_supplied = 0; + if (manifestpath[0] && access(manifestpath, R_OK) == 0) { m = rhizome_read_manifest_file(manifestpath, 0, 0); // no verify + if (!m) + return WHY("Manifest file could not be loaded -- not added to rhizome"); + manifest_file_supplied = 1; } else { m = rhizome_new_manifest(); + if (!m) + return WHY("Manifest struct could not be allocated -- not added to rhizome"); } - /* Use the file's basename to fill in a missing "name". */ + /* Fill in a few missing manifest fields, to make it easier to use when adding new files: + - the payload file's basename for "name" + - current time for "date" + */ if (rhizome_manifest_get(m, "name", NULL, 0) == NULL) { const char *name = strrchr(filepath, '/'); name = name ? name + 1 : filepath; rhizome_manifest_set(m, "name", name); } - /* Use current time to fill in a missing "date". */ if (rhizome_manifest_get(m, "date", NULL, 0) == NULL) { rhizome_manifest_set_ll(m, "date", overlay_gettime_ms()); } /* Add the manifest and its associated file to the Rhizome database, generating an "id" in the * process */ - int ret = rhizome_add_manifest(m, filepath, + rhizome_manifest *mout = NULL; + int ret = rhizome_add_manifest(m, &mout, filepath, NULL, // no groups - XXX should allow them 255, // ttl - XXX should read from somewhere - manifestpath[0] != 0, // int verifyP + manifest_file_supplied, // int verifyP 1, // int checkFileP 1 // int signP ); @@ -656,11 +666,13 @@ int app_rhizome_add_file(int argc, char **argv, struct command_line_option *o) } else { /* If successfully added, overwrite the manifest file so that the Java component that is * invoking this command can read it to obtain feedback on the result. */ - if (manifestpath[0] && rhizome_write_manifest_file(m, manifestpath) == -1) { + if (manifestpath[0] && rhizome_write_manifest_file(mout, manifestpath) == -1) { ret = WHY("Could not overwrite manifest file."); } } rhizome_manifest_free(m); + if (mout != m) + rhizome_manifest_free(mout); return ret; } diff --git a/dna.c b/dna.c index d5466a06..521e5194 100644 --- a/dna.c +++ b/dna.c @@ -581,7 +581,7 @@ int main(int argc,char **argv) manifest. A different calling would be required to import an existing pre-signed manifest */ - return rhizome_bundle_import(NULL,optarg, + return rhizome_bundle_import(NULL, NULL, optarg, NULL /* no groups - XXX should allow them */, 255 /* ttl - XXX should read from somewhere, e.g., bar if being imported */, diff --git a/rhizome.c b/rhizome.c index 795e8326..88e35e24 100644 --- a/rhizome.c +++ b/rhizome.c @@ -28,12 +28,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. file and object buffers and lifetimes. */ -int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl, +int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out, char *bundle, + char *groups[], int ttl, int verifyP, int checkFileP, int signP) { + if (m_out) *m_out = NULL; + char filename[1024]; char manifestname[1024]; - if (snprintf(filename, sizeof(filename), "%s/import/file.%s", rhizome_datastore_path, bundle) >= sizeof(filename) || snprintf(manifestname, sizeof(manifestname), "%s/import/manifest.%s", rhizome_datastore_path, bundle) >= sizeof(manifestname)) { return WHY("Manifest bundle name too long"); @@ -51,7 +53,8 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in } /* Add the manifest and its associated file to the Rhizome database. */ - int ret = rhizome_add_manifest(m, filename, groups, ttl, verifyP, checkFileP, signP); + rhizome_manifest *dupm; + int ret = rhizome_add_manifest(m, &dupm, filename, groups, ttl, verifyP, checkFileP, signP); unlink(filename); if (ret == -1) { unlink(manifestname); @@ -62,9 +65,11 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in ret = WHY("Could not write manifest file."); } - /* If the manifest structure was allocated in this function, then this function is responsible for - freeing it */ - if (!m_in) + /* If the manifest structure was allocated in this function, and it is not being returned to the + caller, then this function is responsible for freeing it */ + if (m_out) + *m_out = m; + else if (!m_in) rhizome_manifest_free(m); return ret; @@ -121,7 +126,8 @@ int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], in */ -int rhizome_add_manifest(rhizome_manifest *m, +int rhizome_add_manifest(rhizome_manifest *m_in, + rhizome_manifest **m_out, const char *filename, char *groups[], int ttl, @@ -130,22 +136,20 @@ int rhizome_add_manifest(rhizome_manifest *m, int signP ) { - char *id = NULL; - char hexhash[SHA512_DIGEST_STRING_LENGTH]; - int verifyErrors = 0; + if (m_out) *m_out = NULL; /* Ensure manifest meets basic sanity checks. */ - const char *name = rhizome_manifest_get(m, "name", NULL, 0); + const char *name = rhizome_manifest_get(m_in, "name", NULL, 0); if (name == NULL || !name[0]) return WHY("Manifest missing 'name' field"); - if (rhizome_manifest_get_ll(m, "date") == -1) + if (rhizome_manifest_get_ll(m_in, "date") == -1) return WHY("Manifest missing 'date' field"); /* Keep payload file name handy for later */ - m->dataFileName = strdup(filename); + m_in->dataFileName = strdup(filename); /* Store time to live, clamped to within legal range */ - m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl; + m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl; /* Check payload file is accessible and discover its length, then check that it matches the file size stored in the manifest */ @@ -153,115 +157,121 @@ int rhizome_add_manifest(rhizome_manifest *m, struct stat stat; if (lstat(filename,&stat)) return WHY("Could not stat() payload file"); - m->fileLength = stat.st_size; - long long mfilesize = rhizome_manifest_get_ll(m, "filesize"); - if (mfilesize != -1 && mfilesize != m->fileLength) { - WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m->fileLength); - ++verifyErrors; + m_in->fileLength = stat.st_size; + long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize"); + if (mfilesize != -1 && mfilesize != m_in->fileLength) { + WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m_in->fileLength); + if (verifyP) + return -1; } } + /* Bail out now if errors occurred loading the manifest file, eg signature failed to validate */ + if (verifyP && m_in->errors) + return WHYF("Manifest.errors (%d) is non-zero", m_in->errors); + /* Compute hash of payload unless we know verification has already failed */ - if (checkFileP ? !(verifyP && (m->errors || verifyErrors)) : signP) { + if (checkFileP || signP) { + char hexhash[SHA512_DIGEST_STRING_LENGTH]; if (rhizome_hash_file(filename, hexhash)) return WHY("Could not hash file."); - memcpy(&m->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH); - m->fileHashedP = 1; + memcpy(&m_in->fileHexHash[0], &hexhash[0], SHA512_DIGEST_STRING_LENGTH); + m_in->fileHashedP = 1; } - /* Check that paylod hash matches manifest */ + /* Check that payload hash matches manifest */ if (checkFileP) { - const char *mhexhash = rhizome_manifest_get(m, "filehash", NULL, 0); - if (mhexhash && strcmp(hexhash, mhexhash)) { - WHYF("Manifest.filehash (%s) != actual file hash (%s)", mhexhash, hexhash); - ++verifyErrors; + const char *mhexhash = rhizome_manifest_get(m_in, "filehash", NULL, 0); + if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) { + WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash); + if (verifyP) + return -1; } } - /* If any signature errors were encountered on loading, or manifest is inconsistent with payload, - then bail out now. */ - if (verifyP) { - if (m->errors) - WHYF("Manifest.errors (%d) is non-zero", m->errors); - if (verifyErrors || m->errors) - return WHY("Errors encountered verifying bundle manifest"); - } - /* Fill in the manifest so that duplicate detection can be performed, and to avoid redundant work by rhizome_manifest_finalise() below. */ if (checkFileP) { - rhizome_manifest_set(m, "filehash", hexhash); - rhizome_manifest_set_ll(m, "first_byte", 0); - rhizome_manifest_set_ll(m, "last_byte", m->fileLength); + rhizome_manifest_set(m_in, "filehash", m_in->fileHexHash); + rhizome_manifest_set_ll(m_in, "first_byte", 0); + rhizome_manifest_set_ll(m_in, "last_byte", m_in->fileLength); } /* Check if a manifest is already stored for the same payload with the same details. This catches the case of "dna rhizome add file " on the same file more than once. (Debounce!) */ rhizome_manifest *dupm = NULL; - if (rhizome_find_duplicate(m, &dupm) == -1) + if (rhizome_find_duplicate(m_in, &dupm) == -1) return WHY("Errors encountered searching for duplicate manifest"); if (dupm) { - if (debug & DEBUG_RHIZOME) fprintf(stderr, "Not adding manifest for payload name=\"%s\" hexhash=%s - duplicate found in rhizome store\n", name, hexhash); + if (debug & DEBUG_RHIZOME) + fprintf(stderr, "Not adding manifest for payload name=\"%s\" hexhash=%s - duplicate found in rhizome store\n", name, m_in->fileHexHash); #if 0 /* TODO Upgrade the version of the duplicate? */ - long long version = rhizome_manifest_get_ll(m, "version"); + long long version = rhizome_manifest_get_ll(m_in, "version"); long long dupversion = rhizome_manifest_get_ll(dupm, "version"); if (version > dupversion) { rhizome_manifest_set_ll(dupm, "version", version); ... } #endif - rhizome_manifest_free(dupm); + if (m_out) { + /* Finish completing the manifest */ + if (rhizome_manifest_finalise(dupm, 0)) + return WHY("Failed to finalise manifest.\n"); + *m_out = dupm; + } + else + rhizome_manifest_free(dupm); return 0; } /* Supply manifest version number if missing, so we can do the version check below */ - if (rhizome_manifest_get(m, "version", NULL, 0) == NULL) { - rhizome_manifest_set_ll(m, "version", overlay_gettime_ms()); + if (rhizome_manifest_get(m_in, "version", NULL, 0) == NULL) { + rhizome_manifest_set_ll(m_in, "version", overlay_gettime_ms()); } - /* If the manifest already has an ID, look to see if we possess its private key */ - if ((id = rhizome_manifest_get(m, "id", NULL, 0))) { - rhizome_hex_to_bytes(id, m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2); - if (!rhizome_find_keypair_bytes(m->cryptoSignPublic, m->cryptoSignSecret)) - m->haveSecret=1; - } - - /* Discard the new manifest it is older than the most recent known version with the same ID */ - if (id) { + /* If the manifest already has an ID */ + char *id = NULL; + if ((id = rhizome_manifest_get(m_in, "id", NULL, 0))) { + /* Discard the new manifest it is older than the most recent known version with the same ID */ long long storedversion = sqlite_exec_int64("SELECT version from manifests where id='%s';", id); - if (storedversion > rhizome_manifest_get_ll(m, "version")) { + if (storedversion > rhizome_manifest_get_ll(m_in, "version")) { return WHY("Newer version exists"); } + /* Check if we know its private key */ + rhizome_hex_to_bytes(id, m_in->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2); + if (!rhizome_find_keypair_bytes(m_in->cryptoSignPublic, m_in->cryptoSignSecret)) + m_in->haveSecret=1; } else { /* The manifest had no ID (256 bit random string being a public key in the NaCl CryptoSign crypto system), so create one. */ printf("manifest does not have an id\n"); - rhizome_manifest_createid(m); + rhizome_manifest_createid(m_in); /* The ID is implicit in transit, but we need to store it in the file, so that reimporting manifests on receiver nodes works easily. We might implement something that strips the id variable out of the manifest when sending it, or some other scheme to avoid sending all the extra bytes. */ - id = rhizome_bytes_to_hex(m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); - rhizome_manifest_set(m, "id", id); + id = rhizome_bytes_to_hex(m_in->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES); + rhizome_manifest_set(m_in, "id", id); } /* Add group memberships */ if (groups) { int i; for(i = 0; groups[i]; i++) - rhizome_manifest_add_group(m, groups[i]); + rhizome_manifest_add_group(m_in, groups[i]); } /* Finish completing the manifest */ - if (rhizome_manifest_finalise(m,signP)) + if (rhizome_manifest_finalise(m_in, signP)) return WHY("Failed to finalise manifest.\n"); /* Okay, it is written, and can be put directly into the rhizome database now */ - if (rhizome_store_bundle(m, filename) == -1) + if (rhizome_store_bundle(m_in, filename) == -1) return WHY("rhizome_store_bundle() failed."); + if (m_out) *m_out = m_in; return 0; } diff --git a/rhizome.h b/rhizome.h index d1d8d051..b2006a5c 100644 --- a/rhizome.h +++ b/rhizome.h @@ -182,9 +182,11 @@ int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid); int rhizome_store_file(const char *file,char *hash,int priortity); char *rhizome_safe_encode(unsigned char *in,int len); int rhizome_finish_sqlstatement(sqlite3_stmt *statement); -int rhizome_bundle_import(rhizome_manifest *m_in,char *bundle,char *groups[], int ttl, +int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out, char *bundle, + char *groups[], int ttl, int verifyP, int checkFileP, int signP); -int rhizome_add_manifest(rhizome_manifest *m, const char *filename, char *groups[], int ttl, +int rhizome_add_manifest(rhizome_manifest *m_in, rhizome_manifest **m_out, const char *filename, + char *groups[], int ttl, int verifyP, int checkFileP, int signP); int rhizome_manifest_finalise(rhizome_manifest *m,int signP); char *rhizome_bytes_to_hex(unsigned char *in,int byteCount); diff --git a/rhizome_database.c b/rhizome_database.c index 1b36fb15..ede1497a 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -504,7 +504,11 @@ char *rhizome_safe_encode(unsigned char *in,int len) int rhizome_list_manifests(int limit, int offset) { char sqlcmd[1024]; - int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id"); + int n = snprintf(sqlcmd, sizeof(sqlcmd), + "SELECT files.id, files.length, files.datavalid, manifests.id, manifests.manifest, manifests.version, manifests.inserttime" + " FROM files, filemanifests, manifests WHERE files.id = filemanifests.fileid AND filemanifests.manifestid = manifests.id" + " ORDER BY files.id ASC" + ); if (n >= sizeof(sqlcmd)) return WHY("SQL command too long"); if (limit) { @@ -539,21 +543,23 @@ int rhizome_list_manifests(int limit, int offset) ret = WHY("Incorrect statement column"); break; } - size_t filesize = sqlite3_column_int(statement, 1); const char *manifestblob = (char *) sqlite3_column_blob(statement, 4); size_t manifestblobsize = sqlite3_column_bytes(statement, 4); // must call after sqlite3_column_blob() - //printf("manifest blob = %s\n", manifestblob); rhizome_manifest *m = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0); const char *name = rhizome_manifest_get(m, "name", NULL, 0); - printf("file id = %s\nfile length = %u\nfile datavalid = %u\nfile name = \"%s\"\n\n", + long long date = rhizome_manifest_get_ll(m, "date"); + printf("fileid=%s:manifestid=%s:version=%d:inserttime=%lld:length=%u:datavalid=%u:date=%lld:name=%s\n", sqlite3_column_text(statement, 0), - filesize, + sqlite3_column_text(statement, 3), + sqlite3_column_int(statement, 5), + (long long) sqlite3_column_int64(statement, 6), + sqlite3_column_int(statement, 1), sqlite3_column_int(statement, 2), + date, name ); rhizome_manifest_free(m); } - printf("Found %lu rows\n", (unsigned long) rows); } sqlite3_finalize(statement); return ret; @@ -758,6 +764,9 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) { if (!m->fileHashedP) return WHY("Manifest payload is not hashed"); + const char *name = rhizome_manifest_get(m, "name", NULL, 0); + if (!name) + return WHY("Manifest has no name"); char sqlcmd[1024]; int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT manifests.id, manifests.manifest FROM filemanifests, manifests WHERE filemanifests.fileid = ? AND filemanifests.manifestid = manifests.id"); if (n >= sizeof(sqlcmd)) @@ -766,12 +775,10 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) sqlite3_stmt *statement; const char *cmdtail; if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) { - sqlite3_finalize(statement); ret = WHY(sqlite3_errmsg(rhizome_db)); } else { - if (debug & DEBUG_RHIZOME) fprintf(stderr, "fileHaxHash = \"%s\"\n", m->fileHexHash); + if (debug & DEBUG_RHIZOME) fprintf(stderr, "fileHexHash = \"%s\"\n", m->fileHexHash); sqlite3_bind_text(statement, 1, m->fileHexHash, -1, SQLITE_STATIC); - const char *name = rhizome_manifest_get(m, "name", NULL, 0); size_t rows = 0; while (sqlite3_step(statement) == SQLITE_ROW) { ++rows; @@ -793,12 +800,20 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() rhizome_manifest *mq = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0); const char *nameq = rhizome_manifest_get(mq, "name", NULL, 0); - /* No need to compare "filehash" here, but we do so as a precaution. */ + const char *filehashq = rhizome_manifest_get(mq, "filehash", NULL, 0); + long long lengthq = rhizome_manifest_get_ll(mq, "filesize"); if (debug & DEBUG_RHIZOME) fprintf(stderr, "Consider manifest.id=%s manifest.name=\"%s\"\n", manifestid, nameq); - if ( !strcmp(nameq, name) - && !strncmp(rhizome_manifest_get(mq, "filehash", NULL, 0), m->fileHexHash, SHA512_DIGEST_STRING_LENGTH)) { + /* No need to compare "filehash" or "filesize" here, but we do so as a precaution if present */ + if ( nameq && !strcmp(nameq, name) + && (!filehashq || strncmp(filehashq, m->fileHexHash, SHA512_DIGEST_STRING_LENGTH) == 0) + && (lengthq == -1 || lengthq == m->fileLength) + ) { + memcpy(mq->fileHexHash, m->fileHexHash, SHA512_DIGEST_STRING_LENGTH); + mq->fileHashedP = 1; + mq->fileLength = m->fileLength; *found = mq; ret = 1; + if (debug & DEBUG_RHIZOME) fprintf(stderr, "found\n"); break; } rhizome_manifest_free(mq); diff --git a/rhizome_fetch.c b/rhizome_fetch.c index efe447cc..bfaca9b8 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -393,7 +393,7 @@ int rhizome_queue_manifest_import(rhizome_manifest *m, rhizome_datastore_path, rhizome_manifest_get(m,"id",NULL,0)); if (!rhizome_write_manifest_file(m,filename)) { - rhizome_bundle_import(m,rhizome_manifest_get(m,"id",NULL,0), + rhizome_bundle_import(m, NULL, rhizome_manifest_get(m,"id",NULL,0), NULL /* no additional groups */, m->ttl-1 /* TTL */, 1 /* do verify */, @@ -522,17 +522,15 @@ int rhizome_fetch_poll() q->manifest->finalised=1; q->manifest->manifest_bytes=q->manifest->manifest_all_bytes; if (!rhizome_write_manifest_file(q->manifest,filename)) { - rhizome_bundle_import(q->manifest, - rhizome_manifest_get(q->manifest, - "id",NULL,0), - NULL /* no additional groups */, - q->manifest->ttl-1 /* TTL */, - 1 /* do verify */, - 1 /* do check hash of file */, - 0 /* do not sign it, just keep existing - signatures */); + rhizome_bundle_import(q->manifest, NULL, + rhizome_manifest_get(q->manifest, "id", NULL, 0), + NULL /* no additional groups */, + q->manifest->ttl - 1 /* TTL */, + 1 /* do verify */, + 1 /* do check hash of file */, + 0 /* do not sign it, just keep existing signatures */); q->manifest=NULL; - } + } } } break; diff --git a/tests/dna_rhizome b/tests/dna_rhizome index d7b5ecb0..afe942f3 100755 --- a/tests/dna_rhizome +++ b/tests/dna_rhizome @@ -26,9 +26,29 @@ setup_dna_rhizome() { executeOk $dna config set debug rhizome } -assert_rhizome_list_empty() { +assert_rhizome_list() { executeOk $dna rhizome list - assertStdoutIs -e "Found 0 rows\n" + assertStdoutLineCount '==' $# + assertStdoutGrep --matches=$# '^fileid=.*:name=' + local filename + for filename; do + local filehash='[^:]\+' + # If there is a manifest file that looks like it matches this payload + # file, then use its file hash to check the rhizome list output. + if [ -r "$filename.manifest" ]; then + local name=$(sed -n -e '/^name=/s///p' "$filename.manifest") + if [ "$name" == "$filename" ]; then + filehash=$(sed -n -e '/^filehash=/s///p' "$filename.manifest") + fi + fi + assertStdoutGrep --matches=1 "^\(.*:\)\?fileid=$filehash:.*:name=$filename\$" + done +} + +strip_signatures() { + for file; do + cat -v "$file" | sed -e '/^^@/,$d' >"tmp.$file" && mv -f "tmp.$file" "$file" + done } doc_InitialEmptyList="Initial rhizome list is empty" @@ -36,15 +56,47 @@ setup_InitialEmptyList() { setup_dna_rhizome } test_InitialEmptyList() { - assert_rhizome_list_empty + assert_rhizome_list +} + +doc_AddNoManifest="Rhizome add with no manifest file" +setup_AddNoManifest() { + setup_dna_rhizome + assert_rhizome_list + echo "A test file" >file1 + echo "Another test file" >file2 +} +test_AddNoManifest() { + executeOk $dna rhizome add file file1 +} + +doc_AddNonExistManifest="Rhizome add with non-existent manifest file" +setup_AddNonExistManifest() { + setup_dna_rhizome + assert_rhizome_list + echo "A test file" >file1 + echo "Another test file" >file2 +} +test_AddNonExistManifest() { + assert --error-on-fail [ ! -e file1.manifest ] + executeOk $dna rhizome add file file1 file1.manifest + assert [ -r file1.manifest ] + tfw_cat -v file1.manifest + assertGrep file1.manifest '^name=file1$' + assertGrep file1.manifest '^date=[0-9]\+$' + assertGrep file1.manifest '^version=[0-9]\+$' + assertGrep file1.manifest '^id=[0-9a-fA-F]\+$' + assertGrep file1.manifest "^filesize=$(cat file1 | wc --bytes)\$" + assertGrep file1.manifest "^first_byte=0$" + assertGrep file1.manifest "^last_byte=$(cat file1 | wc --bytes)\$" } doc_AddManifest="Rhizome add with manifest" setup_AddManifest() { setup_dna_rhizome - assert_rhizome_list_empty + assert_rhizome_list echo "A test file" >file1 - echo 'name=wah' >file1.manifest + echo -e 'name=wah\ndate=12345' >file1.manifest echo "Another test file" >file2 } test_AddManifest() { @@ -53,7 +105,7 @@ test_AddManifest() { tfw_cat --stderr tfw_cat -v file1.manifest assertGrep file1.manifest '^name=wah$' - assertGrep file1.manifest '^date=[0-9]\+$' + assertGrep file1.manifest '^date=12345$' assertGrep file1.manifest '^version=[0-9]\+$' assertGrep file1.manifest '^id=[0-9a-fA-F]\+$' assertGrep file1.manifest "^filesize=$(cat file1 | wc --bytes)\$" @@ -64,55 +116,86 @@ test_AddManifest() { doc_AddThenList="Rhizome list contains one file after one add" setup_AddThenList() { setup_dna_rhizome - assert_rhizome_list_empty + assert_rhizome_list echo "A test file" >file1 echo "Another test file" >file2 } test_AddThenList() { # Add first file - executeOk $dna rhizome add file file1 - executeOk $dna rhizome list - assertStdoutGrep "^Found 1 rows$" - assertStdoutLineCount '==' 6 - assertStdoutGrep --matches=1 '^file name = "file1"$' - assertStdoutGrep --matches=0 '^file name = "file2"$' - replayStdout >add1.stdout + executeOk $dna rhizome add file file1 file1.manifest + assert_rhizome_list file1 # Add second file - executeOk $dna rhizome add file file2 - executeOk $dna rhizome list - assertStdoutGrep "^Found 2 rows$" - assertStdoutLineCount '==' 11 - assertStdoutGrep --matches=1 '^file name = "file1"$' - assertStdoutGrep --matches=1 '^file name = "file2"$' + executeOk $dna rhizome add file file2 file2.manifest + assert_rhizome_list file1 file2 } doc_AddDuplicate="Rhizome add of same file" setup_AddDuplicate() { setup_dna_rhizome - assert_rhizome_list_empty + assert_rhizome_list echo "A test file" >file1 echo "Another test file" >file2 + echo "A test file, second version" >file1_2 # Add first file - executeOk $dna rhizome add file file1 + executeOk $dna rhizome add file file1 file1.manifest # Add second file - executeOk $dna rhizome add file file2 + executeOk $dna rhizome add file file2 file2.manifest # Make sure they are both in the list. - executeOk $dna rhizome list - assertStdoutGrep "^Found 2 rows$" - assertStdoutGrep --matches=1 '^file name = "file1"$' - assertStdoutGrep --matches=1 '^file name = "file2"$' + assert_rhizome_list file1 file2 } test_AddDuplicate() { - # Add first file again (nothing should change except its date). - executeOk $dna config get debug - tfw_cat --stdout - executeOk $dna rhizome add file file1 - tfw_cat --stderr - executeOk $dna rhizome list - assertStdoutGrep "^Found 2 rows$" - assertStdoutLineCount '==' 11 - assertStdoutGrep --matches=1 '^file name = "file1"$' - assertStdoutGrep --matches=1 '^file name = "file2"$' + # Add first file again - nothing should change in its manifests, and it + # should appear that the add command succeeded (with perhaps some grumbling + # on stderr). + executeOk $dna rhizome add file file1 file1.manifestA + assert [ -s file1.manifestA ] + assert_rhizome_list file1 file2 + strip_signatures file1.manifest file1.manifestA + assert diff file1.manifest file1.manifestA + # Repeat for second file. + executeOk $dna rhizome add file file2 file2.manifestA + assert [ -s file2.manifestA ] + assert_rhizome_list file1 file2 + strip_signatures file2.manifest file2.manifestA + assert diff file2.manifest file2.manifestA +} + +doc_AddMismatched="Rhizome add fails on mismatched manifest and payload" +setup_AddMismatched() { + setup_AddDuplicate +} +test_AddMismatched() { + # Try to add another file using an existing manifest, should fail and leave + # the manifest file unchanged. + cp file1.manifest file1_2.manifest + execute $dna rhizome add file file1_2 file1_2.manifest + assertExitStatus '!=' 0 + assert cmp file1.manifest file1_2.manifest + # And rhizome store should be unchanged. + assert_rhizome_list file1 file2 +} + +doc_AddUpdate="Rhizome add new version of existing file" +setup_AddUpdate() { + setup_AddDuplicate +} +test_AddUpdate() { + # Change payload associated with first manifest. + cp file1.manifest file1_2.manifest + strip_signatures file1_2.manifest + sed -i -e '/^date=/d' -e '/^filehash=/d' -e '/^filesize=/d' file1_2.manifest + tfw_cat file1_2.manifest + assertGrep --matches=0 --error-on-fail file1_2.manifest '^filehash=' + executeOk $dna rhizome add file file1_2 file1_2.manifest + # The new manifest must now have a different filehash from the original. + filehash1=$(sed -n -e '/^filehash=/s///p' file1.manifest) + filehash1_2=$(sed -n -e '/^filehash=/s///p' file1_2.manifest) + assert [ -n "$filehash1" ] + assert [ -n "$filehash1_2" ] + assert [ "$filehash1" != "$filehash1_2" ] + # Rhizome store contents reflect new payload. + mv -f file1_2.manifest file1.manifest + assert_rhizome_list file1 file2 } runTests "$@"