diff --git a/commandline.c b/commandline.c index a06c1b0a..66c66801 100644 --- a/commandline.c +++ b/commandline.c @@ -1460,8 +1460,8 @@ int app_rhizome_add_file(const struct cli_parsed *parsed, struct cli_context *co /* Don't verify the manifest, because it will fail if it is incomplete. This is okay, because we fill in any missing bits and sanity check before trying to write it out. However, we do insist that whatever we load is - valid and not malformed. */ - if (rhizome_read_manifest_file(m, manifestpath, 0) == -1 || m->malformed) { + parsed okay and not malformed. */ + if (rhizome_read_manifest_from_file(m, manifestpath) || m->malformed) { rhizome_manifest_free(m); keyring_free(keyring); return WHY("Manifest file could not be loaded -- not added to rhizome"); @@ -1634,7 +1634,7 @@ int app_rhizome_append_manifest(const struct cli_parsed *parsed, struct cli_cont if (!m) return WHY("Out of manifests."); int ret = -1; - if ( rhizome_read_manifest_file(m, manifestpath, 0) != -1 + if ( rhizome_read_manifest_from_file(m, manifestpath) != -1 && rhizome_manifest_validate(m) && rhizome_manifest_verify(m) ) { diff --git a/rhizome.c b/rhizome.c index db4caee7..70cf093c 100644 --- a/rhizome.c +++ b/rhizome.c @@ -104,21 +104,18 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, manifest_path ? alloca_str_toprint(manifest_path) : "NULL", filepath ? alloca_str_toprint(filepath) : "NULL"); - unsigned char buffer[MAX_MANIFEST_BYTES]; size_t buffer_len = 0; + int ret = 0; // manifest has been appended to the end of the file. if (strcmp(manifest_path, filepath)==0){ unsigned char marker[4]; - int ret=0; FILE *f = fopen(filepath, "r"); if (f == NULL) return WHYF_perror("Could not open manifest file %s for reading.", filepath); - if (fseek(f, -sizeof(marker), SEEK_END)) ret=WHY_perror("Unable to seek to end of file"); - if (ret==0){ ret = fread(marker, 1, sizeof(marker), f); if (ret==sizeof(marker)) @@ -126,39 +123,37 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, else ret=WHY_perror("Unable to read end of manifest marker"); } - if (ret==0){ if (marker[2]!=0x41 || marker[3]!=0x10) ret=WHYF("Expected 0x4110 marker at end of file"); } - if (ret==0){ buffer_len = read_uint16(marker); if (buffer_len < 1 || buffer_len > MAX_MANIFEST_BYTES) ret=WHYF("Invalid manifest length %zu", buffer_len); } - if (ret==0){ if (fseek(f, -(buffer_len+sizeof(marker)), SEEK_END)) ret=WHY_perror("Unable to seek to end of file"); } - - if (ret==0){ - ssize_t nread = fread(buffer, 1, buffer_len, f); - if ((size_t)nread != buffer_len) - ret=WHY_perror("Unable to read manifest contents"); + if (ret == 0 && fread(m->manifestdata, buffer_len, 1, f) != 1) { + if (ferror(f)) + ret = WHYF("fread(%p,%zu,1,%s) error", m->manifestdata, buffer_len, alloca_str_toprint(filepath)); + else if (feof(f)) + ret = WHYF("fread(%p,%zu,1,%s) hit end of file", m->manifestdata, buffer_len, alloca_str_toprint(filepath)); } - fclose(f); - - if (ret) - return ret; - - manifest_path=(char*)buffer; + } else { + ssize_t size = read_whole_file(manifest_path, m->manifestdata, sizeof m->manifestdata); + if (size == -1) + ret = -1; + buffer_len = (size_t) size; } - - if (rhizome_read_manifest_file(m, manifest_path, buffer_len) == -1) - return WHY("could not read manifest file"); + if (ret) + return ret; + m->manifest_all_bytes = buffer_len; + if (rhizome_manifest_parse(m) == -1) + return WHY("could not parse manifest file"); if (!rhizome_manifest_validate(m)) return WHY("manifest is invalid"); if (!rhizome_manifest_verify(m)) diff --git a/rhizome.h b/rhizome.h index 2a41f310..1eb9ab3f 100644 --- a/rhizome.h +++ b/rhizome.h @@ -455,8 +455,12 @@ int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char int rhizome_manifest_selfsign(rhizome_manifest *m); int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority); int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp); -int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferPAndSize); +int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename); int rhizome_manifest_validate(rhizome_manifest *m); +int rhizome_manifest_parse(rhizome_manifest *m); +int rhizome_manifest_verify(rhizome_manifest *m); +int rhizome_manifest_check_sanity(rhizome_manifest *m_in); + int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out); void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m); @@ -479,8 +483,6 @@ void rhizome_find_bundle_author_and_secret(rhizome_manifest *m); int rhizome_lookup_author(rhizome_manifest *m); void rhizome_authenticate_author(rhizome_manifest *m); -int rhizome_manifest_verify(rhizome_manifest *m); -int rhizome_manifest_check_sanity(rhizome_manifest *m_in); int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate); int rhizome_add_manifest(rhizome_manifest *m_in,int ttl); diff --git a/rhizome_bundle.c b/rhizome_bundle.c index 4298b0ef..2bfe04a0 100644 --- a/rhizome_bundle.c +++ b/rhizome_bundle.c @@ -519,7 +519,7 @@ int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifes * * @author Andrew Bettison */ -static int rhizome_manifest_parse(rhizome_manifest *m) +int rhizome_manifest_parse(rhizome_manifest *m) { IN(); assert(m->manifest_all_bytes <= sizeof m->manifestdata); @@ -789,27 +789,13 @@ int rhizome_manifest_validate(rhizome_manifest *m) return ret; } -int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferP) +int rhizome_read_manifest_from_file(rhizome_manifest *m, const char *filename) { - if (!m) - return WHY("Null manifest"); - if (bufferP>sizeof(m->manifestdata)) - return WHY("Buffer too big"); - - if (bufferP) { - m->manifest_all_bytes=bufferP; - memcpy(m->manifestdata, filename, m->manifest_all_bytes); - } else { - ssize_t bytes = read_whole_file(filename, m->manifestdata, sizeof m->manifestdata); - if (bytes == -1) - return -1; - m->manifest_all_bytes = (size_t) bytes; - } - switch (rhizome_manifest_parse(m)) { - case 0: return 0; - case -1: return -1; - default: return WHY("Invalid manifest"); - } + ssize_t bytes = read_whole_file(filename, m->manifestdata, sizeof m->manifestdata); + if (bytes == -1) + return -1; + m->manifest_all_bytes = (size_t) bytes; + return rhizome_manifest_parse(m); } int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out) diff --git a/rhizome_database.c b/rhizome_database.c index 011be4be..b37cc05a 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -174,24 +174,28 @@ void verify_bundles() sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;"); while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) { sqlite3_int64 rowid = sqlite3_column_int64(statement, 0); - const void *manifest = sqlite3_column_blob(statement, 1); - size_t manifest_length = sqlite3_column_bytes(statement, 1); - rhizome_manifest *m=rhizome_new_manifest(); - int ret = -1; - if ( rhizome_read_manifest_file(m, manifest, manifest_length) == 0 - && rhizome_manifest_validate(m) - && rhizome_manifest_verify(m) - ) { - assert(m->finalised); - // Store it again, to ensure that MANIFESTS columns are up to date. - ret = rhizome_store_bundle(m); + const void *blob = sqlite3_column_blob(statement, 1); + size_t blob_length = sqlite3_column_bytes(statement, 1); + rhizome_manifest *m = rhizome_new_manifest(); + if (m) { + memcpy(m->manifestdata, blob, blob_length); + m->manifest_all_bytes = blob_length; + int ret = -1; + if ( rhizome_manifest_parse(m) != -1 + && rhizome_manifest_validate(m) + && rhizome_manifest_verify(m) + ) { + assert(m->finalised); + // Store it again, to ensure that MANIFESTS columns are up to date. + ret = rhizome_store_bundle(m); + } + if (ret) { + if (config.debug.rhizome) + DEBUGF("Removing invalid manifest entry @%lld", rowid); + sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END); + } + rhizome_manifest_free(m); } - if (ret) { - if (config.debug.rhizome) - DEBUGF("Removing invalid manifest entry @%lld", rowid); - sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END); - } - rhizome_manifest_free(m); } sqlite3_finalize(statement); } @@ -1613,7 +1617,9 @@ int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c) rhizome_manifest *m = c->manifest = rhizome_new_manifest(); if (m == NULL) RETURN(-1); - if ( rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1 + memcpy(m->manifestdata, manifestblob, manifestblobsize); + m->manifest_all_bytes = manifestblobsize; + if ( rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m) ) { WHYF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); @@ -1750,7 +1756,9 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found) const unsigned char *q_manifestid = sqlite3_column_text(statement, 0); const char *manifestblob = (char *) sqlite3_column_blob(statement, 1); size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() - if ( rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1 + memcpy(blob_m->manifestdata, manifestblob, manifestblobsize); + blob_m->manifest_all_bytes = manifestblobsize; + if ( rhizome_manifest_parse(blob_m) == -1 || !rhizome_manifest_validate(blob_m) ) { WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid); @@ -1794,7 +1802,9 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement) const char *q_author = (const char *) sqlite3_column_text(statement, 4); size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob() uint64_t q_rowid = sqlite3_column_int64(statement, 5); - if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1 || !rhizome_manifest_validate(m)) + memcpy(m->manifestdata, q_blob, q_blobsize); + m->manifest_all_bytes = q_blobsize; + if (rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m)) return WHYF("Manifest bid=%s in database but invalid", q_id); if (q_author) { sid_t author; diff --git a/rhizome_direct.c b/rhizome_direct.c index 6f903614..312007ee 100644 --- a/rhizome_direct.c +++ b/rhizome_direct.c @@ -438,10 +438,15 @@ rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix, size_t if (manifestblobsize<1||manifestblobsize>1024) goto error; const char *manifestblob = (char *) sqlite3_column_blob(statement, 0); - if (!manifestblob) goto error; + if (!manifestblob) + goto error; - rhizome_manifest *m=rhizome_new_manifest(); - if ( rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1 + rhizome_manifest *m = rhizome_new_manifest(); + if (!m) + goto error; + memcpy(m->manifestdata, manifestblob, manifestblobsize); + m->manifest_all_bytes = manifestblobsize; + if ( rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m) ) { rhizome_manifest_free(m); diff --git a/rhizome_direct_http.c b/rhizome_direct_http.c index 2479fcb5..5d5621fc 100644 --- a/rhizome_direct_http.c +++ b/rhizome_direct_http.c @@ -207,7 +207,7 @@ static int rhizome_direct_addfile_end(struct http_request *hr) rhizome_direct_clear_temporary_files(r); return 0; } - if (manifestTemplate[0] && rhizome_read_manifest_file(m, manifestTemplate, 0) == -1) { + if (manifestTemplate[0] && rhizome_read_manifest_from_file(m, manifestTemplate) == -1) { WHY("Manifest template read failed"); rhizome_manifest_free(m); rhizome_direct_clear_temporary_files(r); diff --git a/rhizome_fetch.c b/rhizome_fetch.c index 149a9860..dd773723 100644 --- a/rhizome_fetch.c +++ b/rhizome_fetch.c @@ -1321,7 +1321,9 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot) call schedule queued items. */ rhizome_manifest *m = rhizome_new_manifest(); if (m) { - if ( rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1 + memcpy(m->manifestdata, slot->manifest_buffer, (size_t)slot->manifest_bytes); + m->manifest_all_bytes = (size_t)slot->manifest_bytes; + if ( rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m) ) { DEBUGF("Couldn't read manifest"); diff --git a/rhizome_packetformats.c b/rhizome_packetformats.c index 8ea6027b..d70cd739 100644 --- a/rhizome_packetformats.c +++ b/rhizome_packetformats.c @@ -350,7 +350,9 @@ int overlay_rhizome_saw_advertisements(struct decode_context *context, struct ov // The manifest looks potentially interesting, so now do a full parse and validation. if ((m = rhizome_new_manifest()) == NULL) goto next; - if ( rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1 + memcpy(m->manifestdata, data, manifest_length); + m->manifest_all_bytes = manifest_length; + if ( rhizome_manifest_parse(m) == -1 || !rhizome_manifest_validate(m) ) { WARN("Malformed manifest");