ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-01-29 15:43:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Re-add database cleanup

Browse Source
This commit is contained in:
Jeremy Lakeman 2012-12-31 15:09:12 +10:30
parent 898cddfcd3
commit a492c05d64
6 changed files with 28 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
rhizome.h
View File

@ -129,7 +129,6 @@ typedef struct rhizome_manifest {
/* When finalised, we keep the filehash and maximum priority due to any
group membership handy */
long long fileLength;
int fileHashedP;
char fileHexHash[SHA512_DIGEST_STRING_LENGTH];
int fileHighestPriority;
/* Absolute path of the file associated with the manifest */

1
rhizome_bundle.c
View File

@ -175,7 +175,6 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bu
/* Force to upper case to avoid case sensitive comparison problems later. */
str_toupper_inplace(m->values[m->var_count]);
strcpy(m->fileHexHash, m->values[m->var_count]);
m->fileHashedP = 1;
}
} else if (strcasecmp(var, "BK") == 0) {
if (!rhizome_str_is_bundle_key(value)) {

53
rhizome_database.c
View File

@ -222,15 +222,8 @@ int rhizome_opendb()
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);");
// We can't delete a file that is being transferred in another process at this very moment...
// FIXME, reinstate with a check for insert time
/* Clean out half-finished entries from the database
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE filehash IS NULL;");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILEBLOBS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE FILEBLOBS.id = FILES.id);");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE filehash != '' AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);");
sqlite_exec_void("DELETE FROM FILES WHERE datavalid=0;");
*/
// TODO don't cleanup before every command line operation...
rhizome_cleanup();
RETURN(0);
}
@ -590,6 +583,21 @@ long long rhizome_database_used_bytes()
return db_page_size * (db_page_count - db_free_page_count);
}
void rhizome_cleanup()
{
// clean out unreferenced files
// TODO keep updating inserttime for *very* long transfers?
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=0;", gettime_ms() - 300000)) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < %lld AND datavalid=1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);", gettime_ms() - 1000)) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
if (sqlite_exec_void("DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );")) {
WARNF("delete failed: %s", sqlite3_errmsg(rhizome_db));
}
}
int rhizome_make_space(int group_priority, long long bytes)
{
/* Asked for impossibly large amount */
@ -600,6 +608,8 @@ int rhizome_make_space(int group_priority, long long bytes)
if (db_used == -1)
return -1;
rhizome_cleanup();
/* If there is already enough space now, then do nothing more */
if (db_used<=(config.rhizome.database_size-bytes-65536))
return 0;
@ -689,7 +699,6 @@ int rhizome_drop_stored_file(const char *id,int maximum_priority)
return 0;
}
/*
Store the specified manifest into the sqlite database.
We assume that sufficient space has been made for us.
@ -734,14 +743,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
rhizome_manifest_to_bar(m,bar);
/* Store the file (but not if it is already in the database) */
// TODO encrypted payloads - pass encryption key here. Filehash should be of
// encrypted data.
// We should add the file in the same transaction, but closing the blob seems
// to cause some issues.
char filehash[RHIZOME_FILEHASH_STRLEN + 1];
if (m->fileLength > 0) {
if (!m->fileHashedP)
return WHY("Manifest payload hash unknown");
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
@ -777,19 +780,8 @@ int rhizome_store_bundle(rhizome_manifest *m)
sqlite3_finalize(stmt);
stmt = NULL;
// we might need to leave the old file around for a bit
// clean out unreferenced files first
// TODO remove old payload?
// FIXME where is the ? parameter bound????
if (sqlite_exec_void("DELETE FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);")) {
WHYF("delete failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
goto rollback;
}
if (sqlite_exec_void("DELETE FROM FILEBLOBS WHERE NOT EXISTS ( SELECT 1 FROM FILES WHERE FILES.id = FILEBLOBS.id );")) {
WHYF("delete failed, %s: %s", sqlite3_errmsg(rhizome_db), sqlite3_sql(stmt));
goto rollback;
}
if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
int closed=rhizome_manifest_get_ll(m,"closedgroup");
if (closed<1) closed=0;
@ -1063,8 +1055,6 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
strbuf_puts(b, "SELECT id, manifest, version, author FROM manifests WHERE ");
if (m->fileLength != 0) {
if (!m->fileHashedP)
return WHY("Manifest payload is not hashed");
strbuf_puts(b, "filehash = ?");
} else
strbuf_puts(b, "filesize = 0");
@ -1196,7 +1186,6 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
}
memcpy(blob_m->cryptoSignPublic, manifest_id, RHIZOME_MANIFEST_ID_BYTES);
memcpy(blob_m->fileHexHash, m->fileHexHash, RHIZOME_FILEHASH_STRLEN + 1);
blob_m->fileHashedP = 1;
blob_m->fileLength = m->fileLength;
blob_m->version = q_version;
*found = blob_m;
@ -1279,13 +1268,11 @@ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp)
ret = WHY("Manifest is missing 'filehash' field");
else {
memcpy(m->fileHexHash, blob_filehash, RHIZOME_FILEHASH_STRLEN + 1);
m->fileHashedP = 1;
}
} else {
if (blob_filehash != NULL)
WARN("Manifest contains spurious 'filehash' field -- ignored");
m->fileHexHash[0] = '\0';
m->fileHashedP = 0;
}
long long blob_version = rhizome_manifest_get_ll(m, "version");
if (blob_version == -1)

12
rhizome_fetch.c
View File

@ -707,14 +707,8 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
}
}
if (!m->fileHashedP)
return WHY("Manifest missing filehash");
// If the payload is already available, no need to fetch, so import now.
long long gotfile = 0;
if (sqlite_exec_int64(&gotfile, "SELECT COUNT(*) FROM FILES WHERE ID='%s' and datavalid=1;", m->fileHexHash) != 1)
return WHY("select failed");
if (gotfile) {
if (rhizome_exists(m->fileHexHash)){
if (config.debug.rhizome_rx)
DEBUGF(" fetch not started - payload already present, so importing instead");
if (rhizome_add_manifest(m, m->ttl-1) == -1)
@ -1384,8 +1378,8 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, char *buffer, int byt
RETURN(-1);
} else {
int ret=sqlite_exec_void_retry(&retry,
"UPDATE FILES SET datavalid=1 WHERE id='%s'",
slot->manifest->fileHexHash);
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
gettime_ms(), slot->manifest->fileHexHash);
if (ret!=SQLITE_OK)
if (config.debug.rhizome_rx)
DEBUGF("error marking row valid: %s",sqlite3_errmsg(rhizome_db));

10
rhizome_store.c
View File

@ -235,8 +235,8 @@ int rhizome_finish_write(struct rhizome_write *write){
goto failure;
}
if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET datavalid=1 WHERE id='%s'",
write->id)!=SQLITE_OK){
"UPDATE FILES SET inserttime=%lld, datavalid=1 WHERE id='%s'",
gettime_ms(), write->id)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure;
}
@ -252,8 +252,8 @@ int rhizome_finish_write(struct rhizome_write *write){
sqlite_exec_void_retry(&retry,"DELETE FROM FILES WHERE id='%s';",hash_out);
if (sqlite_exec_void_retry(&retry,
"UPDATE FILES SET datavalid=1, id='%s' WHERE id='%s'",
hash_out, write->id)!=SQLITE_OK){
"UPDATE FILES SET id='%s', inserttime=%lld, datavalid=1 WHERE id='%s'",
hash_out, gettime_ms(), write->id)!=SQLITE_OK){
WHYF("Failed to update files: %s", sqlite3_errmsg(rhizome_db));
goto failure;
}
@ -318,7 +318,6 @@ int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
if (m->fileLength == 0){
m->fileHexHash[0] = '\0';
rhizome_manifest_del(m, "filehash");
m->fileHashedP = 0;
}
return 0;
}
@ -342,7 +341,6 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
if (rhizome_finish_write(&write))
return -1;
m->fileHashedP = 1;
strlcpy(m->fileHexHash, write.id, SHA512_DIGEST_STRING_LENGTH);
rhizome_manifest_set(m, "filehash", m->fileHexHash);
return 0;

1
serval.h
View File

@ -530,6 +530,7 @@ int serval_packetvisualise(XPRINTF xpf, const char *message, const unsigned char
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int rhizome_opendb();
void rhizome_cleanup();
int parseCommandLine(const char *argv0, int argc, const char *const *argv);