Ensure orphan payloads or manifests are cleaned up

commandline.c

@@ -1939,6 +1939,13 @@ int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context *conte
   if (config.debug.verbose)
     DEBUG_cli_parsed(parsed);
   int verify = cli_arg(parsed, "verify", NULL, NULL, NULL) == 0;
+  
+  /* Ensure the Rhizome database exists and is open */
+  if (create_serval_instance_dir() == -1)
+    return -1;
+  if (rhizome_opendb() == -1)
+    return -1;
+  
   if (verify)
     verify_bundles();
   struct rhizome_cleanup_report report;
@@ -1950,6 +1957,8 @@ int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context *conte
   cli_put_long(context, report.deleted_orphan_files, "\n");
   cli_field_name(context, "deleted_orphan_fileblobs", ":");
   cli_put_long(context, report.deleted_orphan_fileblobs, "\n");
+  cli_field_name(context, "deleted_orphan_manifests", ":");
+  cli_put_long(context, report.deleted_orphan_manifests, "\n");
   return 0;
 }
 

rhizome.h

@@ -332,6 +332,7 @@ struct rhizome_cleanup_report {
     unsigned deleted_stale_incoming_files;
     unsigned deleted_orphan_files;
     unsigned deleted_orphan_fileblobs;
+    unsigned deleted_orphan_manifests;
 };
 
 int rhizome_cleanup(struct rhizome_cleanup_report *report);

rhizome_database.c

@@ -1238,7 +1238,7 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
 
   // Remove external payload files for old, unreferenced payloads.
   statement = sqlite_prepare_bind(&retry,
-      "SELECT id FROM FILES WHERE inserttime < ? AND datavalid = 1 AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
+      "SELECT id FROM FILES WHERE inserttime < ? AND NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);",
       INT64, insert_horizon_no_manifest, END);
   while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
     candidates++;
@@ -1271,11 +1271,19 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report)
   if ((ret = rhizome_delete_orphan_fileblobs_retry(&retry)) > 0 && report)
     report->deleted_orphan_fileblobs += ret;
 
+  // delete manifests that no longer have payload files
+  ret = sqlite_exec_void_retry_loglevel(LOG_LEVEL_WARN, &retry,
+      "DELETE FROM MANIFESTS WHERE inserttime < ? AND filesize > 0 AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);",
+      INT64, insert_horizon_no_manifest, END);
+  if (report && ret > 0)
+    report->deleted_orphan_manifests += ret;
+  
   if (config.debug.rhizome && report)
-    DEBUGF("report deleted_stale_incoming_files=%u deleted_orphan_files=%u deleted_orphan_fileblobs=%u",
+    DEBUGF("report deleted_stale_incoming_files=%u deleted_orphan_files=%u deleted_orphan_fileblobs=%u deleted_orphan_manifests=%u",
 	report->deleted_stale_incoming_files,
 	report->deleted_orphan_files,
-	report->deleted_orphan_fileblobs
+	report->deleted_orphan_fileblobs,
+	report->deleted_orphan_manifests
       );
   RETURN(0);
   OUT();

tests/rhizomeops

@@ -1177,6 +1177,12 @@ setup_delete() {
       extract_manifest_filehash HASH$i file$i.manifest
    done
 }
+rhizome_clean() {
+   executeOk_servald rhizome clean
+   extract_stdout_keyvalue deleted_files 'deleted_orphan_files' '[0-9]\+'
+   extract_stdout_keyvalue deleted_fileblobs 'deleted_orphan_fileblobs' '[0-9]\+'
+   extract_stdout_keyvalue deleted_manifests 'deleted_orphan_manifests' '[0-9]\+'
+}
 
 doc_DeleteManifest="Delete a manifest from store"
 setup_DeleteManifest() {
@@ -1190,6 +1196,10 @@ test_DeleteManifest() {
    execute --exit-status=1 --stderr $servald rhizome export manifest "$BID2"
    executeOk_servald rhizome export file "$HASH2" file2x
    assert diff file2 file2x
+   rhizome_clean
+   assert [ $deleted_files = 1 ]
+   assert [ $deleted_fileblobs = 1 ]
+   assert [ $deleted_manifests = 0 ]
 }
 
 doc_DeletePayload="Delete a payload from store"
@@ -1203,6 +1213,12 @@ test_DeletePayload() {
    assert_rhizome_list file{1..4}
    executeOk_servald rhizome export manifest "$BID3"
    execute --exit-status=1 --stderr $servald rhizome export file "$HASH3" file3x
+   rhizome_clean
+   assert [ $deleted_files = 0 ]
+   assert [ $deleted_fileblobs = 0 ]
+   assert [ $deleted_manifests = 1 ]
+   executeOk_servald rhizome list
+   assert_rhizome_list file{1,2,4}
 }
 
 doc_DeleteBundle="Delete a bundle from store"
@@ -1216,6 +1232,10 @@ test_DeleteBundle() {
    assert_rhizome_list file{1..3}
    execute --exit-status=1 --stderr $servald rhizome export manifest "$BID4"
    execute --exit-status=1 --stderr $servald rhizome export file "$HASH4" file4x
+   rhizome_clean
+   assert [ $deleted_files = 0 ]
+   assert [ $deleted_fileblobs = 0 ]
+   assert [ $deleted_manifests = 0 ]
 }
 
 doc_DeleteFile="Delete a file from store"
@@ -1229,6 +1249,12 @@ test_DeleteFile() {
    assert_rhizome_list file{1..4}
    executeOk_servald rhizome export manifest "$BID1"
    execute --exit-status=1 --stderr $servald rhizome export file "$HASH1" file1x
+   rhizome_clean
+   assert [ $deleted_files = 0 ]
+   assert [ $deleted_fileblobs = 0 ]
+   assert [ $deleted_manifests = 1 ]
+   executeOk_servald rhizome list
+   assert_rhizome_list file{2..4}
 }
 
 runTests "$@"