diff --git a/doc/REST-API-Rhizome.md b/doc/REST-API-Rhizome.md index d3a77f9a..a6062f0a 100644 --- a/doc/REST-API-Rhizome.md +++ b/doc/REST-API-Rhizome.md @@ -1145,26 +1145,27 @@ Fetch on the current disk usage of the rhizome store. The results will be a single json object with the following fields; +* `file_count` - the number of file payloads currently stored. + +* `internal_bytes` - the total size of all payloads stored inside the + sqlite database + * `external_bytes` - the total size of all payloads larger than rhizome.max_blob_size, that have been stored outside of sqlite, in the rhizome blob folder. -* `db_page_size` - the size of disk pages returned by sqlite. +* `overhead_bytes` - the total disk space used by manifests or sqlite. -* `db_total_pages` - the number of disk pages in the sqlite database file. +* `used_bytes` - the total bytes of space used in the sqlite database, and + in payloads stored outside of sqlite. -* `db_available_pages` - the number of disk pages in the sqlite database file - that have been allocated but are not currently in use. - -* `content_bytes` - the total bytes of space used in the sqlite database, and - in payloads stored outside of sqlite. This should be equal to; - db_page_size * (db_total_pages - db_available_pages) + external_bytes - -* `content_limit_bytes` - the calculated storage limit that is being applied. - This will be the smallest of the configured rhizome.database_size or the +* `available_bytes` - the remaining space for storing additional file payloads. + Limited by the smallest of the configured rhizome.database_size or the maximum we can store while keeping rhizome.min_free_space available for other uses. +* `reclaimable_bytes` - space allocated by sqlite that can be reclaimed. + * `filesystem_bytes` - the measured total size of the filesystem where the rhizome store is located. diff --git a/java-api/src/org/servalproject/json/JsonField.java b/java-api/src/org/servalproject/json/JsonField.java index 48be40e7..9627d79b 100644 --- a/java-api/src/org/servalproject/json/JsonField.java +++ b/java-api/src/org/servalproject/json/JsonField.java @@ -1,5 +1,8 @@ package org.servalproject.json; +import java.util.HashMap; +import java.util.Map; + public class JsonField { public final String name; public final boolean required; @@ -10,4 +13,26 @@ public class JsonField { this.required = required; this.factory = factory; } + + public static MapBuilder mapBuilder(){ + return new MapBuilder(); + } + + public static class MapBuilder { + private Map fields = new HashMap<>(); + + public MapBuilder addField(String name, boolean required, Class type){ + fields.put(name, new JsonField(name, required, new JsonObjectHelper.ConstructorFactory(type))); + return this; + } + + public MapBuilder addField(String name, boolean required, JsonObjectHelper.Factory factory){ + fields.put(name, new JsonField(name, required, factory)); + return this; + } + + public Map build(){ + return fields; + } + } } diff --git a/java-api/src/org/servalproject/json/JsonObjectHelper.java b/java-api/src/org/servalproject/json/JsonObjectHelper.java index c6107b1e..d1dd4046 100644 --- a/java-api/src/org/servalproject/json/JsonObjectHelper.java +++ b/java-api/src/org/servalproject/json/JsonObjectHelper.java @@ -87,14 +87,14 @@ public class JsonObjectHelper { } public abstract static class ObjectFactory implements Factory{ - protected final Map columnMap = new HashMap<>(); + private final Map columnMap; - protected void add(String name, boolean required, Factory factory){ - columnMap.put(name, new JsonField(name, required, factory)); + protected ObjectFactory(Map columnMap) { + this.columnMap = columnMap; } @Override - public Object create(JsonParser parser, JsonParser.ValueType type) throws IOException, JsonParser.JsonParseException { + public T create(JsonParser parser, JsonParser.ValueType type) throws IOException, JsonParser.JsonParseException { if (type == JsonParser.ValueType.Null) return null; diff --git a/java-api/src/org/servalproject/servaldna/ServalDClient.java b/java-api/src/org/servalproject/servaldna/ServalDClient.java index ab2d9730..8bbc8aa6 100644 --- a/java-api/src/org/servalproject/servaldna/ServalDClient.java +++ b/java-api/src/org/servalproject/servaldna/ServalDClient.java @@ -21,6 +21,7 @@ package org.servalproject.servaldna; import org.servalproject.codec.Base64; +import org.servalproject.json.JsonParser; import org.servalproject.servaldna.keyring.KeyringCommon; import org.servalproject.servaldna.keyring.KeyringIdentity; import org.servalproject.servaldna.keyring.KeyringIdentityList; @@ -35,6 +36,7 @@ import org.servalproject.servaldna.meshms.MeshMSMessageList; import org.servalproject.servaldna.meshms.MeshMSStatus; import org.servalproject.servaldna.rhizome.RhizomeBundleList; import org.servalproject.servaldna.rhizome.RhizomeCommon; +import org.servalproject.servaldna.rhizome.RhizomeDiskStatus; import org.servalproject.servaldna.rhizome.RhizomeEncryptionException; import org.servalproject.servaldna.rhizome.RhizomeException; import org.servalproject.servaldna.rhizome.RhizomeFakeManifestException; @@ -115,6 +117,10 @@ public class ServalDClient implements ServalDHttpConnectionFactory { return list; } + public RhizomeDiskStatus rhizomeDiskStatus() throws ServalDInterfaceException, IOException, JsonParser.JsonParseException { + return RhizomeCommon.rhizomeStatus(this); + } + public RhizomeBundleList rhizomeListBundles() throws ServalDInterfaceException, IOException { RhizomeBundleList list = new RhizomeBundleList(this); diff --git a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java index a797fe23..2b139b2f 100644 --- a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java +++ b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java @@ -21,9 +21,11 @@ package org.servalproject.servaldna.rhizome; +import org.servalproject.json.JsonParser; import org.servalproject.servaldna.BundleId; import org.servalproject.servaldna.BundleSecret; import org.servalproject.servaldna.ContentType; +import org.servalproject.servaldna.HttpRequest; import org.servalproject.servaldna.PostHelper; import org.servalproject.servaldna.ServalDHttpConnectionFactory; import org.servalproject.servaldna.ServalDInterfaceException; @@ -354,6 +356,12 @@ public class RhizomeCommon } } + public static RhizomeDiskStatus rhizomeStatus(ServalDHttpConnectionFactory connector) throws ServalDInterfaceException, IOException, JsonParser.JsonParseException { + HttpRequest request = new HttpRequest("GET", "/restful/rhizome/storestatus.json"); + request.connect(connector); + return RhizomeDiskStatus.factory.create(request.parser, request.parser.parse()); + } + public static String quoteString(String unquoted) { if (unquoted == null) diff --git a/java-api/src/org/servalproject/servaldna/rhizome/RhizomeDiskStatus.java b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeDiskStatus.java new file mode 100644 index 00000000..96117564 --- /dev/null +++ b/java-api/src/org/servalproject/servaldna/rhizome/RhizomeDiskStatus.java @@ -0,0 +1,97 @@ +package org.servalproject.servaldna.rhizome; + +import org.servalproject.json.JsonField; +import org.servalproject.json.JsonObjectHelper; + +import java.util.Map; +import java.util.UUID; + +public class RhizomeDiskStatus { + public final String rhizomeDir; + public final UUID rhizomeUUID; + public final long fileCount; + public final long internalBytes; + public final long externalBytes; + public final long overheadBytes; + public final long usedBytes; + public final long availableBytes; + public final long reclaimableBytes; + public final long filesystemBytes; + public final long filesystemFreeBytes; + + public RhizomeDiskStatus( + String rhizomeDir, + UUID rhizomeUUID, + long fileCount, + long internalBytes, + long externalBytes, + long overheadBytes, + long usedBytes, + long availableBytes, + long reclaimableBytes, + long filesystemBytes, + long filesystemFreeBytes + ) { + this.rhizomeDir = rhizomeDir; + this.rhizomeUUID = rhizomeUUID; + this.fileCount = fileCount; + this.internalBytes = internalBytes; + this.externalBytes = externalBytes; + this.overheadBytes = overheadBytes; + this.usedBytes = usedBytes; + this.availableBytes = availableBytes; + this.reclaimableBytes = reclaimableBytes; + this.filesystemBytes = filesystemBytes; + this.filesystemFreeBytes = filesystemFreeBytes; + } + + @Override + public String toString() { + return "RhizomeDiskStatus{" + + "rhizomeDir='" + rhizomeDir + '\'' + + ", rhizomeUUID=" + rhizomeUUID + + ", fileCount=" + fileCount + + ", internalBytes=" + internalBytes + + ", externalBytes=" + externalBytes + + ", overheadBytes=" + overheadBytes + + ", usedBytes=" + usedBytes + + ", availableBytes=" + availableBytes + + ", reclaimableBytes=" + reclaimableBytes + + ", filesystemBytes=" + filesystemBytes + + ", filesystemFreeBytes=" + filesystemFreeBytes + + '}'; + } + + private static Map fields = JsonField.mapBuilder() + .addField("rhizome_dir", true, JsonObjectHelper.StringFactory) + .addField("rhizome_uuid", true, JsonObjectHelper.StringFactory) + .addField("file_count", true, JsonObjectHelper.LongFactory) + .addField("internal_bytes", true, JsonObjectHelper.LongFactory) + .addField("external_bytes", true, JsonObjectHelper.LongFactory) + .addField("overhead_bytes", true, JsonObjectHelper.LongFactory) + .addField("used_bytes", true, JsonObjectHelper.LongFactory) + .addField("available_bytes", true, JsonObjectHelper.LongFactory) + .addField("reclaimable_bytes", true, JsonObjectHelper.LongFactory) + .addField("filesystem_bytes", true, JsonObjectHelper.LongFactory) + .addField("filesystem_free_bytes", true, JsonObjectHelper.LongFactory) + .build(); + + public static JsonObjectHelper.ObjectFactory factory = new JsonObjectHelper.ObjectFactory(fields) { + @Override + public RhizomeDiskStatus create(Map values) { + return new RhizomeDiskStatus( + (String)values.get("rhizome_dir"), + UUID.fromString((String)values.get("rhizome_uuid")), + (Long)values.get("file_count"), + (Long)values.get("internal_bytes"), + (Long)values.get("external_bytes"), + (Long)values.get("overhead_bytes"), + (Long)values.get("used_bytes"), + (Long)values.get("available_bytes"), + (Long)values.get("reclaimable_bytes"), + (Long)values.get("filesystem_bytes"), + (Long)values.get("filesystem_free_bytes") + ); + } + }; +} diff --git a/java-api/test/org/servalproject/test/Rhizome.java b/java-api/test/org/servalproject/test/Rhizome.java index b6a8d13a..fc8f6be5 100644 --- a/java-api/test/org/servalproject/test/Rhizome.java +++ b/java-api/test/org/servalproject/test/Rhizome.java @@ -34,6 +34,7 @@ import org.servalproject.servaldna.ServerControl; import org.servalproject.servaldna.BundleId; import org.servalproject.servaldna.BundleSecret; import org.servalproject.servaldna.SubscriberId; +import org.servalproject.servaldna.rhizome.RhizomeDiskStatus; import org.servalproject.servaldna.rhizome.RhizomeManifest; import org.servalproject.servaldna.rhizome.RhizomeIncompleteManifest; import org.servalproject.servaldna.rhizome.RhizomeImportStatus; @@ -282,6 +283,22 @@ public class Rhizome { System.exit(0); } + private static void rhizome_disk_status() { + try { + ServalDClient client = new ServerControl().getRestfulClient();; + RhizomeDiskStatus status = client.rhizomeDiskStatus(); + System.out.println(status.toString()); + System.exit(0); + } catch (ServalDInterfaceException e) { + System.out.println(e.toString()); + } catch (JsonParser.JsonParseException e) { + System.out.println(e.toString()); + } catch (IOException e) { + System.out.println(e.toString()); + } + System.exit(1); + } + public static void main(String... args) { if (args.length < 1) @@ -308,6 +325,8 @@ public class Rhizome { ); else if (methodName.equals("rhizome-import")) rhizome_import(args[1], args[2]); + else if (methodName.equals("rhizome-disk-status")) + rhizome_disk_status(); } catch (Exception e) { e.printStackTrace(); System.exit(1); @@ -315,4 +334,5 @@ public class Rhizome { System.err.println("No such command: " + methodName); System.exit(1); } + } diff --git a/rhizome_cli.c b/rhizome_cli.c index dbb19f00..0e5b880e 100644 --- a/rhizome_cli.c +++ b/rhizome_cli.c @@ -492,9 +492,17 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context verify_bundles(); } struct rhizome_cleanup_report report; + if (clean && rhizome_cleanup(&report) == -1) + return -1; + if (rhizome_store_space_usage(&report.space_used)!=RHIZOME_PAYLOAD_STATUS_EMPTY) + return -1; + + cli_field_name(context, "rhizome_dir", ":"); + cli_put_string(context, rhizome_database.dir_path, "\n"); + cli_field_name(context, "rhizome_uuid", ":"); + cli_put_string(context, alloca_uuid_str(rhizome_database.uuid), "\n"); + if (clean){ - if (rhizome_cleanup(&report) == -1) - return -1; cli_field_name(context, "deleted_stale_incoming_files", ":"); cli_put_long(context, report.deleted_stale_incoming_files, "\n"); cli_field_name(context, "deleted_orphan_files", ":"); @@ -504,8 +512,6 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context cli_field_name(context, "deleted_orphan_manifests", ":"); cli_put_long(context, report.deleted_orphan_manifests, "\n"); } - if (rhizome_store_space_usage(&report.space_used)!=RHIZOME_PAYLOAD_STATUS_EMPTY) - return -1; cli_field_name(context, "file_count", ":"); cli_put_long(context, report.space_used.file_count, "\n"); cli_field_name(context, "file_size_bytes", ":"); @@ -520,7 +526,9 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context ((report.space_used.content_limit_bytes == UINT64_MAX) ? 0 : report.space_used.content_bytes), "\n"); cli_field_name(context, "reclaimable_bytes", ":"); cli_put_long(context, report.space_used.db_available_pages * report.space_used.db_page_size, "\n"); - cli_field_name(context, "free_space_bytes", ":"); + cli_field_name(context, "filesystem_bytes", ":"); + cli_put_long(context, report.space_used.filesystem_bytes, "\n"); + cli_field_name(context, "filesystem_free_bytes", ":"); cli_put_long(context, report.space_used.filesystem_free_bytes, "\n"); return 0; } diff --git a/rhizome_restful.c b/rhizome_restful.c index dedcebba..4a263688 100644 --- a/rhizome_restful.c +++ b/rhizome_restful.c @@ -1258,14 +1258,17 @@ static int rhizome_disk_status_content_chunk(struct http_request *hr, strbuf b) strbuf_json_string(b, alloca_uuid_str(rhizome_database.uuid)); strbuf_puts(b, ",\n"); - strbuf_sprintf(b, "\"external_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.external_bytes); - strbuf_sprintf(b, "\"db_page_size\":%"PRIu64",\n", r->u.status.rhizome_space.db_page_size); - strbuf_sprintf(b, "\"db_total_pages\":%"PRIu64",\n", r->u.status.rhizome_space.db_total_pages); - strbuf_sprintf(b, "\"db_available_pages\":%"PRIu64",\n", r->u.status.rhizome_space.db_available_pages); - strbuf_sprintf(b, "\"content_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.content_bytes); - strbuf_sprintf(b, "\"content_limit_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.content_limit_bytes); - strbuf_sprintf(b, "\"filesystem_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.filesystem_bytes); - strbuf_sprintf(b, "\"filesystem_free_bytes\":%"PRIu64"\n}", r->u.status.rhizome_space.filesystem_free_bytes); + struct rhizome_space_report *space = &r->u.status.rhizome_space; + + strbuf_sprintf(b, "\"file_count\":%"PRIu64",\n", space->file_count); + strbuf_sprintf(b, "\"internal_bytes\":%"PRIu64",\n", space->internal_bytes); + strbuf_sprintf(b, "\"external_bytes\":%"PRIu64",\n", space->external_bytes); + strbuf_sprintf(b, "\"overhead_bytes\":%"PRIu64",\n", space->content_bytes - (space->external_bytes + space->internal_bytes)); + strbuf_sprintf(b, "\"used_bytes\":%"PRIu64",\n", space->content_bytes); + strbuf_sprintf(b, "\"available_bytes\":%"PRIu64",\n", space->content_limit_bytes - space->content_bytes); + strbuf_sprintf(b, "\"reclaimable_bytes\":%"PRIu64",\n", space->db_available_pages * space->db_page_size); + strbuf_sprintf(b, "\"filesystem_bytes\":%"PRIu64",\n", space->filesystem_bytes); + strbuf_sprintf(b, "\"filesystem_free_bytes\":%"PRIu64"\n}", space->filesystem_free_bytes); return 0; } diff --git a/tests/rhizomejava b/tests/rhizomejava index 0270bf90..de9db5da 100755 --- a/tests/rhizomejava +++ b/tests/rhizomejava @@ -50,6 +50,17 @@ teardown() { report_all_servald_servers } +doc_RhizomeDiskStatus="Java API Rhizome disk status" +setup_RhizomeDiskStatus() { + setup + executeOk_servald config set rhizome.max_blob_size 1024 + rhizome_add_files --size=512 file1 --size=2048 file2 +} +test_RhizomeDiskStatus() { + executeJavaOk --stderr org.servalproject.test.Rhizome rhizome-disk-status + tfw_cat --stdout --stderr +} + doc_RhizomeList="Java API Rhizome list 100 bundles" setup_RhizomeList() { setup diff --git a/tests/rhizomerestful b/tests/rhizomerestful index a20ba721..ffbb9a08 100755 --- a/tests/rhizomerestful +++ b/tests/rhizomerestful @@ -1159,6 +1159,11 @@ doc_RhizomeStatus="REST API Rhizome storage status" setup_RhizomeStatus() { export SERVALD_FAKE_SPACE_LIMIT=2M setup + executeOk_servald config \ + set rhizome.max_blob_size 1024 \ + set rhizome.min_free_space 65536 \ + sync + rhizome_add_files --size=512 file1 --size=2048 file2 } test_RhizomeStatus() { rest_request GET "/restful/rhizome/storestatus.json"