ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-19 21:27:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add Java API for restful storage information request

Browse Source
This commit is contained in:
Jeremy Lakeman 2018-06-06 16:56:51 +09:30
parent 4aeb6545ad
commit 18bc162361
11 changed files with 212 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
doc/REST-API-Rhizome.md
View File

@ -1145,26 +1145,27 @@ Fetch on the current disk usage of the rhizome store.
The results will be a single json object with the following fields;
* `file_count` - the number of file payloads currently stored.
* `internal_bytes` - the total size of all payloads stored inside the
sqlite database
* `external_bytes` - the total size of all payloads larger than
rhizome.max_blob_size, that have been stored outside of sqlite, in the
rhizome blob folder.
* `db_page_size` - the size of disk pages returned by sqlite.
* `overhead_bytes` - the total disk space used by manifests or sqlite.
* `db_total_pages` - the number of disk pages in the sqlite database file.
* `used_bytes` - the total bytes of space used in the sqlite database, and
in payloads stored outside of sqlite.
* `db_available_pages` - the number of disk pages in the sqlite database file
that have been allocated but are not currently in use.
* `content_bytes` - the total bytes of space used in the sqlite database, and
in payloads stored outside of sqlite. This should be equal to;
db_page_size * (db_total_pages - db_available_pages) + external_bytes
* `content_limit_bytes` - the calculated storage limit that is being applied.
This will be the smallest of the configured rhizome.database_size or the
* `available_bytes` - the remaining space for storing additional file payloads.
Limited by the smallest of the configured rhizome.database_size or the
maximum we can store while keeping rhizome.min_free_space available for
other uses.
* `reclaimable_bytes` - space allocated by sqlite that can be reclaimed.
* `filesystem_bytes` - the measured total size of the filesystem where the
rhizome store is located.

25
java-api/src/org/servalproject/json/JsonField.java
View File

@ -1,5 +1,8 @@
package org.servalproject.json;
import java.util.HashMap;
import java.util.Map;
public class JsonField {
public final String name;
public final boolean required;
@ -10,4 +13,26 @@ public class JsonField {
this.required = required;
this.factory = factory;
}
public static MapBuilder mapBuilder(){
return new MapBuilder();
}
public static class MapBuilder {
private Map<String, JsonField> fields = new HashMap<>();
public MapBuilder addField(String name, boolean required, Class<?> type){
fields.put(name, new JsonField(name, required, new JsonObjectHelper.ConstructorFactory(type)));
return this;
}
public MapBuilder addField(String name, boolean required, JsonObjectHelper.Factory factory){
fields.put(name, new JsonField(name, required, factory));
return this;
}
public Map<String, JsonField> build(){
return fields;
}
}
}

8
java-api/src/org/servalproject/json/JsonObjectHelper.java
View File

@ -87,14 +87,14 @@ public class JsonObjectHelper {
}
public abstract static class ObjectFactory<T> implements Factory{
protected final Map<String, JsonField> columnMap = new HashMap<>();
private final Map<String, JsonField> columnMap;
protected void add(String name, boolean required, Factory factory){
columnMap.put(name, new JsonField(name, required, factory));
protected ObjectFactory(Map<String, JsonField> columnMap) {
this.columnMap = columnMap;
}
@Override
public Object create(JsonParser parser, JsonParser.ValueType type) throws IOException, JsonParser.JsonParseException {
public T create(JsonParser parser, JsonParser.ValueType type) throws IOException, JsonParser.JsonParseException {
if (type == JsonParser.ValueType.Null)
return null;

6
java-api/src/org/servalproject/servaldna/ServalDClient.java
View File

@ -21,6 +21,7 @@
package org.servalproject.servaldna;
import org.servalproject.codec.Base64;
import org.servalproject.json.JsonParser;
import org.servalproject.servaldna.keyring.KeyringCommon;
import org.servalproject.servaldna.keyring.KeyringIdentity;
import org.servalproject.servaldna.keyring.KeyringIdentityList;
@ -35,6 +36,7 @@ import org.servalproject.servaldna.meshms.MeshMSMessageList;
import org.servalproject.servaldna.meshms.MeshMSStatus;
import org.servalproject.servaldna.rhizome.RhizomeBundleList;
import org.servalproject.servaldna.rhizome.RhizomeCommon;
import org.servalproject.servaldna.rhizome.RhizomeDiskStatus;
import org.servalproject.servaldna.rhizome.RhizomeEncryptionException;
import org.servalproject.servaldna.rhizome.RhizomeException;
import org.servalproject.servaldna.rhizome.RhizomeFakeManifestException;
@ -115,6 +117,10 @@ public class ServalDClient implements ServalDHttpConnectionFactory {
return list;
}
public RhizomeDiskStatus rhizomeDiskStatus() throws ServalDInterfaceException, IOException, JsonParser.JsonParseException {
return RhizomeCommon.rhizomeStatus(this);
}
public RhizomeBundleList rhizomeListBundles() throws ServalDInterfaceException, IOException
{
RhizomeBundleList list = new RhizomeBundleList(this);

8
java-api/src/org/servalproject/servaldna/rhizome/RhizomeCommon.java
View File

@ -21,9 +21,11 @@
package org.servalproject.servaldna.rhizome;
import org.servalproject.json.JsonParser;
import org.servalproject.servaldna.BundleId;
import org.servalproject.servaldna.BundleSecret;
import org.servalproject.servaldna.ContentType;
import org.servalproject.servaldna.HttpRequest;
import org.servalproject.servaldna.PostHelper;
import org.servalproject.servaldna.ServalDHttpConnectionFactory;
import org.servalproject.servaldna.ServalDInterfaceException;
@ -354,6 +356,12 @@ public class RhizomeCommon
}
}
public static RhizomeDiskStatus rhizomeStatus(ServalDHttpConnectionFactory connector) throws ServalDInterfaceException, IOException, JsonParser.JsonParseException {
HttpRequest request = new HttpRequest("GET", "/restful/rhizome/storestatus.json");
request.connect(connector);
return RhizomeDiskStatus.factory.create(request.parser, request.parser.parse());
}
public static String quoteString(String unquoted)
{
if (unquoted == null)

97
java-api/src/org/servalproject/servaldna/rhizome/RhizomeDiskStatus.java Normal file
View File

@ -0,0 +1,97 @@
package org.servalproject.servaldna.rhizome;
import org.servalproject.json.JsonField;
import org.servalproject.json.JsonObjectHelper;
import java.util.Map;
import java.util.UUID;
public class RhizomeDiskStatus {
public final String rhizomeDir;
public final UUID rhizomeUUID;
public final long fileCount;
public final long internalBytes;
public final long externalBytes;
public final long overheadBytes;
public final long usedBytes;
public final long availableBytes;
public final long reclaimableBytes;
public final long filesystemBytes;
public final long filesystemFreeBytes;
public RhizomeDiskStatus(
String rhizomeDir,
UUID rhizomeUUID,
long fileCount,
long internalBytes,
long externalBytes,
long overheadBytes,
long usedBytes,
long availableBytes,
long reclaimableBytes,
long filesystemBytes,
long filesystemFreeBytes
) {
this.rhizomeDir = rhizomeDir;
this.rhizomeUUID = rhizomeUUID;
this.fileCount = fileCount;
this.internalBytes = internalBytes;
this.externalBytes = externalBytes;
this.overheadBytes = overheadBytes;
this.usedBytes = usedBytes;
this.availableBytes = availableBytes;
this.reclaimableBytes = reclaimableBytes;
this.filesystemBytes = filesystemBytes;
this.filesystemFreeBytes = filesystemFreeBytes;
}
@Override
public String toString() {
return "RhizomeDiskStatus{" +
"rhizomeDir='" + rhizomeDir + '\'' +
", rhizomeUUID=" + rhizomeUUID +
", fileCount=" + fileCount +
", internalBytes=" + internalBytes +
", externalBytes=" + externalBytes +
", overheadBytes=" + overheadBytes +
", usedBytes=" + usedBytes +
", availableBytes=" + availableBytes +
", reclaimableBytes=" + reclaimableBytes +
", filesystemBytes=" + filesystemBytes +
", filesystemFreeBytes=" + filesystemFreeBytes +
'}';
}
private static Map<String, JsonField> fields = JsonField.mapBuilder()
.addField("rhizome_dir", true, JsonObjectHelper.StringFactory)
.addField("rhizome_uuid", true, JsonObjectHelper.StringFactory)
.addField("file_count", true, JsonObjectHelper.LongFactory)
.addField("internal_bytes", true, JsonObjectHelper.LongFactory)
.addField("external_bytes", true, JsonObjectHelper.LongFactory)
.addField("overhead_bytes", true, JsonObjectHelper.LongFactory)
.addField("used_bytes", true, JsonObjectHelper.LongFactory)
.addField("available_bytes", true, JsonObjectHelper.LongFactory)
.addField("reclaimable_bytes", true, JsonObjectHelper.LongFactory)
.addField("filesystem_bytes", true, JsonObjectHelper.LongFactory)
.addField("filesystem_free_bytes", true, JsonObjectHelper.LongFactory)
.build();
public static JsonObjectHelper.ObjectFactory<RhizomeDiskStatus> factory = new JsonObjectHelper.ObjectFactory<RhizomeDiskStatus>(fields) {
@Override
public RhizomeDiskStatus create(Map<String, Object> values) {
return new RhizomeDiskStatus(
(String)values.get("rhizome_dir"),
UUID.fromString((String)values.get("rhizome_uuid")),
(Long)values.get("file_count"),
(Long)values.get("internal_bytes"),
(Long)values.get("external_bytes"),
(Long)values.get("overhead_bytes"),
(Long)values.get("used_bytes"),
(Long)values.get("available_bytes"),
(Long)values.get("reclaimable_bytes"),
(Long)values.get("filesystem_bytes"),
(Long)values.get("filesystem_free_bytes")
);
}
};
}

20
java-api/test/org/servalproject/test/Rhizome.java
View File

@ -34,6 +34,7 @@ import org.servalproject.servaldna.ServerControl;
import org.servalproject.servaldna.BundleId;
import org.servalproject.servaldna.BundleSecret;
import org.servalproject.servaldna.SubscriberId;
import org.servalproject.servaldna.rhizome.RhizomeDiskStatus;
import org.servalproject.servaldna.rhizome.RhizomeManifest;
import org.servalproject.servaldna.rhizome.RhizomeIncompleteManifest;
import org.servalproject.servaldna.rhizome.RhizomeImportStatus;
@ -282,6 +283,22 @@ public class Rhizome {
System.exit(0);
}
private static void rhizome_disk_status() {
try {
ServalDClient client = new ServerControl().getRestfulClient();;
RhizomeDiskStatus status = client.rhizomeDiskStatus();
System.out.println(status.toString());
System.exit(0);
} catch (ServalDInterfaceException e) {
System.out.println(e.toString());
} catch (JsonParser.JsonParseException e) {
System.out.println(e.toString());
} catch (IOException e) {
System.out.println(e.toString());
}
System.exit(1);
}
public static void main(String... args)
{
if (args.length < 1)
@ -308,6 +325,8 @@ public class Rhizome {
);
else if (methodName.equals("rhizome-import"))
rhizome_import(args[1], args[2]);
else if (methodName.equals("rhizome-disk-status"))
rhizome_disk_status();
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
@ -315,4 +334,5 @@ public class Rhizome {
System.err.println("No such command: " + methodName);
System.exit(1);
}
}

18
rhizome_cli.c
View File

@ -492,9 +492,17 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context
verify_bundles();
}
struct rhizome_cleanup_report report;
if (clean && rhizome_cleanup(&report) == -1)
return -1;
if (rhizome_store_space_usage(&report.space_used)!=RHIZOME_PAYLOAD_STATUS_EMPTY)
return -1;
cli_field_name(context, "rhizome_dir", ":");
cli_put_string(context, rhizome_database.dir_path, "\n");
cli_field_name(context, "rhizome_uuid", ":");
cli_put_string(context, alloca_uuid_str(rhizome_database.uuid), "\n");
if (clean){
if (rhizome_cleanup(&report) == -1)
return -1;
cli_field_name(context, "deleted_stale_incoming_files", ":");
cli_put_long(context, report.deleted_stale_incoming_files, "\n");
cli_field_name(context, "deleted_orphan_files", ":");
@ -504,8 +512,6 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context
cli_field_name(context, "deleted_orphan_manifests", ":");
cli_put_long(context, report.deleted_orphan_manifests, "\n");
}
if (rhizome_store_space_usage(&report.space_used)!=RHIZOME_PAYLOAD_STATUS_EMPTY)
return -1;
cli_field_name(context, "file_count", ":");
cli_put_long(context, report.space_used.file_count, "\n");
cli_field_name(context, "file_size_bytes", ":");
@ -520,7 +526,9 @@ static int app_rhizome_clean(const struct cli_parsed *parsed, struct cli_context
((report.space_used.content_limit_bytes == UINT64_MAX) ? 0 : report.space_used.content_bytes), "\n");
cli_field_name(context, "reclaimable_bytes", ":");
cli_put_long(context, report.space_used.db_available_pages * report.space_used.db_page_size, "\n");
cli_field_name(context, "free_space_bytes", ":");
cli_field_name(context, "filesystem_bytes", ":");
cli_put_long(context, report.space_used.filesystem_bytes, "\n");
cli_field_name(context, "filesystem_free_bytes", ":");
cli_put_long(context, report.space_used.filesystem_free_bytes, "\n");
return 0;
}

19
rhizome_restful.c
View File

@ -1258,14 +1258,17 @@ static int rhizome_disk_status_content_chunk(struct http_request *hr, strbuf b)
strbuf_json_string(b, alloca_uuid_str(rhizome_database.uuid));
strbuf_puts(b, ",\n");
strbuf_sprintf(b, "\"external_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.external_bytes);
strbuf_sprintf(b, "\"db_page_size\":%"PRIu64",\n", r->u.status.rhizome_space.db_page_size);
strbuf_sprintf(b, "\"db_total_pages\":%"PRIu64",\n", r->u.status.rhizome_space.db_total_pages);
strbuf_sprintf(b, "\"db_available_pages\":%"PRIu64",\n", r->u.status.rhizome_space.db_available_pages);
strbuf_sprintf(b, "\"content_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.content_bytes);
strbuf_sprintf(b, "\"content_limit_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.content_limit_bytes);
strbuf_sprintf(b, "\"filesystem_bytes\":%"PRIu64",\n", r->u.status.rhizome_space.filesystem_bytes);
strbuf_sprintf(b, "\"filesystem_free_bytes\":%"PRIu64"\n}", r->u.status.rhizome_space.filesystem_free_bytes);
struct rhizome_space_report *space = &r->u.status.rhizome_space;
strbuf_sprintf(b, "\"file_count\":%"PRIu64",\n", space->file_count);
strbuf_sprintf(b, "\"internal_bytes\":%"PRIu64",\n", space->internal_bytes);
strbuf_sprintf(b, "\"external_bytes\":%"PRIu64",\n", space->external_bytes);
strbuf_sprintf(b, "\"overhead_bytes\":%"PRIu64",\n", space->content_bytes - (space->external_bytes + space->internal_bytes));
strbuf_sprintf(b, "\"used_bytes\":%"PRIu64",\n", space->content_bytes);
strbuf_sprintf(b, "\"available_bytes\":%"PRIu64",\n", space->content_limit_bytes - space->content_bytes);
strbuf_sprintf(b, "\"reclaimable_bytes\":%"PRIu64",\n", space->db_available_pages * space->db_page_size);
strbuf_sprintf(b, "\"filesystem_bytes\":%"PRIu64",\n", space->filesystem_bytes);
strbuf_sprintf(b, "\"filesystem_free_bytes\":%"PRIu64"\n}", space->filesystem_free_bytes);
return 0;
}

11
tests/rhizomejava
View File

@ -50,6 +50,17 @@ teardown() {
report_all_servald_servers
}
doc_RhizomeDiskStatus="Java API Rhizome disk status"
setup_RhizomeDiskStatus() {
setup
executeOk_servald config set rhizome.max_blob_size 1024
rhizome_add_files --size=512 file1 --size=2048 file2
}
test_RhizomeDiskStatus() {
executeJavaOk --stderr org.servalproject.test.Rhizome rhizome-disk-status
tfw_cat --stdout --stderr
}
doc_RhizomeList="Java API Rhizome list 100 bundles"
setup_RhizomeList() {
setup

5
tests/rhizomerestful
View File

@ -1159,6 +1159,11 @@ doc_RhizomeStatus="REST API Rhizome storage status"
setup_RhizomeStatus() {
export SERVALD_FAKE_SPACE_LIMIT=2M
setup
executeOk_servald config \
set rhizome.max_blob_size 1024 \
set rhizome.min_free_space 65536 \
sync
rhizome_add_files --size=512 file1 --size=2048 file2
}
test_RhizomeStatus() {
rest_request GET "/restful/rhizome/storestatus.json"