mirror of
https://github.com/servalproject/serval-dna.git
synced 2024-12-18 20:57:56 +00:00
Add "rhizome extract file" command with tests
This commit is contained in:
Andrew Bettison
parent
6e959fd34c
commit
7ba15ccdd7
@ -1061,9 +1061,31 @@ int app_rhizome_extract_manifest(int argc, const char *const *argv, struct comma
|
||||
return ret;
|
||||
}
|
||||
|
||||
int cli_fileid(const char *arg)
|
||||
{
|
||||
return rhizome_str_is_file_hash(arg);
|
||||
}
|
||||
|
||||
int app_rhizome_extract_file(int argc, const char *const *argv, struct command_line_option *o)
|
||||
{
|
||||
return WHY("Not implemented");
|
||||
const char *fileid, *filepath;
|
||||
if (cli_arg(argc, argv, o, "fileid", &fileid, cli_fileid, NULL)
|
||||
|| cli_arg(argc, argv, o, "filepath", &filepath, NULL, "") == -1)
|
||||
return -1;
|
||||
/* Ensure the Rhizome database exists and is open */
|
||||
if (create_serval_instance_dir() == -1)
|
||||
return -1;
|
||||
rhizome_datastore_path = serval_instancepath();
|
||||
rhizome_opendb();
|
||||
/* Extract the file from the database */
|
||||
int ret = rhizome_retrieve_file(fileid, filepath);
|
||||
switch (ret) {
|
||||
case 0: ret = 1; break;
|
||||
case 1: ret = 0; break;
|
||||
case -1: break;
|
||||
default: ret = WHYF("Unsupported return value %d", ret); break;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int cli_uint(const char *arg)
|
||||
|
||||
@ -160,8 +160,10 @@ extern sqlite3 *rhizome_db;
|
||||
|
||||
int rhizome_opendb();
|
||||
int rhizome_manifest_createid(rhizome_manifest *m);
|
||||
int rhizome_strn_is_manifest_id(const char *id);
|
||||
int rhizome_str_is_manifest_id(const char *id);
|
||||
int rhizome_strn_is_manifest_id(const char *text);
|
||||
int rhizome_str_is_manifest_id(const char *text);
|
||||
int rhizome_strn_is_file_hash(const char *text);
|
||||
int rhizome_str_is_file_hash(const char *text);
|
||||
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename);
|
||||
int rhizome_manifest_sign(rhizome_manifest *m);
|
||||
int rhizome_drop_stored_file(char *id,int maximum_priority);
|
||||
@ -216,7 +218,8 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar);
|
||||
char nybltochar(int n);
|
||||
int rhizome_queue_manifest_import(rhizome_manifest *m,struct sockaddr_in *peerip);
|
||||
int rhizome_list_manifests(int limit, int offset);
|
||||
int rhizome_retrieve_manifest(const char *id, rhizome_manifest **mp);
|
||||
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp);
|
||||
int rhizome_retrieve_file(const char *fileid, const char *filepath);
|
||||
|
||||
#define RHIZOME_DONTVERIFY 0
|
||||
#define RHIZOME_VERIFY 1
|
||||
|
||||
@ -147,6 +147,21 @@ rhizome_manifest *rhizome_read_manifest_file(const char *filename, int bufferP,
|
||||
return m;
|
||||
}
|
||||
|
||||
int rhizome_strn_is_file_hash(const char *text)
|
||||
{
|
||||
int i;
|
||||
for (i = 0; i != SHA512_DIGEST_LENGTH * 2; ++i)
|
||||
if (!isxdigit(text[i]))
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
int rhizome_str_is_file_hash(const char *text)
|
||||
{
|
||||
size_t len = strlen(text);
|
||||
return len == SHA512_DIGEST_LENGTH * 2 && rhizome_strn_is_file_hash(text);
|
||||
}
|
||||
|
||||
int rhizome_hash_file(const char *filename,char *hash_out)
|
||||
{
|
||||
/* Gnarf! NaCl's crypto_hash() function needs the whole file passed in in one
|
||||
|
||||
@ -842,15 +842,13 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
|
||||
}
|
||||
|
||||
/* Retrieve a manifest from the database, given its manifest ID.
|
||||
|
||||
Returns 1 if manifest is found (new manifest is allocated and assigned to *m, caller is
|
||||
responsible for freeing).
|
||||
|
||||
Returns 0 if manifest is not found (*m is unchanged).
|
||||
|
||||
Returns -1 on error (*m is unchanged).
|
||||
*
|
||||
* Returns 1 if manifest is found (if mp != NULL then a new manifest struct is allocated, made
|
||||
* finalisable and * assigned to *mp, caller is responsible for freeing).
|
||||
* Returns 0 if manifest is not found (*mp is unchanged).
|
||||
* Returns -1 on error (*mp is unchanged).
|
||||
*/
|
||||
int rhizome_retrieve_manifest(const char *id, rhizome_manifest **mp)
|
||||
int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp)
|
||||
{
|
||||
char sqlcmd[1024];
|
||||
int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT id, manifest, version, inserttime FROM manifests WHERE id = ?");
|
||||
@ -864,7 +862,7 @@ int rhizome_retrieve_manifest(const char *id, rhizome_manifest **mp)
|
||||
sqlite3_finalize(statement);
|
||||
ret = WHY(sqlite3_errmsg(rhizome_db));
|
||||
} else {
|
||||
sqlite3_bind_text(statement, 1, id, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES * 2, SQLITE_STATIC);
|
||||
sqlite3_bind_text(statement, 1, manifestid, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES * 2, SQLITE_STATIC);
|
||||
while (sqlite3_step(statement) == SQLITE_ROW) {
|
||||
if (!( sqlite3_column_count(statement) == 4
|
||||
&& sqlite3_column_type(statement, 0) == SQLITE_TEXT
|
||||
@ -877,28 +875,33 @@ int rhizome_retrieve_manifest(const char *id, rhizome_manifest **mp)
|
||||
}
|
||||
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
|
||||
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
|
||||
m = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0);
|
||||
if (m == NULL) {
|
||||
ret = WHY("Invalid manifest blob from database");
|
||||
} else {
|
||||
rhizome_hex_to_bytes(id, m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2);
|
||||
const char *filehashq = rhizome_manifest_get(m, "filehash", NULL, 0);
|
||||
if (filehashq == NULL)
|
||||
ret = WHY("Manifest is missing filehash line");
|
||||
else {
|
||||
memcpy(m->fileHexHash, filehashq, SHA512_DIGEST_STRING_LENGTH);
|
||||
m->fileHashedP = 1;
|
||||
if (mp) {
|
||||
m = rhizome_read_manifest_file(manifestblob, manifestblobsize, 0);
|
||||
if (m == NULL) {
|
||||
ret = WHY("Invalid manifest blob from database");
|
||||
} else {
|
||||
ret = 1;
|
||||
rhizome_hex_to_bytes(manifestid, m->cryptoSignPublic, crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES*2);
|
||||
const char *filehashq = rhizome_manifest_get(m, "filehash", NULL, 0);
|
||||
if (filehashq == NULL)
|
||||
ret = WHY("Manifest is missing filehash line");
|
||||
else {
|
||||
memcpy(m->fileHexHash, filehashq, SHA512_DIGEST_STRING_LENGTH);
|
||||
m->fileHashedP = 1;
|
||||
}
|
||||
long long versionq = rhizome_manifest_get_ll(m, "version");
|
||||
if (versionq == -1)
|
||||
ret = WHY("Manifest is missing version line");
|
||||
else
|
||||
m->version = versionq;
|
||||
long long lengthq = rhizome_manifest_get_ll(m, "filesize");
|
||||
if (lengthq == -1)
|
||||
ret = WHY("Manifest is missing filesize line");
|
||||
else
|
||||
m->fileLength = lengthq;
|
||||
}
|
||||
long long versionq = rhizome_manifest_get_ll(m, "version");
|
||||
if (versionq == -1)
|
||||
ret = WHY("Manifest is missing version line");
|
||||
else
|
||||
m->version = versionq;
|
||||
long long lengthq = rhizome_manifest_get_ll(m, "filesize");
|
||||
if (lengthq == -1)
|
||||
ret = WHY("Manifest is missing filesize line");
|
||||
else
|
||||
m->fileLength = lengthq;
|
||||
}
|
||||
if (ret == 1) {
|
||||
cli_puts("manifestid"); cli_delim(":");
|
||||
cli_puts((const char *)sqlite3_column_text(statement, 0)); cli_delim("\n");
|
||||
cli_puts("version"); cli_delim(":");
|
||||
@ -907,13 +910,72 @@ int rhizome_retrieve_manifest(const char *id, rhizome_manifest **mp)
|
||||
cli_printf("%lld", (long long) sqlite3_column_int64(statement, 3)); cli_delim("\n");
|
||||
// Could write the manifest blob to the CLI output here, but that would require the output to
|
||||
// support byte[] fields as well as String fields.
|
||||
ret = 1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
if (ret > 0)
|
||||
if (mp && ret == 1)
|
||||
*mp = m;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Retrieve a file from the database, given its file hash.
|
||||
*
|
||||
* Returns 1 if file is found (contents are written to filepath if given).
|
||||
* Returns 0 if file is not found.
|
||||
* Returns -1 on error.
|
||||
*/
|
||||
int rhizome_retrieve_file(const char *fileid, const char *filepath)
|
||||
{
|
||||
char sqlcmd[1024];
|
||||
int n = snprintf(sqlcmd, sizeof(sqlcmd), "SELECT id, data, length FROM files WHERE id = ? AND datavalid != 0");
|
||||
if (n >= sizeof(sqlcmd))
|
||||
return WHY("SQL command too long");
|
||||
sqlite3_stmt *statement;
|
||||
const char *cmdtail;
|
||||
int ret = 0;
|
||||
if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) {
|
||||
sqlite3_finalize(statement);
|
||||
ret = WHY(sqlite3_errmsg(rhizome_db));
|
||||
} else {
|
||||
sqlite3_bind_text(statement, 1, fileid, SHA512_DIGEST_LENGTH * 2, SQLITE_STATIC);
|
||||
while (sqlite3_step(statement) == SQLITE_ROW) {
|
||||
if (!( sqlite3_column_count(statement) == 3
|
||||
&& sqlite3_column_type(statement, 0) == SQLITE_TEXT
|
||||
&& sqlite3_column_type(statement, 1) == SQLITE_BLOB
|
||||
&& sqlite3_column_type(statement, 2) == SQLITE_INTEGER
|
||||
)) {
|
||||
ret = WHY("Incorrect statement column");
|
||||
break;
|
||||
}
|
||||
const char *fileblob = (char *) sqlite3_column_blob(statement, 1);
|
||||
size_t fileblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
|
||||
long long length = sqlite3_column_int64(statement, 2);
|
||||
if (fileblobsize != length)
|
||||
ret = WHY("File length does not match blob size");
|
||||
else {
|
||||
int fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0775);
|
||||
if (fd == -1) {
|
||||
ret = WHYF("Cannot open %s for write/create", filepath);
|
||||
perror("open");
|
||||
} else if (write(fd, fileblob, length) != length) {
|
||||
ret = WHYF("Error writing %lld bytes to %s ", (long long) length, filepath);
|
||||
perror("write");
|
||||
} else if (close(fd) == -1) {
|
||||
ret = WHYF("Error flushing to %s ", filepath);
|
||||
perror("close");
|
||||
} else {
|
||||
ret = 1;
|
||||
cli_puts("filehash"); cli_delim(":");
|
||||
cli_puts((const char *)sqlite3_column_text(statement, 0)); cli_delim("\n");
|
||||
cli_puts("length"); cli_delim(":");
|
||||
cli_printf("%lld", length); cli_delim("\n");
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
sqlite3_finalize(statement);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -160,6 +160,7 @@ executeOk() {
|
||||
echo "# executeOk $*"
|
||||
_tfw_getopts executeok "$@"
|
||||
_tfw_opt_exit_status=0
|
||||
_tfw_dump_on_fail --stderr
|
||||
shift $_tfw_getopts_shift
|
||||
_tfw_execute "$@"
|
||||
}
|
||||
@ -499,12 +500,27 @@ _tfw_assert() {
|
||||
return 0
|
||||
}
|
||||
|
||||
declare -a _tfw_opt_dump_on_fail
|
||||
|
||||
_tfw_dump_on_fail() {
|
||||
for arg; do
|
||||
local _found=false
|
||||
local _f
|
||||
for _f in "${_tfw_opt_dump_on_fail[@]}"; do
|
||||
if [ "$_f" = "$arg" ]; then
|
||||
_found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
$_found || _tfw_opt_dump_on_fail+=("$arg")
|
||||
done
|
||||
}
|
||||
|
||||
_tfw_getopts() {
|
||||
local context="$1"
|
||||
shift
|
||||
_tfw_message=
|
||||
_tfw_dump_stdout_on_fail=false
|
||||
_tfw_dump_stderr_on_fail=false
|
||||
_tfw_opt_dump_on_fail=()
|
||||
_tfw_opt_error_on_fail=false
|
||||
_tfw_opt_exit_status=
|
||||
_tfw_opt_matches=
|
||||
@ -512,8 +528,9 @@ _tfw_getopts() {
|
||||
_tfw_getopts_shift=0
|
||||
while [ $# -ne 0 ]; do
|
||||
case "$context:$1" in
|
||||
*:--stdout) _tfw_dump_stdout_on_fail=true;;
|
||||
*:--stderr) _tfw_dump_stderr_on_fail=true;;
|
||||
*:--stdout) _tfw_dump_on_fail --stdout;;
|
||||
*:--stderr) _tfw_dump_on_fail --stderr;;
|
||||
assert*:--dump-on-fail=*) _tfw_dump_on_fail "${1#*=}";;
|
||||
execute:--exit-status=*) _tfw_opt_exit_status="${1#*=}";;
|
||||
assert*:--error-on-fail) _tfw_opt_error_on_fail=true;;
|
||||
assert*:--message=*) _tfw_message="${1#*=}";;
|
||||
@ -779,8 +796,7 @@ _tfw_backtrace() {
|
||||
_tfw_failexit() {
|
||||
# When exiting a test case due to a failure, log any diagnostic output that
|
||||
# has been requested.
|
||||
$_tfw_dump_stdout_on_fail && tfw_cat --stdout
|
||||
$_tfw_dump_stderr_on_fail && tfw_cat --stderr
|
||||
tfw_cat "${_tfw_opt_dump_on_fail[@]}"
|
||||
# A failure during setup or teardown is treated as an error.
|
||||
case $_tfw_phase in
|
||||
testcase)
|
||||
|
||||
@ -67,28 +67,28 @@ strip_signatures() {
|
||||
done
|
||||
}
|
||||
|
||||
extract_manifest_id() {
|
||||
extract_manifest() {
|
||||
local _var="$1"
|
||||
local _manifestfile="$2"
|
||||
local _id=$(sed -n -e '/^id=[0-9a-fA-F]\{64\}$/s/^id=//p' "$_manifestfile")
|
||||
assert --message="$_manifestfile contains valid 'id=' line" [ -n "$_id" ]
|
||||
[ -n "$_var" ] && eval $_var=$_id
|
||||
local _label="$3"
|
||||
local _rexp="$4"
|
||||
local _value=$(sed -n -e "/^$_label=$_rexp\$/s/^$_label=//p" "$_manifestfile")
|
||||
assert --message="$_manifestfile contains valid '$_label=' line" \
|
||||
--dump-on-fail="$_manifestfile" \
|
||||
[ -n "$_value" ]
|
||||
[ -n "$_var" ] && eval $_var=$_value
|
||||
}
|
||||
|
||||
extract_manifest_version() {
|
||||
local _var="$1"
|
||||
local _manifestfile="$2"
|
||||
local _version=$(sed -n -e '/^version=[0-9]\{1,\}$/s/^version=//p' "$_manifestfile")
|
||||
assert --message="$_manifestfile contains valid 'version=' line" [ -n "$_version" ]
|
||||
[ -n "$_var" ] && eval $_var=$_version
|
||||
extract_manifest_id() {
|
||||
extract_manifest "$1" "$2" id '[0-9a-fA-F]\{64\}'
|
||||
}
|
||||
|
||||
extract_manifest_filehash() {
|
||||
local _var="$1"
|
||||
local _manifestfile="$2"
|
||||
local _filehash=$(sed -n -e '/^filehash=[0-9a-fA-F]\{1,\}$/s/^filehash=//p' "$_manifestfile")
|
||||
assert --message="$_manifestfile contains valid 'filehash=' line" [ -n "$_filehash" ]
|
||||
eval $_var=$_filehash
|
||||
extract_manifest "$1" "$2" filehash '[0-9a-fA-F]\{128\}'
|
||||
}
|
||||
|
||||
extract_manifest_version() {
|
||||
extract_manifest "$1" "$2" version '[0-9]\{1,\}'
|
||||
}
|
||||
|
||||
doc_InitialEmptyList="Initial list is empty"
|
||||
@ -216,6 +216,50 @@ test_ExtractManifestInvalidID() {
|
||||
assert [ ! -e foo.manifest ]
|
||||
}
|
||||
|
||||
doc_AddThenExtractFile="Extract file after one add"
|
||||
setup_AddThenExtractFile() {
|
||||
setup_dna_rhizome
|
||||
echo "A test file" >file1
|
||||
executeOk $dna rhizome add file file1 file1.manifest
|
||||
assert_rhizome_list file1
|
||||
extract_manifest_filehash filehash file1.manifest
|
||||
}
|
||||
test_AddThenExtractFile() {
|
||||
executeOk $dna rhizome extract file $filehash file1x
|
||||
assert cmp file1 file1x
|
||||
local size=$(( $(cat file1 | wc -c) + 0 ))
|
||||
assertStdoutLineCount '==' 2
|
||||
assertStdoutGrep --matches=1 "^filehash:$filehash$"
|
||||
assertStdoutGrep --matches=1 "^length:$size$"
|
||||
}
|
||||
|
||||
doc_ExtractMissingFile="Extract non-existent file"
|
||||
setup_ExtractMissingFile() {
|
||||
setup_dna_rhizome
|
||||
filehash=0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF
|
||||
}
|
||||
test_ExtractMissingFile() {
|
||||
execute --exit-status=1 $dna rhizome extract file $filehash foo
|
||||
assertStdoutLineCount '==' 0
|
||||
assert [ ! -e foo ]
|
||||
}
|
||||
|
||||
doc_ExtractFileInvalidID="Extract file using invalid ID"
|
||||
setup_ExtractFileInvalidID() {
|
||||
setup_dna_rhizome
|
||||
}
|
||||
test_ExtractFileInvalidID() {
|
||||
execute --exit-status=255 $dna rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEx foo
|
||||
assertStdoutLineCount '==' 0
|
||||
assert [ ! -e foo ]
|
||||
execute --exit-status=255 $dna rhizome extract file 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDE foo
|
||||
assertStdoutLineCount '==' 0
|
||||
assert [ ! -e foo ]
|
||||
execute --exit-status=255 $dna rhizome extract file '' foo
|
||||
assertStdoutLineCount '==' 0
|
||||
assert [ ! -e foo ]
|
||||
}
|
||||
|
||||
doc_AddDuplicate="Add same manifest detects duplicate"
|
||||
setup_AddDuplicate() {
|
||||
setup_dna_rhizome
|
||||
|
||||
Loading…
Reference in New Issue
Block a user