ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improve Rhizome DB path configuration

Browse Source 
Change the default 'rhizome.datastore_path' to "rhizome", ie, a sub-
directory of the instance directory.

Add a new erename() operation to "os.h".

Clean up the code around the fix 39f582cca (two commits ago) for a bug
introduced in 2014: the 'rhizome.db' file was always being created in
the instance directory, regardless of the setting of the
'rhizome.datastore_path' config option.

Whenever the Rhizome database is opened (by the server or by a CLI
invocation), if the database file does not exist but there is a
'rhizome.db' file in the legacy location, then move that file (and any
accompanying "blob" and "hash" sub-directories) to the configured
location.

Update copyright notices.
This commit is contained in:
Andrew Bettison 2018-04-17 12:01:02 +09:30
parent 9ba27b28a0
commit faa0f392c5
13 changed files with 236 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
conf_schema.h
View File

@ -448,12 +448,12 @@ ATOM(bool_t, enable, 1, boolean,, "If true, server opens
ATOM(bool_t, fetch, 1, boolean,, "If false, no new bundles will be fetched from peers")
ATOM(bool_t, reliable_clock, 1, boolean,, "If false, consider the system clock to be unreliable for generating timestamps")
ATOM(bool_t, clean_on_open, 0, boolean,, "If true, Rhizome database is cleaned at start of every command")
STRING(256, datastore_path, "", str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory")
STRING(256, datastore_path, RHIZOME_DEFAULT_PATH, str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory")
ATOM(uint64_t, database_size, UINT64_MAX, uint64_scaled,, "Maximum size of database in bytes")
ATOM(uint64_t, min_free_space, 100*1024*1024, uint64_scaled,, "Minimum free space to preserve on the disk")
ATOM(uint32_t, max_blob_size, 128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs")
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.")
ATOM(uint32_t, fetch_delay_ms, 50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.")
ATOM(uint32_t, fetch_delay_ms, 50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
SUB_STRUCT(rhizome_direct, direct,)
SUB_STRUCT(rhizome_api, api,)
SUB_STRUCT(rhizome_http, http,)

65
instance.c
View File

@ -1,6 +1,7 @@
/*
Serval DNA instance paths
Copyright (C) 2012-2015 Serval Project Inc.
Copyright (C) 2016-2018 Flinders University
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -23,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <stdlib.h>
#include "instance.h"
#include "conf.h"
#include "str.h"
#include "os.h"
#include "strbuf.h"
@ -105,20 +107,21 @@ void set_instance_path(const char *path)
know_instancepath = 1;
}
static int vformf_basepath(struct __sourceloc __whence, strbuf b, const char *basepath, const char *fmt, va_list ap)
static int vformf_path(struct __sourceloc __whence, strbuf b, const char *fhs_path, const char *configured_path, const char *fmt, va_list ap)
{
if (fmt)
strbuf_va_vprintf(b, fmt, ap);
if (!strbuf_overrun(b) && (strbuf_len(b) == 0 || strbuf_str(b)[0] != '/')) {
strbuf_reset(b);
strbuf_puts(b, basepath);
if (fmt) {
if (strbuf_substr(b, -1)[0] != '/')
strbuf_putc(b, '/');
strbuf_va_vprintf(b, fmt, ap);
}
const char *ipath = instance_path();
strbuf_path_join(b, ipath ? ipath : fhs_path, NULL);
if (configured_path)
strbuf_path_join(b, configured_path, NULL);
assert(strbuf_str(b)[0] == '/');
int fmt_overrun = 0;
if (fmt) {
strbuf sb = strbuf_alloca(strbuf_size(b));
strbuf_va_vprintf(sb, fmt, ap);
strbuf_path_join(b, strbuf_str(sb), NULL);
fmt_overrun = strbuf_overrun(sb);
}
if (!strbuf_overrun(b))
if (!strbuf_overrun(b) && !fmt_overrun)
return 1;
WHYF("instance path overflow (strlen %lu, sizeof buffer %lu): %s",
(unsigned long)strbuf_count(b),
@ -127,28 +130,11 @@ static int vformf_basepath(struct __sourceloc __whence, strbuf b, const char *ba
return 0;
}
int _formf_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *basepath, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_basepath(__whence, strbuf_local(buf, bufsiz), basepath, fmt, ap);
va_end(ap);
return ret;
}
static int vformf_path(struct __sourceloc __whence, strbuf b, const char *syspath, const char *fmt, va_list ap)
{
const char *ipath = instance_path();
if (!ipath)
ipath=syspath;
return vformf_basepath(__whence, b, ipath, fmt, ap);
}
int _formf_serval_etc_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_ETC_PATH, fmt, ap);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_ETC_PATH, NULL, fmt, ap);
va_end(ap);
return ret;
}
@ -164,7 +150,7 @@ int _formf_serval_run_path(struct __sourceloc __whence, char *buf, size_t bufsiz
int _vformf_serval_run_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, va_list ap)
{
return vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH, fmt, ap);
return vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH, NULL, fmt, ap);
}
strbuf strbuf_system_log_path(strbuf sb)
@ -188,7 +174,7 @@ int _formf_serval_cache_path(struct __sourceloc __whence, char *buf, size_t bufs
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_CACHE_PATH, fmt, ap);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_CACHE_PATH, NULL, fmt, ap);
va_end(ap);
return ret;
}
@ -197,7 +183,16 @@ int _formf_rhizome_store_path(struct __sourceloc __whence, char *buf, size_t buf
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), RHIZOME_STORE_PATH, fmt, ap);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), RHIZOME_STORE_PATH, config.rhizome.datastore_path, fmt, ap);
va_end(ap);
return ret;
}
int _formf_rhizome_store_legacy_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), RHIZOME_STORE_PATH, NULL, fmt, ap);
va_end(ap);
return ret;
}
@ -206,7 +201,7 @@ int _formf_serval_tmp_path(struct __sourceloc __whence, char *buf, size_t bufsiz
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_TMP_PATH, fmt, ap);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_TMP_PATH, NULL, fmt, ap);
va_end(ap);
return ret;
}
@ -215,7 +210,7 @@ int _formf_servald_proc_path(struct __sourceloc __whence, char *buf, size_t bufs
{
va_list ap;
va_start(ap, fmt);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH "/proc", fmt, ap);
int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH "/proc", NULL, fmt, ap);
va_end(ap);
return ret;
}

43
instance.h
View File

@ -1,7 +1,7 @@
/*
Serval DNA instance paths
Copyright (C) 2014-2015 Serval Project Inc.
Copyright (C) 2016 Flinders University
Copyright (C) 2016-2018 Flinders University
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -31,8 +31,14 @@ void set_instance_path(const char *path);
int create_serval_instance_dir();
int _formf_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *basepath, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,5,6)));
/* Macros for forming the absolute paths of various files into a given char*
* buffer. Evaluates to true if the pathname fits into the provided buffer,
* false (0) otherwise (after logging an error).
*
* The "legacy" path functions/macros are exclusively used to provide
* compatibility with earlier versions of Serval DNA, and may be deleted once
* no longer needed.
*/
int _formf_serval_etc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _formf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _vformf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list);
@ -40,27 +46,30 @@ int _formf_serval_cache_path(struct __sourceloc, char *buf, size_t bufsiz, const
int _formf_serval_tmp_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _formf_servald_proc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _formf_rhizome_store_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
int _formf_rhizome_store_legacy_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5)));
#define formf_path(buf,bufsz,basepath,fmt,...) _formf_path(__WHENCE__, buf, bufsz, basepath, fmt, ##__VA_ARGS__)
#define formf_serval_etc_path(buf,bufsz,fmt,...) _formf_serval_etc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_serval_run_path(buf,bufsz,fmt,...) _formf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define vformf_serval_run_path(buf,bufsz,fmt,ap) _vformf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ap)
#define formf_serval_cache_path(buf,bufsz,fmt,...) _formf_serval_cache_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_serval_tmp_path(buf,bufsz,fmt,...) _formf_serval_tmp_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_servald_proc_path(buf,bufsz,fmt,...) _formf_servald_proc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_rhizome_store_path(buf,bufsz,fmt,...) _formf_rhizome_store_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_serval_etc_path(buf,bufsz,fmt,...) _formf_serval_etc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_serval_run_path(buf,bufsz,fmt,...) _formf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define vformf_serval_run_path(buf,bufsz,fmt,ap) _vformf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ap)
#define formf_serval_cache_path(buf,bufsz,fmt,...) _formf_serval_cache_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_serval_tmp_path(buf,bufsz,fmt,...) _formf_serval_tmp_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_servald_proc_path(buf,bufsz,fmt,...) _formf_servald_proc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_rhizome_store_path(buf,bufsz,fmt,...) _formf_rhizome_store_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
#define formf_rhizome_store_legacy_path(buf,bufsz,fmt,...) _formf_rhizome_store_legacy_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__)
/* Handy macros for forming the absolute paths of various files, using a char[]
* buffer whose declaration is in scope (so that sizeof(buf) will work).
* Evaluates to true if the pathname fits into the provided buffer, false (0)
* otherwise (after logging an error).
*/
#define FORMF_SERVAL_ETC_PATH(buf,fmt,...) formf_serval_etc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_RUN_PATH(buf,fmt,...) formf_serval_run_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_CACHE_PATH(buf,fmt,...) formf_serval_cache_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_TMP_PATH(buf,fmt,...) formf_serval_tmp_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVALD_PROC_PATH(buf,fmt,...) formf_servald_proc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_ETC_PATH(buf,fmt,...) formf_serval_etc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_RUN_PATH(buf,fmt,...) formf_serval_run_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define VFORMF_SERVAL_RUN_PATH(buf,fmt,ap) vformf_serval_run_path(buf, sizeof(buf), fmt, ap)
#define FORMF_SERVAL_CACHE_PATH(buf,fmt,...) formf_serval_cache_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVAL_TMP_PATH(buf,fmt,...) formf_serval_tmp_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_SERVALD_PROC_PATH(buf,fmt,...) formf_servald_proc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_RHIZOME_STORE_PATH(buf,fmt,...) formf_rhizome_store_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
#define FORMF_RHIZOME_STORE_LEGACY_PATH(buf,fmt,...) formf_rhizome_store_legacy_path(buf, sizeof(buf), fmt, ##__VA_ARGS__)
strbuf strbuf_system_log_path(strbuf sb);
strbuf strbuf_serval_log_path(strbuf sb);

36
os.c
View File

@ -1,6 +1,8 @@
/*
Serval Distributed Numbering Architecture (DNA)
Copyright (C) 2010 Paul Gardner-Stephen
/*
Serval DNA operating system services
Copyright (C) 2010 Paul Gardner-Stephen
Copyright (C) 2012-2015 Serval Project Inc.
Copyright (C) 2016-2018 Flinders University
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -114,6 +116,15 @@ int _mkdirsn(struct __sourceloc whence, const char *path, size_t len, mode_t mod
return -1;
}
int _erename(struct __sourceloc __whence, const char *oldpath, const char *newpath, int log_level)
{
if (log_level != LOG_LEVEL_SILENT)
LOGF(log_level, "rename %s -> %s", alloca_str_toprint(oldpath), alloca_str_toprint(newpath));
if (rename(oldpath, newpath) == -1)
return WHYF_perror("rename(%s,%s)", alloca_str_toprint(oldpath), alloca_str_toprint(newpath));
return 0;
}
time_ms_t gettime_ms()
{
struct timeval nowtv;
@ -221,6 +232,7 @@ int get_file_meta(const char *path, struct file_meta *metap)
return WHYF_perror("stat(%s)", path);
*metap = FILE_META_NONEXIST;
} else {
metap->mode = st.st_mode;
metap->size = st.st_size;
metap->mtime.tv_sec = st.st_mtime;
// Truncate to whole seconds to ensure that this code will work on file systems that only have
@ -323,6 +335,24 @@ int alter_file_meta(const char *path, const struct file_meta *origp, struct file
return 1;
}
int file_exists(const char *path)
{
struct file_meta meta;
return get_file_meta(path, &meta) != -1 && is_file_meta_exists(&meta);
}
int file_exists_is_regular(const char *path)
{
struct file_meta meta;
return get_file_meta(path, &meta) != -1 && is_file_meta_regular(&meta);
}
int file_exists_is_directory(const char *path)
{
struct file_meta meta;
return get_file_meta(path, &meta) != -1 && !is_file_meta_directory(&meta);
}
ssize_t get_self_executable_path(char *buf, size_t len)
{
#if defined(linux)

38
os.h
View File

@ -1,6 +1,7 @@
/*
Serval DNA native Operating System interface
Copyright (C) 2012 Serval Project Inc.
Copyright (C) 2012-2014 Serval Project Inc.
Copyright (C) 2017-2018 Flinders University
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -25,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#endif
#include <sys/types.h> // for off64_t
#include <sys/stat.h> // for S_IFMT etc.
#include <stdio.h> // for NULL
#include <stdlib.h>
#include <stdint.h> // for int64_t
@ -106,7 +108,9 @@ __SERVAL_DNA__OS_INLINE off64_t lseek64(int fd, off64_t offset, int whence) {
# endif
#endif
/* The "e" variants log the error before returning -1.
/* Functions to create a directory and any missing parent directories. The "e"
* variants log the error before returning -1. The "_info" variants log all
* created directories at INFO level.
*/
typedef void MKDIR_LOG_FUNC(struct __sourceloc, const char *, mode_t, void *);
MKDIR_LOG_FUNC log_info_mkdir;
@ -130,6 +134,12 @@ int _emkdirsn(struct __sourceloc, const char *path, size_t len, mode_t mode, MKD
#define emkdirs_info(path, mode) emkdirs_log((path), (mode), log_info_mkdir, NULL)
#define emkdirsn_info(path, len, mode) emkdirsn_log((path), (len), (mode), log_info_mkdir, NULL)
/* Function to rename a file, logging any error before returning -1.
*/
int _erename(struct __sourceloc, const char *oldpath, const char *newpath, int log_level);
#define erename(oldpath, newpath) _erename(__WHENCE__, (oldpath), (newpath), LOG_LEVEL_SILENT)
#define erename_info(oldpath, newpath) _erename(__WHENCE__, (oldpath), (newpath), LOG_LEVEL_INFO)
/* Read the symbolic link into the supplied buffer and add a terminating nul.
* Logs an ERROR and returns -1 if the buffer is too short to hold the link
* content and the terminating nul. If readlink(2) returns an error, then logs
@ -185,17 +195,30 @@ int malloc_read_whole_file(const char *path, unsigned char **bufp, size_t *sizp)
struct file_meta {
struct timespec mtime;
off_t size;
mode_t mode;
};
#define FILE_META_UNKNOWN ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = -1 })
#define FILE_META_UNKNOWN ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = -1, .mode = -1 })
// A non-existent file is treated as size == 0 and an impossible modification
// time, so that cmp_file_meta() will not compare it as equal with any existing
// file.
#define FILE_META_NONEXIST ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = 0 })
#define FILE_META_NONEXIST ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = 0, .mode = 0 })
__SERVAL_DNA__OS_INLINE int is_file_meta_nonexist(const struct file_meta *m) {
return m->mtime.tv_sec == -1 && m->mtime.tv_nsec == -1 && m->size == 0;
return m->mtime.tv_sec == -1 && m->mtime.tv_nsec == -1 && m->size == 0 && m->mode == 0;
}
__SERVAL_DNA__OS_INLINE int is_file_meta_exists(const struct file_meta *m) {
return m->mtime.tv_sec != -1;
}
__SERVAL_DNA__OS_INLINE int is_file_meta_regular(const struct file_meta *m) {
return is_file_meta_exists(m) && (m->mode & S_IFMT) == S_IFREG;
}
__SERVAL_DNA__OS_INLINE int is_file_meta_directory(const struct file_meta *m) {
return is_file_meta_exists(m) && (m->mode & S_IFMT) == S_IFDIR;
}
int get_file_meta(const char *path, struct file_meta *metap);
@ -205,6 +228,11 @@ int cmp_file_meta(const struct file_meta *a, const struct file_meta *b);
// by bumping the file's modification time or altering its inode.
int alter_file_meta(const char *path, const struct file_meta *origp, struct file_meta *metap);
// Convenience functions based on get_file_meta().
int file_exists(const char *path);
int file_exists_is_regular(const char *path);
int file_exists_is_directory(const char *path);
/* Fill the given buffer with the nul-terminated absolute path of the calling
* process's executable. Logs an error and returns -1 if the executable cannot
* be determined or the supplied buffer is too short. Otherwise returns the

8
rhizome.h
View File

@ -1,5 +1,6 @@
/*
Serval DNA Rhizome file distribution
Copyright (C) 2016-2018 Flinders University
Copyright (C) 2010-2014 Serval Project Inc.
Copyright (C) 2010 Paul Gardner-Stephen
@ -337,19 +338,18 @@ int rhizome_configure();
int rhizome_enabled();
int rhizome_fetch_delay_ms();
#define RHIZOME_DEFAULT_PATH "rhizome"
#define RHIZOME_BLOB_SUBDIR "blob"
#define RHIZOME_HASH_SUBDIR "hash"
struct rhizome_database{
struct rhizome_database {
char dir_path[1024];
sqlite3 *db;
serval_uuid_t uuid;
char folder[1024];
};
extern __thread struct rhizome_database rhizome_database;
#define FORMF_RHIZOME_STORE_PATH(buf,fmt,...) formf_path((buf), sizeof(buf), rhizome_database.folder, (fmt), ##__VA_ARGS__)
int rhizome_opendb();
int rhizome_close_db();
void verify_bundles();

1
rhizome_cli.c
View File

@ -1,6 +1,7 @@
/*
Serval DNA - Rhizome command line interface
Copyright (C) 2014 Serval Project Inc.
Copyright (C) 2016-2017 Flinders University
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License

147
rhizome_database.c
View File

@ -1,6 +1,7 @@
/*
Serval DNA - Rhizome database operations
Copyright (C) 2012-2014 Serval Project Inc.
Copyright (C) 2016-2018 Flinders University
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -37,7 +38,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
__thread struct rhizome_database rhizome_database = {NULL,{{{0}}},{0}};
__thread struct rhizome_database rhizome_database = {
.dir_path = "",
.db = NULL,
.uuid = SERVAL_UUID_INVALID
};
static time_ms_t rhizomeRetryLimit = -1;
int is_debug_rhizome()
@ -235,65 +241,82 @@ int rhizome_opendb()
IN();
if (sodium_init()==-1)
if (sodium_init() == -1)
RETURN(WHY("Failed to initialise libsodium"));
if (!formf_rhizome_store_path(rhizome_database.folder, sizeof rhizome_database.folder, "%s", config.rhizome.datastore_path))
RETURN (-1);
DEBUGF(rhizome, "Rhizome datastore path = %s", alloca_str_toprint(rhizome_database.folder));
if (emkdirs_info(rhizome_database.folder, 0700)==-1)
RETURN (-1);
char dbpath[1024];
if (!FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db"))
// Work out the absolute path of the directory that contains all the database
// files/subdirectories.
if (!FORMF_RHIZOME_STORE_PATH(rhizome_database.dir_path, "%s", ""))
RETURN(-1);
struct file_meta meta;
if (get_file_meta(dbpath, &meta) == -1)
DEBUGF(rhizome, "Rhizome store directory path = %s", alloca_str_toprint(rhizome_database.dir_path));
// Work out the absolute paths of the database file and subdirectories.
char dbpath[sizeof rhizome_database.dir_path];
char blobpath[sizeof rhizome_database.dir_path];
char hashpath[sizeof rhizome_database.dir_path];
char temppath[sizeof rhizome_database.dir_path];
int db_exists = 0;
{
struct file_meta dbmeta;
if ( !FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db")
|| !FORMF_RHIZOME_STORE_PATH(blobpath, RHIZOME_BLOB_SUBDIR)
|| !FORMF_RHIZOME_STORE_PATH(hashpath, RHIZOME_HASH_SUBDIR)
|| !FORMF_RHIZOME_STORE_PATH(temppath, "sqlite3tmp")
|| get_file_meta(dbpath, &dbmeta) == -1)
RETURN(-1);
db_exists = is_file_meta_exists(&dbmeta);
}
// Create missing store directory.
if (emkdirs_info(rhizome_database.dir_path, 0700) == -1)
RETURN(-1);
if (meta.mtime.tv_sec == -1 && config.rhizome.datastore_path[0]){
// Move the database after datastore path is set for the first time
// This is mostly here to transparently fix a bug where we were ignoring the config value
char src[1024];
char dest[1024];
if (formf_rhizome_store_path(src, sizeof src, "rhizome.db")
&& strcmp(dbpath, src)!=0
&& get_file_meta(src, &meta)==0
&& meta.mtime.tv_sec != -1){
INFOF("Moving rhizome store from %s to %s", src, dbpath);
if (rename(src, dbpath))
WHYF_perror("rename(%s, %s)", src, dbpath);
if (formf_rhizome_store_path(src, sizeof src, RHIZOME_BLOB_SUBDIR)
&& FORMF_RHIZOME_STORE_PATH(dest, RHIZOME_BLOB_SUBDIR)
&& rename(src, dest)
&& errno!=ENOENT){
WHYF_perror("rename(%s, %s)", src, dest);
}
if (formf_rhizome_store_path(src, sizeof src, RHIZOME_HASH_SUBDIR)
&& FORMF_RHIZOME_STORE_PATH(dest, RHIZOME_HASH_SUBDIR)
&& rename(src, dest)
&& errno!=ENOENT){
WHYF_perror("rename(%s, %s)", src, dest);
}
// If the database file does not exist, then SQLite will create it. However, in 2014 a bug was
// introduced that always created the database file in the instance directory, regardless of the
// setting of the 'rhizome.datastore_path' config option. This bug has now been fixed, so in
// order to preserve the Rhizome data store when upgrading from an earlier (buggy) version, if
// there is a database file at this "legacy" location, then move it into the correct, configured
// location, instead of creating a fresh (empty) database file.
if (!db_exists) {
char legacy_dbpath[sizeof rhizome_database.dir_path];
if (FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dbpath, "rhizome.db")
&& strcmp(legacy_dbpath, dbpath) != 0
&& file_exists(legacy_dbpath)
) {
INFOF("Recover legacy Rhizome SQLite database");
// Move the legacy database file to its correct location.
if (erename_info(legacy_dbpath, dbpath) == -1)
RETURN(-1);
// Move any legacy "blob" and "hash" subdirectories too, if they are present.
char legacy_dirpath[sizeof rhizome_database.dir_path];
if ( !file_exists(blobpath)
&& FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dirpath, RHIZOME_BLOB_SUBDIR)
&& file_exists(legacy_dirpath))
erename_info(legacy_dirpath, blobpath);
if ( !file_exists(hashpath)
&& FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dirpath, RHIZOME_HASH_SUBDIR)
&& file_exists(legacy_dirpath))
erename_info(legacy_dirpath, hashpath);
}
else
INFOF("Creating Rhizome SQLite database: %s", dbpath);
}
if (!sqlite3_temp_directory){
char tmp[1024];
if (!FORMF_RHIZOME_STORE_PATH(tmp, "sqlite3tmp"))
RETURN(-1);
if (emkdirs_info(tmp, 0700) == -1)
RETURN(-1);
sqlite3_temp_directory = sqlite3_mprintf("%s", tmp);
}
// Create missing sub-directories.
if ( emkdirs_info(blobpath, 0700) == -1
|| emkdirs_info(hashpath, 0700) == -1
|| emkdirs_info(temppath, 0700) == -1)
RETURN(-1);
sqlite3_config(SQLITE_CONFIG_LOG,sqlite_log,NULL);
// Inform SQLite of its temporary directory.
assert(!sqlite3_temp_directory);
sqlite3_temp_directory = sqlite3_mprintf("%s", temppath);
if (sqlite3_open(dbpath,&rhizome_database.db)){
// Set up SQLite logging.
sqlite3_config(SQLITE_CONFIG_LOG, sqlite_log, NULL);
// Open the SQLite database file, creating it if necessary.
if (sqlite3_open(dbpath, &rhizome_database.db)){
RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_database.db)));
}
sqlite3_trace_v2(rhizome_database.db, SQLITE_TRACE_STMT, sqlite_trace_callback, NULL);
@ -304,7 +327,7 @@ int rhizome_opendb()
/* Read Rhizome configuration */
DEBUGF(rhizome, "Rhizome will use %"PRIu64"B of storage for its database.", (uint64_t) config.rhizome.database_size);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
uint64_t version;
@ -358,7 +381,7 @@ int rhizome_opendb()
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
}
if (version<2 && meta.mtime.tv_sec != -1){
if (version<2 && db_exists){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END);
@ -369,28 +392,28 @@ int rhizome_opendb()
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
}
if (version<4 && meta.mtime.tv_sec != -1){
if (version<4 && db_exists){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
}
if (version<5 && meta.mtime.tv_sec != -1){
if (version<5 && db_exists){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
}
if (version<6 && meta.mtime.tv_sec != -1){
if (version<6 && db_exists){
// we've always been at war with eurasia
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPLIST; ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPMEMBERSHIPS; ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS VERIFICATIONS; ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS FILEMANIFESTS;", END);
}
if (version<7 && meta.mtime.tv_sec != -1){
if (version<7 && db_exists){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE FILES ADD COLUMN last_verified integer;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=7;", END);
}
if (version<8){
if (meta.mtime.tv_sec != -1)
if (db_exists)
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN manifest_hash text collate nocase;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFEST_HASH ON MANIFESTS(manifest_hash);", END);
@ -448,7 +471,7 @@ int rhizome_close_db()
IN();
if (rhizome_database.db) {
rhizome_cache_close();
if (!sqlite3_get_autocommit(rhizome_database.db)){
WHY("Uncommitted transaction!");
sqlite_exec_void("ROLLBACK;", END);
@ -461,11 +484,11 @@ int rhizome_close_db()
int r = sqlite3_close(rhizome_database.db);
if (r != SQLITE_OK)
RETURN(WHYF("Failed to close sqlite database, %s",sqlite3_errmsg(rhizome_database.db)));
}
rhizome_database.db=NULL;
if (sqlite3_temp_directory)
rhizome_database.db = NULL;
assert(sqlite3_temp_directory);
sqlite3_free(sqlite3_temp_directory);
sqlite3_temp_directory=NULL;
sqlite3_temp_directory = NULL;
}
RETURN(0);
OUT();
}

4
rhizome_store.c
View File

@ -195,8 +195,8 @@ static uint64_t store_get_free_space()
#if defined(HAVE_SYS_STATVFS_H) || (defined(HAVE_SYS_STAT_H) && defined(HAVE_SYS_VFS_H))
else {
struct statvfs stats;
if (statvfs(rhizome_database.folder, &stats)==-1)
WARNF_perror("statvfs(%s)", rhizome_database.folder);
if (statvfs(rhizome_database.dir_path, &stats)==-1)
WARNF_perror("statvfs(%s)", alloca_str_toprint(rhizome_database.dir_path));
else
space = stats.f_frsize * (uint64_t)stats.f_bavail;
}

2
serval_uuid.h
View File

@ -69,6 +69,8 @@ typedef struct serval_uuid {
} u;
} serval_uuid_t;
#define SERVAL_UUID_INVALID { .u = { .record = { .clock_seq_hi_and_reserved = 0 } } }
enum serval_uuid_version {
UUID_VERSION_UNSUPPORTED = 0,
UUID_VERSION_TIME_BASED = 1,

2
socket.c
View File

@ -53,7 +53,7 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *add
addr->local.sun_family = AF_UNIX;
va_list ap;
va_start(ap, fmt);
int r = vformf_serval_run_path(addr->local.sun_path, sizeof addr->local.sun_path, fmt, ap);
int r = VFORMF_SERVAL_RUN_PATH(addr->local.sun_path, fmt, ap);
va_end(ap);
if (!r)
return WHY("socket name overflow");

2
testdefs_rhizome.sh
View File

@ -235,7 +235,7 @@ strip_signatures() {
get_external_blob_path(){
local _var="$1"
local _hash="$2"
local _filepath="$SERVALINSTANCE_PATH/blob/${_hash:0:2}/${_hash:2:2}/${_hash:4}"
local _filepath="$SERVALINSTANCE_PATH/rhizome/blob/${_hash:0:2}/${_hash:2:2}/${_hash:4}"
if [ -n "$_var" ]; then
eval "$_var=\$_filepath"
fi

26
tests/rhizomeops
View File

@ -3,6 +3,7 @@
# Tests for Serval rhizome command-line operations.
#
# Copyright 2012-2015 Serval Project, Inc.
# Copyright 2016-2018 Flinders University
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@ -310,14 +311,17 @@ setup_MoveDatabaseFolder() {
executeOk_servald config set rhizome.max_blob_size 0
echo "A test file" >file1
executeOk_servald rhizome add file "$SIDA" file1 file1.manifest
assert --stderr [ -e "$SERVALINSTANCE_PATH/rhizome/rhizome.db" ]
executeOk mv "$SERVALINSTANCE_PATH/rhizome/"* "$SERVALINSTANCE_PATH"
executeOk rmdir "$SERVALINSTANCE_PATH/rhizome/"
}
test_MoveDatabaseFolder() {
executeOk_servald config \
set rhizome.datastore_path "$instance_dir/moved_rhizome"
executeOk_servald config set rhizome.datastore_path "moved"
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author="$SIDA" file1
assert [ ! -e "$instance_dir/rhizome.db" ]
assert [ -e "$instance_dir/moved_rhizome/rhizome.db" ]
assert --stderr [ ! -e "$SERVALINSTANCE_PATH/rhizome/rhizome.db" ]
assert --stderr [ -e "$SERVALINSTANCE_PATH/moved/rhizome.db" ]
tfw_cat --stderr
}
doc_CleanVerify="Verify all bundles"
@ -1147,10 +1151,10 @@ test_JournalAppendNoHash() {
assert_stdout_add_file file1
extract_stdout_manifestid BID
extract_stdout_filehash HASH
assert [ $(find "$SERVALINSTANCE_PATH/hash" -type f| wc -l) -eq 1 ]
assert [ $(find "$SERVALINSTANCE_PATH/rhizome/hash" -type f| wc -l) -eq 1 ]
executeOk_servald rhizome journal append "$SIDA" "$BID" file2
tfw_cat --stdout --stderr
assert [ $(find "$SERVALINSTANCE_PATH/hash" -type f| wc -l) -eq 2 ]
assert [ $(find "$SERVALINSTANCE_PATH/rhizome/hash" -type f| wc -l) -eq 2 ]
assertStderrGrep 'Reusing journal'
executeOk_servald rhizome extract file "$BID" filex
tfw_cat --stdout --stderr
@ -1178,7 +1182,7 @@ setup_JournalAppendSharedPayload() {
extract_stdout_filehash HASH12
get_external_blob_path blob_file12 "$HASH12"
assert cmp file12 "$blob_file12"
assert [ $(find "$SERVALINSTANCE_PATH/blob" -type f| wc -l) -eq 2 ]
assert [ $(find "$SERVALINSTANCE_PATH/rhizome/blob" -type f| wc -l) -eq 2 ]
}
test_JournalAppendSharedPayload() {
executeOk_servald rhizome journal append "$SIDA" "" file1
@ -1186,14 +1190,14 @@ test_JournalAppendSharedPayload() {
assert_stdout_add_file file1
extract_stdout_filehash addedhash
assert [ "$addedhash" = "$HASH1" ]
assert [ $(find "$SERVALINSTANCE_PATH/blob" -type f| wc -l) -eq 2 ]
assert [ $(find "$SERVALINSTANCE_PATH/rhizome/blob" -type f| wc -l) -eq 2 ]
extract_stdout_manifestid BID
executeOk_servald rhizome journal append "$SIDA" "$BID" file2
tfw_cat --stdout --stderr
assert_stdout_add_file file12 !name
extract_stdout_filehash addedhash
assert [ "$addedhash" = "$HASH12" ]
assert [ $(find "$SERVALINSTANCE_PATH/blob" -type f| wc -l) -eq 2 ]
assert [ $(find "$SERVALINSTANCE_PATH/rhizome/blob" -type f| wc -l) -eq 2 ]
}
doc_JournalAddCreate="Cannot create a journal using file add"
@ -1478,8 +1482,8 @@ setup_ImportOwnBundle() {
extract_manifest_filehash filehash fileB.manifest
extract_manifest_BK BK fileB.manifest
extract_manifest_date date fileB.manifest
assert [ -e "$SERVALINSTANCE_PATH/rhizome.db" ]
rm -f "$SERVALINSTANCE_PATH/rhizome.db"
assert [ -e "$SERVALINSTANCE_PATH/rhizome/rhizome.db" ]
rm -f "$SERVALINSTANCE_PATH/rhizome/rhizome.db"
executeOk_servald rhizome list
assert_rhizome_list
}