diff --git a/conf_schema.h b/conf_schema.h index 0b2c13e1..c0a5dde6 100644 --- a/conf_schema.h +++ b/conf_schema.h @@ -448,12 +448,12 @@ ATOM(bool_t, enable, 1, boolean,, "If true, server opens ATOM(bool_t, fetch, 1, boolean,, "If false, no new bundles will be fetched from peers") ATOM(bool_t, reliable_clock, 1, boolean,, "If false, consider the system clock to be unreliable for generating timestamps") ATOM(bool_t, clean_on_open, 0, boolean,, "If true, Rhizome database is cleaned at start of every command") -STRING(256, datastore_path, "", str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory") +STRING(256, datastore_path, RHIZOME_DEFAULT_PATH, str_nonempty,, "Path of rhizome storage directory, absolute or relative to instance directory") ATOM(uint64_t, database_size, UINT64_MAX, uint64_scaled,, "Maximum size of database in bytes") ATOM(uint64_t, min_free_space, 100*1024*1024, uint64_scaled,, "Minimum free space to preserve on the disk") ATOM(uint32_t, max_blob_size, 128 * 1024, uint32_scaled,, "Store payloads larger than this in files not SQLite blobs") -ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.") -ATOM(uint32_t, fetch_delay_ms, 50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch") +ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.") +ATOM(uint32_t, fetch_delay_ms, 50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch") SUB_STRUCT(rhizome_direct, direct,) SUB_STRUCT(rhizome_api, api,) SUB_STRUCT(rhizome_http, http,) diff --git a/instance.c b/instance.c index 66716f7f..62942239 100644 --- a/instance.c +++ b/instance.c @@ -1,6 +1,7 @@ /* Serval DNA instance paths Copyright (C) 2012-2015 Serval Project Inc. +Copyright (C) 2016-2018 Flinders University This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -23,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include #include "instance.h" +#include "conf.h" #include "str.h" #include "os.h" #include "strbuf.h" @@ -105,20 +107,21 @@ void set_instance_path(const char *path) know_instancepath = 1; } -static int vformf_basepath(struct __sourceloc __whence, strbuf b, const char *basepath, const char *fmt, va_list ap) +static int vformf_path(struct __sourceloc __whence, strbuf b, const char *fhs_path, const char *configured_path, const char *fmt, va_list ap) { - if (fmt) - strbuf_va_vprintf(b, fmt, ap); - if (!strbuf_overrun(b) && (strbuf_len(b) == 0 || strbuf_str(b)[0] != '/')) { - strbuf_reset(b); - strbuf_puts(b, basepath); - if (fmt) { - if (strbuf_substr(b, -1)[0] != '/') - strbuf_putc(b, '/'); - strbuf_va_vprintf(b, fmt, ap); - } + const char *ipath = instance_path(); + strbuf_path_join(b, ipath ? ipath : fhs_path, NULL); + if (configured_path) + strbuf_path_join(b, configured_path, NULL); + assert(strbuf_str(b)[0] == '/'); + int fmt_overrun = 0; + if (fmt) { + strbuf sb = strbuf_alloca(strbuf_size(b)); + strbuf_va_vprintf(sb, fmt, ap); + strbuf_path_join(b, strbuf_str(sb), NULL); + fmt_overrun = strbuf_overrun(sb); } - if (!strbuf_overrun(b)) + if (!strbuf_overrun(b) && !fmt_overrun) return 1; WHYF("instance path overflow (strlen %lu, sizeof buffer %lu): %s", (unsigned long)strbuf_count(b), @@ -127,28 +130,11 @@ static int vformf_basepath(struct __sourceloc __whence, strbuf b, const char *ba return 0; } -int _formf_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *basepath, const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - int ret = vformf_basepath(__whence, strbuf_local(buf, bufsiz), basepath, fmt, ap); - va_end(ap); - return ret; -} - -static int vformf_path(struct __sourceloc __whence, strbuf b, const char *syspath, const char *fmt, va_list ap) -{ - const char *ipath = instance_path(); - if (!ipath) - ipath=syspath; - return vformf_basepath(__whence, b, ipath, fmt, ap); -} - int _formf_serval_etc_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_ETC_PATH, fmt, ap); + int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_ETC_PATH, NULL, fmt, ap); va_end(ap); return ret; } @@ -164,7 +150,7 @@ int _formf_serval_run_path(struct __sourceloc __whence, char *buf, size_t bufsiz int _vformf_serval_run_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, va_list ap) { - return vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH, fmt, ap); + return vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH, NULL, fmt, ap); } strbuf strbuf_system_log_path(strbuf sb) @@ -188,7 +174,7 @@ int _formf_serval_cache_path(struct __sourceloc __whence, char *buf, size_t bufs { va_list ap; va_start(ap, fmt); - int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_CACHE_PATH, fmt, ap); + int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_CACHE_PATH, NULL, fmt, ap); va_end(ap); return ret; } @@ -197,7 +183,16 @@ int _formf_rhizome_store_path(struct __sourceloc __whence, char *buf, size_t buf { va_list ap; va_start(ap, fmt); - int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), RHIZOME_STORE_PATH, fmt, ap); + int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), RHIZOME_STORE_PATH, config.rhizome.datastore_path, fmt, ap); + va_end(ap); + return ret; +} + +int _formf_rhizome_store_legacy_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), RHIZOME_STORE_PATH, NULL, fmt, ap); va_end(ap); return ret; } @@ -206,7 +201,7 @@ int _formf_serval_tmp_path(struct __sourceloc __whence, char *buf, size_t bufsiz { va_list ap; va_start(ap, fmt); - int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_TMP_PATH, fmt, ap); + int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_TMP_PATH, NULL, fmt, ap); va_end(ap); return ret; } @@ -215,7 +210,7 @@ int _formf_servald_proc_path(struct __sourceloc __whence, char *buf, size_t bufs { va_list ap; va_start(ap, fmt); - int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH "/proc", fmt, ap); + int ret = vformf_path(__whence, strbuf_local(buf, bufsiz), SERVAL_RUN_PATH "/proc", NULL, fmt, ap); va_end(ap); return ret; } diff --git a/instance.h b/instance.h index 3d7f466b..12ea807f 100644 --- a/instance.h +++ b/instance.h @@ -1,7 +1,7 @@ /* Serval DNA instance paths Copyright (C) 2014-2015 Serval Project Inc. -Copyright (C) 2016 Flinders University +Copyright (C) 2016-2018 Flinders University This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -31,8 +31,14 @@ void set_instance_path(const char *path); int create_serval_instance_dir(); -int _formf_path(struct __sourceloc __whence, char *buf, size_t bufsiz, const char *basepath, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,5,6))); - +/* Macros for forming the absolute paths of various files into a given char* + * buffer. Evaluates to true if the pathname fits into the provided buffer, + * false (0) otherwise (after logging an error). + * + * The "legacy" path functions/macros are exclusively used to provide + * compatibility with earlier versions of Serval DNA, and may be deleted once + * no longer needed. + */ int _formf_serval_etc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _vformf_serval_run_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list); @@ -40,27 +46,30 @@ int _formf_serval_cache_path(struct __sourceloc, char *buf, size_t bufsiz, const int _formf_serval_tmp_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_servald_proc_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); int _formf_rhizome_store_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); +int _formf_rhizome_store_legacy_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, ...) __attribute__((__ATTRIBUTE_format(printf,4,5))); -#define formf_path(buf,bufsz,basepath,fmt,...) _formf_path(__WHENCE__, buf, bufsz, basepath, fmt, ##__VA_ARGS__) - -#define formf_serval_etc_path(buf,bufsz,fmt,...) _formf_serval_etc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) -#define formf_serval_run_path(buf,bufsz,fmt,...) _formf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) -#define vformf_serval_run_path(buf,bufsz,fmt,ap) _vformf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ap) -#define formf_serval_cache_path(buf,bufsz,fmt,...) _formf_serval_cache_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) -#define formf_serval_tmp_path(buf,bufsz,fmt,...) _formf_serval_tmp_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) -#define formf_servald_proc_path(buf,bufsz,fmt,...) _formf_servald_proc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) -#define formf_rhizome_store_path(buf,bufsz,fmt,...) _formf_rhizome_store_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define formf_serval_etc_path(buf,bufsz,fmt,...) _formf_serval_etc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define formf_serval_run_path(buf,bufsz,fmt,...) _formf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define vformf_serval_run_path(buf,bufsz,fmt,ap) _vformf_serval_run_path(__WHENCE__, buf, bufsz, fmt, ap) +#define formf_serval_cache_path(buf,bufsz,fmt,...) _formf_serval_cache_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define formf_serval_tmp_path(buf,bufsz,fmt,...) _formf_serval_tmp_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define formf_servald_proc_path(buf,bufsz,fmt,...) _formf_servald_proc_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define formf_rhizome_store_path(buf,bufsz,fmt,...) _formf_rhizome_store_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) +#define formf_rhizome_store_legacy_path(buf,bufsz,fmt,...) _formf_rhizome_store_legacy_path(__WHENCE__, buf, bufsz, fmt, ##__VA_ARGS__) /* Handy macros for forming the absolute paths of various files, using a char[] * buffer whose declaration is in scope (so that sizeof(buf) will work). * Evaluates to true if the pathname fits into the provided buffer, false (0) * otherwise (after logging an error). */ -#define FORMF_SERVAL_ETC_PATH(buf,fmt,...) formf_serval_etc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) -#define FORMF_SERVAL_RUN_PATH(buf,fmt,...) formf_serval_run_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) -#define FORMF_SERVAL_CACHE_PATH(buf,fmt,...) formf_serval_cache_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) -#define FORMF_SERVAL_TMP_PATH(buf,fmt,...) formf_serval_tmp_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) -#define FORMF_SERVALD_PROC_PATH(buf,fmt,...) formf_servald_proc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define FORMF_SERVAL_ETC_PATH(buf,fmt,...) formf_serval_etc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define FORMF_SERVAL_RUN_PATH(buf,fmt,...) formf_serval_run_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define VFORMF_SERVAL_RUN_PATH(buf,fmt,ap) vformf_serval_run_path(buf, sizeof(buf), fmt, ap) +#define FORMF_SERVAL_CACHE_PATH(buf,fmt,...) formf_serval_cache_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define FORMF_SERVAL_TMP_PATH(buf,fmt,...) formf_serval_tmp_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define FORMF_SERVALD_PROC_PATH(buf,fmt,...) formf_servald_proc_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define FORMF_RHIZOME_STORE_PATH(buf,fmt,...) formf_rhizome_store_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) +#define FORMF_RHIZOME_STORE_LEGACY_PATH(buf,fmt,...) formf_rhizome_store_legacy_path(buf, sizeof(buf), fmt, ##__VA_ARGS__) strbuf strbuf_system_log_path(strbuf sb); strbuf strbuf_serval_log_path(strbuf sb); diff --git a/os.c b/os.c index 2f3c9117..5a0e2516 100644 --- a/os.c +++ b/os.c @@ -1,6 +1,8 @@ -/* -Serval Distributed Numbering Architecture (DNA) -Copyright (C) 2010 Paul Gardner-Stephen +/* +Serval DNA operating system services +Copyright (C) 2010 Paul Gardner-Stephen +Copyright (C) 2012-2015 Serval Project Inc. +Copyright (C) 2016-2018 Flinders University This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -114,6 +116,15 @@ int _mkdirsn(struct __sourceloc whence, const char *path, size_t len, mode_t mod return -1; } +int _erename(struct __sourceloc __whence, const char *oldpath, const char *newpath, int log_level) +{ + if (log_level != LOG_LEVEL_SILENT) + LOGF(log_level, "rename %s -> %s", alloca_str_toprint(oldpath), alloca_str_toprint(newpath)); + if (rename(oldpath, newpath) == -1) + return WHYF_perror("rename(%s,%s)", alloca_str_toprint(oldpath), alloca_str_toprint(newpath)); + return 0; +} + time_ms_t gettime_ms() { struct timeval nowtv; @@ -221,6 +232,7 @@ int get_file_meta(const char *path, struct file_meta *metap) return WHYF_perror("stat(%s)", path); *metap = FILE_META_NONEXIST; } else { + metap->mode = st.st_mode; metap->size = st.st_size; metap->mtime.tv_sec = st.st_mtime; // Truncate to whole seconds to ensure that this code will work on file systems that only have @@ -323,6 +335,24 @@ int alter_file_meta(const char *path, const struct file_meta *origp, struct file return 1; } +int file_exists(const char *path) +{ + struct file_meta meta; + return get_file_meta(path, &meta) != -1 && is_file_meta_exists(&meta); +} + +int file_exists_is_regular(const char *path) +{ + struct file_meta meta; + return get_file_meta(path, &meta) != -1 && is_file_meta_regular(&meta); +} + +int file_exists_is_directory(const char *path) +{ + struct file_meta meta; + return get_file_meta(path, &meta) != -1 && !is_file_meta_directory(&meta); +} + ssize_t get_self_executable_path(char *buf, size_t len) { #if defined(linux) diff --git a/os.h b/os.h index 1d138602..a5a9dd52 100644 --- a/os.h +++ b/os.h @@ -1,6 +1,7 @@ /* Serval DNA native Operating System interface -Copyright (C) 2012 Serval Project Inc. +Copyright (C) 2012-2014 Serval Project Inc. +Copyright (C) 2017-2018 Flinders University This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -25,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #endif #include // for off64_t +#include // for S_IFMT etc. #include // for NULL #include #include // for int64_t @@ -106,7 +108,9 @@ __SERVAL_DNA__OS_INLINE off64_t lseek64(int fd, off64_t offset, int whence) { # endif #endif -/* The "e" variants log the error before returning -1. +/* Functions to create a directory and any missing parent directories. The "e" + * variants log the error before returning -1. The "_info" variants log all + * created directories at INFO level. */ typedef void MKDIR_LOG_FUNC(struct __sourceloc, const char *, mode_t, void *); MKDIR_LOG_FUNC log_info_mkdir; @@ -130,6 +134,12 @@ int _emkdirsn(struct __sourceloc, const char *path, size_t len, mode_t mode, MKD #define emkdirs_info(path, mode) emkdirs_log((path), (mode), log_info_mkdir, NULL) #define emkdirsn_info(path, len, mode) emkdirsn_log((path), (len), (mode), log_info_mkdir, NULL) +/* Function to rename a file, logging any error before returning -1. + */ +int _erename(struct __sourceloc, const char *oldpath, const char *newpath, int log_level); +#define erename(oldpath, newpath) _erename(__WHENCE__, (oldpath), (newpath), LOG_LEVEL_SILENT) +#define erename_info(oldpath, newpath) _erename(__WHENCE__, (oldpath), (newpath), LOG_LEVEL_INFO) + /* Read the symbolic link into the supplied buffer and add a terminating nul. * Logs an ERROR and returns -1 if the buffer is too short to hold the link * content and the terminating nul. If readlink(2) returns an error, then logs @@ -185,17 +195,30 @@ int malloc_read_whole_file(const char *path, unsigned char **bufp, size_t *sizp) struct file_meta { struct timespec mtime; off_t size; + mode_t mode; }; -#define FILE_META_UNKNOWN ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = -1 }) +#define FILE_META_UNKNOWN ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = -1, .mode = -1 }) // A non-existent file is treated as size == 0 and an impossible modification // time, so that cmp_file_meta() will not compare it as equal with any existing // file. -#define FILE_META_NONEXIST ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = 0 }) +#define FILE_META_NONEXIST ((struct file_meta){ .mtime = { .tv_sec = -1, .tv_nsec = -1 }, .size = 0, .mode = 0 }) __SERVAL_DNA__OS_INLINE int is_file_meta_nonexist(const struct file_meta *m) { - return m->mtime.tv_sec == -1 && m->mtime.tv_nsec == -1 && m->size == 0; + return m->mtime.tv_sec == -1 && m->mtime.tv_nsec == -1 && m->size == 0 && m->mode == 0; +} + +__SERVAL_DNA__OS_INLINE int is_file_meta_exists(const struct file_meta *m) { + return m->mtime.tv_sec != -1; +} + +__SERVAL_DNA__OS_INLINE int is_file_meta_regular(const struct file_meta *m) { + return is_file_meta_exists(m) && (m->mode & S_IFMT) == S_IFREG; +} + +__SERVAL_DNA__OS_INLINE int is_file_meta_directory(const struct file_meta *m) { + return is_file_meta_exists(m) && (m->mode & S_IFMT) == S_IFDIR; } int get_file_meta(const char *path, struct file_meta *metap); @@ -205,6 +228,11 @@ int cmp_file_meta(const struct file_meta *a, const struct file_meta *b); // by bumping the file's modification time or altering its inode. int alter_file_meta(const char *path, const struct file_meta *origp, struct file_meta *metap); +// Convenience functions based on get_file_meta(). +int file_exists(const char *path); +int file_exists_is_regular(const char *path); +int file_exists_is_directory(const char *path); + /* Fill the given buffer with the nul-terminated absolute path of the calling * process's executable. Logs an error and returns -1 if the executable cannot * be determined or the supplied buffer is too short. Otherwise returns the diff --git a/rhizome.h b/rhizome.h index 76e8513a..4f3913b8 100644 --- a/rhizome.h +++ b/rhizome.h @@ -1,5 +1,6 @@ /* Serval DNA Rhizome file distribution +Copyright (C) 2016-2018 Flinders University Copyright (C) 2010-2014 Serval Project Inc. Copyright (C) 2010 Paul Gardner-Stephen @@ -337,19 +338,18 @@ int rhizome_configure(); int rhizome_enabled(); int rhizome_fetch_delay_ms(); +#define RHIZOME_DEFAULT_PATH "rhizome" #define RHIZOME_BLOB_SUBDIR "blob" #define RHIZOME_HASH_SUBDIR "hash" -struct rhizome_database{ +struct rhizome_database { + char dir_path[1024]; sqlite3 *db; serval_uuid_t uuid; - char folder[1024]; }; extern __thread struct rhizome_database rhizome_database; -#define FORMF_RHIZOME_STORE_PATH(buf,fmt,...) formf_path((buf), sizeof(buf), rhizome_database.folder, (fmt), ##__VA_ARGS__) - int rhizome_opendb(); int rhizome_close_db(); void verify_bundles(); diff --git a/rhizome_cli.c b/rhizome_cli.c index a262b5eb..d869721f 100644 --- a/rhizome_cli.c +++ b/rhizome_cli.c @@ -1,6 +1,7 @@ /* Serval DNA - Rhizome command line interface Copyright (C) 2014 Serval Project Inc. + Copyright (C) 2016-2017 Flinders University This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License diff --git a/rhizome_database.c b/rhizome_database.c index dec6e800..daca5a0e 100644 --- a/rhizome_database.c +++ b/rhizome_database.c @@ -1,6 +1,7 @@ /* Serval DNA - Rhizome database operations Copyright (C) 2012-2014 Serval Project Inc. +Copyright (C) 2016-2018 Flinders University This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -37,7 +38,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. static int rhizome_delete_manifest_retry(sqlite_retry_state *retry, const rhizome_bid_t *bidp); -__thread struct rhizome_database rhizome_database = {NULL,{{{0}}},{0}}; +__thread struct rhizome_database rhizome_database = { + .dir_path = "", + .db = NULL, + .uuid = SERVAL_UUID_INVALID + }; + static time_ms_t rhizomeRetryLimit = -1; int is_debug_rhizome() @@ -235,65 +241,82 @@ int rhizome_opendb() IN(); - if (sodium_init()==-1) + if (sodium_init() == -1) RETURN(WHY("Failed to initialise libsodium")); - if (!formf_rhizome_store_path(rhizome_database.folder, sizeof rhizome_database.folder, "%s", config.rhizome.datastore_path)) - RETURN (-1); - DEBUGF(rhizome, "Rhizome datastore path = %s", alloca_str_toprint(rhizome_database.folder)); - if (emkdirs_info(rhizome_database.folder, 0700)==-1) - RETURN (-1); - char dbpath[1024]; - - if (!FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db")) + // Work out the absolute path of the directory that contains all the database + // files/subdirectories. + if (!FORMF_RHIZOME_STORE_PATH(rhizome_database.dir_path, "%s", "")) RETURN(-1); - - struct file_meta meta; - if (get_file_meta(dbpath, &meta) == -1) + DEBUGF(rhizome, "Rhizome store directory path = %s", alloca_str_toprint(rhizome_database.dir_path)); + + // Work out the absolute paths of the database file and subdirectories. + char dbpath[sizeof rhizome_database.dir_path]; + char blobpath[sizeof rhizome_database.dir_path]; + char hashpath[sizeof rhizome_database.dir_path]; + char temppath[sizeof rhizome_database.dir_path]; + int db_exists = 0; + { + struct file_meta dbmeta; + if ( !FORMF_RHIZOME_STORE_PATH(dbpath, "rhizome.db") + || !FORMF_RHIZOME_STORE_PATH(blobpath, RHIZOME_BLOB_SUBDIR) + || !FORMF_RHIZOME_STORE_PATH(hashpath, RHIZOME_HASH_SUBDIR) + || !FORMF_RHIZOME_STORE_PATH(temppath, "sqlite3tmp") + || get_file_meta(dbpath, &dbmeta) == -1) + RETURN(-1); + db_exists = is_file_meta_exists(&dbmeta); + } + + // Create missing store directory. + if (emkdirs_info(rhizome_database.dir_path, 0700) == -1) RETURN(-1); - if (meta.mtime.tv_sec == -1 && config.rhizome.datastore_path[0]){ - // Move the database after datastore path is set for the first time - // This is mostly here to transparently fix a bug where we were ignoring the config value - char src[1024]; - char dest[1024]; - if (formf_rhizome_store_path(src, sizeof src, "rhizome.db") - && strcmp(dbpath, src)!=0 - && get_file_meta(src, &meta)==0 - && meta.mtime.tv_sec != -1){ - - INFOF("Moving rhizome store from %s to %s", src, dbpath); - if (rename(src, dbpath)) - WHYF_perror("rename(%s, %s)", src, dbpath); - - if (formf_rhizome_store_path(src, sizeof src, RHIZOME_BLOB_SUBDIR) - && FORMF_RHIZOME_STORE_PATH(dest, RHIZOME_BLOB_SUBDIR) - && rename(src, dest) - && errno!=ENOENT){ - WHYF_perror("rename(%s, %s)", src, dest); - } - - if (formf_rhizome_store_path(src, sizeof src, RHIZOME_HASH_SUBDIR) - && FORMF_RHIZOME_STORE_PATH(dest, RHIZOME_HASH_SUBDIR) - && rename(src, dest) - && errno!=ENOENT){ - WHYF_perror("rename(%s, %s)", src, dest); - } + // If the database file does not exist, then SQLite will create it. However, in 2014 a bug was + // introduced that always created the database file in the instance directory, regardless of the + // setting of the 'rhizome.datastore_path' config option. This bug has now been fixed, so in + // order to preserve the Rhizome data store when upgrading from an earlier (buggy) version, if + // there is a database file at this "legacy" location, then move it into the correct, configured + // location, instead of creating a fresh (empty) database file. + if (!db_exists) { + char legacy_dbpath[sizeof rhizome_database.dir_path]; + if (FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dbpath, "rhizome.db") + && strcmp(legacy_dbpath, dbpath) != 0 + && file_exists(legacy_dbpath) + ) { + INFOF("Recover legacy Rhizome SQLite database"); + // Move the legacy database file to its correct location. + if (erename_info(legacy_dbpath, dbpath) == -1) + RETURN(-1); + // Move any legacy "blob" and "hash" subdirectories too, if they are present. + char legacy_dirpath[sizeof rhizome_database.dir_path]; + if ( !file_exists(blobpath) + && FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dirpath, RHIZOME_BLOB_SUBDIR) + && file_exists(legacy_dirpath)) + erename_info(legacy_dirpath, blobpath); + if ( !file_exists(hashpath) + && FORMF_RHIZOME_STORE_LEGACY_PATH(legacy_dirpath, RHIZOME_HASH_SUBDIR) + && file_exists(legacy_dirpath)) + erename_info(legacy_dirpath, hashpath); } + else + INFOF("Creating Rhizome SQLite database: %s", dbpath); } - if (!sqlite3_temp_directory){ - char tmp[1024]; - if (!FORMF_RHIZOME_STORE_PATH(tmp, "sqlite3tmp")) - RETURN(-1); - if (emkdirs_info(tmp, 0700) == -1) - RETURN(-1); - sqlite3_temp_directory = sqlite3_mprintf("%s", tmp); - } + // Create missing sub-directories. + if ( emkdirs_info(blobpath, 0700) == -1 + || emkdirs_info(hashpath, 0700) == -1 + || emkdirs_info(temppath, 0700) == -1) + RETURN(-1); - sqlite3_config(SQLITE_CONFIG_LOG,sqlite_log,NULL); + // Inform SQLite of its temporary directory. + assert(!sqlite3_temp_directory); + sqlite3_temp_directory = sqlite3_mprintf("%s", temppath); - if (sqlite3_open(dbpath,&rhizome_database.db)){ + // Set up SQLite logging. + sqlite3_config(SQLITE_CONFIG_LOG, sqlite_log, NULL); + + // Open the SQLite database file, creating it if necessary. + if (sqlite3_open(dbpath, &rhizome_database.db)){ RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_database.db))); } sqlite3_trace_v2(rhizome_database.db, SQLITE_TRACE_STMT, sqlite_trace_callback, NULL); @@ -304,7 +327,7 @@ int rhizome_opendb() /* Read Rhizome configuration */ DEBUGF(rhizome, "Rhizome will use %"PRIu64"B of storage for its database.", (uint64_t) config.rhizome.database_size); - + sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT; uint64_t version; @@ -358,7 +381,7 @@ int rhizome_opendb() sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END); } - if (version<2 && meta.mtime.tv_sec != -1){ + if (version<2 && db_exists){ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN service text;", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN name text;", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN sender text collate nocase;", END); @@ -369,28 +392,28 @@ int rhizome_opendb() sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END); } - if (version<4 && meta.mtime.tv_sec != -1){ + if (version<4 && db_exists){ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END); } - if (version<5 && meta.mtime.tv_sec != -1){ + if (version<5 && db_exists){ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END); } - if (version<6 && meta.mtime.tv_sec != -1){ + if (version<6 && db_exists){ // we've always been at war with eurasia sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPLIST; ", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS GROUPMEMBERSHIPS; ", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS VERIFICATIONS; ", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DROP TABLE IF EXISTS FILEMANIFESTS;", END); } - if (version<7 && meta.mtime.tv_sec != -1){ + if (version<7 && db_exists){ sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE FILES ADD COLUMN last_verified integer;", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=7;", END); } if (version<8){ - if (meta.mtime.tv_sec != -1) + if (db_exists) sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN manifest_hash text collate nocase;", END); sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFEST_HASH ON MANIFESTS(manifest_hash);", END); @@ -448,7 +471,7 @@ int rhizome_close_db() IN(); if (rhizome_database.db) { rhizome_cache_close(); - + if (!sqlite3_get_autocommit(rhizome_database.db)){ WHY("Uncommitted transaction!"); sqlite_exec_void("ROLLBACK;", END); @@ -461,11 +484,11 @@ int rhizome_close_db() int r = sqlite3_close(rhizome_database.db); if (r != SQLITE_OK) RETURN(WHYF("Failed to close sqlite database, %s",sqlite3_errmsg(rhizome_database.db))); - } - rhizome_database.db=NULL; - if (sqlite3_temp_directory) + rhizome_database.db = NULL; + assert(sqlite3_temp_directory); sqlite3_free(sqlite3_temp_directory); - sqlite3_temp_directory=NULL; + sqlite3_temp_directory = NULL; + } RETURN(0); OUT(); } diff --git a/rhizome_store.c b/rhizome_store.c index 01c028f1..4d050a0f 100644 --- a/rhizome_store.c +++ b/rhizome_store.c @@ -195,8 +195,8 @@ static uint64_t store_get_free_space() #if defined(HAVE_SYS_STATVFS_H) || (defined(HAVE_SYS_STAT_H) && defined(HAVE_SYS_VFS_H)) else { struct statvfs stats; - if (statvfs(rhizome_database.folder, &stats)==-1) - WARNF_perror("statvfs(%s)", rhizome_database.folder); + if (statvfs(rhizome_database.dir_path, &stats)==-1) + WARNF_perror("statvfs(%s)", alloca_str_toprint(rhizome_database.dir_path)); else space = stats.f_frsize * (uint64_t)stats.f_bavail; } diff --git a/serval_uuid.h b/serval_uuid.h index 451e03e0..dac1e589 100644 --- a/serval_uuid.h +++ b/serval_uuid.h @@ -69,6 +69,8 @@ typedef struct serval_uuid { } u; } serval_uuid_t; +#define SERVAL_UUID_INVALID { .u = { .record = { .clock_seq_hi_and_reserved = 0 } } } + enum serval_uuid_version { UUID_VERSION_UNSUPPORTED = 0, UUID_VERSION_TIME_BASED = 1, diff --git a/socket.c b/socket.c index 36cbcff1..14e77912 100644 --- a/socket.c +++ b/socket.c @@ -53,7 +53,7 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *add addr->local.sun_family = AF_UNIX; va_list ap; va_start(ap, fmt); - int r = vformf_serval_run_path(addr->local.sun_path, sizeof addr->local.sun_path, fmt, ap); + int r = VFORMF_SERVAL_RUN_PATH(addr->local.sun_path, fmt, ap); va_end(ap); if (!r) return WHY("socket name overflow"); diff --git a/testdefs_rhizome.sh b/testdefs_rhizome.sh index 4c140e15..7a20341d 100644 --- a/testdefs_rhizome.sh +++ b/testdefs_rhizome.sh @@ -235,7 +235,7 @@ strip_signatures() { get_external_blob_path(){ local _var="$1" local _hash="$2" - local _filepath="$SERVALINSTANCE_PATH/blob/${_hash:0:2}/${_hash:2:2}/${_hash:4}" + local _filepath="$SERVALINSTANCE_PATH/rhizome/blob/${_hash:0:2}/${_hash:2:2}/${_hash:4}" if [ -n "$_var" ]; then eval "$_var=\$_filepath" fi diff --git a/tests/rhizomeops b/tests/rhizomeops index 250a5bba..f0e4ac1c 100755 --- a/tests/rhizomeops +++ b/tests/rhizomeops @@ -3,6 +3,7 @@ # Tests for Serval rhizome command-line operations. # # Copyright 2012-2015 Serval Project, Inc. +# Copyright 2016-2018 Flinders University # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -310,14 +311,17 @@ setup_MoveDatabaseFolder() { executeOk_servald config set rhizome.max_blob_size 0 echo "A test file" >file1 executeOk_servald rhizome add file "$SIDA" file1 file1.manifest + assert --stderr [ -e "$SERVALINSTANCE_PATH/rhizome/rhizome.db" ] + executeOk mv "$SERVALINSTANCE_PATH/rhizome/"* "$SERVALINSTANCE_PATH" + executeOk rmdir "$SERVALINSTANCE_PATH/rhizome/" } test_MoveDatabaseFolder() { - executeOk_servald config \ - set rhizome.datastore_path "$instance_dir/moved_rhizome" + executeOk_servald config set rhizome.datastore_path "moved" executeOk_servald rhizome list assert_rhizome_list --fromhere=1 --author="$SIDA" file1 - assert [ ! -e "$instance_dir/rhizome.db" ] - assert [ -e "$instance_dir/moved_rhizome/rhizome.db" ] + assert --stderr [ ! -e "$SERVALINSTANCE_PATH/rhizome/rhizome.db" ] + assert --stderr [ -e "$SERVALINSTANCE_PATH/moved/rhizome.db" ] + tfw_cat --stderr } doc_CleanVerify="Verify all bundles" @@ -1147,10 +1151,10 @@ test_JournalAppendNoHash() { assert_stdout_add_file file1 extract_stdout_manifestid BID extract_stdout_filehash HASH - assert [ $(find "$SERVALINSTANCE_PATH/hash" -type f| wc -l) -eq 1 ] + assert [ $(find "$SERVALINSTANCE_PATH/rhizome/hash" -type f| wc -l) -eq 1 ] executeOk_servald rhizome journal append "$SIDA" "$BID" file2 tfw_cat --stdout --stderr - assert [ $(find "$SERVALINSTANCE_PATH/hash" -type f| wc -l) -eq 2 ] + assert [ $(find "$SERVALINSTANCE_PATH/rhizome/hash" -type f| wc -l) -eq 2 ] assertStderrGrep 'Reusing journal' executeOk_servald rhizome extract file "$BID" filex tfw_cat --stdout --stderr @@ -1178,7 +1182,7 @@ setup_JournalAppendSharedPayload() { extract_stdout_filehash HASH12 get_external_blob_path blob_file12 "$HASH12" assert cmp file12 "$blob_file12" - assert [ $(find "$SERVALINSTANCE_PATH/blob" -type f| wc -l) -eq 2 ] + assert [ $(find "$SERVALINSTANCE_PATH/rhizome/blob" -type f| wc -l) -eq 2 ] } test_JournalAppendSharedPayload() { executeOk_servald rhizome journal append "$SIDA" "" file1 @@ -1186,14 +1190,14 @@ test_JournalAppendSharedPayload() { assert_stdout_add_file file1 extract_stdout_filehash addedhash assert [ "$addedhash" = "$HASH1" ] - assert [ $(find "$SERVALINSTANCE_PATH/blob" -type f| wc -l) -eq 2 ] + assert [ $(find "$SERVALINSTANCE_PATH/rhizome/blob" -type f| wc -l) -eq 2 ] extract_stdout_manifestid BID executeOk_servald rhizome journal append "$SIDA" "$BID" file2 tfw_cat --stdout --stderr assert_stdout_add_file file12 !name extract_stdout_filehash addedhash assert [ "$addedhash" = "$HASH12" ] - assert [ $(find "$SERVALINSTANCE_PATH/blob" -type f| wc -l) -eq 2 ] + assert [ $(find "$SERVALINSTANCE_PATH/rhizome/blob" -type f| wc -l) -eq 2 ] } doc_JournalAddCreate="Cannot create a journal using file add" @@ -1478,8 +1482,8 @@ setup_ImportOwnBundle() { extract_manifest_filehash filehash fileB.manifest extract_manifest_BK BK fileB.manifest extract_manifest_date date fileB.manifest - assert [ -e "$SERVALINSTANCE_PATH/rhizome.db" ] - rm -f "$SERVALINSTANCE_PATH/rhizome.db" + assert [ -e "$SERVALINSTANCE_PATH/rhizome/rhizome.db" ] + rm -f "$SERVALINSTANCE_PATH/rhizome/rhizome.db" executeOk_servald rhizome list assert_rhizome_list }