ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2025-04-26 22:10:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source 
Conflicts:
	rhizome_fetch.c
	rhizome_http.c
	testframework.sh
This commit is contained in:
Jeremy Lakeman 2012-07-17 15:39:55 +09:30
commit 1d53726b7b
17 changed files with 849 additions and 487 deletions

42
commandline.c

@ -1108,23 +1108,43 @@ int app_rhizome_add_file(int argc, const char *const *argv, struct command_line_
ret = WHY("Could not overwrite manifest file.");
service = rhizome_manifest_get(mwritten, "service", NULL, 0);
if (service) {
cli_puts("service"); cli_delim(":"); cli_puts(service); cli_delim("\n");
cli_puts("service");
cli_delim(":");
cli_puts(service);
cli_delim("\n");
}
{
char bid[RHIZOME_MANIFEST_ID_STRLEN + 1];
rhizome_bytes_to_hex_upper(mwritten->cryptoSignPublic, bid, RHIZOME_MANIFEST_ID_BYTES);
cli_puts("manifestid"); cli_delim(":"); cli_puts(bid); cli_delim("\n");
cli_puts("manifestid");
cli_delim(":");
cli_puts(bid);
cli_delim("\n");
}
{
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(mwritten->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
cli_puts("secret"); cli_delim(":"); cli_puts(secret); cli_delim("\n");
cli_puts("secret");
cli_delim(":");
cli_puts(secret);
cli_delim("\n");
}
cli_puts("filesize");
cli_delim(":");
cli_printf("%lld", mwritten->fileLength);
cli_delim("\n");
if (mwritten->fileLength != 0) {
cli_puts("filehash");
cli_delim(":");
cli_puts(mwritten->fileHexHash);
cli_delim("\n");
}
cli_puts("filehash"); cli_delim(":"); cli_puts(mwritten->fileHexHash); cli_delim("\n");
cli_puts("filesize"); cli_delim(":"); cli_printf("%lld", mwritten->fileLength); cli_delim("\n");
const char *name = rhizome_manifest_get(mwritten, "name", NULL, 0);
if (name) {
cli_puts("name"); cli_delim(":"); cli_puts(name); cli_delim("\n");
cli_puts("name");
cli_delim(":");
cli_puts(name);
cli_delim("\n");
}
rhizome_manifest_free(m);
if (mout != m)
@ -1178,14 +1198,16 @@ int app_rhizome_import_bundle(int argc, const char *const *argv, struct command_
cli_puts(alloca_tohex(m->cryptoSignPublic, RHIZOME_MANIFEST_ID_BYTES));
cli_delim("\n");
}
cli_puts("filehash");
cli_delim(":");
cli_puts(m->fileHexHash);
cli_delim("\n");
cli_puts("filesize");
cli_delim(":");
cli_printf("%lld", m->fileLength);
cli_delim("\n");
if (m->fileLength != 0) {
cli_puts("filehash");
cli_delim(":");
cli_puts(m->fileHexHash);
cli_delim("\n");
}
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
if (name) {
cli_puts("name");

23
dna_helper.c

@ -88,11 +88,20 @@ parseDnaReply(unsigned char *bytes, int count,
return 0;
}
void dna_helper_monitor(int fd)
{
return;
}
struct sched_ent dna_helper_sched;
static int
dna_helper_start(const char *command, const char *arg) {
int stdin_fds[2], stdout_fds[2];
pid_t pid;
DEBUG("Starting DNA helper");
if (pipe(stdin_fds))
return WHY_perror("pipe");
@ -102,7 +111,7 @@ dna_helper_start(const char *command, const char *arg) {
return WHY_perror("pipe");
}
if ((pid = fork()) != 0) {
if ((pid = fork()) == 0) {
/* Child, should exec() to become helper after installing file descriptors. */
if (dup2(stdin_fds[1], 0)) /* replace stdin */
exit(-1);
@ -111,6 +120,7 @@ dna_helper_start(const char *command, const char *arg) {
if (dup2(stdout_fds[0], 2)) /* replace stderr */
exit(-1);
execl(command, command, arg, NULL);
DEBUG("execl() failed");
abort(); /* Can't get here */
} else {
if (pid == -1) {
@ -125,6 +135,11 @@ dna_helper_start(const char *command, const char *arg) {
/* Parent, should put file descriptors into place for use */
dna_helper_stdin = stdin_fds[0];
dna_helper_stdout = stdout_fds[1];
/* Need to watch dna_helper_stdout */
// XXX need to initialise structure before calling watch
// XXX watch(&dna_helper_sched);
// XXX need to add unwatch() when we detect that the process has died.
return 0;
}
}
@ -145,6 +160,7 @@ dna_helper_enqueue(char *did, unsigned char *requestorSid) {
/* Check if we have a helper configured. If not, then set
dna_helper_stdin to magic value of -2 so that we don't waste time
in future looking up the dna helper configuration value. */
DEBUG("We have no DNA helper configured");
dna_helper_stdin = -2;
return -1;
}
@ -154,9 +170,11 @@ dna_helper_enqueue(char *did, unsigned char *requestorSid) {
/* Okay, so we have a helper configured.
Run it */
if (dna_helper_start(dna_helper, dna_helper_arg) < 0)
if (dna_helper_start(dna_helper, dna_helper_arg) < 0) {
/* Something broke, bail out */
DEBUG("Failed to start dna helper");
return -1;
}
}
/* Write request to dna helper.
@ -186,6 +204,7 @@ dna_helper_enqueue(char *did, unsigned char *requestorSid) {
close(dna_helper_stdout);
dna_helper_stdin = -1;
dna_helper_stdout = -1;
DEBUG("Sigpipe encountered");
return -1;
}

34
fdqueue.c

@ -30,23 +30,16 @@ struct sched_ent *next_alarm=NULL;
struct sched_ent *next_deadline=NULL;
struct profile_total poll_stats={NULL,0,"Idle (in poll)",0,0,0};
void list_alarms(){
long long now=overlay_gettime_ms();
struct sched_ent *alarm = next_alarm;
void list_alarms() {
DEBUG("Alarms;");
long long now = overlay_gettime_ms();
struct sched_ent *alarm;
for (alarm = next_alarm; alarm; alarm = alarm->_next)
DEBUGF("%s in %lldms", (alarm->stats ? alarm->stats->name : "Unnamed"), alarm->alarm - now);
DEBUG("File handles;");
int i;
INFO("Alarms;");
while(alarm){
INFOF("%s in %lldms",
(alarm->stats?alarm->stats->name:"Unnamed"),
alarm->alarm - now);
alarm = alarm->_next;
}
INFO("File handles;");
for (i=0;i<fdcount;i++)
INFOF("%s watching #%d",
(fd_callbacks[i]->stats?fd_callbacks[i]->stats->name:"Unnamed"),
fds[i].fd);
for (i = 0; i < fdcount; ++i)
DEBUGF("%s watching #%d", (fd_callbacks[i]->stats ? fd_callbacks[i]->stats->name : "Unnamed"), fds[i].fd);
}
int deadline(struct sched_ent *alarm){
@ -136,9 +129,11 @@ int watch(struct sched_ent *alarm){
if (alarm->_poll_index>=0 && fd_callbacks[alarm->_poll_index]==alarm){
// updating event flags
INFOF("Updating watch %s, #%d for %d", (alarm->stats?alarm->stats->name:"Unnamed"), alarm->poll.fd, alarm->poll.events);
if (debug & DEBUG_IO)
DEBUGF("Updating watch %s, #%d for %d", (alarm->stats?alarm->stats->name:"Unnamed"), alarm->poll.fd, alarm->poll.events);
}else{
INFOF("Adding watch %s, #%d for %d", (alarm->stats?alarm->stats->name:"Unnamed"), alarm->poll.fd, alarm->poll.events);
if (debug & DEBUG_IO)
DEBUGF("Adding watch %s, #%d for %d", (alarm->stats?alarm->stats->name:"Unnamed"), alarm->poll.fd, alarm->poll.events);
if (fdcount>=MAX_WATCHED_FDS)
return WHY("Too many file handles to watch");
fd_callbacks[fdcount]=alarm;
@ -165,7 +160,8 @@ int unwatch(struct sched_ent *alarm){
fds[fdcount].fd=-1;
fd_callbacks[fdcount]=NULL;
alarm->_poll_index=-1;
INFOF("%s stopped watching #%d for %d", (alarm->stats?alarm->stats->name:"Unnamed"), alarm->poll.fd, alarm->poll.events);
if (debug & DEBUG_IO)
DEBUGF("%s stopped watching #%d for %d", (alarm->stats?alarm->stats->name:"Unnamed"), alarm->poll.fd, alarm->poll.events);
return 0;
}

4
log.c

@ -266,7 +266,9 @@ char *toprint(char *dstStr, size_t dstChars, const unsigned char *srcBuf, size_t
strbuf_puts(b, "\\r");
else if (*srcBuf == '\t')
strbuf_puts(b, "\\t");
else if (isprint(*srcBuf))
else if (*srcBuf == '\\')
strbuf_puts(b, "\\\\");
else if (*srcBuf >= ' ' && *srcBuf <= '~')
strbuf_putc(b, *srcBuf);
else
strbuf_sprintf(b, "\\x%02x", *srcBuf);

1
overlay_mdp.c

@ -563,6 +563,7 @@ int overlay_saw_mdp_frame(overlay_mdp_frame *mdp,long long now)
when results become available, so this function will return
immediately, so as not to cause blockages and delays in servald.
*/
DEBUGF("Asking DNA helper to resolve '%s'",did);
dna_helper_enqueue(did,mdp->out.src.sid);
}
RETURN(0);

75
rhizome.c

@ -83,7 +83,7 @@ int rhizome_bundle_import(rhizome_manifest *m_in, rhizome_manifest **m_out,
}
/* Add the manifest and its associated file to the Rhizome database. */
m->dataFileName=strdup(filename);
m->dataFileName = strdup(filename);
if (rhizome_manifest_check_file(m))
return WHY("File does not belong to manifest");
int ret=rhizome_manifest_check_duplicate(m,NULL);
@ -207,15 +207,19 @@ int rhizome_manifest_bind_file(rhizome_manifest *m_in,const char *filename,int e
DEBUGF("filename=%s, fileLength=%lld", filename, m_in->fileLength);
rhizome_manifest_set_ll(m_in,"filesize",m_in->fileLength);
/* Compute hash of payload */
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,filename, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
/* Store hash of payload */
rhizome_manifest_set(m_in, "filehash", m_in->fileHexHash);
m_in->fileHashedP=1;
/* Compute hash of non-empty payload */
if (m_in->fileLength != 0) {
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,filename, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
rhizome_manifest_set(m_in, "filehash", m_in->fileHexHash);
m_in->fileHashedP = 1;
} else {
m_in->fileHexHash[0] = '\0';
rhizome_manifest_del(m_in, "filehash");
m_in->fileHashedP = 0;
}
return 0;
}
@ -237,34 +241,43 @@ int rhizome_manifest_check_file(rhizome_manifest *m_in)
/* Check payload file is accessible and discover its length, then check that it matches
the file size stored in the manifest */
long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize");
m_in->fileLength = 0;
if (m_in->dataFileName[0]) {
struct stat stat;
if (lstat(m_in->dataFileName,&stat))
return WHYF("Could not stat() payload file '%s'",m_in->dataFileName);
m_in->fileLength = stat.st_size;
} else
m_in->fileLength = 0;
if (lstat(m_in->dataFileName,&stat) == -1) {
if (errno != ENOENT || mfilesize != 0)
return WHYF_perror("stat(%s)", m_in->dataFileName);
} else {
m_in->fileLength = stat.st_size;
}
}
if (debug & DEBUG_RHIZOME)
DEBUGF("filename=%s, fileLength=%lld", m_in->dataFileName, m_in->fileLength);
long long mfilesize = rhizome_manifest_get_ll(m_in, "filesize");
if (mfilesize != -1 && mfilesize != m_in->fileLength) {
WHYF("Manifest.filesize (%lld) != actual file size (%lld)", mfilesize, m_in->fileLength);
return -1;
}
/* Compute hash of payload */
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,m_in->dataFileName, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
m_in->fileHashedP = 1;
/* Check that payload hash matches manifest */
/* If payload is empty, ensure manifest has not file hash, otherwis compute the hash of the
payload and check that it matches manifest. */
const char *mhexhash = rhizome_manifest_get(m_in, "filehash", NULL, 0);
if (!mhexhash) return WHY("manifest contains no file hash");
if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) {
WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash);
return -1;
if (m_in->fileLength != 0) {
char hexhashbuf[RHIZOME_FILEHASH_STRLEN + 1];
if (rhizome_hash_file(m_in,m_in->dataFileName, hexhashbuf))
return WHY("Could not hash file.");
memcpy(&m_in->fileHexHash[0], &hexhashbuf[0], sizeof hexhashbuf);
m_in->fileHashedP = 1;
if (!mhexhash) return WHY("manifest contains no file hash");
if (mhexhash && strcmp(m_in->fileHexHash, mhexhash)) {
WHYF("Manifest.filehash (%s) does not match payload hash (%s)", mhexhash, m_in->fileHexHash);
return -1;
}
} else {
if (mhexhash != NULL) {
WHYF("Manifest.filehash (%s) should be absent for empty payload", mhexhash);
return -1;
}
}
return 0;
@ -324,8 +337,6 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
/* If the manifest already has an ID */
char id[SID_STRLEN + 1];
char ofilehash[RHIZOME_FILEHASH_STRLEN + 1];
ofilehash[0] = '\0';
if (rhizome_manifest_get(m_in, "id", id, SID_STRLEN + 1)) {
str_toupper_inplace(id);
/* Discard the new manifest unless it is newer than the most recent known version with the same ID */
@ -346,10 +357,6 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
default:
return WHY("Select found too many rows!");
}
strbuf b = strbuf_local(ofilehash, sizeof ofilehash);
sqlite_exec_strbuf(b, "SELECT filehash from manifests where id='%s';", id);
if (strbuf_overrun(b))
return WHYF("fileid too long: '%s'", strbuf_str(b));
} else {
/* no manifest ID */
return WHY("Manifest does not have an ID");

6
rhizome.h

@ -163,6 +163,10 @@ int rhizome_str_is_file_hash(const char *text);
#define alloca_tohex_bid(bid) alloca_tohex((bid), RHIZOME_MANIFEST_ID_BYTES)
int http_header_complete(const char *buf, size_t len, size_t tail);
int str_startswith(char *str, const char *substring, char **afterp);
int strcase_startswith(char *str, const char *substring, char **afterp);
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename);
int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(const char *id,int maximum_priority);
@ -173,6 +177,7 @@ char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out
long long rhizome_manifest_get_ll(rhizome_manifest *m, const char *var);
int rhizome_manifest_set_ll(rhizome_manifest *m,char *var,long long value);
int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value);
int rhizome_manifest_del(rhizome_manifest *m, const char *var);
long long rhizome_file_size(char *filename);
void _rhizome_manifest_free(const char *sourcefile,const char *funcname,int line,
rhizome_manifest *m);
@ -248,4 +253,3 @@ int rhizome_ignore_manifest_check(rhizome_manifest *m,
int rhizome_suggest_queue_manifest_import(rhizome_manifest *m,
struct sockaddr_in *peerip);

78
rhizome_bundle.c

@ -281,6 +281,10 @@ int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, int bu
WARNF("Missing filehash field");
m->errors++;
}
if (have_filehash && m->fileLength == 0) {
WARNF("Spurious filehash field");
m->errors++;
}
// TODO Determine group membership here.
@ -370,13 +374,33 @@ double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_
return default_value;
}
/* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_manifest_del(rhizome_manifest *m, const char *var)
{
int ret = 0;
int i;
for (i = 0; i < m->var_count; ++i)
if (strcmp(m->vars[i], var) == 0) {
free(m->vars[i]);
free(m->values[i]);
--m->var_count;
m->finalised = 0;
ret = 1;
break;
}
for (; i < m->var_count; ++i) {
m->vars[i] = m->vars[i + 1];
m->values[i] = m->values[i + 1];
}
return ret;
}
int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value)
{
if (!m)
return WHY("m == NULL");
int i;
if (!m) return -1;
for(i=0;i<m->var_count;i++)
if (!strcmp(m->vars[i],var)) {
free(m->values[i]);
@ -384,14 +408,12 @@ int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value
m->finalised=0;
return 0;
}
if (m->var_count>=MAX_MANIFEST_VARS) return -1;
if (m->var_count >= MAX_MANIFEST_VARS)
return WHY("no more manifest vars");
m->vars[m->var_count]=strdup(var);
m->values[m->var_count]=strdup(value);
m->var_count++;
m->finalised=0;
return 0;
}
@ -622,27 +644,31 @@ int rhizome_manifest_dump(rhizome_manifest *m, const char *msg)
int rhizome_manifest_finalise(rhizome_manifest *m)
{
/* set fileHexHash */
if (!m->fileHashedP) {
if (rhizome_hash_file(m,m->dataFileName,m->fileHexHash))
return WHY("rhizome_hash_file() failed during finalisation of manifest.");
m->fileHashedP=1;
/* set fileLength and "filesize" var */
if (m->dataFileName[0]) {
struct stat stat;
if (lstat(m->dataFileName, &stat)) {
WHY_perror("lstat");
return WHY("Could not stat() associated file");
}
m->fileLength = stat.st_size;
} else
m->fileLength = 0;
rhizome_manifest_set_ll(m, "filesize", m->fileLength);
/* set fileLength */
if (m->dataFileName[0]) {
struct stat stat;
if (lstat(m->dataFileName, &stat)) {
WHY_perror("lstat");
return WHY("Could not stat() associated file");
}
m->fileLength = stat.st_size;
} else
m->fileLength = 0;
/* set fileHexHash and "filehash" var */
if (m->fileLength != 0) {
if (!m->fileHashedP) {
if (rhizome_hash_file(m, m->dataFileName, m->fileHexHash))
return WHY("rhizome_hash_file() failed during finalisation of manifest.");
m->fileHashedP = 1;
}
rhizome_manifest_set(m, "filehash", m->fileHexHash);
} else {
m->fileHexHash[0] = '\0';
m->fileHashedP = 0;
rhizome_manifest_del(m, "filehash");
}
/* Set file hash and size information */
rhizome_manifest_set(m,"filehash",m->fileHexHash);
rhizome_manifest_set_ll(m,"filesize",m->fileLength);
/* set fileHighestPriority based on group associations.
XXX - Should probably be set as groups are added */

118
rhizome_database.c

@ -165,7 +165,7 @@ int rhizome_opendb()
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE filehash IS NULL;");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM FILES WHERE NOT EXISTS( SELECT 1 FROM MANIFESTS WHERE MANIFESTS.filehash = FILES.id);");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);");
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "DELETE FROM MANIFESTS WHERE filehash != '' AND NOT EXISTS( SELECT 1 FROM FILES WHERE MANIFESTS.filehash = FILES.id);");
RETURN(0);
}
@ -527,21 +527,14 @@ int rhizome_store_bundle(rhizome_manifest *m)
} else {
/* We don't have the secret for this manifest, so only allow updates if
the self-signature is valid */
if (!m->selfSigned) {
WHY("*** Insert into manifests failed (-2).");
if (!m->selfSigned)
return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt.");
}
}
char manifestid[RHIZOME_MANIFEST_ID_STRLEN + 1];
rhizome_manifest_get(m, "id", manifestid, sizeof manifestid);
str_toupper_inplace(manifestid);
char filehash[RHIZOME_FILEHASH_STRLEN + 1];
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
char *err;
int sql_ret;
sqlite3_stmt *stmt;
@ -550,14 +543,21 @@ int rhizome_store_bundle(rhizome_manifest *m)
unsigned char bar[RHIZOME_BAR_BYTES];
rhizome_manifest_to_bar(m,bar);
// we should add the file in the same transaction, but closing the blob seems to cause some issues.
/* Store the file */
// TODO encrypted payloads - pass encryption key here
if (m->fileLength>0){
if (rhizome_store_file(m,NULL)){
char filehash[RHIZOME_FILEHASH_STRLEN + 1];
if (m->fileLength > 0) {
if (!m->fileHashedP)
return WHY("Manifest payload hash unknown");
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
// TODO encrypted payloads - pass encryption key here
// We should add the file in the same transaction, but closing the blob seems to cause some issues.
if (rhizome_store_file(m, NULL)) {
WHY("Could not store file");
sql_ret = 1;
}
} else {
filehash[0] = '\0';
}
if (!rhizome_db) rhizome_opendb();
@ -697,6 +697,7 @@ int rhizome_list_manifests(const char *service, const char *sender_sid, const ch
break;
}
const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0);
DEBUGF("id = %s", q_manifestid);
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
long long q_version = sqlite3_column_int64(statement, 2);
@ -976,8 +977,6 @@ int rhizome_update_file_priority(const char *fileid)
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
int checkVersionP)
{
if (!m->fileHashedP)
return WHY("Manifest payload is not hashed");
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
const char *name = NULL;
const char *sender = NULL;
@ -996,29 +995,37 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
return WHYF("Unsupported service '%s'", service);
}
char sqlcmd[1024];
char *s = sqlcmd;
s += snprintf(s, &sqlcmd[sizeof sqlcmd] - s,
"SELECT id, manifest, version FROM manifests"
" WHERE filehash = ?"
);
if (checkVersionP && s < &sqlcmd[sizeof sqlcmd])
s += snprintf(s, sqlcmd + sizeof(sqlcmd) - s, " AND version = ?");
if (s >= &sqlcmd[sizeof sqlcmd])
return WHY("SQL command too long");
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
strbuf_puts(b, "SELECT id, manifest, version FROM manifests WHERE ");
if (m->fileLength != 0) {
if (!m->fileHashedP)
return WHY("Manifest payload is not hashed");
strbuf_puts(b, "filehash = ?");
} else
strbuf_puts(b, "filesize = 0");
if (checkVersionP)
strbuf_puts(b, " AND version = ?");
if (strbuf_overrun(b))
return WHYF("SQL command too long: %s", strbuf_str(b));
int ret = 0;
sqlite3_stmt *statement;
const char *cmdtail;
if (debug&DEBUG_RHIZOME) DEBUGF("sql query: %s",sqlcmd);
if (debug & DEBUG_RHIZOME)
DEBUGF("sql query: %s", sqlcmd);
if (sqlite3_prepare_v2(rhizome_db, sqlcmd, strlen(sqlcmd) + 1, &statement, &cmdtail) != SQLITE_OK) {
ret = WHY(sqlite3_errmsg(rhizome_db));
} else {
int field = 1;
char filehash[RHIZOME_FILEHASH_STRLEN + 1];
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
if (debug & DEBUG_RHIZOME) DEBUGF("filehash=\"%s\"", filehash);
sqlite3_bind_text(statement, 1, filehash, -1, SQLITE_STATIC);
if (m->fileLength != 0) {
strncpy(filehash, m->fileHexHash, sizeof filehash);
str_toupper_inplace(filehash);
if (debug & DEBUG_RHIZOME)
DEBUGF("filehash=\"%s\"", filehash);
sqlite3_bind_text(statement, field++, filehash, -1, SQLITE_STATIC);
}
if (checkVersionP)
sqlite3_bind_int64(statement, 2, m->version);
sqlite3_bind_int64(statement, field++, m->version);
size_t rows = 0;
while (sqlite3_step(statement) == SQLITE_ROW) {
++rows;
@ -1071,16 +1078,24 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found,
q_manifestid, q_version, blob_version);
++inconsistent;
}
if (!blob_filehash && strcasecmp(blob_filehash, m->fileHexHash)) {
WARNF("MANIFESTS row id=%s joined to FILES row id=%s has inconsistent blob: blob.filehash=%s -- skipped",
q_manifestid, m->fileHexHash, blob_filehash);
++inconsistent;
}
if (blob_filesize != -1 && blob_filesize != m->fileLength) {
WARNF("MANIFESTS row id=%s joined to FILES row id=%s has inconsistent blob: known file size %lld, blob.filesize=%lld -- skipped",
q_manifestid, m->fileHexHash, m->fileLength, blob_filesize);
++inconsistent;
}
if (m->fileLength != 0) {
if (!blob_filehash && strcasecmp(blob_filehash, m->fileHexHash)) {
WARNF("MANIFESTS row id=%s joined to FILES row id=%s has inconsistent blob: blob.filehash=%s -- skipped",
q_manifestid, m->fileHexHash, blob_filehash);
++inconsistent;
}
} else {
if (!blob_filehash) {
WARNF("MANIFESTS row id=%s joined to FILES row id=%s has inconsistent blob: blob.filehash should be absent -- skipped",
q_manifestid, m->fileHexHash);
++inconsistent;
}
}
if (checkVersionP && q_version != m->version) {
WARNF("SELECT query with version=%lld returned incorrect row: manifests.version=%lld -- skipped", m->version, q_version);
++inconsistent;
@ -1182,23 +1197,30 @@ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp)
const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
if (blob_service == NULL)
ret = WHY("Manifest is missing 'service' field");
long long filesizeq = rhizome_manifest_get_ll(m, "filesize");
if (filesizeq == -1)
ret = WHY("Manifest is missing 'filesize' field");
else
m->fileLength = filesizeq;
const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
if (blob_filehash == NULL)
ret = WHY("Manifest is missing 'filehash' field");
else {
memcpy(m->fileHexHash, blob_filehash, RHIZOME_FILEHASH_STRLEN + 1);
m->fileHashedP = 1;
if (m->fileLength != 0) {
if (blob_filehash == NULL)
ret = WHY("Manifest is missing 'filehash' field");
else {
memcpy(m->fileHexHash, blob_filehash, RHIZOME_FILEHASH_STRLEN + 1);
m->fileHashedP = 1;
}
} else {
if (blob_filehash != NULL)
WARN("Manifest contains spurious 'filehash' field");
m->fileHexHash[0] = '\0';
m->fileHashedP = 0;
}
long long blob_version = rhizome_manifest_get_ll(m, "version");
if (blob_version == -1)
ret = WHY("Manifest is missing 'version' field");
else
m->version = blob_version;
long long filesizeq = rhizome_manifest_get_ll(m, "filesize");
if (filesizeq == -1)
ret = WHY("Manifest is missing 'filesize' field");
else
m->fileLength = filesizeq;
if (ret == 1) {
cli_puts("service"); cli_delim(":");
cli_puts(blob_service); cli_delim("\n");
@ -1208,10 +1230,12 @@ int rhizome_retrieve_manifest(const char *manifestid, rhizome_manifest **mp)
cli_printf("%lld", (long long) sqlite3_column_int64(statement, 2)); cli_delim("\n");
cli_puts("inserttime"); cli_delim(":");
cli_printf("%lld", (long long) sqlite3_column_int64(statement, 3)); cli_delim("\n");
cli_puts("filehash"); cli_delim(":");
cli_puts(m->fileHexHash); cli_delim("\n");
cli_puts("filesize"); cli_delim(":");
cli_printf("%lld", (long long) m->fileLength); cli_delim("\n");
if (m->fileLength != 0) {
cli_puts("filehash"); cli_delim(":");
cli_puts(m->fileHexHash); cli_delim("\n");
}
// Could write the manifest blob to the CLI output here, but that would require the output to
// support byte[] fields as well as String fields.
}

327
rhizome_fetch.c

@ -381,20 +381,41 @@ int rhizome_position_candidate(int position)
return 0;
}
void rhizome_import_received_bundle(struct rhizome_manifest *m)
{
// TODO: We already have the manifest struct in memory, should import the bundle
// directly from that, not by writing it to a file and re-reading it!
const char *id = rhizome_manifest_get(m, "id", NULL, 0);
if (id == NULL) {
WHY("Manifest missing ID");
return;
}
if (create_rhizome_import_dir() == -1)
return;
char filename[1024];
if (!FORM_RHIZOME_IMPORT_PATH(filename, "manifest.%s", id))
return;
/* Do really write the manifest unchanged */
m->finalised = 1;
m->manifest_bytes = m->manifest_all_bytes;
if (debug & DEBUG_RHIZOME_RX) {
DEBUGF("manifest bid=%s len=%d has %d signatories", id, m->manifest_bytes, m->sig_count);
dump("manifest", m->manifestdata, m->manifest_all_bytes);
}
if (rhizome_write_manifest_file(m, filename) != -1)
rhizome_bundle_import(m, NULL, id, m->ttl - 1 /* TTL */);
}
/* Verifies manifests as late as possible to avoid wasting time. */
int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, struct sockaddr_in *peerip)
{
IN();
/* must free manifest when done with it */
char *id=rhizome_manifest_get(m,"id",NULL,0);
long long filesize=rhizome_manifest_get_ll(m,"filesize");
char *id = rhizome_manifest_get(m, "id", NULL, 0);
int priority=100; /* normal priority */
int i;
m->version = rhizome_manifest_get_ll(m,"version");
if (debug & DEBUG_RHIZOME_RX)
DEBUGF("Considering manifest import bid=%s version=%lld size=%lld priority=%d:", id, m->version, filesize, priority);
DEBUGF("Considering manifest import bid=%s version=%lld size=%lld priority=%d:", id, m->version, m->fileLength, priority);
if (rhizome_manifest_version_cache_lookup(m)) {
if (debug & DEBUG_RHIZOME_RX)
@ -409,7 +430,20 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, struct sockaddr_i
DEBUGF(" is new (have version %lld)", stored_version);
}
if (m->fileLength == 0) {
if (rhizome_manifest_verify(m) != 0) {
WHY("Error verifying manifest when considering for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m, peerip, 60000);
rhizome_manifest_free(m);
RETURN(-1);
}
rhizome_import_received_bundle(m);
RETURN(0);
}
/* work out where to put it in the list */
int i;
for(i=0;i<candidate_count;i++)
{
/* If this manifest is already in the list, stop.
@ -421,11 +455,9 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, struct sockaddr_i
/* duplicate.
XXX - Check versions! We should replace older with newer,
and then update position in queue based on size */
long long list_version=rhizome_manifest_get_ll(candidates[i].manifest, "version");
long long this_version=rhizome_manifest_get_ll(m,"version");
if (list_version>=this_version) {
/* this version is older than the one in the list,
so don't list this one */
long long list_version = rhizome_manifest_get_ll(candidates[i].manifest, "version");
if (list_version >= m->version) {
/* this version is older than the one in the list, so don't list this one */
rhizome_manifest_free(m);
RETURN(0);
} else {
@ -451,7 +483,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, struct sockaddr_i
the list down. */
if (candidates[i].priority>priority
||(candidates[i].priority==priority
&&candidates[i].size>filesize))
&&candidates[i].size>m->fileLength))
break;
}
if (i>=MAX_CANDIDATES) {
@ -482,7 +514,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, struct sockaddr_i
bytes);
/* put new candidate in */
candidates[i].manifest=m;
candidates[i].size=filesize;
candidates[i].size=m->fileLength;
candidates[i].priority=priority;
candidates[i].peer=*peerip;
@ -770,37 +802,11 @@ void rhizome_write_content(rhizome_file_fetch_record *q, char *buffer, int bytes
/* got all of file */
if (debug & DEBUG_RHIZOME_RX)
DEBUGF("Received all of file via rhizome -- now to import it");
{
fclose(q->file);
q->file = NULL;
// TODO: We already have the manifest struct in memory, should import the bundle
// directly from that, not by writing it to a file and re-reading it!
const char *id = rhizome_manifest_get(q->manifest, "id", NULL, 0);
if (id == NULL)
{ WHY("Manifest missing ID"); return; }
if (create_rhizome_import_dir() == -1)
return;
char filename[1024];
if (!FORM_RHIZOME_IMPORT_PATH(filename,"manifest.%s", id))
return;
/* Do really write the manifest unchanged */
if (debug & DEBUG_RHIZOME_RX) {
DEBUGF("manifest has %d signatories",q->manifest->sig_count);
DEBUGF("manifest bid=%s len=%d",
rhizome_manifest_get(q->manifest, "id", NULL, 0),
q->manifest->manifest_bytes);
dump("manifest",&q->manifest->manifestdata[0],
q->manifest->manifest_all_bytes);
}
q->manifest->finalised=1;
q->manifest->manifest_bytes=q->manifest->manifest_all_bytes;
if (rhizome_write_manifest_file(q->manifest,filename) != -1) {
rhizome_bundle_import(q->manifest, NULL, id,
q->manifest->ttl - 1 /* TTL */);
}
rhizome_manifest_free(q->manifest);
q->manifest = NULL;
}
fclose(q->file);
q->file = NULL;
rhizome_import_received_bundle(q->manifest);
rhizome_manifest_free(q->manifest);
q->manifest = NULL;
rhizome_fetch_close(q);
return;
}
@ -822,128 +828,131 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
return;
}
switch(q->state)
{
switch(q->state) {
case RHIZOME_FETCH_CONNECTING:
case RHIZOME_FETCH_SENDINGHTTPREQUEST:
rhizome_fetch_write(q);
break;
case RHIZOME_FETCH_RXFILE:
/* Keep reading until we have the promised amount of data */
sigPipeFlag=0;
errno=0;
char buffer[8192];
int bytes=read(q->alarm.poll.fd,buffer,8192);
/* If we got some data, see if we have found the end of the HTTP request */
if (bytes>0)
rhizome_write_content(q, buffer, bytes);
break;
case RHIZOME_FETCH_RXHTTPHEADERS:
/* Keep reading until we have two CR/LFs in a row */
sigPipeFlag=0;
errno=0;
bytes=read(q->alarm.poll.fd,&q->request[q->request_len],
1024-q->request_len-1);
if (sigPipeFlag||((bytes==0)&&(errno==0))) {
/* broken pipe, so close connection */
if (debug&DEBUG_RHIZOME)
DEBUG("Closing rhizome fetch connection due to sigpipe");
rhizome_fetch_close(q);
return;
}
/* If we got some data, see if we have found the end of the HTTP request */
if (bytes>0) {
int lfcount=0;
int i=q->request_len-160;
// reset timeout
unschedule(&q->alarm);
q->alarm.alarm=overlay_gettime_ms() + RHIZOME_IDLE_TIMEOUT;
q->alarm.deadline = q->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
schedule(&q->alarm);
if (i<0) i=0;
q->request_len+=bytes;
if (q->request_len<1024)
q->request[q->request_len]=0;
for(;i<(q->request_len+bytes);i++)
{
switch(q->request[i]) {
case '\n': lfcount++; break;
case '\r': /* ignore CR */ break;
case 0: /* ignore NUL (telnet inserts them) */ break;
default: lfcount=0; break;
}
if (lfcount==2) break;
}
if (debug&DEBUG_RHIZOME)
dump("http reply headers",(unsigned char *)q->request,lfcount==2?i:q->request_len);
if (lfcount==2) {
/* We have the response headers, so parse.
(we may also have some bytes of content, so we need to be a little
careful) */
/* Terminate string at end of headers */
q->request[i]=0;
/* Get HTTP result code */
char *s=strstr(q->request,"HTTP/1.0 ");
if (!s) {
if (debug&DEBUG_RHIZOME) DEBUGF("HTTP response lacked HTTP/1.0 response code.");
rhizome_fetch_close(q);
return;
}
int http_response_code=strtoll(&s[9],NULL,10);
if (http_response_code!=200) {
if (debug&DEBUG_RHIZOME) DEBUGF("Rhizome web server returned %d != 200 OK",http_response_code);
rhizome_fetch_close(q);
return;
}
/* Get content length */
s=strstr(q->request,"Content-length: ");
if (!s) {
if (debug&DEBUG_RHIZOME)
DEBUGF("Missing Content-Length: header.");
rhizome_fetch_close(q);
return;
}
q->file_len=strtoll(&s[16],NULL,10);
if (q->file_len<0) {
if (debug&DEBUG_RHIZOME)
DEBUGF("Illegal file size (%d).",q->file_len);
rhizome_fetch_close(q);
return;
}
/* Okay, we have both, and are all set.
File is already open, so just write out any initial bytes of the
file we read, and update state flag.
*/
q->state=RHIZOME_FETCH_RXFILE;
int fileRxBytes=q->request_len-(i+1);
if (fileRxBytes>0)
rhizome_write_content(q, &q->request[i+1], fileRxBytes);
case RHIZOME_FETCH_RXFILE: {
/* Keep reading until we have the promised amount of data */
char buffer[8192];
sigPipeFlag = 0;
int bytes = read_nonblock(q->alarm.poll.fd, buffer, sizeof buffer);
/* If we got some data, see if we have found the end of the HTTP request */
if (bytes > 0) {
rhizome_write_content(q, buffer, bytes);
} else {
if (debug & DEBUG_RHIZOME_RX)
DEBUG("Empty read, closing connection");
rhizome_fetch_close(q);
return;
}
}
if (sigPipeFlag) {
if (debug & DEBUG_RHIZOME_RX)
DEBUG("Received SIGPIPE, closing connection");
rhizome_fetch_close(q);
return;
}
}
break;
case RHIZOME_FETCH_RXHTTPHEADERS: {
/* Keep reading until we have two CR/LFs in a row */
sigPipeFlag = 0;
int bytes = read_nonblock(q->alarm.poll.fd, &q->request[q->request_len], 1024 - q->request_len - 1);
/* If we got some data, see if we have found the end of the HTTP reply */
if (bytes > 0) {
// reset timeout
unschedule(&q->alarm);
q->alarm.alarm = overlay_gettime_ms() + RHIZOME_IDLE_TIMEOUT;
q->alarm.deadline = q->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
schedule(&q->alarm);
q->request_len += bytes;
if (http_header_complete(q->request, q->request_len, bytes + 4)) {
if (debug & DEBUG_RHIZOME_RX)
DEBUGF("Got HTTP reply: %s", alloca_toprint(160, (unsigned char *)q->request, q->request_len));
/* We have all the reply headers, so parse them, taking care of any following bytes of
content. */
char *p = NULL;
if (!str_startswith(q->request, "HTTP/1.0 ", &p)) {
if (debug&DEBUG_RHIZOME_RX)
DEBUGF("Malformed HTTP reply: missing HTTP/1.0 preamble");
rhizome_fetch_close(q);
return;
}
int http_response_code = 0;
char *nump;
for (nump = p; isdigit(*p); ++p)
http_response_code = http_response_code * 10 + *p - '0';
if (p == nump || *p != ' ') {
if (debug&DEBUG_RHIZOME_RX)
DEBUGF("Malformed HTTP reply: missing decimal status code");
rhizome_fetch_close(q);
return;
}
if (http_response_code != 200) {
if (debug & DEBUG_RHIZOME_RX)
DEBUGF("Failed HTTP request: rhizome server returned %d != 200 OK", http_response_code);
rhizome_fetch_close(q);
return;
}
// This loop will terminate, because http_header_complete() above found at least
// "\n\n" at the end of the header, and probably "\r\n\r\n".
while (*p++ != '\n')
;
// Iterate over header lines until the last blank line.
long long content_length = -1;
while (*p != '\r' && *p != '\n') {
if (strcase_startswith(p, "Content-Length:", &p)) {
while (*p == ' ')
++p;
content_length = 0;
for (nump = p; isdigit(*p); ++p)
content_length = content_length * 10 + *p - '0';
if (p == nump || (*p != '\r' && *p != '\n')) {
if (debug & DEBUG_RHIZOME_RX) {
DEBUGF("Invalid HTTP reply: malformed Content-Length header");
rhizome_fetch_close(q);
return;
}
}
}
while (*p++ != '\n')
;
}
if (*p == '\r')
++p;
++p; // skip '\n' at end of blank line
if (content_length == -1) {
if (debug & DEBUG_RHIZOME_RX)
DEBUGF("Invalid HTTP reply: missing Content-Length header");
rhizome_fetch_close(q);
return;
}
q->file_len = content_length;
/* We have all we need. The file is already open, so just write out any initial bytes of
the body we read.
*/
q->state = RHIZOME_FETCH_RXFILE;
int content_bytes = q->request + q->request_len - p;
if (content_bytes > 0)
rhizome_write_content(q, p, content_bytes);
}
} else {
if (debug & DEBUG_RHIZOME_RX)
DEBUG("Empty read, closing connection");
rhizome_fetch_close(q);
return;
}
if (sigPipeFlag) {
if (debug & DEBUG_RHIZOME_RX)
DEBUG("Received SIGPIPE, closing connection");
rhizome_fetch_close(q);
return;
}
}
break;
default:
if (debug&DEBUG_RHIZOME)
if (debug & DEBUG_RHIZOME_RX)
DEBUG("Closing rhizome fetch connection due to illegal/unimplemented state.");
rhizome_fetch_close(q);
return;

331
rhizome_http.c

@ -72,13 +72,12 @@ typedef struct rhizome_http_request {
} rhizome_http_request;
int rhizome_server_free_http_request(rhizome_http_request *r);
int rhizome_server_http_send_bytes(rhizome_http_request *r);
int rhizome_server_parse_http_request(rhizome_http_request *r);
int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char *response);
int rhizome_server_http_response_header(rhizome_http_request *r,int result,
char *mime_type,unsigned long long bytes);
int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column);
static int rhizome_server_free_http_request(rhizome_http_request *r);
static int rhizome_server_http_send_bytes(rhizome_http_request *r);
static int rhizome_server_parse_http_request(rhizome_http_request *r);
static int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response);
static int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, unsigned long long bytes);
static int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column);
#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32
@ -226,7 +225,7 @@ void rhizome_client_poll(struct sched_ent *alarm)
/* Keep reading until we have two CR/LFs in a row */
r->request[r->request_length] = '\0';
sigPipeFlag=0;
int bytes = read_nonblock(r->alarm.poll.fd, &r->request[r->request_length], RHIZOME_HTTP_REQUEST_MAXLEN - r->request_length - 1);
int bytes = read_nonblock(r->alarm.poll.fd, &r->request[r->request_length], RHIZOME_HTTP_REQUEST_MAXLEN - r->request_length);
/* If we got some data, see if we have found the end of the HTTP request */
if (bytes > 0) {
// reset inactivity timer
@ -234,23 +233,8 @@ void rhizome_client_poll(struct sched_ent *alarm)
r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
unschedule(&r->alarm);
schedule(&r->alarm);
int i = r->request_length - 160;
if (i<0) i=0;
r->request_length+=bytes;
if (r->request_length<RHIZOME_HTTP_REQUEST_MAXLEN)
r->request[r->request_length]=0;
if (0)
dump("request", (unsigned char *)r->request,r->request_length);
int lfcount;
for(lfcount = 0; lfcount < 2 && i < r->request_length + bytes; ++i) {
switch (r->request[i]) {
case '\n': ++lfcount; break;
case '\r': break;
case '\0': break; // ignore NUL (telnet inserts them)
default: lfcount = 0; break;
}
}
if (lfcount == 2) {
r->request_length += bytes;
if (http_header_complete(r->request, r->request_length, bytes + 4)) {
/* We have the request. Now parse it to see if we can respond to it */
rhizome_server_parse_http_request(r);
}
@ -275,7 +259,6 @@ void rhizome_client_poll(struct sched_ent *alarm)
return;
}
void rhizome_server_poll(struct sched_ent *alarm)
{
struct sockaddr addr;
@ -284,7 +267,8 @@ void rhizome_server_poll(struct sched_ent *alarm)
while ((sock = accept(rhizome_server_socket, &addr, &addr_len)) != -1) {
if (addr.sa_family == AF_INET) {
struct sockaddr_in *peerip = (struct sockaddr_in *)&addr;
INFOF("HTTP ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u",
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("HTTP ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u",
addr_len, peerip->sin_family, peerip->sin_port,
((unsigned char*)&peerip->sin_addr.s_addr)[0],
((unsigned char*)&peerip->sin_addr.s_addr)[1],
@ -292,41 +276,49 @@ void rhizome_server_poll(struct sched_ent *alarm)
((unsigned char*)&peerip->sin_addr.s_addr)[3]
);
} else {
INFOF("HTTP ACCEPT addrlen=%u family=%u data=%s",
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("HTTP ACCEPT addrlen=%u family=%u data=%s",
addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data));
}
rhizome_http_request *request = calloc(sizeof(rhizome_http_request),1);
/* We are now trying to read the HTTP request */
request->request_type=RHIZOME_HTTP_REQUEST_RECEIVING;
request->alarm.function = rhizome_client_poll;
connection_stats.name="rhizome_client_poll";
request->alarm.stats=&connection_stats;
request->alarm.poll.fd=sock;
request->alarm.poll.events=POLLIN;
request->alarm.alarm = overlay_gettime_ms()+RHIZOME_IDLE_TIMEOUT;
request->alarm.deadline = request->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
// watch for the incoming http request
watch(&request->alarm);
// set an inactivity timeout to close the connection
schedule(&request->alarm);
rhizome_http_request *request = calloc(sizeof(rhizome_http_request), 1);
if (request == NULL) {
WHYF_perror("calloc(%u, 1)", sizeof(rhizome_http_request));
WHY("Cannot respond to request, out of memory");
} else {
/* We are now trying to read the HTTP request */
request->request_type=RHIZOME_HTTP_REQUEST_RECEIVING;
request->alarm.function = rhizome_client_poll;
connection_stats.name="rhizome_client_poll";
request->alarm.stats=&connection_stats;
request->alarm.poll.fd=sock;
request->alarm.poll.events=POLLIN;
request->alarm.alarm = overlay_gettime_ms()+RHIZOME_IDLE_TIMEOUT;
request->alarm.deadline = request->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
// watch for the incoming http request
watch(&request->alarm);
// set an inactivity timeout to close the connection
schedule(&request->alarm);
}
}
if (errno != EAGAIN) {
WARN_perror("accept");
}
}
int rhizome_server_free_http_request(rhizome_http_request *r)
static int rhizome_server_free_http_request(rhizome_http_request *r)
{
unwatch(&r->alarm);
unschedule(&r->alarm);
close(r->alarm.poll.fd);
if (r->buffer&&r->buffer_size) free(r->buffer);
if (r->blob) sqlite3_blob_close(r->blob);
if (r->buffer)
free(r->buffer);
if (r->blob)
sqlite3_blob_close(r->blob);
free(r);
return 0;
}
int rhizome_server_sql_query_http_response(rhizome_http_request *r,
static int rhizome_server_sql_query_http_response(rhizome_http_request *r,
char *column,char *table,char *query_body,
int bytes_per_row,int dehexP)
{
@ -339,10 +331,17 @@ int rhizome_server_sql_query_http_response(rhizome_http_request *r,
the body, although encryption is not yet implemented here.
*/
if (r->buffer) { free(r->buffer); r->buffer=NULL; }
r->buffer_size=16384;
r->buffer=malloc(r->buffer_size);
if (!r->buffer) return WHY_perror("malloc");
if (r->buffer == NULL || r->buffer_size < 16384) {
if (r->buffer)
free(r->buffer);
r->buffer_size = 16384;
r->buffer = malloc(r->buffer_size);
if (r->buffer == NULL) {
r->buffer_size = 0;
WHY_perror("malloc");
return WHY("Cannot send response, out of memory");
}
}
r->buffer_length=0;
r->buffer_offset=0;
r->source_record_size=bytes_per_row;
@ -351,10 +350,9 @@ int rhizome_server_sql_query_http_response(rhizome_http_request *r,
/* Work out total response length */
long long response_bytes=256+r->source_count*r->source_record_size;
rhizome_server_http_response_header(r,200,"servalproject.org/rhizome-list",
response_bytes);
rhizome_server_http_response_header(r, 200, "servalproject.org/rhizome-list", response_bytes);
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("headers consumed %d bytes",r->buffer_length);
DEBUGF("headers consumed %d bytes", r->buffer_length);
/* Clear and prepare response header */
bzero(&r->buffer[r->buffer_length],256);
@ -389,7 +387,7 @@ int rhizome_server_sql_query_http_response(rhizome_http_request *r,
return rhizome_server_sql_query_fill_buffer(r, table, column);
}
int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column)
static int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, char *column)
{
unsigned char blob_value[r->source_record_size*2+1];
@ -486,37 +484,90 @@ int rhizome_server_sql_query_fill_buffer(rhizome_http_request *r, char *table, c
return 0;
}
static int strcmp_prefix(char *str, const char *prefix, char **afterp)
int http_header_complete(const char *buf, size_t len, size_t tail)
{
while (*prefix && *str && *prefix == *str)
++prefix, ++str;
if (*prefix)
return (unsigned char)*str - (unsigned char)*prefix;
if (afterp)
*afterp = str;
return 0;
const char *bufend = buf + len;
if (tail < len)
buf = bufend - tail;
int count = 0;
for (; count < 2 && buf != bufend; ++buf) {
switch (*buf) {
case '\n': ++count; break;
case '\r': break;
case '\0': break; // ignore NUL (telnet inserts them)
default: count = 0; break;
}
}
return count == 2;
}
int rhizome_server_parse_http_request(rhizome_http_request *r)
/* Check if a given string starts with a given sub-string. If so, return 1 and, if afterp is not
NULL, set *afterp to point to the character immediately following the substring. Otherwise
return 0.
This function is used to parse HTTP headers and responses, which are typically not
nul-terminated, but are held in a buffer which has an associated length. To avoid this function
running past the end of the buffer, the caller must ensure that the buffer contains a sub-string
that is not part of the sub-string being sought, eg, "\r\n\r\n" as detected by
http_header_complete(). This guarantees that this function will return nonzero before running
past the end of the buffer.
@author Andrew Bettison <andrew@servalproject.com>
*/
int str_startswith(char *str, const char *substring, char **afterp)
{
while (*substring && *substring == *str)
++substring, ++str;
if (*substring)
return 0;
if (afterp)
*afterp = str;
return 1;
}
/* Case-insensitive form of str_startswith().
*/
int strcase_startswith(char *str, const char *substring, char **afterp)
{
while (*substring && *str && toupper(*substring) == toupper(*str))
++substring, ++str;
if (*substring)
return 0;
if (afterp)
*afterp = str;
return 1;
}
static int rhizome_server_parse_http_request(rhizome_http_request *r)
{
/* Switching to writing, so update the call-back */
r->alarm.poll.events=POLLOUT;
watch(&r->alarm);
/* Clear request type flags */
r->request_type=0;
char path[1024];
if (sscanf(r->request, "GET %1024s HTTP/1.%*1[01]%*[\r\n]", path) != 1) {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (unsigned char *)r->request, r->request_length));
rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
} else {
// Start building up a response.
r->request_type = 0;
// Parse the HTTP "GET" line.
char *path = NULL;
size_t pathlen = 0;
if (str_startswith(r->request, "GET ", &path)) {
char *p;
// This loop is guaranteed to terminate before the end of the buffer, because we know that the
// buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block.
for (p = path; !isspace(*p); ++p)
;
pathlen = p - path;
if ( str_startswith(p, " HTTP/1.", &p)
&& (str_startswith(p, "0", &p) || str_startswith(p, "1", &p))
&& (str_startswith(p, "\r\n", &p) || str_startswith(p, "\n", &p))
)
path[pathlen] = '\0';
else
path = NULL;
}
if (path) {
char *id = NULL;
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("GET %s", path);
DEBUGF("GET %s", alloca_toprint(1024, (unsigned char *)path, pathlen));
if (strcmp(path, "/favicon.ico") == 0) {
r->request_type = RHIZOME_HTTP_REQUEST_FAVICON;
rhizome_server_http_response_header(r, 200, "image/vnd.microsoft.icon", favicon_len);
rhizome_server_http_response_header(r, 200, "image/vnd.microsoft.icon", favicon_len);
} else if (strcmp(path, "/rhizome/groups") == 0) {
/* Return the list of known groups */
rhizome_server_sql_query_http_response(r, "id", "groups", "from groups", 32, 1);
@ -526,7 +577,7 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
} else if (strcmp(path, "/rhizome/bars") == 0) {
/* Return the list of known BARs */
rhizome_server_sql_query_http_response(r, "bar", "manifests", "from manifests", 32, 0);
} else if (strcmp_prefix(path, "/rhizome/file/", &id) == 0) {
} else if (str_startswith(path, "/rhizome/file/", &id)) {
/* Stream the specified payload */
if (!rhizome_str_is_file_hash(id)) {
rhizome_server_simple_http_response(r, 400, "<html><h1>Invalid payload ID</h1></html>\r\n");
@ -546,12 +597,16 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
r->request_type |= RHIZOME_HTTP_REQUEST_BLOB;
}
}
} else if (strcmp_prefix(path, "/rhizome/manifest/", &id) == 0) {
/* TODO: Stream the specified manifest */
} else if (str_startswith(path, "/rhizome/manifest/", &id)) {
// TODO: Stream the specified manifest
rhizome_server_simple_http_response(r, 500, "<html><h1>Not implemented</h1></html>\r\n");
} else {
rhizome_server_simple_http_response(r, 404, "<html><h1>Not found</h1></html>\r\n");
}
} else {
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (unsigned char *)r->request, r->request_length));
rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
}
/* Try sending data immediately. */
@ -562,43 +617,89 @@ int rhizome_server_parse_http_request(rhizome_http_request *r)
/* Return appropriate message for HTTP response codes, both known and unknown. */
#define A_VALUE_GREATER_THAN_FOUR (2+3)
char *httpResultString(int id) {
switch (id) {
case 200: return "OK"; break;
case 206: return "Partial Content"; break;
case 404: return "Not found"; break;
default:
case A_VALUE_GREATER_THAN_FOUR:
if (id>4) return "A suffusion of yellow";
/* The following MUST be the longest string returned by this function */
else return "THE JUDGEMENT OF KING WEN: Chun Signifies Difficulties At Outset, As Of Blade Of Grass Pushing Up Against Stone.";
static const char *httpResultString(int response_code) {
switch (response_code) {
case 200: return "OK";
case 206: return "Partial Content";
case 404: return "Not found";
case 500: return "Internal server error";
default: return "A suffusion of yellow";
}
}
int rhizome_server_simple_http_response(rhizome_http_request *r,int result, char *response)
struct http_response {
unsigned int result_code;
const char * content_type;
unsigned long long content_length;
const char * body;
};
static strbuf strbuf_build_http_response(strbuf sb, const struct http_response *h)
{
if (r->buffer) free(r->buffer);
r->buffer_size=strlen(response)+strlen("HTTP/1.0 000 \r\n\r\nContent-type: text/html\r\nContent-length: 0000\r\n\r\n")+strlen(httpResultString(result))+strlen(response)+100;
strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", h->result_code, httpResultString(h->result_code));
strbuf_sprintf(sb, "Content-type: %s\r\n", h->content_type);
strbuf_sprintf(sb, "Content-length: %llu\r\n", h->content_length);
strbuf_puts(sb, "\r\n");
if (h->body)
strbuf_puts(sb, h->body);
return sb;
}
r->buffer=(unsigned char *)malloc(r->buffer_size);
snprintf((char *)r->buffer,r->buffer_size,"HTTP/1.0 %03d %s\r\nContent-type: text/html\r\nContent-length: %d\r\n\r\n%s",result,httpResultString(result),(int)strlen(response),response);
r->buffer_size=strlen((char *)r->buffer)+1;
r->buffer_length=r->buffer_size-1;
r->buffer_offset=0;
r->request_type=RHIZOME_HTTP_REQUEST_FROMBUFFER;
static int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h)
{
strbuf b = strbuf_local((char *) r->buffer, r->buffer_size);
strbuf_build_http_response(b, h);
if (r->buffer == NULL || strbuf_overrun(b)) {
// Need a bigger buffer
if (r->buffer)
free(r->buffer);
r->buffer_size = strbuf_count(b) + 1;
r->buffer = malloc(r->buffer_size);
if (r->buffer == NULL) {
WHYF_perror("malloc(%u)", r->buffer_size);
r->buffer_size = 0;
return WHY("Cannot send response, out of memory");
}
strbuf_init(b, (char *) r->buffer, r->buffer_size);
strbuf_build_http_response(b, h);
if (strbuf_overrun(b))
return WHYF("Bug! Cannot send response, buffer not big enough");
}
r->buffer_length = strbuf_len(b);
r->buffer_offset = 0;
r->request_type |= RHIZOME_HTTP_REQUEST_FROMBUFFER;
if (debug & DEBUG_RHIZOME_TX)
DEBUGF("Sending HTTP response: %s", alloca_toprint(120, r->buffer, r->buffer_length));
return 0;
}
static int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response)
{
struct http_response hr;
hr.result_code = result;
hr.content_type = "text/html";
hr.content_length = strlen(response);
hr.body = response;
return rhizome_server_set_response(r, &hr);
}
static int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, unsigned long long bytes)
{
struct http_response hr;
hr.result_code = result;
hr.content_type = mime_type;
hr.content_length = bytes;
hr.body = NULL;
return rhizome_server_set_response(r, &hr);
}
/*
return codes:
1: connection still open.
0: connection finished.
<0: an error occurred.
*/
int rhizome_server_http_send_bytes(rhizome_http_request *r)
static int rhizome_server_http_send_bytes(rhizome_http_request *r)
{
// keep writing until the write would block or we run out of data
while(r->request_type){
@ -703,33 +804,3 @@ int rhizome_server_http_send_bytes(rhizome_http_request *r)
if (!r->request_type) return rhizome_server_free_http_request(r);
return 1;
}
int rhizome_server_http_response_header(rhizome_http_request *r,int result,
char *mime_type,unsigned long long bytes)
{
int min_buff = strlen("HTTP/1.0 000 \r\nContent-type: \r\nContent-length: \r\n\r\n")
+strlen(httpResultString(result))
+strlen(mime_type)+20;
if (min_buff+bytes > 65536){
min_buff = 65536;
}else{
min_buff += bytes;
}
if (r->buffer_size < min_buff) {
if (r->buffer)
free(r->buffer);
r->buffer=(unsigned char *)malloc(min_buff);
r->buffer_size=min_buff;
}
snprintf((char *)r->buffer,r->buffer_size,"HTTP/1.0 %03d %s\r\nContent-type: %s\r\nContent-length: %lld\r\n\r\n",result,httpResultString(result),mime_type,bytes);
r->buffer_length=strlen((char *)r->buffer);
r->buffer_offset=0;
r->request_type|=RHIZOME_HTTP_REQUEST_FROMBUFFER;
return 0;
}

2
serval.h

@ -795,7 +795,7 @@ char *catv(const char *data, char *buf, size_t len);
int dump(char *name, unsigned char *addr, size_t len);
char *toprint(char *dstStr, size_t dstChars, const unsigned char *srcBuf, size_t srcBytes);
#define alloca_toprint(dstlen,buf,len) toprint((char *)alloca((dstlen) + 1), (dstlen) + 1, (buf), (len) + 1)
#define alloca_toprint(dstlen,buf,len) toprint((char *)alloca((dstlen) + 1), (dstlen) + 1, (buf), (len))
#define alloca_tohex(buf,len) tohex((char *)alloca((len)*2+1), (buf), (len))
#define alloca_tohex_sid(sid) alloca_tohex((sid), SID_SIZE)

20
serval_packetvisualise.c

@ -14,6 +14,26 @@ char *indent(int n)
int senderSet=0;
unsigned char senderAddress[32];
#ifdef STANDALONE
int main(int argc,char **argv)
{
int i,n;
int len;
unsigned char buff[8192];
for(n=0;n<1024;n++)
{
int i;
len=random()%8192;
for(i=0;i<len;i++) buff[i]=random()&0xff;
serval_packetvisualise(stdout,"Fuzz Test",buff,len);
}
return 0;
}
#endif
int serval_packetvisualise_renderaddress(FILE *f,unsigned char *packet,int *ofs,int senderP)
{

41
testdefs_rhizome.sh

@ -28,6 +28,8 @@ rexp_date='[0-9]\{1,\}'
# Utility function:
# - create N identities in the current instance (I)
# - set variable SID to SID of first identity
# - set variable SID{I} to SID of first identity, eg, SIDA
# - set variables SID{I}{1..N} to SIDs of identities, eg, SIDA1, SIDA2...
# - set variables DID{I}{1..N} to DIDs of identities, eg, DIDA1, DIDA2...
# - set variables NAME{I}{1..N} to names of identities, eg, NAMEA1, NAMEA2...
@ -47,6 +49,10 @@ create_rhizome_identities() {
local namevar=NAME$instance_name$i
extract_stdout_keyvalue $sidvar sid "$rexp_sid"
tfw_log "$sidvar=${!sidvar}"
if [ $i -eq 1 ]; then
SID="${!sidvar}"
eval SID$instance_name="$SID"
fi
extract_stdout_keyvalue_optional DID$instance_name$i did "$rexp_did" && tfw_log "$didvar=${!didvar}"
extract_stdout_keyvalue_optional NAME$instance_name$i name ".*" && tfw_log "$namevar=${!namevar}"
done
@ -74,15 +80,19 @@ assert_manifest_complete() {
assertGrep "$manifest" "^BK=$rexp_bundlekey\$"
assertGrep "$manifest" "^date=$rexp_date\$"
assertGrep "$manifest" "^version=$rexp_version\$"
assertGrep "$manifest" "^filehash=$rexp_filehash\$"
assertGrep "$manifest" "^filesize=$rexp_filesize\$"
if grep -q '^filesize=0$' "$manifest"; then
assertGrep --matches=0 "$manifest" "^filehash="
else
assertGrep "$manifest" "^filehash=$rexp_filehash\$"
fi
if grep -q '^service=file$' "$manifest"; then
assertGrep "$manifest" "^name="
fi
}
assert_rhizome_list() {
assertStdoutLineCount '==' $(($# + 2))
assertStdoutLineCount --stderr '==' $(($# + 2))
assertStdoutIs --stderr --line=1 -e '11\n'
assertStdoutIs --stderr --line=2 -e 'service:id:version:date:.inserttime:.selfsigned:filesize:filehash:sender:recipient:name\n'
local filename
@ -105,7 +115,7 @@ assert_stdout_add_file() {
if replayStdout | grep -q '^service:file$'; then
opt_name=true
fi
fieldnames='service|manifestid|secret|filehash|filesize|name'
fieldnames='service|manifestid|secret|filesize|filehash|name'
for arg; do
case "$arg" in
!+($fieldnames))
@ -125,8 +135,12 @@ assert_stdout_add_file() {
${opt_service:-true} && assertStdoutGrep --matches=1 "^service:$re_service\$"
${opt_manifestid:-true} && assertStdoutGrep --matches=1 "^manifestid:$re_manifestid\$"
${opt_secret:-true} && assertStdoutGrep --matches=1 "^secret:$re_secret\$"
${opt_filehash:-true} && assertStdoutGrep --matches=1 "^filehash:$re_filehash\$"
${opt_filesize:-true} && assertStdoutGrep --matches=1 "^filesize:$re_filesize\$"
if replayStdout | grep -q '^filesize:0$'; then
assertStdoutGrep --matches=0 "^filehash:"
else
${opt_filehash:-true} && assertStdoutGrep --matches=1 "^filehash:$re_filehash\$"
fi
}
assert_stdout_import_bundle() {
@ -137,16 +151,19 @@ assert_stdout_import_bundle() {
unpack_manifest_for_grep() {
local filename="$1"
re_service="$rexp_service"
if [ -n "$filename" ]; then
re_filesize=$(( $(cat "$filename" | wc -c) + 0 ))
else
re_filesize=0
fi
compute_filehash re_filehash "$filename"
re_manifestid="$rexp_manifestid"
re_version="$rexp_version"
re_secret="$rexp_bundlesecret"
re_name=$(escape_grep_basic "${filename##*/}")
local filesize=$(sed -n -e '/^filesize=/s///p' "$filename.manifest" 2>/dev/null)
if [ "$filesize" = 0 ]; then
re_filesize=0
re_filehash=
re_name="\($re_name\)\{0,1\}"
else
re_filesize=$(( $(cat "$filename" | wc -c) + 0 ))
compute_filehash re_filehash "$filename"
fi
# If there is a manifest file that looks like it matches this payload
# file, then use its file hash to check the rhizome list '' output.
local filehash=$(sed -n -e '/^filehash=/s///p' "$filename.manifest" 2>/dev/null)
@ -220,6 +237,10 @@ extract_manifest_BK() {
extract_manifest "$1" "$2" BK "$rexp_bundlekey"
}
extract_manifest_filesize() {
extract_manifest "$1" "$2" filesize "$rexp_filesize"
}
extract_manifest_filehash() {
extract_manifest "$1" "$2" filehash "$rexp_filehash"
}

98
testframework.sh

@ -65,6 +65,11 @@ Options:
-E, --stop-on-error Do not execute any tests after an ERROR occurs
-F, --stop-on-failure Do not execute any tests after a FAIL occurs
--filter=PREFIX Only execute tests whose names start with PREFIX
--filter=N Only execute test number N
--filter=M-N Only execute tests with numbers in range M-N inclusive
--filter=-N Only execute tests with numbers <= N
--filter=N- Only execute tests with numbers >= N
--filter=M,N,... Only execute tests with number M or N or ...
"
}
@ -155,6 +160,7 @@ runTests() {
_tfw_find_tests "${filters[@]}"
# Iterate through all test cases, starting a new test whenever the number of
# running tests is less than the job limit.
_tfw_testcount=0
_tfw_passcount=0
_tfw_failcount=0
_tfw_errorcount=0
@ -162,8 +168,12 @@ runTests() {
_tfw_running_pids=()
_tfw_test_number_watermark=0
local testNumber
local testPosition=0
for ((testNumber = 1; testNumber <= ${#_tfw_tests[*]}; ++testNumber)); do
testName="${_tfw_tests[$(($testNumber - 1))]}"
[ -z "$testName" ] && continue
let ++testPosition
let ++_tfw_testcount
# Wait for any existing child process to finish.
while [ $njobs -ne 0 -a ${#_tfw_running_pids[*]} -ge $njobs ]; do
_tfw_harvest_processes
@ -172,11 +182,11 @@ runTests() {
$_tfw_stop_on_error && [ $_tfw_errorcount -ne 0 ] && break
$_tfw_stop_on_failure && [ $_tfw_failcount -ne 0 ] && break
# Start the next test in a child process.
_tfw_echo_intro $testNumber $testName
_tfw_echo_intro $testPosition $testNumber $testName
[ $njobs -ne 1 ] && echo
(
_tfw_unique=$!
echo "$testNumber $testName" >"$_tfw_results_dir/$_tfw_unique"
echo "$testPosition $testNumber $testName" >"$_tfw_results_dir/$_tfw_unique"
_tfw_tmp=/tmp/_tfw-$_tfw_unique
trap '_tfw_status=$?; rm -rf "$_tfw_tmp"; exit $_tfw_status' EXIT SIGHUP SIGINT SIGTERM
local start_time=$(_tfw_timestamp)
@ -208,7 +218,7 @@ runTests() {
1) result=FAIL;;
0) result=PASS;;
esac
echo "$testNumber $testName $result" >"$_tfw_results_dir/$_tfw_unique"
echo "$testPosition $testNumber $testName $result" >"$_tfw_results_dir/$_tfw_unique"
{
echo "Name: $testName"
echo "Result: $result"
@ -238,14 +248,14 @@ runTests() {
rm -rf "$_tfw_tmpdir"
trap - EXIT SIGHUP SIGINT SIGTERM
# Echo result summary and exit with success if no failures or errors.
s=$([ ${#_tfw_tests[*]} -eq 1 ] || echo s)
echo "${#_tfw_tests[*]} test$s, $_tfw_passcount pass, $_tfw_failcount fail, $_tfw_errorcount error"
s=$([ $_tfw_testcount -eq 1 ] || echo s)
echo "$_tfw_testcount test$s, $_tfw_passcount pass, $_tfw_failcount fail, $_tfw_errorcount error"
[ $_tfw_fatalcount -eq 0 -a $_tfw_failcount -eq 0 -a $_tfw_errorcount -eq 0 ]
}
_tfw_echo_intro() {
local docvar="doc_$2"
echo -n "$1. ${!docvar:-$2}..."
local docvar="doc_$3"
echo -n "$2. ${!docvar:-$3}..."
[ $1 -gt $_tfw_test_number_watermark ] && _tfw_test_number_watermark=$1
}
@ -263,9 +273,10 @@ _tfw_harvest_processes() {
surviving_pids+=($pid)
elif [ -s "$_tfw_results_dir/$pid" ]; then
set -- $(<"$_tfw_results_dir/$pid")
local testNumber="$1"
local testName="$2"
local result="$3"
local testPosition="$1"
local testNumber="$2"
local testName="$3"
local result="$4"
case "$result" in
ERROR)
let _tfw_errorcount=_tfw_errorcount+1
@ -287,16 +298,16 @@ _tfw_harvest_processes() {
_tfw_echo_result "$result"
echo
elif lines=$($_tfw_tput lines); then
local travel=$(($_tfw_test_number_watermark - $testNumber + 1))
local travel=$(($_tfw_test_number_watermark - $testPosition + 1))
if [ $travel -gt 0 -a $travel -lt $lines ] && $_tfw_tput cuu $travel ; then
_tfw_echo_intro $testNumber $testName
_tfw_echo_intro $testPosition $testNumber $testName
echo -n " "
_tfw_echo_result "$result"
echo
$_tfw_tput cud $(($_tfw_test_number_watermark - $testNumber))
travel=$(($_tfw_test_number_watermark - $testPosition))
[ $travel -gt 0 ] && $_tfw_tput cud $travel
fi
else
_tfw_echo_intro $testNumber $testName
echo -n "$testNumber. ... "
_tfw_echo_result "$result"
echo
@ -1043,27 +1054,76 @@ _tfw_checkTerminfo() {
}
# Return a list of test names in the _tfw_tests array variable, in the order
# that the test_TestName functions were defined.
# that the test_TestName functions were defined. Test names must start with
# an alphabetic character (not numeric or '_').
_tfw_find_tests() {
_tfw_tests=()
_tfw_shopt -s extdebug
local name
for name in $(builtin declare -F |
sed -n -e '/^declare -f test_./s/^declare -f test_//p' |
sed -n -e '/^declare -f test_[A-Za-z]/s/^declare -f test_//p' |
while read name; do builtin declare -F "test_$name"; done |
sort --key 2,2n --key 3,3 |
sed -e 's/^test_//' -e 's/[ ].*//')
do
local number=$((${#_tfw_tests[*]} + 1))
local testName=
if [ $# -eq 0 ]; then
_tfw_tests+=("$name")
testName="$name"
else
local filter
for filter; do
case "$name" in
"$filter"*) _tfw_tests+=("$name"); break;;
case "$filter" in
+([0-9]))
if [ $number -eq $filter ]; then
testName="$name"
break
fi
;;
+([0-9])*(,+([0-9])))
local oIFS="$IFS"
IFS=,
local -a numbers=($filter)
IFS="$oIFS"
local n
for n in ${numbers[*]}; do
if [ $number -eq $n ]; then
testName="$name"
break 2
fi
done
;;
+([0-9])-)
local start=${filter%-}
if [ $number -ge $start ]; then
testName="$name"
break
fi
;;
-+([0-9]))
local end=${filter#-}
if [ $number -le $end ]; then
testName="$name"
break
fi
;;
+([0-9])-+([0-9]))
local start=${filter%-*}
local end=${filter#*-}
if [ $number -ge $start -a $number -le $end ]; then
testName="$name"
break
fi
;;
*)
case "$name" in
"$filter"*) testName="$name"; break;;
esac
;;
esac
done
fi
_tfw_tests+=("$testName")
done
_tfw_shopt_restore
}

2
tests/rhizomeops

@ -140,6 +140,8 @@ test_AddEmpty() {
assertGrep .manifest '^service=file$'
assertGrep .manifest '^name=$'
assertGrep .manifest '^filesize=0$'
executeOk_servald rhizome list ''
assert_rhizome_list ''
}
doc_AddThenList="List contains one file after one add"

134
tests/rhizomeprotocol

@ -63,45 +63,123 @@ bundle_received_by() {
return 0
}
doc_FileTransferNew="One new bundle single transfer"
setup_FileTransferNew() {
setup_common() {
setup_servald
setup_rhizome
assert_no_servald_processes
start_servald_instances +A +B
echo 'File one' >file1
set_instance +A
executeOk_servald rhizome add file '' '' file1 file1.manifest
extract_manifest_id BID file1.manifest
extract_manifest_version VERSION file1.manifest
set_instance +B
}
test_FileTransferNew() {
wait_until bundle_received_by +B
executeOk_servald rhizome list ''
assert_rhizome_list file1!
}
doc_FileMultiTransferNew="One new bundle multi transfer"
setup_FileMultiTransferNew() {
setup_servald
setup_rhizome
assert_no_servald_processes
start_servald_instances +A +B +C +D +E
echo 'File one' >file1
set_instance +A
executeOk_servald rhizome add file '' '' file1 file1.manifest
extract_manifest_id BID file1.manifest
extract_manifest_version VERSION file1.manifest
extract_manifest_vars() {
local manifest="${1?}"
extract_manifest_id BID "$manifest"
extract_manifest_version VERSION "$manifest"
extract_manifest_filesize FILESIZE "$manifest"
FILEHASH=
if [ "$FILESIZE" != '0' ]; then
extract_manifest_filehash FILEHASH "$manifest"
fi
}
test_FileMultiTransferNew() {
add_file() {
local name="$1"
[ -e "$name" ] || echo 'File $name' >"$name"
local sidvar="SID$instance_name"
executeOk_servald rhizome add file "${!sidvar}" '' "$name" "$name.manifest"
executeOk_servald rhizome list ''
assert_rhizome_list "$name"
extract_manifest_vars "$name.manifest"
}
update_file() {
local orig_name="$1"
local new_name="$2"
[ -e "$new_name" ] || echo 'File $new_name' >"$new_name"
local sidvar="SID$instance_name"
[ "$new_name" != "$orig_name" ] && cp "$orig_name.manifest" "$new_name.manifest"
sed -i -e '/^date=/d;/^filehash=/d;/^filesize=/d;/^version=/d;/^name=/d' "$new_name.manifest"
executeOk_servald rhizome add file "${!sidvar}" '' "$new_name" "$new_name.manifest"
executeOk_servald rhizome list ''
assert_rhizome_list "$new_name"
extract_manifest_vars "$new_name.manifest"
}
doc_FileTransfer="New bundle and update transfer to one node"
setup_FileTransfer() {
setup_common
set_instance +A
add_file file1
start_servald_instances +A +B
}
test_FileTransfer() {
wait_until bundle_received_by +B
set_instance +B
assert_received file1
set_instance +A
update_file file1 file2
set_instance +B
wait_until bundle_received_by +B
assert_received file2
}
doc_FileTransferBig="Big new bundle transfers to one node"
setup_FileTransferBig() {
setup_common
set_instance +A
dd if=/dev/urandom of=file1 bs=1k count=1k 2>&1
echo x >>file1
ls -l file1
add_file file1
start_servald_instances +A +B
}
test_FileTransferBig() {
wait_until bundle_received_by +B
set_instance +B
assert_received file1
}
assert_received() {
local name="${1?}"
executeOk_servald rhizome list ''
assert_rhizome_list "$name!"
local _hash
if [ -s "$name" ]; then
extract_manifest_filehash _hash "$name.manifest"
executeOk_servald rhizome extract file "$_hash" extracted
assert cmp "$name" extracted
fi
}
doc_FileTransferMulti="New bundle transfers to four nodes"
setup_FileTransferMulti() {
setup_common
set_instance +A
add_file file1
start_servald_instances +A +B +C +D +E
}
test_FileTransferMulti() {
wait_until bundle_received_by +B +C +D +E
local I
for I in +B +C +D +E; do
set_instance $I
executeOk_servald rhizome list ''
assert_rhizome_list file1!
assert_received file1
done
}
doc_FileTransferDelete="Payload deletion transfers to one node"
setup_FileTransferDelete() {
setup_common
set_instance +A
add_file file1
start_servald_instances +A +B
wait_until bundle_received_by +B
set_instance +A
>file1_2
update_file file1 file1_2
}
test_FileTransferDelete() {
wait_until bundle_received_by +B
set_instance +B
assert_received file1_2
}
runTests "$@"