ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-30 01:48:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'naf4' into 'development'

Browse Source 
Rewritten HTTP request parsing and response buffering
This commit is contained in:
Andrew Bettison 2013-10-28 18:20:40 +10:30
commit 9d54c629b2
40 changed files with 3317 additions and 1518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
commandline.c
View File

@ -483,7 +483,7 @@ int app_echo(const struct cli_parsed *parsed, struct cli_context *context)
DEBUGF("echo:argv[%d]=\"%s\"", i, arg);
if (escapes) {
unsigned char buf[strlen(arg)];
size_t len = strn_fromprint(buf, sizeof buf, arg, '\0', NULL);
size_t len = strn_fromprint(buf, sizeof buf, arg, 0, '\0', NULL);
cli_write(context, buf, len);
} else
cli_puts(context, arg);

2
conf.h
View File

@ -233,8 +233,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "serval.h"
#include "rhizome.h"
typedef char bool_t;
#define CONFIG_FILE_MAX_SIZE (32 * 1024)
#define INTERFACE_NAME_STRLEN 40

16
conf_schema.h
View File

@ -233,6 +233,7 @@ ATOM(bool_t, dnaresponses, 0, boolean,, "")
ATOM(bool_t, dnahelper, 0, boolean,, "")
ATOM(bool_t, queues, 0, boolean,, "")
ATOM(bool_t, timing, 0, boolean,, "")
ATOM(bool_t, httpd, 0, boolean,, "")
ATOM(bool_t, io, 0, boolean,, "")
ATOM(bool_t, verbose_io, 0, boolean,, "")
ATOM(bool_t, interactive_io, 0, boolean,, "")
@ -261,6 +262,7 @@ ATOM(bool_t, slipbytestream, 0, boolean,, "")
ATOM(bool_t, packetconstruction, 0, boolean,, "")
ATOM(bool_t, rhizome, 0, boolean,, "")
ATOM(bool_t, rhizome_bind, 0, boolean,, "")
ATOM(bool_t, rhizome_httpd, 0, boolean,, "")
ATOM(bool_t, rhizome_tx, 0, boolean,, "")
ATOM(bool_t, rhizome_rx, 0, boolean,, "")
ATOM(bool_t, rhizome_ads, 0, boolean,, "")
@ -375,6 +377,15 @@ STRUCT(rhizome_direct)
SUB_STRUCT(peerlist, peer,)
END_STRUCT
STRUCT(user)
STRING(50, password, "", str,, "Authentication password")
END_STRUCT
ARRAY(userlist,)
KEY_STRING(25, str)
VALUE_SUB_STRUCT(user)
END_ARRAY(10)
STRUCT(rhizome_api_addfile)
STRING(64, uri_path, "", absolute_path,, "URI path for HTTP add-file request")
ATOM(struct in_addr, allow_host, hton_in_addr(INADDR_LOOPBACK), in_addr,, "IP address of host allowed to make HTTP add-file request")
@ -383,8 +394,13 @@ ATOM(sid_t, default_author, SID_ANY, sid,, "Author of ad
ATOM(rhizome_bk_t, bundle_secret_key, RHIZOME_BK_NONE, rhizome_bk,, "Secret key of add-file bundle to try if sender not given")
END_STRUCT
STRUCT(rhizome_api_restful)
SUB_STRUCT(userlist, users,)
END_STRUCT
STRUCT(rhizome_api)
SUB_STRUCT(rhizome_api_addfile, addfile,)
SUB_STRUCT(rhizome_api_restful, restful,)
END_STRUCT
STRUCT(rhizome_http)

3
constants.h
View File

@ -220,5 +220,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#define UNLOCK_CHALLENGE (0xF1)
#define UNLOCK_RESPONSE (0xF2)
// should there be a types.h to hold this?
typedef char bool_t;
#endif // __SERVALD_CONSTANTS_H

6
directory_service.c
View File

@ -1,9 +1,11 @@
#include "constants.h"
#include "mdp_client.h"
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#include <stdio.h>
#include <unistd.h>
#include "constants.h"
#include "mdp_client.h"
#include "str.h"
struct item{

2
fakeradio.c
View File

@ -4,7 +4,9 @@
#include <stdlib.h>
#include <stdint.h>
#include <fcntl.h>
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#include <errno.h>
#include <time.h>
#include <sys/time.h>

3
fdqueue.c
View File

@ -17,8 +17,7 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <poll.h>
#include "serval.h"
#include "fdqueue.h"
#include "conf.h"
#include "str.h"
#include "strbuf.h"

92
fdqueue.h Normal file
View File

@ -0,0 +1,92 @@
/*
Serval DNA file descriptor queue
Copyright (C) 2012-2013 Serval Project, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef __SERVALDNA__FDQUEUE_H
#define __SERVALDNA__FDQUEUE_H
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#include "os.h"
#include "log.h"
struct profile_total {
struct profile_total *_next;
int _initialised;
const char *name;
time_ms_t max_time;
time_ms_t total_time;
time_ms_t child_time;
int calls;
};
struct call_stats{
time_ms_t enter_time;
time_ms_t child_time;
struct profile_total *totals;
struct call_stats *prev;
};
struct sched_ent;
typedef void (*ALARM_FUNCP) (struct sched_ent *alarm);
struct sched_ent{
struct sched_ent *_next;
struct sched_ent *_prev;
ALARM_FUNCP function;
void *context;
struct pollfd poll;
// when we should first consider the alarm
time_ms_t alarm;
// the order we will prioritise the alarm
time_ms_t deadline;
struct profile_total *stats;
int _poll_index;
};
int is_scheduled(const struct sched_ent *alarm);
int _schedule(struct __sourceloc, struct sched_ent *alarm);
int _unschedule(struct __sourceloc, struct sched_ent *alarm);
int _watch(struct __sourceloc, struct sched_ent *alarm);
int _unwatch(struct __sourceloc, struct sched_ent *alarm);
#define schedule(alarm) _schedule(__WHENCE__, alarm)
#define unschedule(alarm) _unschedule(__WHENCE__, alarm)
#define watch(alarm) _watch(__WHENCE__, alarm)
#define unwatch(alarm) _unwatch(__WHENCE__, alarm)
int fd_poll();
/* function timing routines */
int fd_clearstats();
int fd_showstats();
int fd_checkalarms();
int fd_func_enter(struct __sourceloc, struct call_stats *this_call);
int fd_func_exit(struct __sourceloc, struct call_stats *this_call);
void dump_stack(int log_level);
#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \
struct call_stats _this_call={.totals=&_aggregate_stats}; \
fd_func_enter(__HERE__, &_this_call);
#define OUT() fd_func_exit(__HERE__, &_this_call)
#define RETURN(X) do { OUT(); return (X); } while (0);
#define RETURNNULL do { OUT(); return (NULL); } while (0);
#endif // __SERVALDNA__FDQUEUE_H

2
headerfiles.mk
View File

@ -18,6 +18,8 @@ HDRS= fifo.h \
crypto.h \
log.h \
net.h \
fdqueue.h \
http_server.h \
xprintf.h \
constants.h \
monitor-client.h \

1762
http_server.c Normal file
View File

File diff suppressed because it is too large Load Diff

156
http_server.h Normal file
View File

@ -0,0 +1,156 @@
/*
Serval DNA - HTTP Server API
Copyright (C) 2013 Serval Project, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef __SERVALDNA__HTTP_SERVER_H
#define __SERVALDNA__HTTP_SERVER_H
#include <limits.h>
#include "constants.h"
#include "strbuf.h"
#include "fdqueue.h"
/* Generic HTTP request handling.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
extern const char HTTP_VERB_GET[];
extern const char HTTP_VERB_POST[];
extern const char HTTP_VERB_PUT[];
extern const char HTTP_VERB_HEAD[];
extern const char HTTP_VERB_DELETE[];
extern const char HTTP_VERB_TRACE[];
extern const char HTTP_VERB_OPTIONS[];
extern const char HTTP_VERB_CONNECT[];
extern const char HTTP_VERB_PATCH[];
typedef uint64_t http_size_t;
#define PRIhttp_size_t PRIu64
struct http_request;
struct http_range {
enum http_range_type { NIL = 0, CLOSED, OPEN, SUFFIX } type;
http_size_t first; // only for CLOSED or OPEN
http_size_t last; // only for CLOSED or SUFFIX
};
unsigned http_range_close(struct http_range *dst, const struct http_range *src, unsigned nranges, http_size_t content_length);
http_size_t http_range_bytes(const struct http_range *range, unsigned nranges);
#define CONTENT_LENGTH_UNKNOWN UINT64_MAX
struct http_request_headers {
http_size_t content_length;
const char *content_type;
const char *content_subtype;
const char *boundary;
unsigned short content_range_count;
struct http_range content_ranges[5];
};
struct http_response_headers {
http_size_t content_length;
http_size_t content_range_start; // range_end = range_start + content_length - 1
http_size_t resource_length; // size of entire resource
const char *content_type; // "type/subtype"
const char *boundary;
};
typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *);
struct http_response {
uint16_t result_code;
struct http_response_headers header;
const char *content;
HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content
};
#define MIME_FILENAME_MAXLEN 127
struct mime_content_disposition {
char type[64];
char name[64];
char filename[MIME_FILENAME_MAXLEN + 1];
http_size_t size;
time_t creation_date;
time_t modification_date;
time_t read_date;
};
struct http_mime_handler {
void (*handle_mime_preamble)(struct http_request *, const char *, size_t);
void (*handle_mime_part_start)(struct http_request *);
void (*handle_mime_content_disposition)(struct http_request *, const struct mime_content_disposition *);
void (*handle_mime_header)(struct http_request *, const char *label, const char *, size_t);
void (*handle_mime_body)(struct http_request *, const char *, size_t);
void (*handle_mime_part_end)(struct http_request *);
void (*handle_mime_epilogue)(struct http_request *, const char *, size_t);
};
struct http_request;
void http_request_init(struct http_request *r, int sockfd);
void http_request_free_response_buffer(struct http_request *r);
int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
void http_request_finalise(struct http_request *r);
void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR);
void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
struct http_request {
struct sched_ent alarm; // MUST BE FIRST ELEMENT
enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase;
bool_t *debug_flag;
bool_t *disable_tx_flag;
time_ms_t initiate_time; // time connection was initiated
time_ms_t idle_timeout; // disconnect if no bytes received for this long
struct sockaddr_in client_sockaddr_in;
HTTP_REQUEST_PARSER parser; // current parser function
HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed
HTTP_REQUEST_PARSER handle_headers; // called after all headers are parsed
HTTP_REQUEST_PARSER handle_content_end; // called after all content is received
enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state;
struct http_mime_handler form_data; // called to parse multipart/form-data body
void (*finalise)(struct http_request *);
void (*free)(void*);
const char *verb; // points to nul terminated static string, "GET", "PUT", etc.
const char *path; // points into buffer; nul terminated
uint8_t version_major; // m from from HTTP/m.n
uint8_t version_minor; // n from HTTP/m.n
struct http_request_headers request_header;
const char *received; // start of received data in buffer[]
const char *end; // end of received data in buffer[]
const char *parsed; // start of unparsed data in buffer[]
const char *cursor; // for parsing
http_size_t request_content_remaining;
struct http_response response;
http_size_t response_length; // total response bytes (header + content)
http_size_t response_sent; // for counting up to response_length
char *response_buffer;
size_t response_buffer_size;
size_t response_buffer_length;
size_t response_buffer_sent;
void (*response_free_buffer)(void*);
char buffer[8 * 1024];
};
#endif // __SERVALDNA__HTTP_SERVER_H

4
keyring.c
View File

@ -675,7 +675,7 @@ static int load_did_name(keypair *kp, const char *text)
return WHY("duplicate DID");
const char *e = NULL;
bzero(kp->private_key, kp->private_key_len);
strn_fromprint(kp->private_key, kp->private_key_len, t, '"', &e);
strn_fromprint(kp->private_key, kp->private_key_len, t, 0, '"', &e);
if (*e != '"')
return WHY("malformed DID quoted string");
t = e + 1;
@ -685,7 +685,7 @@ static int load_did_name(keypair *kp, const char *text)
return WHY("duplicate Name");
const char *e = NULL;
bzero(kp->public_key, kp->public_key_len);
strn_fromprint(kp->public_key, kp->public_key_len, t, '"', &e);
strn_fromprint(kp->public_key, kp->public_key_len, t, 0, '"', &e);
if (*e != '"')
return WHY("malformed Name quoted string");
t = e + 1;

4
log.h
View File

@ -153,8 +153,8 @@ struct strbuf;
#define DEBUG(X) DEBUGF("%s", (X))
#define DEBUGF_perror(F,...) LOGF_perror(LOG_LEVEL_DEBUG, F, ##__VA_ARGS__)
#define DEBUG_perror(X) DEBUGF_perror("%s", (X))
#define D DEBUG("D")
#define T { if (config.debug.trace) DEBUG("T"); }
#define D (DEBUG("D"), 1)
#define T (config.debug.trace ? DEBUG("T") : 1)
#define DEBUG_argv(X,ARGC,ARGV) logArgv(LOG_LEVEL_DEBUG, __WHENCE__, (X), (ARGC), (ARGV))
#define dump(X,A,N) logDump(LOG_LEVEL_DEBUG, __WHENCE__, (X), (const unsigned char *)(A), (size_t)(N))

52
meshms.c
View File

@ -49,7 +49,7 @@ struct ply_read{
// details of the current record
uint64_t record_end_offset;
uint16_t record_length;
int buffer_size;
size_t buffer_size;
char type;
// raw record data
unsigned char *buffer;
@ -78,8 +78,7 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m)
alloca_tohex(keyring->contexts[cn]->identities[in]
->keypairs[kp]->private_key, crypto_box_curve25519xsalsa20poly1305_SECRETKEYBYTES));
int ret = rhizome_get_bundle_from_seed(m, seed);
if (ret<0)
if (rhizome_get_bundle_from_seed(m, seed) == -1)
return -1;
// always consider the content encrypted, we don't need to rely on the manifest itself.
@ -225,9 +224,9 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome
if (rhizome_retrieve_manifest(bid, m))
return -1;
int ret = rhizome_open_decrypt_read(m, NULL, &ply->read);
if (ret>0)
if (ret == 1)
WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (ret)
if (ret != 0)
return ret;
ply->read.offset = ply->read.length = m->fileLength;
return 0;
@ -399,11 +398,11 @@ static int update_conversation(const sid_t *my_sid, struct conversations *conv){
goto end;
ret = ply_find_next(&ply, MESHMS_BLOCK_TYPE_ACK);
if (ret<0)
if (ret == -1)
goto end;
if (ret==0){
if (unpack_uint(ply.buffer, ply.record_length, &previous_ack)<0)
if (unpack_uint(ply.buffer, ply.record_length, &previous_ack) == -1)
previous_ack=0;
}
if (config.debug.meshms)
@ -478,7 +477,7 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
bzero(&buff, sizeof(buff));
int ret = rhizome_open_decrypt_read(m, NULL, &read);
if (ret<0)
if (ret == -1)
goto end;
unsigned char version=0xFF;
@ -527,8 +526,9 @@ end:
return 0;
}
static int write_conversation(struct rhizome_write *write, struct conversations *conv){
int len=0;
static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv)
{
size_t len=0;
if (!conv)
return len;
{
@ -541,14 +541,14 @@ static int write_conversation(struct rhizome_write *write, struct conversations
len+=pack_uint(&buffer[len], conv->read_offset);
len+=pack_uint(&buffer[len], conv->their_size);
int ret=rhizome_write_buffer(write, buffer, len);
if (ret<0)
if (ret == -1)
return ret;
}else{
len+=measure_packed_uint(conv->their_last_message);
len+=measure_packed_uint(conv->read_offset);
len+=measure_packed_uint(conv->their_size);
}
DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %d",
DEBUGF("len %s, %"PRId64", %"PRId64", %"PRId64" = %zu",
alloca_tohex_sid_t(conv->them),
conv->their_last_message,
conv->read_offset,
@ -556,14 +556,14 @@ static int write_conversation(struct rhizome_write *write, struct conversations
len);
}
// write the two child nodes
int ret=write_conversation(write, conv->_left);
if (ret<0)
ssize_t ret = write_conversation(write, conv->_left);
if (ret == -1)
return ret;
len+=ret;
len += (size_t) ret;
ret = write_conversation(write, conv->_right);
if (ret<0)
if (ret == -1)
return ret;
len+=ret;
len += (size_t) ret;
return len;
}
@ -578,22 +578,22 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
// TODO rebalance tree...
// measure the final payload first
int len=write_conversation(NULL, conv);
if (len<0)
ssize_t len=write_conversation(NULL, conv);
if (len == -1)
goto end;
// then write it
m->version++;
rhizome_manifest_set_ll(m,"version",m->version);
m->fileLength = len+1;
m->fileLength = (size_t) len + 1;
rhizome_manifest_set_ll(m,"filesize",m->fileLength);
if (rhizome_write_open_manifest(&write, m))
if (rhizome_write_open_manifest(&write, m) == -1)
goto end;
unsigned char version=1;
if (rhizome_write_buffer(&write, &version, 1)<0)
if (rhizome_write_buffer(&write, &version, 1) == -1)
goto end;
if (write_conversation(&write, conv)<0)
if (write_conversation(&write, conv) == -1)
goto end;
if (rhizome_finish_write(&write))
goto end;
@ -795,7 +795,7 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
// find their last ACK so we know if messages have been received
int r = ply_find_next(&read_theirs, MESHMS_BLOCK_TYPE_ACK);
if (r==0){
if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack)<0)
if (unpack_uint(read_theirs.buffer, read_theirs.record_length, &their_last_ack) == -1)
their_last_ack=0;
else
their_ack_offset = read_theirs.record_end_offset;
@ -822,11 +822,11 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
// read their message list, and insert all messages that are included in the ack range
if (conv->found_their_ply){
int ofs=unpack_uint(read_ours.buffer, read_ours.record_length, (uint64_t*)&read_theirs.read.offset);
if (ofs<0)
if (ofs == -1)
break;
uint64_t end_range;
int x = unpack_uint(read_ours.buffer+ofs, read_ours.record_length - ofs, &end_range);
if (x<0)
if (x == -1)
end_range=0;
else
end_range = read_theirs.read.offset - end_range;

2
monitor-cli.c
View File

@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#include <fcntl.h>
#include "serval.h"

8
overlay.c
View File

@ -133,12 +133,8 @@ schedule(&_sched_##X); }
rhizome_cleanup(NULL);
}
/* Rhizome http server needs to know which callback to attach
to client sockets, so provide it here, along with the name to
appear in time accounting statistics. */
rhizome_http_server_start(rhizome_server_parse_http_request,
"rhizome_server_parse_http_request",
RHIZOME_HTTP_PORT,RHIZOME_HTTP_PORT_MAX);
// start the HTTP server if enabled
rhizome_http_server_start(RHIZOME_HTTP_PORT, RHIZOME_HTTP_PORT_MAX);
// start the dna helper if configured
dna_helper_start();

2
performance_timing.c
View File

@ -17,7 +17,7 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "serval.h"
#include "fdqueue.h"
#include "conf.h"
struct profile_total *stats_head=NULL;

120
rhizome.h
View File

@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "sha2.h"
#include "str.h"
#include "strbuf.h"
#include "http_server.h"
#include "nacl.h"
#include <sys/stat.h>
@ -269,7 +270,7 @@ int rhizome_str_is_bundle_crypt_key(const char *text);
int rhizome_strn_is_file_hash(const char *text);
int rhizome_str_is_file_hash(const char *text);
int http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
typedef struct sqlite_retry_state {
unsigned int limit; // do not retry once elapsed >= limit
@ -471,8 +472,8 @@ struct rhizome_write_buffer
{
struct rhizome_write_buffer *_next;
int64_t offset;
int buffer_size;
int data_size;
size_t buffer_size;
size_t data_size;
unsigned char data[0];
};
@ -487,7 +488,7 @@ struct rhizome_write
int64_t written_offset;
int64_t file_length;
struct rhizome_write_buffer *buffer_list;
int buffer_size;
size_t buffer_size;
int crypt;
unsigned char key[RHIZOME_CRYPT_KEY_BYTES];
@ -502,7 +503,7 @@ struct rhizome_write
struct rhizome_read_buffer{
uint64_t offset;
unsigned char data[RHIZOME_CRYPT_PAGE_SIZE];
int len;
size_t len;
};
struct rhizome_read
@ -525,71 +526,27 @@ struct rhizome_read
int64_t length;
};
typedef struct rhizome_http_request {
struct sched_ent alarm;
time_ms_t initiate_time; /* time connection was initiated */
/* Rhizome-specific HTTP request handling.
*/
typedef struct rhizome_http_request
{
struct http_request http; // MUST BE FIRST ELEMENT
struct sockaddr_in requestor;
/* identify request from others being run.
/* Identify request from others being run.
Monotonic counter feeds it. Only used for debugging when we write
post-<uuid>.log files for multi-part form requests. */
unsigned int uuid;
/* The HTTP request as currently received */
int request_length;
int header_length;
char request[1024];
/* Nature of the request */
int request_type;
/* All of the below are receiving data */
#define RHIZOME_HTTP_REQUEST_RECEIVING -1
#define RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART -2
// callback function to fill the response buffer
int (*generator)(struct rhizome_http_request *r);
/* Local buffer of data to be sent.
If a RHIZOME_HTTP_REQUEST_FROMBUFFER, then the buffer is sent, and when empty
the request is closed.
Else emptying the buffer triggers a request to fetch more data. Only if no
more data is provided do we then close the request. */
unsigned char *buffer;
int buffer_size; // size
int buffer_length; // number of bytes loaded into buffer
int buffer_offset; // where we are between [0,buffer_length)
struct rhizome_read read_state;
/* Path of request (used by POST multipart form requests where
the actual processing of the request does not occur while the
request headers are still available. */
char path[1024];
/* Boundary string for POST multipart form requests */
char boundary_string[1024];
int boundary_string_length;
/* File currently being written to while decoding POST multipart form */
FILE *field_file;
enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part;
int part_fd;
/* Which parts have been received in POST multipart form */
bool_t received_manifest;
bool_t received_data;
/* Name of data file supplied */
char data_file_name[1024];
/* Which fields have been seen in POST multipart form */
int fields_seen;
/* The seen fields bitmap above shares values with the actual Rhizome Direct
state machine. The state numbers (and thus bitmap values for the various
fields) are listed here.
To avoid confusion, we should not use single bit values for states that do
not correspond directly to a particular field.
Doesn't really matter what they are apart from not having exactly one bit set.
In fact, the only reason to not have exactly one bit set is so that we keep as
many bits available for field types as possible.
*/
#define RD_MIME_STATE_MANIFESTHEADERS (1<<0)
#define RD_MIME_STATE_DATAHEADERS (1<<1)
#define RD_MIME_STATE_INITIAL 0
#define RD_MIME_STATE_PARTHEADERS 0xffff0000
#define RD_MIME_STATE_BODY 0xffff0001
char data_file_name[MIME_FILENAME_MAXLEN + 1];
/* The source specification data which are used in different ways by different
request types */
@ -607,15 +564,6 @@ typedef struct rhizome_http_request {
} rhizome_http_request;
struct http_response {
unsigned int result_code;
const char * content_type;
uint64_t content_start;
uint64_t content_end;
uint64_t content_length;
const char * body;
};
int rhizome_received_content(const unsigned char *bidprefix,uint64_t version,
uint64_t offset,int count,unsigned char *bytes,
int type);
@ -629,9 +577,7 @@ int rhizome_server_simple_http_response(rhizome_http_request *r, int result, con
int rhizome_server_http_response(rhizome_http_request *r, int result,
const char *mime_type, const char *body, uint64_t bytes);
int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes);
int rhizome_http_server_start(int (*http_parse_func)(rhizome_http_request *),
const char *http_parse_func_description,
uint16_t port_low, uint16_t port_high);
int rhizome_http_server_start(uint16_t port_low, uint16_t port_high);
int is_rhizome_enabled();
int is_rhizome_mdp_enabled();
@ -654,12 +600,12 @@ typedef struct rhizome_direct_bundle_cursor {
rhizome_bid_t bid_low;
rhizome_bid_t bid_high;
unsigned char *buffer;
int buffer_size;
int buffer_used;
int buffer_offset_bytes;
size_t buffer_size;
size_t buffer_used;
size_t buffer_offset_bytes;
} rhizome_direct_bundle_cursor;
rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size);
rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size);
void rhizome_direct_bundle_iterator_unlimit(rhizome_direct_bundle_cursor *r);
int rhizome_direct_bundle_iterator_pickle_range(rhizome_direct_bundle_cursor *r,
unsigned char *pickled,
@ -723,7 +669,7 @@ rhizome_direct_sync_request
*rhizome_direct_new_sync_request(
void (*transport_specific_dispatch_function)
(struct rhizome_direct_sync_request *),
int buffer_size,int interval, int mode,
size_t buffer_size, int interval, int mode,
void *transport_specific_state);
int rhizome_direct_continue_sync_request(rhizome_direct_sync_request *r);
int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r);
@ -761,7 +707,7 @@ int rhizome_fetch_status_html(struct strbuf *b);
int rhizome_fetch_has_queue_space(unsigned char log2_size);
struct http_response_parts {
int code;
uint16_t code;
char *reason;
int64_t range_start;
int64_t content_length;
@ -774,34 +720,34 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
int rhizome_exists(const rhizome_filehash_t *hashp);
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority);
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size);
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size);
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size);
int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
int rhizome_write_file(struct rhizome_write *write, const char *filename);
int rhizome_fail_write(struct rhizome_write *write);
int rhizome_finish_write(struct rhizome_write *write);
int rhizome_import_file(rhizome_manifest *m, const char *filepath);
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length);
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length);
int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
int rhizome_add_file(rhizome_manifest *m, const char *filepath);
int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size);
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len);
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len);
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset,
const unsigned char *key, const unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, int buffer_length);
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len);
int rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
int rhizome_read_close(struct rhizome_read *read);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length);
int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, int length);
uint64_t fileOffset, unsigned char *buffer, size_t length);
int rhizome_cache_close();
int rhizome_database_filehash_from_id(const rhizome_bid_t *bidp, uint64_t version, rhizome_filehash_t *hashp);

16
rhizome_crypto.c
View File

@ -72,13 +72,13 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
return -1;
int ret=rhizome_retrieve_manifest(&key.Public, m);
if (ret<0)
if (ret == -1)
return -1;
m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID;
m->cryptoSignPublic = key.Public;
bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret);
if (ret>0)
if (ret == 1)
rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
return ret;
}
@ -590,22 +590,22 @@ static void add_nonce(unsigned char *nonce, int64_t value){
/* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries
* for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE)
*/
int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stream_offset,
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset,
const unsigned char *key, const unsigned char *nonce){
if (stream_offset<0)
return WHY("Invalid stream offset");
int64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
int offset=0;
size_t offset=0;
unsigned char block_nonce[crypto_stream_xsalsa20_NONCEBYTES];
bcopy(nonce, block_nonce, sizeof(block_nonce));
add_nonce(block_nonce, nonce_offset);
if (nonce_offset < stream_offset){
int padding = stream_offset & (RHIZOME_CRYPT_PAGE_SIZE -1);
int size = RHIZOME_CRYPT_PAGE_SIZE - padding;
size_t padding = stream_offset & (RHIZOME_CRYPT_PAGE_SIZE -1);
size_t size = RHIZOME_CRYPT_PAGE_SIZE - padding;
if (size>buffer_size)
size=buffer_size;
@ -619,11 +619,11 @@ int rhizome_crypt_xor_block(unsigned char *buffer, int buffer_size, int64_t stre
}
while(offset < buffer_size){
int size = buffer_size - offset;
size_t size = buffer_size - offset;
if (size>RHIZOME_CRYPT_PAGE_SIZE)
size=RHIZOME_CRYPT_PAGE_SIZE;
crypto_stream_xsalsa20_xor(buffer+offset, buffer+offset, size, block_nonce, key);
crypto_stream_xsalsa20_xor(buffer+offset, buffer+offset, (unsigned long long) size, block_nonce, key);
add_nonce(block_nonce, RHIZOME_CRYPT_PAGE_SIZE);
offset+=size;

9
rhizome_direct.c
View File

@ -135,7 +135,7 @@ rhizome_direct_sync_request
*rhizome_direct_new_sync_request(
void (*transport_specific_dispatch_function)
(struct rhizome_direct_sync_request *),
int buffer_size,int interval, int mode, void *state)
size_t buffer_size, int interval, int mode, void *state)
{
assert(mode&3);
@ -260,8 +260,7 @@ int rhizome_direct_conclude_sync_request(rhizome_direct_sync_request *r)
multiple versions of a given bundle introduces only a slight complication.
*/
rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
(unsigned char *buffer,int size, int max_response_bytes)
rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response(unsigned char *buffer,int size, int max_response_bytes)
{
if (size<10) return NULL;
if (size>65536) return NULL;
@ -288,7 +287,7 @@ rhizome_direct_bundle_cursor *rhizome_direct_get_fill_response
}
DEBUGF("unpickled size_high=%"PRId64", limit_size_high=%"PRId64,
c->size_high,c->limit_size_high);
DEBUGF("c->buffer_size=%d",c->buffer_size);
DEBUGF("c->buffer_size=%zu",c->buffer_size);
/* Get our list of BARs for the same cursor range */
int us_count=rhizome_direct_bundle_iterator_fill(c,-1);
@ -534,7 +533,7 @@ int app_rhizome_direct_sync(const struct cli_parsed *parsed, struct cli_context
}
}
rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(int buffer_size)
rhizome_direct_bundle_cursor *rhizome_direct_bundle_iterator(size_t buffer_size)
{
rhizome_direct_bundle_cursor *r=calloc(sizeof(rhizome_direct_bundle_cursor),1);
assert(r!=NULL);

807
rhizome_direct_http.c
View File

File diff suppressed because it is too large Load Diff

16
rhizome_fetch.c
View File

@ -181,7 +181,7 @@ int rhizome_fetch_queue_bytes(){
return bytes;
}
int rhizome_fetch_status_html(struct strbuf *b)
int rhizome_fetch_status_html(strbuf b)
{
int i,j;
for(i=0;i<NQUEUES;i++){
@ -503,8 +503,10 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
rhizome_manifest_free(slot->previous);
slot->previous=NULL;
}else{
assert(slot->previous->fileLength >= slot->manifest->journalTail);
assert(slot->manifest->fileLength > 0);
strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n",
slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength);
slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength - 1);
}
}
@ -1483,7 +1485,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
slot->alarm.deadline = slot->alarm.alarm + config.rhizome.idle_timeout;
schedule(&slot->alarm);
slot->request_len += bytes;
if (http_header_complete(slot->request, slot->request_len, bytes)) {
if (is_http_header_complete(slot->request, slot->request_len, bytes)) {
if (config.debug.rhizome_rx)
DEBUGF("Got HTTP reply: %s", alloca_toprint(160, slot->request, slot->request_len));
/* We have all the reply headers, so parse them, taking care of any following bytes of
@ -1495,9 +1497,9 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
rhizome_fetch_switch_to_mdp(slot);
return;
}
if (parts.code != 200) {
if (parts.code != 200 && parts.code != 206) {
if (config.debug.rhizome_rx)
DEBUGF("Failed HTTP request: rhizome server returned %d != 200 OK", parts.code);
DEBUGF("Failed HTTP request: rhizome server returned %03u", parts.code);
rhizome_fetch_switch_to_mdp(slot);
return;
}
@ -1564,7 +1566,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
This function takes a pointer to a buffer into which the entire HTTP response header has been
read. The caller must have ensured that the buffer contains at least one consecutive pair of
newlines '\n', optionally with carriage returns '\r' preceding and optionally interspersed with
nul characters '\0' (which can originate from telnet). The http_header_complete() function
nul characters '\0' (which can originate from telnet). The is_http_header_complete() function
is useful for this.
This returns pointers to within the supplied buffer, and may overwrite some characters in the
buffer, for example to nul-terminate a string that was terminated by space ' ' or newline '\r'
@ -1576,7 +1578,7 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
int unpack_http_response(char *response, struct http_response_parts *parts)
{
IN();
parts->code = -1;
parts->code = 0;
parts->reason = NULL;
parts->range_start=0;
parts->content_length = -1;

636
rhizome_http.c
View File

@ -23,18 +23,73 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#ifdef HAVE_SYS_FILIO_H
#include <sys/filio.h>
#endif
#include <assert.h>
#include "serval.h"
#include "overlay_address.h"
#include "conf.h"
#include "str.h"
#include "rhizome.h"
#include "http_server.h"
#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32
struct sched_ent server_alarm;
struct profile_total server_stats;
struct http_handler{
const char *path;
int (*parser)(rhizome_http_request *r, const char *remainder);
};
struct profile_total connection_stats;
static int rhizome_status_page(rhizome_http_request *r, const char *remainder);
static int rhizome_file_page(rhizome_http_request *r, const char *remainder);
static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder);
static int interface_page(rhizome_http_request *r, const char *remainder);
static int neighbour_page(rhizome_http_request *r, const char *remainder);
static int fav_icon_header(rhizome_http_request *r, const char *remainder);
static int root_page(rhizome_http_request *r, const char *remainder);
extern int rhizome_direct_import(rhizome_http_request *r, const char *remainder);
extern int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder);
extern int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder);
struct http_handler paths[]={
{"/rhizome/status", rhizome_status_page},
{"/rhizome/file/", rhizome_file_page},
{"/rhizome/import", rhizome_direct_import},
{"/rhizome/enquiry", rhizome_direct_enquiry},
{"/rhizome/manifestbyprefix/", manifest_by_prefix_page},
{"/rhizome/", rhizome_direct_dispatch},
{"/interface/", interface_page},
{"/neighbour/", neighbour_page},
{"/favicon.ico", fav_icon_header},
{"/", root_page},
};
static int rhizome_dispatch(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
INFOF("RHIZOME HTTP SERVER, %s %s", r->http.verb, r->http.path);
r->http.response.content_generator = NULL;
unsigned i;
for (i = 0; i < NELS(paths); ++i) {
const char *remainder;
if (str_startswith(r->http.path, paths[i].path, &remainder)){
int ret = paths[i].parser(r, remainder);
if (ret < 0) {
http_request_simple_response(&r->http, 500, NULL);
return 0;
}
if (ret == 0)
return 0;
}
}
http_request_simple_response(&r->http, 404, NULL);
return 0;
}
struct sched_ent server_alarm;
struct profile_total server_stats = {
.name = "rhizome_server_poll",
};
/*
HTTP server and client code for rhizome transfers and rhizome direct.
@ -47,9 +102,6 @@ static int rhizome_server_socket = -1;
static int request_count=0;
static time_ms_t rhizome_server_last_start_attempt = -1;
int (*rhizome_http_parse_func)(rhizome_http_request *)=NULL;
const char *rhizome_http_parse_func_description="(null)";
// Format icon data using:
// od -vt u1 ~/Downloads/favicon.ico | cut -c9- | sed 's/ */,/g'
unsigned char favicon_bytes[]={
@ -88,9 +140,7 @@ int is_rhizome_http_server_running()
Return 1 if the server is already started successfully.
Return 2 if the server was not started because it is too soon since last failed attempt.
*/
int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *),
const char *parse_func_desc,
uint16_t port_low, uint16_t port_high)
int rhizome_http_server_start(uint16_t port_low, uint16_t port_high)
{
if (rhizome_server_socket != -1)
return 1;
@ -100,7 +150,7 @@ int rhizome_http_server_start(int (*parse_func)(rhizome_http_request *),
if (now < rhizome_server_last_start_attempt + 5000)
return 2;
rhizome_server_last_start_attempt = now;
if (config.debug.rhizome_tx)
if (config.debug.rhizome_httpd)
DEBUGF("Starting rhizome HTTP server");
uint16_t port;
@ -162,14 +212,9 @@ success:
else
INFOF("HTTP SERVER (LIMITED SERVICE), START port=%"PRIu16" fd=%d", port, rhizome_server_socket);
/* Remember which function to call when handling client connections */
rhizome_http_parse_func=parse_func;
rhizome_http_parse_func_description=parse_func_desc;
rhizome_http_server_port = port;
/* Add Rhizome HTTPd server to list of file descriptors to watch */
server_alarm.function = rhizome_server_poll;
server_stats.name="rhizome_server_poll";
server_alarm.stats = &server_stats;
server_alarm.poll.fd = rhizome_server_socket;
server_alarm.poll.events = POLLIN;
@ -178,88 +223,14 @@ success:
}
void rhizome_client_poll(struct sched_ent *alarm)
static void rhizome_server_finalise_http_request(struct http_request *_r)
{
rhizome_http_request *r = (rhizome_http_request *)alarm;
if (alarm->poll.revents == 0 || alarm->poll.revents & (POLLHUP | POLLERR)){
if (config.debug.rhizome_tx)
DEBUGF("Closing connection due to timeout or error %d", alarm->poll.revents);
rhizome_server_free_http_request(r);
return;
rhizome_http_request *r = (rhizome_http_request *) _r;
rhizome_read_close(&r->read_state);
request_count--;
}
if (alarm->poll.revents & POLLIN){
switch(r->request_type)
{
case RHIZOME_HTTP_REQUEST_RECEIVING_MULTIPART:
{
/* Reading multi-part form data. Read some bytes and proces them. */
char buffer[16384];
sigPipeFlag=0;
int bytes = read_nonblock(r->alarm.poll.fd, buffer, 16384);
/* If we got some data, see if we have found the end of the HTTP request */
if (bytes > 0) {
// reset inactivity timer
r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT;
r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
unschedule(&r->alarm);
schedule(&r->alarm);
rhizome_direct_process_post_multipart_bytes(r,buffer,bytes);
}
/* We don't drop the connection on an empty read, because that results
in connections dropping when they shouldn't, including during testing.
The idle timeout should drop the connections instead.
*/
if (sigPipeFlag) {
if (config.debug.rhizome_tx)
DEBUG("Received SIGPIPE, closing connection");
rhizome_server_free_http_request(r);
return;
}
}
break;
case RHIZOME_HTTP_REQUEST_RECEIVING:
/* Keep reading until we have two CR/LFs in a row */
r->request[r->request_length] = '\0';
sigPipeFlag=0;
int bytes = read_nonblock(r->alarm.poll.fd, &r->request[r->request_length], sizeof r->request - r->request_length);
/* If we got some data, see if we have found the end of the HTTP request */
if (bytes > 0) {
// reset inactivity timer
r->alarm.alarm = gettime_ms() + RHIZOME_IDLE_TIMEOUT;
r->alarm.deadline = r->alarm.alarm + RHIZOME_IDLE_TIMEOUT;
unschedule(&r->alarm);
schedule(&r->alarm);
r->request_length += bytes;
r->header_length = http_header_complete(r->request, r->request_length, bytes);
if (r->header_length){
/* We have the request. Now parse it to see if we can respond to it */
if (rhizome_http_parse_func!=NULL)
rhizome_http_parse_func(r);
}
} else {
if (config.debug.rhizome_tx)
DEBUG("Empty read, closing connection");
rhizome_server_free_http_request(r);
return;
}
if (sigPipeFlag) {
if (config.debug.rhizome_tx)
DEBUG("Received SIGPIPE, closing connection");
rhizome_server_free_http_request(r);
return;
}
break;
}
}
if (alarm->poll.revents & POLLOUT){
/* Socket already has request -- so just try to send some data. */
rhizome_server_http_send_bytes(r);
}
return;
}
static int rhizome_dispatch(struct http_request *);
static unsigned int rhizome_http_request_uuid_counter = 0;
@ -269,7 +240,10 @@ void rhizome_server_poll(struct sched_ent *alarm)
struct sockaddr addr;
unsigned int addr_len = sizeof addr;
int sock;
if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) != -1) {
if ((sock = accept(rhizome_server_socket, &addr, &addr_len)) == -1) {
if (errno && errno != EAGAIN)
WARN_perror("accept");
} else {
struct sockaddr_in *peerip=NULL;
if (addr.sa_family == AF_INET) {
peerip = (struct sockaddr_in *)&addr;
@ -285,56 +259,34 @@ void rhizome_server_poll(struct sched_ent *alarm)
addr_len, addr.sa_family, alloca_tohex((unsigned char *)addr.sa_data, sizeof addr.sa_data)
);
}
rhizome_http_request *request = calloc(sizeof(rhizome_http_request), 1);
rhizome_http_request *request = emalloc_zero(sizeof(rhizome_http_request));
if (request == NULL) {
WHYF_perror("calloc(%u, 1)", (int)sizeof(rhizome_http_request));
WHY("Cannot respond to request, out of memory");
WHY("Cannot respond to HTTP request, out of memory");
close(sock);
} else {
request_count++;
request->uuid = rhizome_http_request_uuid_counter++;
if (peerip) request->requestor=*peerip;
else bzero(&request->requestor,sizeof(request->requestor));
request->data_file_name[0]=0;
/* We are now trying to read the HTTP request */
request->request_type=RHIZOME_HTTP_REQUEST_RECEIVING;
request->alarm.function = rhizome_client_poll;
request->data_file_name[0] = '\0';
request->read_state.blob_fd = -1;
request->read_state.blob_rowid = -1;
connection_stats.name="rhizome_client_poll";
request->alarm.stats=&connection_stats;
request->alarm.poll.fd=sock;
request->alarm.poll.events=POLLIN;
request->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT;
request->alarm.deadline = request->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
// watch for the incoming http request
watch(&request->alarm);
// set an inactivity timeout to close the connection
schedule(&request->alarm);
if (peerip)
request->http.client_sockaddr_in = *peerip;
request->http.handle_headers = rhizome_dispatch;
request->http.debug_flag = &config.debug.rhizome_httpd;
request->http.disable_tx_flag = &config.debug.rhizome_nohttptx;
request->http.finalise = rhizome_server_finalise_http_request;
request->http.free = free;
request->http.idle_timeout = RHIZOME_IDLE_TIMEOUT;
http_request_init(&request->http, sock);
}
}
if (errno && errno != EAGAIN)
WARN_perror("accept");
}
if (alarm->poll.revents & (POLLHUP | POLLERR)) {
INFO("Error on tcp listen socket");
}
}
int rhizome_server_free_http_request(rhizome_http_request *r)
{
unwatch(&r->alarm);
unschedule(&r->alarm);
close(r->alarm.poll.fd);
if (r->buffer)
free(r->buffer);
rhizome_read_close(&r->read_state);
free(r);
request_count--;
return 0;
}
int http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
{
IN();
const char *bufend = buf + len;
@ -360,144 +312,158 @@ int http_header_complete(const char *buf, size_t len, size_t read_since_last_cal
OUT();
}
static int neighbour_page(rhizome_http_request *r, const char *remainder, const char *headers)
static int neighbour_page(rhizome_http_request *r, const char *remainder)
{
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
char buf[8*1024];
strbuf b = strbuf_local(buf, sizeof buf);
sid_t neighbour_sid;
if (str_to_sid_t(&neighbour_sid, remainder) == -1)
return -1;
return 1;
struct subscriber *neighbour = find_subscriber(neighbour_sid.binary, sizeof(neighbour_sid.binary), 0);
if (!neighbour)
return 1;
strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
link_neighbour_status_html(b, neighbour);
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b))
return -1;
rhizome_server_simple_http_response(r, 200, buf);
http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
return 0;
}
static int interface_page(rhizome_http_request *r, const char *remainder, const char *headers)
static int interface_page(rhizome_http_request *r, const char *remainder)
{
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
char buf[8*1024];
strbuf b=strbuf_local(buf, sizeof buf);
int index=atoi(remainder);
if (index<0 || index>=OVERLAY_MAX_INTERFACES)
return 1;
strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
interface_state_html(b, &overlay_interfaces[index]);
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b))
return -1;
rhizome_server_simple_http_response(r, 200, buf);
http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
return 0;
}
static int rhizome_status_page(rhizome_http_request *r, const char *remainder, const char *headers)
static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
char buf[32*1024];
struct strbuf b;
strbuf_init(&b, buf, sizeof buf);
strbuf_puts(&b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
strbuf_sprintf(&b, "%d HTTP requests<br>", request_count);
strbuf_sprintf(&b, "%d Bundles transferring via MDP<br>", rhizome_cache_count());
rhizome_fetch_status_html(&b);
strbuf_puts(&b, "</body></html>");
if (strbuf_overrun(&b))
strbuf b = strbuf_local(buf, sizeof buf);
strbuf_puts(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>");
strbuf_sprintf(b, "%d HTTP requests<br>", request_count);
strbuf_sprintf(b, "%d Bundles transferring via MDP<br>", rhizome_cache_count());
rhizome_fetch_status_html(b);
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b))
return -1;
rhizome_server_simple_http_response(r, 200, buf);
http_request_response_static(&r->http, 200, "text/html", buf, strbuf_len(b));
return 0;
}
static int rhizome_file_content(rhizome_http_request *r)
static int rhizome_file_content(struct http_request *hr)
{
int suggested_size=65536;
if (suggested_size > r->read_state.length - r->read_state.offset)
suggested_size = r->read_state.length - r->read_state.offset;
if (suggested_size<=0)
return 0;
if (r->buffer_size < suggested_size){
r->buffer_size = suggested_size;
if (r->buffer)
free(r->buffer);
r->buffer = malloc(r->buffer_size);
}
if (!r->buffer)
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(r->http.response_buffer_sent == 0);
assert(r->http.response_buffer_length == 0);
assert(r->read_state.offset < r->read_state.length);
uint64_t readlen = r->read_state.length - r->read_state.offset;
size_t suggested_size = 64 * 1024;
if (suggested_size > readlen)
suggested_size = readlen;
if (r->http.response_buffer_size < suggested_size)
http_request_set_response_bufsize(&r->http, suggested_size);
if (r->http.response_buffer == NULL)
http_request_set_response_bufsize(&r->http, 1);
if (r->http.response_buffer == NULL)
return -1;
r->buffer_length = rhizome_read(&r->read_state, r->buffer, r->buffer_size);
ssize_t len = rhizome_read(&r->read_state,
(unsigned char *)r->http.response_buffer,
r->http.response_buffer_size);
if (len == -1)
return -1;
assert((size_t) len <= r->http.response_buffer_size);
r->http.response_buffer_length += (size_t) len;
return 0;
}
static int rhizome_file_page(rhizome_http_request *r, const char *remainder, const char *headers)
static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
{
/* Stream the specified payload */
if (!is_rhizome_http_enabled())
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (r->http.request_header.content_range_count > 1) {
// To support byte range sets, eg, Range: bytes=0-100,200-300,400- we would have
// to reply with a multipart/byteranges MIME content.
http_request_simple_response(&r->http, 501, "Not Implemented: Byte range sets");
return 0;
}
rhizome_filehash_t filehash;
if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
return -1;
bzero(&r->read_state, sizeof(r->read_state));
/* Refuse to honour HTTP request if required (used for debugging and
testing transition from HTTP to MDP) */
if (rhizome_open_read(&r->read_state, &filehash))
return 1;
if (r->read_state.length==-1){
if (rhizome_read(&r->read_state, NULL, 0)){
bzero(&r->read_state, sizeof r->read_state);
int n = rhizome_open_read(&r->read_state, &filehash);
if (n == -1) {
http_request_simple_response(&r->http, 500, NULL);
return 0;
}
if (n != 0)
return 1;
if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) {
rhizome_read_close(&r->read_state);
return 1;
}
assert(r->read_state.length != -1);
r->http.response.header.resource_length = r->read_state.length;
if (r->http.request_header.content_range_count > 0) {
assert(r->http.request_header.content_range_count == 1);
struct http_range closed;
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length);
if (n == 0 || http_range_bytes(&closed, 1) == 0) {
http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
return 0;
}
const char *range=str_str((char*)headers,"Range: bytes=",-1);
r->read_state.offset = r->source_index = 0;
if (range){
sscanf(range, "Range: bytes=%"PRId64"-", &r->read_state.offset);
if (0)
DEBUGF("Found range header %"PRId64,r->read_state.offset);
r->http.response.header.content_range_start = closed.first;
r->http.response.header.content_length = closed.last - closed.first + 1;
r->read_state.offset = closed.first;
} else {
r->http.response.header.content_range_start = 0;
r->http.response.header.content_length = r->http.response.header.resource_length;
r->read_state.offset = 0;
}
if (r->read_state.length - r->read_state.offset<=0){
rhizome_server_simple_http_response(r, 200, "");
http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content);
return 0;
}
struct http_response hr;
bzero(&hr, sizeof hr);
hr.result_code = 200;
hr.content_type = "application/binary";
hr.content_start = r->read_state.offset;
hr.content_end = r->read_state.length;
hr.content_length = r->read_state.length;
hr.body = NULL;
r->generator = rhizome_file_content;
rhizome_server_set_response(r, &hr);
return 0;
}
static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder, const char *headers)
static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
rhizome_bid_t prefix;
const char *endp = NULL;
unsigned prefix_len = strn_fromhex(prefix.binary, sizeof prefix.binary, remainder, &endp);
@ -505,27 +471,32 @@ static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainde
return 1; // not found
rhizome_manifest *m = rhizome_new_manifest();
int ret = rhizome_retrieve_manifest_by_prefix(prefix.binary, prefix_len, m);
if (ret==0)
rhizome_server_http_response(r, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
if (ret == -1)
http_request_simple_response(&r->http, 500, NULL);
else if (ret == 0)
http_request_response_static(&r->http, 200, "application/binary", (const char *)m->manifestdata, m->manifest_all_bytes);
rhizome_manifest_free(m);
return ret;
return ret <= 0 ? 0 : 1;
}
static int fav_icon_header(rhizome_http_request *r, const char *remainder, const char *headers)
static int fav_icon_header(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
rhizome_server_http_response(r, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
http_request_response_static(&r->http, 200, "image/vnd.microsoft.icon", (const char *)favicon_bytes, favicon_len);
return 0;
}
static int root_page(rhizome_http_request *r, const char *remainder, const char *headers)
static int root_page(rhizome_http_request *r, const char *remainder)
{
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
char temp[8192];
strbuf b=strbuf_local(temp, sizeof(temp));
strbuf b = strbuf_local(temp, sizeof temp);
strbuf_sprintf(b, "<html><head><meta http-equiv=\"refresh\" content=\"5\" ></head><body>"
"<h1>Hello, I'm %s*</h1><br>"
"Interfaces;<br>",
@ -536,247 +507,16 @@ static int root_page(rhizome_http_request *r, const char *remainder, const char
strbuf_sprintf(b, "<a href=\"/interface/%d\">%d: %s, TX: %d, RX: %d</a><br>",
i, i, overlay_interfaces[i].name, overlay_interfaces[i].tx_count, overlay_interfaces[i].recv_count);
}
strbuf_puts(b, "Neighbours;<br>");
link_neighbour_short_status_html(b, "/neighbour");
if (is_rhizome_http_enabled()){
strbuf_puts(b, "<a href=\"/rhizome/status\">Rhizome Status</a><br>");
}
strbuf_puts(b, "</body></html>");
if (strbuf_overrun(b))
return -1;
rhizome_server_simple_http_response(r, 200, temp);
return 0;
}
struct http_handler{
const char *path;
int (*parser)(rhizome_http_request *r, const char *remainder, const char *headers);
};
struct http_handler paths[]={
{"/rhizome/status", rhizome_status_page},
{"/rhizome/file/", rhizome_file_page},
{"/rhizome/manifestbyprefix/", manifest_by_prefix_page},
{"/interface/", interface_page},
{"/neighbour/", neighbour_page},
{"/favicon.ico", fav_icon_header},
{"/", root_page},
};
int rhizome_direct_parse_http_request(rhizome_http_request *r);
int rhizome_server_parse_http_request(rhizome_http_request *r)
{
// Start building up a response.
// Parse the HTTP "GET" line.
char *path = NULL;
char *headers = NULL;
if (str_startswith(r->request, "POST ", (const char **)&path)) {
return rhizome_direct_parse_http_request(r);
} else if (str_startswith(r->request, "GET ", (const char **)&path)) {
const char *p;
size_t header_length = 0;
size_t pathlen = 0;
// This loop is guaranteed to terminate before the end of the buffer, because we know that the
// buffer contains at least "\n\n" and maybe "\r\n\r\n" at the end of the header block.
for (p = path; !isspace(*p); ++p)
;
pathlen = p - path;
if ( str_startswith(p, " HTTP/1.", &p)
&& (str_startswith(p, "0", &p) || str_startswith(p, "1", &p))
&& (str_startswith(p, "\r\n", (const char **)&headers) || str_startswith(p, "\n", (const char **)&headers))
){
path[pathlen] = '\0';
header_length = r->header_length - (headers - r->request);
headers[header_length] = '\0';
if (strbuf_overrun(b)) {
WHY("HTTP Root page buffer overrun");
http_request_simple_response(&r->http, 500, NULL);
} else
path = NULL;
}
if (!path) {
if (config.debug.rhizome_tx)
DEBUGF("Received malformed HTTP request: %s", alloca_toprint(120, (const char *)r->request, r->request_length));
rhizome_server_simple_http_response(r, 400, "<html><h1>Malformed request</h1></html>\r\n");
http_request_response_static(&r->http, 200, "text/html", temp, strbuf_len(b));
return 0;
}
char *id = NULL;
INFOF("RHIZOME HTTP SERVER, GET %s", path);
int i;
r->generator=NULL;
for (i=0;i<sizeof(paths)/sizeof(struct http_handler);i++){
if (str_startswith(path, paths[i].path, (const char **)&id)){
int ret=paths[i].parser(r, id, headers);
if (ret<0)
rhizome_server_simple_http_response(r, 500, "<html><h1>Internal Error</h1></html>\r\n");
if (ret>0)
rhizome_server_simple_http_response(r, 404, "<html><h1>Not Found</h1></html>\r\n");
/* Try sending data immediately. */
rhizome_server_http_send_bytes(r);
return 0;
}
}
rhizome_server_simple_http_response(r, 404, "<html><h1>Not Found</h1></html>\r\n");
return 0;
}
/* Return appropriate message for HTTP response codes, both known and unknown. */
static const char *httpResultString(int response_code) {
switch (response_code) {
case 200: return "OK";
case 201: return "Created";
case 206: return "Partial Content";
case 404: return "Not found";
case 500: return "Internal server error";
default:
if (response_code<=4)
return "Unknown status code";
else
return "A suffusion of yellow";
}
}
static strbuf strbuf_build_http_response(strbuf sb, const struct http_response *h)
{
strbuf_sprintf(sb, "HTTP/1.0 %03u %s\r\n", h->result_code, httpResultString(h->result_code));
strbuf_sprintf(sb, "Content-type: %s\r\n", h->content_type);
if (h->content_end && h->content_length && (h->content_start!=0 || h->content_end!=h->content_length))
strbuf_sprintf(sb,
"Content-range: bytes %"PRIu64"-%"PRIu64"/%"PRIu64"\r\n"
"Content-length: %"PRIu64"\r\n",
h->content_start, h->content_end, h->content_length, h->content_end - h->content_start);
else if (h->content_length)
strbuf_sprintf(sb, "Content-length: %"PRIu64"\r\n", h->content_length);
strbuf_puts(sb, "\r\n");
return sb;
}
int rhizome_server_set_response(rhizome_http_request *r, const struct http_response *h)
{
r->request_type=0;
if (config.debug.rhizome_nohttptx)
unwatch(&r->alarm);
else{
/* Switching to writing, so update the call-back */
r->alarm.poll.events=POLLOUT;
watch(&r->alarm);
}
strbuf b = strbuf_local((char *) r->buffer, r->buffer_size);
strbuf_build_http_response(b, h);
if (r->buffer == NULL || strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length)) {
// Need a bigger buffer
if (r->buffer)
free(r->buffer);
r->buffer_size = strbuf_count(b) + 1;
if (h->body)
r->buffer_size += h->content_length;
r->buffer = malloc(r->buffer_size);
if (r->buffer == NULL) {
WHYF_perror("malloc(%u)", r->buffer_size);
r->buffer_size = 0;
return WHY("Cannot send response, out of memory");
}
strbuf_init(b, (char *) r->buffer, r->buffer_size);
strbuf_build_http_response(b, h);
if (strbuf_overrun(b) || (h->body && strbuf_remaining(b) < h->content_length))
return WHYF("Bug! Cannot send response, buffer not big enough");
}
r->buffer_length = strbuf_len(b);
if (h->body){
bcopy(h->body, strbuf_end(b), h->content_length);
r->buffer_length+=h->content_length;
}
r->buffer_offset = 0;
if (config.debug.rhizome_tx)
DEBUGF("Sending HTTP response: %s", alloca_toprint(160, (const char *)r->buffer, r->buffer_length));
return 0;
}
int rhizome_server_simple_http_response(rhizome_http_request *r, int result, const char *response)
{
struct http_response hr;
bzero(&hr, sizeof hr);
hr.result_code = result;
hr.content_type = "text/html";
hr.content_length = strlen(response);
hr.body = response;
if (result==400) {
DEBUGF("Rejecting http request as malformed due to: %s",
response);
}
return rhizome_server_set_response(r, &hr);
}
int rhizome_server_http_response(rhizome_http_request *r, int result,
const char *mime_type, const char *body, uint64_t bytes)
{
struct http_response hr;
bzero(&hr, sizeof hr);
hr.result_code = result;
hr.content_type = mime_type;
hr.content_length = bytes;
hr.body = body;
return rhizome_server_set_response(r, &hr);
}
int rhizome_server_http_response_header(rhizome_http_request *r, int result, const char *mime_type, uint64_t bytes)
{
return rhizome_server_http_response(r, result, mime_type, NULL, bytes);
}
/*
return codes:
1: connection still open.
0: connection finished.
<0: an error occurred.
*/
int rhizome_server_http_send_bytes(rhizome_http_request *r)
{
// Don't send anything if disabled for testing HTTP->MDP Rhizome failover
if (config.debug.rhizome_nohttptx)
return 1;
// write one block of buffered data
if(r->buffer_offset < r->buffer_length){
int bytes=r->buffer_length - r->buffer_offset;
bytes=write(r->alarm.poll.fd,&r->buffer[r->buffer_offset],bytes);
if (bytes<0){
// stop writing when the tcp buffer is full
// TODO errors?
return 1;
}
r->buffer_offset+=bytes;
// reset inactivity timer
r->alarm.alarm = gettime_ms()+RHIZOME_IDLE_TIMEOUT;
r->alarm.deadline = r->alarm.alarm+RHIZOME_IDLE_TIMEOUT;
unschedule(&r->alarm);
schedule(&r->alarm);
// allow other alarms to fire and wait for the next POLLOUT
return 1;
}
r->buffer_offset=r->buffer_length=0;
if (r->generator){
r->generator(r);
}
// once we've written the whole buffer, and nothing new has been generated, close the connection
if (!r->buffer_length){
if (config.debug.rhizome_tx)
DEBUG("Closing connection, done");
return rhizome_server_free_http_request(r);
}
return 1;
}

170
rhizome_store.c
View File

@ -1,3 +1,4 @@
#include <assert.h>
#include "serval.h"
#include "rhizome.h"
#include "conf.h"
@ -71,7 +72,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex
DEBUGF("Attempting to put blob for id='%"PRId64"' in %s", write->temp_id, blob_path);
write->blob_fd=open(blob_path, O_CREAT | O_TRUNC | O_WRONLY, 0664);
if (write->blob_fd<0)
if (write->blob_fd == -1)
goto insert_row_fail;
if (config.debug.externalblobs)
@ -111,7 +112,7 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex
}
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) == -1){
if (write->blob_fd>=0){
if (write->blob_fd != -1){
if (config.debug.externalblobs)
DEBUGF("Cancel write to fd %d", write->blob_fd);
close(write->blob_fd);
@ -162,7 +163,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer
// open database locks
static int write_get_lock(struct rhizome_write *write_state){
if (write_state->blob_fd>=0 || write_state->sql_blob)
if (write_state->blob_fd != -1 || write_state->sql_blob)
return 0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
@ -186,14 +187,15 @@ static int write_get_lock(struct rhizome_write *write_state){
}
// write data to disk
static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, int data_size){
static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, size_t data_size)
{
if (data_size<=0)
return 0;
if (file_offset != write_state->written_offset)
WARNF("Writing file data out of order! [%"PRId64",%"PRId64"]", file_offset, write_state->written_offset);
if (write_state->blob_fd>=0) {
if (write_state->blob_fd != -1) {
int ofs=0;
// keep trying until all of the data is written.
lseek(write_state->blob_fd, file_offset, SEEK_SET);
@ -233,7 +235,7 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
// close database locks
static int write_release_lock(struct rhizome_write *write_state){
int ret=0;
if (write_state->blob_fd>=0)
if (write_state->blob_fd != -1)
return 0;
if (write_state->sql_blob){
@ -252,7 +254,8 @@ static int write_release_lock(struct rhizome_write *write_state){
// Write data buffers in any order, the data will be cached and streamed into the database in file order.
// Though there is an upper bound on the amount of cached data
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, int data_size){
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size)
{
if (offset + data_size > write_state->file_length)
data_size = write_state->file_length - offset;
@ -260,12 +263,12 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
int ret=0;
int should_write = 0;
// if we are writing to a file, or already have the sql blob open, write as much as we can.
if (write_state->blob_fd>=0 || write_state->sql_blob){
if (write_state->blob_fd != -1 || write_state->sql_blob){
should_write = 1;
}else{
// cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go.
// (Not perfect if the range overlaps)
int64_t new_size = write_state->written_offset + write_state->buffer_size + data_size;
uint64_t new_size = write_state->written_offset + write_state->buffer_size + data_size;
if (new_size>=write_state->file_length || new_size>=RHIZOME_BUFFER_MAXIMUM_SIZE)
should_write = 1;
}
@ -324,7 +327,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
if (!*ptr || offset < (*ptr)->offset){
// found the insert position in the list
int64_t size = data_size;
size_t size = data_size;
// allow for buffers to overlap, we may need to split the incoming buffer into multiple pieces.
if (*ptr && offset+size > (*ptr)->offset)
@ -348,7 +351,7 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
break;
if (config.debug.rhizome)
DEBUGF("Caching block @%"PRId64", %"PRId64, offset, size);
DEBUGF("Caching block @%"PRId64", %zu", offset, size);
struct rhizome_write_buffer *i = emalloc(size + sizeof(struct rhizome_write_buffer));
if (!i){
ret=-1;
@ -377,7 +380,8 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
return ret;
}
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, int data_size){
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size)
{
return rhizome_random_write(write_state, write_state->file_offset, buffer, data_size);
}
@ -398,8 +402,8 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){
if (write->file_offset + size > write->file_length)
size=write->file_length - write->file_offset;
int r = fread(buffer, 1, size, f);
if (r==-1){
size_t r = fread(buffer, 1, size, f);
if (ferror(f)){
ret = WHY_perror("fread");
goto end;
}
@ -417,7 +421,7 @@ end:
int rhizome_fail_write(struct rhizome_write *write)
{
if (write->blob_fd>=0){
if (write->blob_fd != -1){
if (config.debug.externalblobs)
DEBUGF("Closing and removing fd %d", write->blob_fd);
close(write->blob_fd);
@ -572,12 +576,12 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
}
// store a whole payload from a single buffer
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, int length)
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length)
{
if (m->fileLength<=0)
return 0;
if (length!=m->fileLength)
return WHYF("Expected %"PRId64" bytes, got %d", m->fileLength, length);
return WHYF("Expected %"PRId64" bytes, got %zu", m->fileLength, length);
/* Import the file first, checking the hash as we go */
struct rhizome_write write;
@ -723,61 +727,73 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp
return 0; // file opened
}
static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read *read_state, unsigned char *buffer, size_t bufsz)
{
IN();
if (read_state->blob_fd != -1) {
if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1)
RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset));
if (bufsz == 0)
RETURN(0);
ssize_t rd = read(read_state->blob_fd, buffer, bufsz);
if (rd == -1)
RETURN(WHYF_perror("read(%d,%p,%zu)", read_state->blob_fd, buffer, bufsz));
if (config.debug.externalblobs)
DEBUGF("Read %zu bytes from fd=%d @%"PRIx64, (size_t) rd, read_state->blob_fd, read_state->offset);
RETURN(rd);
}
if (read_state->blob_rowid == -1)
RETURN(WHY("file not open"));
sqlite3_blob *blob = NULL;
int ret;
do {
assert(blob == NULL);
ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob);
} while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_open"));
if (ret != SQLITE_OK) {
assert(blob == NULL);
RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db)));
}
assert(blob != NULL);
if (read_state->length == -1)
read_state->length = sqlite3_blob_bytes(blob);
// A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out
// the length.
size_t bytes_read = 0;
if (buffer && bufsz && read_state->offset < read_state->length) {
bytes_read = read_state->length - read_state->offset;
if (bytes_read > bufsz)
bytes_read = bufsz;
assert(bytes_read > 0);
do {
ret = sqlite3_blob_read(blob, buffer, (int) bytes_read, read_state->offset);
} while (sqlite_code_busy(ret) && sqlite_retry(retry, "sqlite3_blob_read"));
if (ret != SQLITE_OK) {
WHYF("sqlite3_blob_read() failed: %s", sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
RETURN(-1);
}
}
sqlite3_blob_close(blob);
RETURN(bytes_read);
OUT();
}
/* Read content from the store, hashing and decrypting as we go.
Random access is supported, but hashing requires all payload contents to be read sequentially. */
// returns the number of bytes read
int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buffer_length)
ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, size_t buffer_length)
{
IN();
// hash check failed, just return an error
if (read_state->invalid)
RETURN(-1);
int bytes_read = 0;
if (read_state->blob_fd >= 0) {
if (lseek(read_state->blob_fd, read_state->offset, SEEK_SET) == -1)
RETURN(WHYF_perror("lseek(%d,%ld,SEEK_SET)", read_state->blob_fd, (long)read_state->offset));
bytes_read = read(read_state->blob_fd, buffer, buffer_length);
if (bytes_read == -1)
RETURN(WHYF_perror("read(%d,%p,%ld)", read_state->blob_fd, buffer, (long)buffer_length));
if (config.debug.externalblobs)
DEBUGF("Read %d bytes from fd %d @%"PRIx64, bytes_read, read_state->blob_fd, read_state->offset);
} else if (read_state->blob_rowid != -1) {
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
do{
sqlite3_blob *blob = NULL;
int ret = sqlite3_blob_open(rhizome_db, "main", "FILEBLOBS", "data", read_state->blob_rowid, 0 /* read only */, &blob);
if (sqlite_code_busy(ret))
goto again;
else if(ret!=SQLITE_OK)
RETURN(WHYF("sqlite3_blob_open failed: %s",sqlite3_errmsg(rhizome_db)));
if (read_state->length==-1)
read_state->length=sqlite3_blob_bytes(blob);
bytes_read = read_state->length - read_state->offset;
if (bytes_read>buffer_length)
bytes_read=buffer_length;
// allow the caller to do a dummy read, just to work out the length
if (!buffer)
bytes_read=0;
if (bytes_read>0){
ret = sqlite3_blob_read(blob, buffer, bytes_read, read_state->offset);
if (sqlite_code_busy(ret))
goto again;
else if(ret!=SQLITE_OK){
WHYF("sqlite3_blob_read failed: %s",sqlite3_errmsg(rhizome_db));
sqlite3_blob_close(blob);
ssize_t n = rhizome_read_retry(&retry, read_state, buffer, buffer_length);
if (n == -1)
RETURN(-1);
}
}
sqlite3_blob_close(blob);
break;
again:
if (blob) sqlite3_blob_close(blob);
if (!sqlite_retry(&retry, "sqlite3_blob_open"))
RETURN(-1);
} while (1);
} else
RETURN(WHY("file not open"));
size_t bytes_read = (size_t) n;
// hash the payload as we go, but only if we happen to read the payload data in order
if (read_state->hash_offset == read_state->offset && buffer && bytes_read>0){
@ -810,9 +826,9 @@ int rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, int buf
}
/* Read len bytes from read->offset into data, using *buffer to cache any reads */
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, int len)
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len)
{
int bytes_copied=0;
size_t bytes_copied=0;
while (len>0){
// make sure we only attempt to read data that actually exists
@ -822,7 +838,7 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer
// if we can supply either the beginning or end of the data from cache, do that first.
uint64_t ofs=read->offset - buffer->offset;
if (ofs>=0 && ofs<=buffer->len){
int size=len;
size_t size=len;
if (size > buffer->len - ofs)
size = buffer->len - ofs;
if (size>0){
@ -838,7 +854,7 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer
ofs = (read->offset+len) - buffer->offset;
if (ofs>0 && ofs<=buffer->len){
int size=len;
size_t size=len;
if (size > ofs)
size = ofs;
if (size>0){
@ -854,10 +870,12 @@ int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer
// remember the requested read offset so we can put it back
ofs = read->offset;
buffer->offset = read->offset = ofs & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
buffer->len = rhizome_read(read, buffer->data, sizeof(buffer->data));
ssize_t len = rhizome_read(read, buffer->data, sizeof(buffer->data));
read->offset = ofs;
if (buffer->len<=0)
return buffer->len;
buffer->len = 0;
if (len == -1)
return -1;
buffer->len = (size_t) len;
}
return bytes_copied;
}
@ -978,7 +996,7 @@ int rhizome_cache_count()
}
// read a block of data, caching meta data for reuse
int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, int length)
int rhizome_read_cached(const rhizome_bid_t *bidp, uint64_t version, time_ms_t timeout, uint64_t fileOffset, unsigned char *buffer, size_t length)
{
// look for a cached entry
struct cache_entry **ptr = find_entry_location(&root, bidp, version);
@ -1120,17 +1138,17 @@ static int rhizome_pipe(struct rhizome_read *read, struct rhizome_write *write,
unsigned char buffer[RHIZOME_CRYPT_PAGE_SIZE];
while(length>0){
int size=sizeof(buffer);
size_t size=sizeof(buffer);
if (size > length)
size=length;
int r = rhizome_read(read, buffer, size);
if (r<0)
ssize_t r = rhizome_read(read, buffer, size);
if (r == -1)
return r;
length -= r;
length -= (size_t) r;
if (rhizome_write_buffer(write, buffer, r))
if (rhizome_write_buffer(write, buffer, (size_t) r))
return -1;
}
@ -1194,12 +1212,12 @@ failure:
return ret;
}
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, int len)
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len)
{
struct rhizome_write write;
bzero(&write, sizeof write);
int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, len);
int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, (uint64_t) len);
if (ret)
return -1;

4
rhizome_sync.c
View File

@ -439,7 +439,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
if (count){
mdp.out.payload_length = ob_position(b);
if (config.debug.rhizome)
if (config.debug.rhizome_ads)
DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last);
overlay_mdp_dispatch(&mdp,0,NULL,0);
}
@ -449,7 +449,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
int rhizome_sync_announce()
{
int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);
sync_send_response(NULL, 0, HEAD_FLAG, 5);
sqlite_set_tracefunc(oldfunc);
return 0;
}

72
serval.h
View File

@ -1,7 +1,7 @@
/*
Serval Daemon
Serval DNA header file
Copyright (C) 2010-2012 Paul Gardner-Stephen
Copyright (C) 2012 Serval Project Inc.
Copyright (C) 2012-2013 Serval Project, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -90,9 +90,6 @@ struct in_addr {
#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
@ -109,6 +106,7 @@ struct in_addr {
#include <ctype.h>
#include <sys/stat.h>
#include "fdqueue.h"
#include "cli.h"
#include "constants.h"
#include "mem.h"
@ -341,42 +339,6 @@ int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret);
extern int sock;
struct profile_total {
struct profile_total *_next;
int _initialised;
const char *name;
time_ms_t max_time;
time_ms_t total_time;
time_ms_t child_time;
int calls;
};
struct call_stats{
time_ms_t enter_time;
time_ms_t child_time;
struct profile_total *totals;
struct call_stats *prev;
};
struct sched_ent;
typedef void (*ALARM_FUNCP) (struct sched_ent *alarm);
struct sched_ent{
struct sched_ent *_next;
struct sched_ent *_prev;
ALARM_FUNCP function;
void *context;
struct pollfd poll;
// when we should first consider the alarm
time_ms_t alarm;
// the order we will prioritise the alarm
time_ms_t deadline;
struct profile_total *stats;
int _poll_index;
};
struct limit_state{
// length of time for a burst
time_ms_t burst_length;
@ -817,17 +779,6 @@ void sigIoHandler(int signal);
int overlay_mdp_setup_sockets();
int is_scheduled(const struct sched_ent *alarm);
int _schedule(struct __sourceloc, struct sched_ent *alarm);
int _unschedule(struct __sourceloc, struct sched_ent *alarm);
int _watch(struct __sourceloc, struct sched_ent *alarm);
int _unwatch(struct __sourceloc, struct sched_ent *alarm);
#define schedule(alarm) _schedule(__WHENCE__, alarm)
#define unschedule(alarm) _unschedule(__WHENCE__, alarm)
#define watch(alarm) _watch(__WHENCE__, alarm)
#define unwatch(alarm) _unwatch(__WHENCE__, alarm)
int fd_poll();
void overlay_interface_discover(struct sched_ent *alarm);
void overlay_packetradio_poll(struct sched_ent *alarm);
int overlay_packetradio_setup_port(overlay_interface *interface);
@ -846,7 +797,6 @@ int overlay_queue_init();
void monitor_client_poll(struct sched_ent *alarm);
void monitor_poll(struct sched_ent *alarm);
void rhizome_client_poll(struct sched_ent *alarm);
void rhizome_fetch_poll(struct sched_ent *alarm);
void rhizome_server_poll(struct sched_ent *alarm);
@ -858,22 +808,6 @@ time_ms_t limit_next_allowed(struct limit_state *state);
int limit_is_allowed(struct limit_state *state);
int limit_init(struct limit_state *state, int rate_micro_seconds);
/* function timing routines */
int fd_clearstats();
int fd_showstats();
int fd_checkalarms();
int fd_func_enter(struct __sourceloc, struct call_stats *this_call);
int fd_func_exit(struct __sourceloc, struct call_stats *this_call);
void dump_stack(int log_level);
#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0}; \
struct call_stats _this_call={.totals=&_aggregate_stats}; \
fd_func_enter(__HERE__, &_this_call);
#define OUT() fd_func_exit(__HERE__, &_this_call)
#define RETURN(X) do { OUT(); return (X); } while (0);
#define RETURNNULL do { OUT(); return (NULL); } while (0);
int olsr_init_socket(void);
int olsr_send(struct overlay_frame *frame);

1
sourcefiles.mk
View File

@ -14,6 +14,7 @@ SERVAL_SOURCES = \
$(SERVAL_BASE)fdqueue.c \
$(SERVAL_BASE)fifo.c \
$(SERVAL_BASE)golay.c \
$(SERVAL_BASE)http_server.c \
$(SERVAL_BASE)keyring.c \
$(SERVAL_BASE)log.c \
$(SERVAL_BASE)lsif.c \

73
str.c
View File

@ -28,6 +28,7 @@
#include <ctype.h>
#include <assert.h>
#include <limits.h>
#include <errno.h>
const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
@ -122,6 +123,14 @@ char *str_toupper_inplace(char *str)
return str;
}
char *str_tolower_inplace(char *str)
{
register char *s;
for (s = str; *s; ++s)
*s = tolower(*s);
return str;
}
const char *strnchr(const char *s, size_t n, char c)
{
for (; n; --n, ++s) {
@ -227,17 +236,48 @@ char *str_str(char *haystack, const char *needle, int haystack_len)
return NULL;
}
int str_to_int(const char *str, int base, int *result, const char **afterp)
{
if (isspace(*str))
return 0;
const char *end = str;
errno = 0;
long value = strtol(str, (char**)&end, base);
if (afterp)
*afterp = end;
if (errno == ERANGE || end == str || value > INT_MAX || value < INT_MIN || isdigit(*end) || (!afterp && *end))
return 0;
if (result)
*result = value;
return 1;
}
int str_to_uint(const char *str, int base, unsigned *result, const char **afterp)
{
if (isspace(*str))
return 0;
const char *end = str;
errno = 0;
unsigned long value = strtoul(str, (char**)&end, base);
if (afterp)
*afterp = end;
if (errno == ERANGE || end == str || value > UINT_MAX || isdigit(*end) || (!afterp && *end))
return 0;
if (result)
*result = value;
return 1;
}
int str_to_int64(const char *str, int base, int64_t *result, const char **afterp)
{
if (isspace(*str))
return 0;
const char *end = str;
errno = 0;
long long value = strtoll(str, (char**)&end, base);
if (end == str)
return 0;
if (afterp)
*afterp = end;
else if (*end)
if (errno == ERANGE || end == str || isdigit(*end) || (!afterp && *end))
return 0;
if (result)
*result = value;
@ -249,12 +289,11 @@ int str_to_uint64(const char *str, int base, uint64_t *result, const char **afte
if (isspace(*str))
return 0;
const char *end = str;
errno = 0;
unsigned long long value = strtoull(str, (char**)&end, base);
if (end == str)
return 0;
if (afterp)
*afterp = end;
else if (*end)
if (errno == ERANGE || end == str || isdigit(*end) || (!afterp && *end))
return 0;
if (result)
*result = value;
@ -295,8 +334,11 @@ int str_to_int64_scaled(const char *str, int base, int64_t *result, const char *
{
int64_t value;
const char *end = str;
if (!str_to_int64(str, base, &value, &end))
if (!str_to_int64(str, base, &value, &end)) {
if (afterp)
*afterp = end;
return 0;
}
value *= scale_factor(end, &end);
if (afterp)
*afterp = end;
@ -311,8 +353,11 @@ int str_to_uint64_scaled(const char *str, int base, uint64_t *result, const char
{
uint64_t value;
const char *end = str;
if (!str_to_uint64(str, base, &value, &end))
if (!str_to_uint64(str, base, &value, &end)) {
if (afterp)
*afterp = end;
return 0;
}
value *= scale_factor(end, &end);
if (afterp)
*afterp = end;
@ -347,8 +392,11 @@ int str_to_uint64_interval_ms(const char *str, int64_t *result, const char **aft
return 0;
const char *end = str;
unsigned long long value = strtoull(str, (char**)&end, 10) * precision;
if (end == str)
if (end == str) {
if (afterp)
*afterp = end;
return 0;
}
if (end[0] == '.' && isdigit(end[1])) {
++end;
unsigned factor;
@ -407,11 +455,12 @@ size_t toprint_str_len(const char *srcStr, const char quotes[2])
return srcStr ? strbuf_count(strbuf_toprint_quoted(strbuf_local(NULL, 0), quotes, srcStr)) : 4;
}
size_t strn_fromprint(unsigned char *dst, size_t dstlen, const char *src, char endquote, const char **afterp)
size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp)
{
unsigned char *const odst = dst;
unsigned char *const edst = dst + dstlen;
while (*src && *src != endquote && dst < edst) {
unsigned char *const edst = dst + dstsiz;
const char *const esrc = srclen ? src + srclen : NULL;
while (src < esrc && *src && *src != endquote && dst < edst) {
switch (*src) {
case '\\':
++src;

19
str.h
View File

@ -68,6 +68,8 @@ size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, c
#define alloca_tohex(buf,bytes) tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf))
#define alloca_strdup(str) strcpy(alloca(strlen(str) + 1), (str))
__STR_INLINE int hexvalue(char c)
{
switch (c) {
@ -92,13 +94,15 @@ __STR_INLINE int hexvalue(char c)
}
int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value);
char *str_toupper_inplace(char *s);
char *str_tolower_inplace(char *s);
char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcBytes, const char quotes[2]);
char *toprint_str(char *dstStr, ssize_t dstBufSiz, const char *srcStr, const char quotes[2]);
size_t toprint_len(const char *srcBuf, size_t srcBytes, const char quotes[2]);
size_t toprint_str_len(const char *srcStr, const char quotes[2]);
size_t strn_fromprint(unsigned char *dst, size_t dstlen, const char *src, char endquote, const char **afterp);
size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp);
#define alloca_toprint(dstlen,buf,len) toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), "``") + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), "``")
#define alloca_str_toprint_quoted(str, quotes) toprint_str((char *)alloca(toprint_str_len((str), (quotes)) + 1), -1, (str), (quotes))
@ -148,7 +152,7 @@ __STR_INLINE ssize_t str_rindex(const char *s, char c)
* nul-terminated, but are held in a buffer which has an associated length. To avoid this function
* running past the end of the buffer, the caller must ensure that the buffer contains a sub-string
* that is not part of the sub-string being sought, eg, "\r\n\r\n" as detected by
* http_header_complete(). This guarantees that this function will return nonzero before running
* is_http_header_complete(). This guarantees that this function will return nonzero before running
* past the end of the buffer.
*
* @author Andrew Bettison <andrew@servalproject.com>
@ -199,13 +203,16 @@ char *str_str(char *haystack, const char *needle, int haystack_len);
/* Parse a string as an integer in ASCII radix notation in the given 'base' (eg, base=10 means
* decimal).
*
* Return 1 if a valid integer was parsed, storing the value in *result (unless result is NULL) and
* storing a pointer to the immediately succeeding character in *afterp (unless afterp is NULL, in
* which case returns 1 only if the immediately succeeding character is a nul '\0'). Returns 0
* otherwise, leaving *result and *afterp unchanged.
* Returns 1 if a valid integer is parsed, storing the value in *result (unless result is NULL) and
* storing a pointer to the immediately succeeding character in *afterp. If afterp is NULL then
* returns 0 unless the immediately succeeding character is a NUL '\0'. If no integer is parsed or
* if the integer overflows (too many digits), then returns 0, leaving *result unchanged and setting
* setting *afterp to point to the character where parsing failed.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int str_to_int(const char *str, int base, int *result, const char **afterp);
int str_to_uint(const char *str, int base, unsigned *result, const char **afterp);
int str_to_int64(const char *str, int base, int64_t *result, const char **afterp);
int str_to_uint64(const char *str, int base, uint64_t *result, const char **afterp);

82
strbuf_helpers.c
View File

@ -17,14 +17,16 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "strbuf_helpers.h"
#include <poll.h>
#include <ctype.h>
#include <string.h>
#include <time.h>
#include <assert.h>
#include <stdarg.h>
#include <inttypes.h>
#include <sys/wait.h>
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
@ -32,6 +34,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <arpa/inet.h>
#endif
#include <sys/uio.h>
#include "http_server.h"
#include "strbuf_helpers.h"
static inline strbuf _toprint(strbuf sb, char c)
{
@ -309,6 +313,16 @@ strbuf strbuf_append_socket_type(strbuf sb, int type)
return sb;
}
strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr)
{
strbuf_sprintf(sb, " %u.%u.%u.%u",
((unsigned char *) &addr->s_addr)[0],
((unsigned char *) &addr->s_addr)[1],
((unsigned char *) &addr->s_addr)[2],
((unsigned char *) &addr->s_addr)[3]);
return sb;
}
strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen)
{
strbuf_append_socket_domain(sb, addr->sa_family);
@ -332,13 +346,9 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t
break;
case AF_INET: {
const struct sockaddr_in *addr_in = (const struct sockaddr_in *) addr;
strbuf_sprintf(sb, " %u.%u.%u.%u:%u",
((unsigned char *) &addr_in->sin_addr.s_addr)[0],
((unsigned char *) &addr_in->sin_addr.s_addr)[1],
((unsigned char *) &addr_in->sin_addr.s_addr)[2],
((unsigned char *) &addr_in->sin_addr.s_addr)[3],
ntohs(addr_in->sin_port)
);
strbuf_putc(sb, ' ');
strbuf_append_in_addr(sb, &addr_in->sin_addr);
strbuf_sprintf(sb, ":%u", ntohs(addr_in->sin_port));
if (addrlen != sizeof(struct sockaddr_in))
strbuf_sprintf(sb, " (addrlen=%d should be %zd)", (int)addrlen, sizeof(struct sockaddr_in));
}
@ -391,3 +401,57 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt)
strbuf_putc(sb, ']');
return sb;
}
strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels)
{
unsigned i;
int first = 1;
for (i = 0; i != nels; ++i) {
const struct http_range *r = &ranges[i];
switch (r->type) {
case NIL: break;
case CLOSED:
strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-%"PRIhttp_size_t, first ? "" : ",", r->first, r->last);
first = 0;
break;
case OPEN:
strbuf_sprintf(sb, "%s%"PRIhttp_size_t"-", first ? "" : ",", r->first);
first = 0;
break;
case SUFFIX:
strbuf_sprintf(sb, "%s-%"PRIhttp_size_t, first ? "" : ",", r->last);
first = 0;
break;
}
}
return sb;
}
strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd)
{
strbuf_puts(sb, "type=");
strbuf_toprint_quoted(sb, "``", cd->type);
strbuf_puts(sb, " name=");
strbuf_toprint_quoted(sb, "``", cd->name);
strbuf_puts(sb, " filename=");
strbuf_toprint_quoted(sb, "``", cd->filename);
strbuf_puts(sb, " size=");
strbuf_sprintf(sb, "%"PRIhttp_size_t, cd->size);
struct tm tm;
strbuf_puts(sb, " creation_date=");
if (cd->creation_date)
strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->creation_date, &tm));
else
strbuf_puts(sb, "0");
strbuf_puts(sb, " modification_date=");
if (cd->modification_date)
strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->modification_date, &tm));
else
strbuf_puts(sb, "0");
strbuf_puts(sb, " read_date=");
if (cd->read_date)
strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->read_date, &tm));
else
strbuf_puts(sb, "0");
return sb;
}

23
strbuf_helpers.h
View File

@ -67,6 +67,7 @@ strbuf strbuf_path_join(strbuf sb, ...);
* @author Andrew Bettison <andrew@servalproject.com>
*/
strbuf strbuf_append_poll_events(strbuf sb, short events);
#define alloca_poll_events(ev) strbuf_str(strbuf_append_poll_events(strbuf_alloca(200), (ev)))
/* Append a nul-terminated string as a single-quoted shell word which, if
* expanded in a shell command line, would evaluate to the original string.
@ -116,6 +117,14 @@ strbuf strbuf_append_socket_domain(strbuf sb, int domain);
strbuf strbuf_append_socket_type(strbuf sb, int type);
#define alloca_socket_type(type) strbuf_str(strbuf_append_socket_type(strbuf_alloca(15), type))
/* Append a textual description of a struct in_addr (in network order) as IPv4
* quartet "N.N.N.N".
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct in_addr;
strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr);
#define alloca_in_addr(addr) strbuf_str(strbuf_append_in_addr(strbuf_alloca(16), (const struct in_addr *)(addr)))
/* Append a textual description of a struct sockaddr_in.
* @author Andrew Bettison <andrew@servalproject.com>
*/
@ -136,4 +145,18 @@ struct iovec;
strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt);
#define alloca_iovec(iov,cnt) strbuf_str(strbuf_append_iovec(strbuf_alloca(200), (iov), (cnt)))
/* Append a representation of a struct http_range[] array.
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct http_range;
strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels);
#define alloca_http_ranges(ra) strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra)))
/* Append a representation of a struct mime_content_disposition struct.
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct mime_content_disposition;
strbuf strbuf_append_mime_content_disposition(strbuf, const struct mime_content_disposition *);
#define alloca_mime_content_disposition(cd) strbuf_str(strbuf_append_mime_content_disposition(strbuf_alloca(500), (cd)))
#endif //__STRBUF_HELPERS_H__

49
testdefs.sh
View File

@ -732,3 +732,52 @@ has_seen_instances() {
instances_see_each_other() {
foreach_instance "$@" has_seen_instances "$@"
}
# Setup function:
# - ensure that the given version of the curl(1) utility is available
# - remove all proxy settings
setup_curl() {
local minversion="${1?}"
local ver="$(curl --version | tr '\n' ' ')"
case "$ver" in
'')
fail "curl(1) command is not present"
;;
curl\ *\ Protocols:*\ http\ *)
set -- $ver
tfw_cmp_version "$2" 7
case $? in
0|2)
unset http_proxy
unset HTTP_PROXY
unset HTTPS_PROXY
unset ALL_PROXY
return 0
;;
esac
fail "curl(1) version $2 is not adequate (expecting $minversion or higher)"
;;
esac
fail "cannot parse output of curl --version: $ver"
}
# Setup function:
# - ensure that version 1.2 or later of the jq(1) utility is available
setup_jq() {
local minversion="${1?}"
local ver="$(jq --version 2>&1)"
case "$ver" in
'')
fail "jq(1) command is not present"
;;
jq\ version\ *)
set -- $ver
tfw_cmp_version "$3" "$minversion"
case $? in
0|2) return 0;;
esac
fail "jq(1) version $3 is not adequate (need $minversion or higher)"
;;
esac
fail "cannot parse output of jq --version: $ver"
}

14
testframework.sh
View File

@ -840,6 +840,20 @@ tfw_quietly() {
fi
}
# Compare the two arguments as dotted ascii decimal version strings.
# Return 0 if they are equal, 1 if arg1 < arg2, 2 if arg1 > arg2
tfw_cmp_version() {
local IFS=.
local i=0 a=($1) b=($2)
for (( i=0; i < ${#a[@]} || i < ${#b[@]}; ++i )); do
local ai="${a[i]:-0}"
local bi="${b[i]:-0}"
(( 10#$ai < 10#$bi )) && return 1
(( 10#$ai > 10#$bi )) && return 2
done
return 0
}
# Append the contents of a file to the test case's stdout log. A normal 'cat'
# to stdout would also do this, but tfw_cat echoes header and footer delimiter
# lines around to content to help distinguish it, and also works even in a

47
tests/framework Executable file
View File

@ -0,0 +1,47 @@
#!/bin/bash
# Tests for Serval rhizome operations.
#
# Copyright 2012 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
source "${0%/*}/../testframework.sh"
shopt -s extglob
test_tfw_cmp_version() {
execute --exit-status=1 tfw_cmp_version 1 2
execute --exit-status=2 tfw_cmp_version 1.0.1 1.0.0
execute --exit-status=1 tfw_cmp_version 1.0 1.1
execute --exit-status=0 tfw_cmp_version 1 1
execute --exit-status=1 tfw_cmp_version 2.1 2.2
execute --exit-status=2 tfw_cmp_version 3.0.4.10 3.0.4.2
execute --exit-status=1 tfw_cmp_version 4.08 4.08.01
execute --exit-status=2 tfw_cmp_version 3.2.1.9.8144 3.2
execute --exit-status=1 tfw_cmp_version 3.2 3.2.1.9.8144
execute --exit-status=1 tfw_cmp_version 1.2 2.1
execute --exit-status=2 tfw_cmp_version 2.1 1.2
execute --exit-status=0 tfw_cmp_version 5.6.7 5.6.7
execute --exit-status=0 tfw_cmp_version 1.01.1 1.1.1
execute --exit-status=0 tfw_cmp_version 1.1.1 1.01.1
execute --exit-status=0 tfw_cmp_version 1 1.0
execute --exit-status=0 tfw_cmp_version 1.0 1
execute --exit-status=0 tfw_cmp_version 1.0.2.0 1.0.2
execute --exit-status=0 tfw_cmp_version 1..0 1.0
execute --exit-status=0 tfw_cmp_version 1.0 1..0
}
runTests "$@"

13
tests/rhizomechannels
View File

@ -40,24 +40,13 @@ configure_servald_server() {
set log.show_pid on \
set log.show_time on \
set debug.rhizome on \
set debug.rhizome_httpd on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set server.respawn_on_crash off \
set mdp.iftype.wifi.tick_ms 500
}
setup_curl_7() {
case "$(curl --version | tr '\n' ' ')" in
curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;;
'') fail "curl(1) command is not present";;
*) fail "curl(1) version is not adequate (expecting 7 or higher)";;
esac
unset http_proxy
unset HTTP_PROXY
unset HTTPS_PROXY
unset ALL_PROXY
}
setup_common() {
setup_servald
assert_no_servald_processes

140
tests/rhizomehttp Executable file
View File

@ -0,0 +1,140 @@
#!/bin/bash
# Tests for Serval rhizome operations.
#
# Copyright 2013 Serval Project, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
source "${0%/*}/../testframework.sh"
source "${0%/*}/../testdefs.sh"
source "${0%/*}/../testdefs_rhizome.sh"
shopt -s extglob
setup() {
setup_curl 7
setup_jq 1.2
setup_servald
set_instance +A
set_rhizome_config
executeOk_servald config \
set rhizome.api.restful.users.harry.password potter \
set rhizome.api.restful.users.ron.password weasley \
set rhizome.api.restful.users.hermione.password grainger
create_single_identity
echo "$SIDA1" >sids
start_servald_instances +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
}
set_rhizome_config() {
executeOk_servald config \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
}
doc_AuthBasicMissing="Basic Authentication credentials are required"
test_AuthBasicMissing() {
execute --exit-status=67 curl \
--silent --fail --show-error \
--output http.output \
--dump-header http.headers \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
}
teardown_AuthBasicMissing() {
tfw_cat http.headers http.output
}
doc_AuthBasicWrong="Basic Authentication credentials must be correct"
test_AuthBasicWrong() {
execute --exit-status=67 curl \
--silent --fail --show-error \
--output http.output \
--dump-header http.headers \
--basic --user fred:nurks \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
executeOk curl \
--silent --fail --show-error \
--output http.output \
--dump-header http.headers \
--basic --user ron:weasley \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
}
doc_RhizomeList="Fetch full Rhizome bundle list in JSON format"
setup_RhizomeList() {
for n in 1 2 3 4; do
create_file file$n ${n}k
executeOk_servald rhizome add file $SIDA file$n file$n.manifest
done
}
test_RhizomeList() {
executeOk curl \
--silent --fail --show-error \
--output http.output \
--dump-header http.headers \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
}
doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format"
test_RhizomeListSinceJSON() {
:
}
doc_RhizomeManifest="Fetch Rhizome bundle manifest"
test_RhizomeManifest() {
:
}
doc_RhizomePayloadRaw="Fetch Rhizome raw payload"
test_RhizomePayloadRaw() {
:
}
doc_RhizomePayloadDecrypted="Fetch Rhizome decrypted payload"
test_RhizomePayloadDecrypted() {
:
}
doc_RhizomeInsert="Insert new Rhizome bundle"
test_RhizomeInsert() {
:
}
doc_MeshmsListConversations="List MeshMS conversations"
test_MeshmsListConversations() {
:
}
doc_MeshmsListMessages="List all MeshMS messages in a single conversation"
test_MeshmsListMessages() {
:
}
doc_MeshmsListMessagesSince="List MeshMS messages in a single conversation since token"
test_MeshmsListMessagesSince() {
:
}
doc_MeshmsSend="Send MeshMS message"
test_MeshmsSend() {
:
}
runTests "$@"

21
tests/rhizomeops
View File

@ -26,21 +26,24 @@ shopt -s extglob
setup_rhizome() {
set_instance +A
executeOk_servald config \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
set_rhizome_config
create_single_identity
echo "$SIDA1" >sids
set_instance +B
set_rhizome_config
create_identities 4
echo "$SIDB1" >>sids
echo "$SIDB2" >>sids
echo "$SIDB3" >>sids
echo "$SIDB4" >>sids
assert [ $(sort sids | uniq | wc -l) -eq 5 ]
}
set_rhizome_config() {
executeOk_servald config \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
create_identities 4
assert [ $SIDB1 != $SIDA1 ]
assert [ $SIDB2 != $SIDA1 ]
assert [ $SIDB3 != $SIDA1 ]
assert [ $SIDB4 != $SIDA1 ]
}
doc_InitialEmptyList="Initial list is empty"

37
tests/rhizomeprotocol
View File

@ -41,24 +41,14 @@ configure_servald_server() {
set log.console.show_pid on \
set log.console.show_time on \
set debug.rhizome on \
set debug.httpd on \
set debug.rhizome_httpd on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set server.respawn_on_crash off \
set mdp.iftype.wifi.tick_ms 500
}
setup_curl_7() {
case "$(curl --version | tr '\n' ' ')" in
curl\ @(7|8|9|[1-9][0-1]).*\ Protocols:*\ http\ *) ;;
'') fail "curl(1) command is not present";;
*) fail "curl(1) version is not adequate (expecting 7 or higher)";;
esac
unset http_proxy
unset HTTP_PROXY
unset HTTPS_PROXY
unset ALL_PROXY
}
setup_common() {
setup_servald
assert_no_servald_processes
@ -228,6 +218,7 @@ start_radio_instance() {
executeOk_servald config \
set debug.rhizome on \
set debug.rhizome_ads on \
set debug.rhizome_httpd on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set debug.throttling on \
@ -524,12 +515,13 @@ test_CorruptPayload() {
wait_until grep -i "Stored file $FILEHASH" $LOGA
}
doc_HttpFetchRange="Fetch a file range using HTTP GET."
doc_HttpFetchRange="Fetch a file range using HTTP GET"
setup_HttpFetchRange() {
setup_curl_7
setup_curl 7
setup_common
set_instance +A
rhizome_add_file file1
rhizome_add_file file1 100
tail --bytes +33 file1 >file1.tail
start_servald_instances +A
wait_until rhizome_http_server_started +A
get_rhizome_server_port PORTA +A
@ -542,14 +534,16 @@ test_HttpFetchRange() {
--write-out '%{http_code}\n' \
--continue-at 32 \
"http://$addr_localhost:$PORTA/rhizome/file/$FILEHASH"
tfw_cat http.headers http.output
assertGrep http.headers "Content-range:"
assertGrep http.headers "Content-length:"
tfw_cat -v http.headers http.output
assertGrep http.headers "^Content-Range: bytes 32-99/100 $"
assertGrep http.headers "^Content-Length: 68 $"
tfw_cat -v file1.tail http.output
assert cmp file1.tail http.output
}
doc_HttpImport="Import bundle using HTTP POST multi-part form."
setup_HttpImport() {
setup_curl_7
setup_curl 7
setup_common
cat >README.WHYNOTSIPS <<'EOF'
When we were looking at implementing secure calls for OpenBTS it was suggested
@ -599,7 +593,7 @@ test_HttpImport() {
doc_HttpAddLocal="Add file locally using HTTP, returns manifest"
setup_HttpAddLocal() {
setup_curl_7
setup_curl 7
setup_common
set_instance +A
executeOk_servald config \
@ -639,6 +633,7 @@ setup_direct() {
set log.console.level debug \
set log.console.show_time on \
set debug.rhizome on \
set debug.rhizome_httpd on \
set debug.rhizome_tx on \
set debug.rhizome_rx on
rhizome_add_file fileB1 2000
@ -662,6 +657,8 @@ setup_DirectPush() {
setup_common
setup_direct
setup_direct_peer
executeOk ls -l
tfw_cat --stdout
}
test_DirectPush() {
set_instance +B

2
tests/rhizomestress
View File

@ -40,6 +40,7 @@ configure_servald_server() {
set log.file.show_pid on \
set log.file.show_time on \
set debug.rhizome off \
set debug.rhizome_httpd off \
set debug.rhizome_tx off \
set debug.rhizome_rx off \
set server.respawn_on_crash off \
@ -147,6 +148,7 @@ setup_StressRhizomeDirect() {
executeOk_servald config \
set log.file.show_time on \
set debug.rhizome off \
set debug.rhizome_httpd off \
set debug.rhizome_tx off \
set debug.rhizome_rx off \
set server.respawn_on_crash off \

4
vomp_console.c
View File

@ -21,8 +21,10 @@
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <poll.h>
#include <fcntl.h>
#ifdef HAVE_POLL_H
#include <poll.h>
#endif
#include "serval.h"
#include "conf.h"