ExternalVendorCode/serval-dna
1
0
Fork 0
mirror of https://github.com/servalproject/serval-dna.git synced 2024-12-18 20:57:56 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge latest version. Clean up merge conflict.

Browse Source
This commit is contained in:
Petter Reinholdtsen 2013-12-01 00:06:36 +01:00
commit 36a2eed016
87 changed files with 9396 additions and 5136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli.c
View File

@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "serval.h"
#include "rhizome.h"
#include "strbuf_helpers.h"
#include "dataformats.h"
int cli_usage(const struct cli_schema *commands, XPRINTF xpf)
{

848
commandline.c
View File

File diff suppressed because it is too large Load Diff

1
conf_schema.c
View File

@ -32,6 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "strbuf.h"
#include "strbuf_helpers.h"
#include "conf.h"
#include "dataformats.h"
int cf_opt_boolean(bool_t *booleanp, const char *text)
{

14
conf_schema.h
View File

@ -242,10 +242,9 @@ ATOM(bool_t, gateway, 0, boolean,, "")
ATOM(bool_t, keyring, 0, boolean,, "")
ATOM(bool_t, security, 0, boolean,, "")
ATOM(bool_t, mdprequests, 0, boolean,, "")
ATOM(bool_t, mavlink, 0, boolean,, "")
ATOM(bool_t, mavlink_payloads, 0, boolean,, "")
ATOM(bool_t, mavlinkfsm, 0, boolean,, "")
ATOM(bool_t, radio_link, 0, boolean,, "")
ATOM(bool_t, peers, 0, boolean,, "")
ATOM(bool_t, overlaybuffer, 0, boolean,, "")
ATOM(bool_t, overlayframes, 0, boolean,, "")
ATOM(bool_t, overlayabbreviations, 0, boolean,, "")
ATOM(bool_t, overlayrouting, 0, boolean,, "")
@ -261,13 +260,15 @@ ATOM(bool_t, slipdecode, 0, boolean,, "")
ATOM(bool_t, slipbytestream, 0, boolean,, "")
ATOM(bool_t, packetconstruction, 0, boolean,, "")
ATOM(bool_t, rhizome, 0, boolean,, "")
ATOM(bool_t, rhizome_bind, 0, boolean,, "")
ATOM(bool_t, rhizome_manifest, 0, boolean,, "")
ATOM(bool_t, rhizome_sql_bind, 0, boolean,, "")
ATOM(bool_t, rhizome_httpd, 0, boolean,, "")
ATOM(bool_t, rhizome_tx, 0, boolean,, "")
ATOM(bool_t, rhizome_rx, 0, boolean,, "")
ATOM(bool_t, rhizome_ads, 0, boolean,, "")
ATOM(bool_t, rhizome_nohttptx, 0, boolean,, "")
ATOM(bool_t, rhizome_mdp_rx, 0, boolean,, "")
ATOM(bool_t, subscriber, 0, boolean,, "")
ATOM(bool_t, throttling, 0, boolean,, "")
ATOM(bool_t, meshms, 0, boolean,, "")
ATOM(bool_t, manifests, 0, boolean,, "")
@ -396,6 +397,8 @@ END_STRUCT
STRUCT(rhizome_api_restful)
SUB_STRUCT(userlist, users,)
ATOM(uint32_t, newsince_timeout, 60, uint32_time_interval,, "Time to block while reporting new bundles")
ATOM(uint32_t, newsince_poll_ms, 2000, uint32_nonzero,, "Database poll interval while blocked reporting new bundles")
END_STRUCT
STRUCT(rhizome_api)
@ -427,6 +430,7 @@ ATOM(bool_t, external_blobs, 0, boolean,, "Store rhizome bundles
ATOM(uint64_t, rhizome_mdp_block_size, 512, uint64_scaled,, "Rhizome MDP block size.")
ATOM(uint64_t, idle_timeout, RHIZOME_IDLE_TIMEOUT, uint64_scaled,, "Rhizome transfer timeout if no data received.")
ATOM(uint64_t, mdp_stall_timeout, 1000, uint64_scaled,, "Timeout to request more data via mdp.")
ATOM(uint32_t, fetch_delay_ms, 50, uint32_nonzero,, "Delay from receiving first bundle advert to initiating fetch")
SUB_STRUCT(rhizome_direct, direct,)
SUB_STRUCT(rhizome_api, api,)
@ -472,8 +476,6 @@ ATOM(bool_t, debug, 0, boolean,, "If true, log details
ATOM(bool_t, point_to_point, 0, boolean,, "If true, assume there will only be two devices on this interface")
ATOM(bool_t, ctsrts, 0, boolean,, "If true, enable CTS/RTS hardware handshaking")
ATOM(int32_t, uartbps, 57600, int32_rs232baudrate,, "Speed of serial UART link speed (which may be different to serial device link speed)")
ATOM(int32_t, throttle, 0, int32_nonneg,, "Limit transmit speed of serial interface (bytes per second)")
ATOM(int32_t, burst_size, 0, int32_nonneg,, "Write no more than this many bytes at a time to a serial interface")
END_STRUCT
ARRAY(interface_list, NO_DUPLICATES)

103
context1.c Normal file
View File

@ -0,0 +1,103 @@
/*******************************************************************************
* The BYTE UNIX Benchmarks - Release 3
* Module: context1.c SID: 3.3 5/15/91 19:30:18
*
*******************************************************************************
* Bug reports, patches, comments, suggestions should be sent to:
*
* Ben Smith, Rick Grehan or Tom Yager
* ben@bytepb.byte.com rick_g@bytepb.byte.com tyager@bytepb.byte.com
*
*******************************************************************************
* Modification Log:
* $Header: context1.c,v 3.4 87/06/22 14:22:59 kjmcdonell Beta $
* August 28, 1990 - changed timing routines--now returns total number of
* iterations in specified time period
* October 22, 1997 - code cleanup to remove ANSI C compiler warnings
* Andy Kahn <kahn@zk3.dec.com>
*
******************************************************************************/
char SCCSid[] = "@(#) @(#)context1.c:3.3 -- 5/15/91 19:30:18";
/*
* Context switching via synchronized unbuffered pipe i/o
*
*/
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include "timeit.c"
unsigned long iter;
int stop_timing=0;
void report()
{
fprintf(stderr, "%lu context switches per second.\n", iter);
stop_timing=1;
}
void context_switch_test(int duration)
{
unsigned long check;
int p1[2], p2[2];
/* set up alarm call */
iter = 0;
wake_me(duration, report);
if (pipe(p1) || pipe(p2)) {
perror("pipe create failed");
exit(1);
}
if (fork()) { /* parent process */
/* master, write p1 & read p2 */
close(p1[0]); close(p2[1]);
while (!stop_timing) {
if (write(p1[1], (char *)&iter, sizeof(iter)) != sizeof(iter)) {
if ((errno != 0) && (errno != EINTR))
perror("master write failed");
exit(1);
}
if (read(p2[0], (char *)&check, sizeof(check)) != sizeof(check)) {
if ((errno != 0) && (errno != EINTR))
perror("master read failed");
exit(1);
}
if (check != iter) {
fprintf(stderr, "Master sync error: expect %lu, got %lu\n",
iter, check);
exit(2);
}
iter++;
}
}
else { /* child process */
unsigned long iter1;
iter1 = 0;
/* slave, read p1 & write p2 */
close(p1[1]); close(p2[0]);
while (!stop_timing) {
if (read(p1[0], (char *)&check, sizeof(check)) != sizeof(check)) {
if ((errno != 0) && (errno != EINTR))
perror("slave read failed");
exit(1);
}
if (check != iter1) {
fprintf(stderr, "Slave sync error: expect %lu, got %lu\n",
iter, check);
exit(2);
}
if (write(p2[1], (char *)&iter1, sizeof(iter1)) != sizeof(check)) {
if ((errno != 0) && (errno != EINTR))
perror("slave write failed");
exit(1);
}
iter1++;
}
}
}

1
crypto.c
View File

@ -3,6 +3,7 @@
#include "serval.h"
#include "overlay_address.h"
#include "crypto.h"
#include "keyring.h"
// verify a signature against a public sas key.
int crypto_verify_signature(unsigned char *sas_key,

23
dataformats.c
View File

@ -17,10 +17,11 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <ctype.h>
#include "serval.h"
#include "rhizome.h"
#include "str.h"
#include <ctype.h>
#include "dataformats.h"
int cmp_sid_t(const sid_t *a, const sid_t *b)
{
@ -116,6 +117,11 @@ int strn_to_rhizome_filehash_t(rhizome_filehash_t *hashp, const char *hex, const
return 0;
}
int str_to_rhizome_bk_t(rhizome_bk_t *bkp, const char *hex)
{
return fromhexstr(bkp->binary, hex, sizeof bkp->binary);
}
int rhizome_strn_is_manifest_id(const char *id)
{
return is_xsubstring(id, RHIZOME_MANIFEST_ID_STRLEN);
@ -156,6 +162,15 @@ int rhizome_str_is_file_hash(const char *hash)
return is_xstring(hash, RHIZOME_FILEHASH_STRLEN);
}
int rhizome_str_is_manifest_service(const char *text)
{
if (text[0] == '\0')
return 0;
while (*text && (isalnum(*text) || *text == '_' || *text == '.'))
++text;
return *text == '\0';
}
int str_is_did(const char *did)
{
size_t len = 0;
@ -200,7 +215,7 @@ void write_uint16(unsigned char *o,uint16_t v)
{ *(o++)=v&0xff; v=v>>8; }
}
uint64_t read_uint64(unsigned char *o)
uint64_t read_uint64(const unsigned char *o)
{
int i;
uint64_t v=0;
@ -208,7 +223,7 @@ uint64_t read_uint64(unsigned char *o)
return v;
}
uint32_t read_uint32(unsigned char *o)
uint32_t read_uint32(const unsigned char *o)
{
int i;
uint32_t v=0;
@ -216,7 +231,7 @@ uint32_t read_uint32(unsigned char *o)
return v;
}
uint16_t read_uint16(unsigned char *o)
uint16_t read_uint16(const unsigned char *o)
{
int i;
uint16_t v=0;

26
dataformats.h Normal file
View File

@ -0,0 +1,26 @@
#ifndef __SERVALD_DATA_FORMATS_H
#define __SERVALD_DATA_FORMATS_H
int str_is_subscriber_id(const char *sid);
int strn_is_subscriber_id(const char *sid, size_t *lenp);
int str_is_did(const char *did);
int strn_is_did(const char *did, size_t *lenp);
int rhizome_strn_is_manifest_id(const char *text);
int rhizome_str_is_manifest_id(const char *text);
int rhizome_strn_is_bundle_key(const char *text);
int rhizome_str_is_bundle_key(const char *text);
int rhizome_strn_is_bundle_crypt_key(const char *text);
int rhizome_str_is_bundle_crypt_key(const char *text);
int rhizome_strn_is_file_hash(const char *text);
int rhizome_str_is_file_hash(const char *text);
int rhizome_str_is_manifest_service(const char *text);
void write_uint64(unsigned char *o,uint64_t v);
void write_uint16(unsigned char *o,uint16_t v);
void write_uint32(unsigned char *o,uint32_t v);
uint64_t read_uint64(const unsigned char *o);
uint32_t read_uint32(const unsigned char *o);
uint16_t read_uint16(const unsigned char *o);
#endif

3
directory_client.c
View File

@ -16,6 +16,7 @@
#include "str.h"
#include "overlay_address.h"
#include "conf.h"
#include "keyring.h"
struct subscriber *directory_service;
@ -51,7 +52,7 @@ static void directory_send(struct subscriber *directory_service, const sid_t *si
// Used by tests
INFOF("Sending directory registration for %s*, %s, %s to %s*",
alloca_tohex_sid_t_trunc(*sidp, 14), did, name, alloca_tohex_sid_t_trunc(directory_service->sid, 14));
overlay_mdp_dispatch(&request, 0, NULL, 0);
overlay_mdp_dispatch(&request, NULL);
}
// send a registration packet for each unlocked identity

1
dna_helper.c
View File

@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "strbuf.h"
#include "strbuf_helpers.h"
#include "overlay_address.h"
#include "dataformats.h"
/*
The challenge with making an interface for calling an external program to

112
fakeradio.c
View File

@ -339,42 +339,107 @@ int transfer_bytes(struct radio_state *radios)
log_time();
fprintf(stderr, "Transferring %d byte packet from %s to %s\n", bytes, t->name, r->name);
}
int i, j;
for (i=0;i<bytes && r->rxb_len<sizeof(r->rxbuffer);i++){
char byte = t->txbuffer[i];
// introduce bit errors
for(j=0;j<8;j++) {
if (random()<ber) {
byte^=(1<<j);
fprintf(stderr,"Flipped a bit\n");
int dropped=0;
// preamble length in bits that must arrive intact
#define PREAMBLE_LENGTH (20+8)
// simulate the probability of a bit error in the packet pre-amble and drop the whole packet
for (i=0;i<PREAMBLE_LENGTH;i++){
if (random()<ber)
dropped=1;
}
if (dropped){
fprintf(stderr,"Dropped the whole radio packet due to bit flip in the pre-amble\n");
}else{
for (i=0;i<bytes && r->rxb_len<sizeof(r->rxbuffer);i++){
char byte = t->txbuffer[i];
// introduce bit errors
for(j=0;j<8;j++) {
if (random()<ber) {
byte^=(1<<j);
fprintf(stderr,"Flipped a bit\n");
}
}
r->rxbuffer[r->rxb_len++]=byte;
}
r->rxbuffer[r->rxb_len++]=byte;
}
if (bytes>0 && bytes < t->txb_len)
bcopy(&t->txbuffer[bytes], t->txbuffer, t->txb_len - bytes);
t->txb_len-=bytes;
if (bytes==0 || --t->tx_count<=0){
// swap who's turn it is to transmit
transmitter = receiver;
r->tx_count=6;
}
// set the wait time for the next transmission
next_transmit_time = gettime_ms() + (bytes+10)/chars_per_ms;
next_transmit_time = gettime_ms() + 5 + bytes/chars_per_ms;
if (bytes==0 || --t->tx_count<=0){
// swap who's turn it is to transmit after sending 3 packets or running out of data.
transmitter = receiver;
r->tx_count=3;
// add Tx->Rx change time (it's about 40ms between receiving empty packets)
next_transmit_time+=15;
}
return bytes;
}
int calc_ber(double target_packet_fraction)
{
int byte_count=220+32;
int max_error_bytes=16;
int ber;
int p;
int byte;
int bit;
// 9,000,000 gives a packet delivery rate of ~99%
// so no point starting smaller than that.
// Only ~30,000,000 reduces packet delivery rate to
// ~1%, so the search range is fairly narrow.
ber=0;
if (target_packet_fraction<=0.9) ber=6900000;
if (target_packet_fraction<=0.5) ber=16900000;
if (target_packet_fraction<=0.25) ber=20600000;
if (target_packet_fraction<=0.1) ber=23400000;
if (target_packet_fraction<=0.05) ber=28600000;
for(;ber<0x70ffffff;ber+=100000)
{
int packet_errors=0;
for(p=0;p<1000;p++) {
int byte_errors=0;
int dropped = 0;
for (byte=0;byte<PREAMBLE_LENGTH;byte++){
if (random()<ber){
dropped = 1;
break;
}
}
if (!dropped){
for(byte=0;byte<byte_count;byte++) {
for(bit=0;bit<8;bit++) if (random()<ber) { byte_errors++; break; }
if (byte_errors>max_error_bytes) { dropped=1; break; }
}
}
if (dropped)
packet_errors++;
}
if (packet_errors>=((1.0-target_packet_fraction)*1000)) break;
}
fprintf(stderr,"ber magic value=%d\n",ber);
return ber;
}
int main(int argc,char **argv)
{
if (1 < argc) {
if (argc>=1) {
chars_per_ms=atol(argv[1]);
if (2 < argc)
ber=atol(argv[2]);
if (argc>=2)
ber=calc_ber(atof(argv[2]));
}
fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms);
fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF);
struct pollfd fds[2];
struct radio_state radios[2];
@ -382,18 +447,21 @@ int main(int argc,char **argv)
bzero(&radios,sizeof radios);
int i;
radios[0].name="left";
radios[1].name="right";
for (i=0;i<2;i++){
radios[i].fd=posix_openpt(O_RDWR|O_NOCTTY);
grantpt(radios[i].fd);
unlockpt(radios[i].fd);
fcntl(radios[i].fd,F_SETFL,fcntl(radios[i].fd, F_GETFL, NULL)|O_NONBLOCK);
fprintf(stdout,"%s\n",ptsname(radios[i].fd));
fprintf(stdout,"%s:%s\n", radios[i].name, ptsname(radios[i].fd));
fds[i].fd = radios[i].fd;
}
radios[0].name="left";
radios[1].name="right";
fflush(stdout);
fprintf(stderr, "Sending %d bytes per ms\n", chars_per_ms);
fprintf(stderr, "Introducing %f%% bit errors\n", (ber * 100.0) / 0xFFFFFFFF);
while(1) {
// what events do we need to poll for? how long can we block?
int64_t now = gettime_ms();

36
fdqueue.c
View File

@ -85,13 +85,15 @@ int is_scheduled(const struct sched_ent *alarm)
// on calling .poll.revents will be zero.
int _schedule(struct __sourceloc __whence, struct sched_ent *alarm)
{
time_ms_t now = gettime_ms();
if (config.debug.io)
DEBUGF("schedule(alarm=%s) called from %s() %s:%d",
alloca_alarm_name(alarm),
__whence.function,__whence.file,__whence.line);
DEBUGF("schedule(alarm=%s) alarm=%.3f deadline=%.3f",
alloca_alarm_name(alarm),
(double)(alarm->alarm - now) / 1000,
(double)(alarm->deadline - now) / 1000
);
if (!alarm->stats)
WARNF("schedule() called from %s() %s:%d without supplying an alarm name",
__whence.function,__whence.file,__whence.line);
WARN("schedule() called without supplying an alarm name");
struct sched_ent *node = next_alarm, *last = NULL;
@ -100,18 +102,16 @@ int _schedule(struct __sourceloc __whence, struct sched_ent *alarm)
if (!alarm->function)
return WHY("Can't schedule if you haven't set the function pointer");
time_ms_t now = gettime_ms();
if (alarm->deadline < alarm->alarm)
alarm->deadline = alarm->alarm;
if (now - alarm->deadline > 1000){
// 1000ms ago? thats silly, if you keep doing it noone else will get a turn.
WHYF("Alarm %s tried to schedule a deadline %"PRId64"ms ago, from %s() %s:%d",
FATALF("Alarm %s tried to schedule a deadline %"PRId64"ms ago",
alloca_alarm_name(alarm),
(now - alarm->deadline),
__whence.function,__whence.file,__whence.line);
(now - alarm->deadline)
);
}
// if the alarm has already expired, move straight to the deadline queue
@ -168,8 +168,7 @@ int _watch(struct __sourceloc __whence, struct sched_ent *alarm)
if (config.debug.io)
DEBUGF("watch(alarm=%s)", alloca_alarm_name(alarm));
if (!alarm->stats)
WARNF("watch() called from %s() %s:%d without supplying an alarm name",
__whence.function,__whence.file,__whence.line);
WARN("watch() called without supplying an alarm name");
if (!alarm->function)
return WHY("Can't watch if you haven't set the function pointer");
@ -177,10 +176,10 @@ int _watch(struct __sourceloc __whence, struct sched_ent *alarm)
if (alarm->_poll_index>=0 && fd_callbacks[alarm->_poll_index]==alarm){
// updating event flags
if (config.debug.io)
DEBUGF("Updating watch %s, #%d for %d", alloca_alarm_name(alarm), alarm->poll.fd, alarm->poll.events);
DEBUGF("Updating watch %s, #%d for %s", alloca_alarm_name(alarm), alarm->poll.fd, alloca_poll_events(alarm->poll.events));
}else{
if (config.debug.io)
DEBUGF("Adding watch %s, #%d for %d", alloca_alarm_name(alarm), alarm->poll.fd, alarm->poll.events);
DEBUGF("Adding watch %s, #%d for %s", alloca_alarm_name(alarm), alarm->poll.fd, alloca_poll_events(alarm->poll.events));
if (fdcount>=MAX_WATCHED_FDS)
return WHY("Too many file handles to watch");
fd_callbacks[fdcount]=alarm;
@ -213,7 +212,7 @@ int _unwatch(struct __sourceloc __whence, struct sched_ent *alarm)
fd_callbacks[fdcount]=NULL;
alarm->_poll_index=-1;
if (config.debug.io)
DEBUGF("%s stopped watching #%d for %d", alloca_alarm_name(alarm), alarm->poll.fd, alarm->poll.events);
DEBUGF("%s stopped watching #%d for %s", alloca_alarm_name(alarm), alarm->poll.fd, alloca_poll_events(alarm->poll.events));
return 0;
}
@ -225,8 +224,8 @@ static void call_alarm(struct sched_ent *alarm, int revents)
struct call_stats call_stats;
call_stats.totals = alarm->stats;
if (config.debug.io) DEBUGF("Calling alarm/callback %p ('%s')",
alarm, alloca_alarm_name(alarm));
if (config.debug.io)
DEBUGF("Calling alarm/callback %p %s", alarm, alloca_alarm_name(alarm));
if (call_stats.totals)
fd_func_enter(__HERE__, &call_stats);
@ -237,7 +236,8 @@ static void call_alarm(struct sched_ent *alarm, int revents)
if (call_stats.totals)
fd_func_exit(__HERE__, &call_stats);
if (config.debug.io) DEBUGF("Alarm %p returned",alarm);
if (config.debug.io)
DEBUGF("Alarm %p returned",alarm);
OUT();
}

6
fdqueue.h
View File

@ -80,13 +80,15 @@ int fd_checkalarms();
int fd_func_enter(struct __sourceloc, struct call_stats *this_call);
int fd_func_exit(struct __sourceloc, struct call_stats *this_call);
void dump_stack(int log_level);
unsigned fd_depth();
#define IN() static struct profile_total _aggregate_stats={NULL,0,__FUNCTION__,0,0,0,0}; \
struct call_stats _this_call={.totals=&_aggregate_stats}; \
fd_func_enter(__HERE__, &_this_call);
#define OUT() fd_func_exit(__HERE__, &_this_call)
#define RETURN(X) do { OUT(); return (X); } while (0);
#define RETURNNULL do { OUT(); return (NULL); } while (0);
#define RETURN(X) do { OUT(); return (X); } while (0)
#define RETURNNULL(X) do { X; OUT(); return (NULL); } while (0)
#define RETURNVOID do { OUT(); return; } while (0)
#endif // __SERVALDNA__FDQUEUE_H

45
golay.c
View File

@ -1,13 +1,14 @@
#define POLY 0xAE3 /* or use the other polynomial, 0xC75 */
#include <inttypes.h>
static unsigned long golay(unsigned long cw)
static uint32_t golay(uint32_t cw)
/* This function calculates [23,12] Golay codewords.
The format of the returned longint is
[checkbits(11),data(12)]. */
{
int i;
unsigned long c;
uint32_t c;
cw&=0xfffl;
c=cw; /* save original codeword */
for (i=1; i<=12; i++) /* examine each data bit */
@ -19,16 +20,16 @@ static unsigned long golay(unsigned long cw)
return((cw<<12)|c); /* assemble codeword */
}
static int parity(unsigned long cw)
static int parity(uint32_t cw)
/* This function checks the overall parity of codeword cw.
If parity is even, 0 is returned, else 1. */
{
unsigned char p;
uint8_t p;
/* XOR the bytes of the codeword */
p=*(unsigned char*)&cw;
p^=*((unsigned char*)&cw+1);
p^=*((unsigned char*)&cw+2);
p=cw & 0xFF;
p^=(cw>>8) & 0xFF;
p^=(cw>>16) & 0xFF;
/* XOR the halves of the intermediate result */
p=p ^ (p>>4);
@ -39,9 +40,9 @@ static int parity(unsigned long cw)
return(p & 1);
}
int golay_encode(unsigned char *data)
int golay_encode(uint8_t *data)
{
unsigned long cw = data[0] | (data[1]<<8) | (data[2]<<16);
uint32_t cw = data[0] | (data[1]<<8) | (data[2]<<16);
cw = golay(cw);
if (parity(cw))
cw|=0x800000l;
@ -51,7 +52,7 @@ int golay_encode(unsigned char *data)
return 0;
}
static unsigned long syndrome(unsigned long cw)
static uint32_t syndrome(uint32_t cw)
/* This function calculates and returns the syndrome
of a [23,12] Golay codeword. */
{
@ -66,7 +67,7 @@ static unsigned long syndrome(unsigned long cw)
return(cw<<12); /* value pairs with upper bits of cw */
}
static int weight(unsigned long cw)
static int weight(uint32_t cw)
/* This function calculates the weight of
23 bit codeword cw. */
{
@ -88,7 +89,7 @@ static int weight(unsigned long cw)
return(bits);
}
static unsigned long rotate_left(unsigned long cw, int n)
static uint32_t rotate_left(uint32_t cw, int n)
/* This function rotates 23 bit codeword cw left by n bits. */
{
int i;
@ -107,7 +108,7 @@ static unsigned long rotate_left(unsigned long cw, int n)
return(cw & 0x7fffffl);
}
static unsigned long rotate_right(unsigned long cw, int n)
static uint32_t rotate_right(uint32_t cw, int n)
/* This function rotates 23 bit codeword cw right by n bits. */
{
int i;
@ -126,20 +127,20 @@ static unsigned long rotate_right(unsigned long cw, int n)
return(cw & 0x7fffffl);
}
static unsigned long correct(unsigned long cw, int *errs)
static uint32_t correct(uint32_t cw, int *errs)
/* This function corrects Golay [23,12] codeword cw, returning the
corrected codeword. This function will produce the corrected codeword
for three or fewer errors. It will produce some other valid Golay
codeword for four or more errors, possibly not the intended
one. *errs is set to the number of bit errors corrected. */
{
unsigned char
uint8_t
w; /* current syndrome limit weight, 2 or 3 */
unsigned long
uint32_t
mask; /* mask for bit flipping */
int
i,j; /* index */
unsigned long
uint32_t
s, /* calculated syndrome */
cwsaver; /* saves initial value of cw */
@ -187,17 +188,17 @@ static unsigned long correct(unsigned long cw, int *errs)
return(cwsaver); /* return original if no corrections */
} /* correct */
int golay_decode(int *errs, const unsigned char *data)
int golay_decode(int *errs, const uint8_t *data)
/* This function decodes codeword *cw , error correction is attempted,
with *errs set to the number of bits corrected, and returning 0 if
no errors exist, or 1 if parity errors exist. */
{
unsigned long cw = data[0] | (data[1]<<8) | (data[2]<<16);
unsigned long parity_bit=cw & 0x800000l;
uint32_t cw = data[0] | (data[1]<<8) | (data[2]<<16);
uint32_t parity_bit=cw & 0x800000l;
cw&=~0x800000l; /* remove parity bit for correction */
cw=correct(cw, errs); /* correct up to three bits */
cw|=parity_bit;
if (parity(cw))
return -1;
*errs++;
return cw&0xFFF;
} /* decode */

10
golay.h
View File

@ -1,2 +1,8 @@
int golay_encode(unsigned char *data);
int golay_decode(int *errs, unsigned char *data);
#ifndef __SERVALD_GOLAY_H
#define __SERVALD_GOLAY_H
int golay_encode(uint8_t *data);
int golay_decode(int *errs, uint8_t *data);
#endif

5
headerfiles.mk
View File

@ -5,17 +5,21 @@ HDRS= fifo.h \
overlay_packet.h \
rhizome.h \
serval.h \
keyring.h \
socket.h \
cli.h \
str.h \
rotbuf.h \
mem.h \
os.h \
uuid.h \
strbuf.h \
strbuf_helpers.h \
sha2.h \
conf.h \
conf_schema.h \
crypto.h \
dataformats.h \
log.h \
net.h \
fdqueue.h \
@ -24,4 +28,5 @@ HDRS= fifo.h \
constants.h \
monitor-client.h \
mdp_client.h \
radio_link.h \
sqlite-amalgamation-3070900/sqlite3.h

779
http_server.c
View File

File diff suppressed because it is too large Load Diff

94
http_server.h
View File

@ -56,13 +56,35 @@ http_size_t http_range_bytes(const struct http_range *range, unsigned nranges);
#define CONTENT_LENGTH_UNKNOWN UINT64_MAX
struct mime_content_type {
char type[64];
char subtype[64];
char multipart_boundary[71];
char charset[31];
};
struct http_client_authorization {
enum http_authorization_scheme { NOAUTH = 0, BASIC } scheme;
union {
struct http_client_credentials_basic {
const char *user;
const char *password;
} basic;
} credentials;
};
struct http_www_authenticate {
enum http_authorization_scheme scheme;
const char *realm;
};
struct http_request_headers {
http_size_t content_length;
const char *content_type;
const char *content_subtype;
const char *boundary;
struct mime_content_type content_type;
unsigned short content_range_count;
struct http_range content_ranges[5];
struct http_client_authorization authorization;
};
struct http_response_headers {
@ -71,15 +93,21 @@ struct http_response_headers {
http_size_t resource_length; // size of entire resource
const char *content_type; // "type/subtype"
const char *boundary;
struct http_www_authenticate www_authenticate;
};
typedef int (*HTTP_CONTENT_GENERATOR)(struct http_request *);
struct http_content_generator_result {
size_t generated;
size_t need;
};
typedef int (HTTP_CONTENT_GENERATOR)(struct http_request *, unsigned char *, size_t, struct http_content_generator_result *);
struct http_response {
uint16_t result_code;
struct http_response_headers header;
const char *content;
HTTP_CONTENT_GENERATOR content_generator; // callback to produce more content
HTTP_CONTENT_GENERATOR *content_generator; // callback to produce more content
};
#define MIME_FILENAME_MAXLEN 127
@ -94,11 +122,16 @@ struct mime_content_disposition {
time_t read_date;
};
struct mime_part_headers {
http_size_t content_length;
struct mime_content_type content_type;
struct mime_content_disposition content_disposition;
};
struct http_mime_handler {
void (*handle_mime_preamble)(struct http_request *, const char *, size_t);
void (*handle_mime_part_start)(struct http_request *);
void (*handle_mime_content_disposition)(struct http_request *, const struct mime_content_disposition *);
void (*handle_mime_header)(struct http_request *, const char *label, const char *, size_t);
void (*handle_mime_part_header)(struct http_request *, const struct mime_part_headers *);
void (*handle_mime_body)(struct http_request *, const char *, size_t);
void (*handle_mime_part_end)(struct http_request *);
void (*handle_mime_epilogue)(struct http_request *, const char *, size_t);
@ -110,46 +143,67 @@ void http_request_init(struct http_request *r, int sockfd);
void http_request_free_response_buffer(struct http_request *r);
int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
void http_request_finalise(struct http_request *r);
void http_request_pause_response(struct http_request *r, time_ms_t until);
void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR);
void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *);
void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);
typedef int (*HTTP_REQUEST_PARSER)(struct http_request *);
struct http_request {
struct sched_ent alarm; // MUST BE FIRST ELEMENT
enum http_request_phase { RECEIVE, TRANSMIT, DONE } phase;
bool_t *debug_flag;
bool_t *disable_tx_flag;
time_ms_t initiate_time; // time connection was initiated
time_ms_t idle_timeout; // disconnect if no bytes received for this long
struct sockaddr_in client_sockaddr_in;
HTTP_REQUEST_PARSER parser; // current parser function
HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed
HTTP_REQUEST_PARSER handle_headers; // called after all headers are parsed
HTTP_REQUEST_PARSER handle_content_end; // called after all content is received
enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state;
struct http_mime_handler form_data; // called to parse multipart/form-data body
// The following control the lifetime of this struct.
enum http_request_phase { RECEIVE, TRANSMIT, PAUSE, DONE } phase;
void (*finalise)(struct http_request *);
void (*free)(void*);
// These can be set up to point to config flags, to allow debug to be
// enabled indpendently for different instances HTTP server instances
// that use this code.
bool_t *debug_flag;
bool_t *disable_tx_flag;
// The following are used for parsing the HTTP request.
time_ms_t initiate_time; // time connection was initiated
time_ms_t idle_timeout; // disconnect if no bytes received for this long
struct sockaddr_in client_sockaddr_in; // caller may supply this
// The parsed HTTP request is accumulated into the following fields.
const char *verb; // points to nul terminated static string, "GET", "PUT", etc.
const char *path; // points into buffer; nul terminated
uint8_t version_major; // m from from HTTP/m.n
uint8_t version_minor; // n from HTTP/m.n
struct http_request_headers request_header;
// Parsing is done by setting 'parser' to point to a series of parsing
// functions as the parsing state progresses.
HTTP_REQUEST_PARSER parser; // current parser function
// The caller may set these up, and they are invoked by the parser as request
// parsing reaches different stages.
HTTP_REQUEST_PARSER handle_first_line; // called after first line is parsed
HTTP_REQUEST_PARSER handle_headers; // called after all HTTP headers are parsed
HTTP_REQUEST_PARSER handle_content_end; // called after all content is received
// The following are used for managing the buffer during RECEIVE phase.
const char *received; // start of received data in buffer[]
const char *end; // end of received data in buffer[]
const char *parsed; // start of unparsed data in buffer[]
const char *cursor; // for parsing
http_size_t request_content_remaining;
// The following are used for parsing a multipart body.
enum mime_state { START, PREAMBLE, HEADER, BODY, EPILOGUE } form_data_state;
struct http_mime_handler form_data;
struct mime_part_headers part_header;
http_size_t part_body_length;
// The following are used for constructing the response that will be sent in
// TRANSMIT phase.
struct http_response response;
// The following are used during TRANSMIT phase to control buffering and
// sending.
http_size_t response_length; // total response bytes (header + content)
http_size_t response_sent; // for counting up to response_length
char *response_buffer;
size_t response_buffer_need;
size_t response_buffer_size;
size_t response_buffer_length;
size_t response_buffer_sent;
void (*response_free_buffer)(void*);
// This buffer is used during RECEIVE and TRANSMIT phase.
char buffer[8 * 1024];
};

295
keyring.c
View File

@ -29,6 +29,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "overlay_address.h"
#include "crypto.h"
#include "overlay_packet.h"
#include "keyring.h"
#include "dataformats.h"
static void keyring_free_keypair(keypair *kp);
static void keyring_free_context(keyring_context *c);
@ -223,7 +225,8 @@ void keyring_free(keyring_file *k)
/* Wipe everything, just to be sure. */
bzero(k,sizeof(keyring_file));
free(k);
return;
}
@ -272,6 +275,7 @@ static void keyring_free_context(keyring_context *c)
}
if (c->KeyRingSalt) {
bzero(c->KeyRingSalt,c->KeyRingSaltLen);
free(c->KeyRingSalt);
c->KeyRingSalt = NULL;
c->KeyRingSaltLen = 0;
}
@ -427,6 +431,7 @@ static const char *keytype_str(unsigned ktype, const char *unknown)
case KEYTYPE_CRYPTOSIGN: return "CRYPTOSIGN";
case KEYTYPE_RHIZOME: return "RHIZOME";
case KEYTYPE_DID: return "DID";
case KEYTYPE_PUBLIC_TAG: return "PUBLIC_TAG";
default: return unknown;
}
}
@ -437,7 +442,7 @@ struct keytype {
size_t packed_size;
void (*creator)(keypair *);
int (*packer)(const keypair *, struct rotbuf *);
int (*unpacker)(keypair *, struct rotbuf *);
int (*unpacker)(keypair *, struct rotbuf *, int);
void (*dumper)(const keypair *, XPRINTF, int);
int (*loader)(keypair *, const char *);
};
@ -483,6 +488,12 @@ static int pack_private_only(const keypair *kp, struct rotbuf *rb)
return 0;
}
static int pack_public_only(const keypair *kp, struct rotbuf *rb)
{
rotbuf_putbuf(rb, kp->public_key, kp->public_key_len);
return 0;
}
static int pack_private_public(const keypair *kp, struct rotbuf *rb)
{
rotbuf_putbuf(rb, kp->private_key, kp->private_key_len);
@ -617,20 +628,36 @@ static int load_unknown(keypair *kp, const char *text)
return 0;
}
static int unpack_private_public(keypair *kp, struct rotbuf *rb)
static int unpack_private_public(keypair *kp, struct rotbuf *rb, int key_length)
{
rotbuf_getbuf(rb, kp->private_key, kp->private_key_len);
rotbuf_getbuf(rb, kp->public_key, kp->public_key_len);
return 0;
}
static int unpack_private_only(keypair *kp, struct rotbuf *rb)
static int unpack_private_only(keypair *kp, struct rotbuf *rb, int key_length)
{
if (!kp->private_key){
kp->private_key_len = key_length;
if ((kp->private_key = emalloc(kp->private_key_len))==NULL)
return -1;
}
rotbuf_getbuf(rb, kp->private_key, kp->private_key_len);
return 0;
}
static int unpack_cryptobox(keypair *kp, struct rotbuf *rb)
static int unpack_public_only(keypair *kp, struct rotbuf *rb, int key_length)
{
if (!kp->public_key){
kp->public_key_len = key_length;
if ((kp->public_key = emalloc(kp->public_key_len))==NULL)
return -1;
}
rotbuf_getbuf(rb, kp->public_key, kp->public_key_len);
return 0;
}
static int unpack_cryptobox(keypair *kp, struct rotbuf *rb, int key_length)
{
rotbuf_getbuf(rb, kp->private_key, kp->private_key_len);
if (!rb->wrap)
@ -646,9 +673,9 @@ static int pack_did_name(const keypair *kp, struct rotbuf *rb)
return pack_private_public(kp, rb);
}
static int unpack_did_name(keypair *kp, struct rotbuf *rb)
static int unpack_did_name(keypair *kp, struct rotbuf *rb, int key_length)
{
if (unpack_private_public(kp, rb) == -1)
if (unpack_private_public(kp, rb, key_length) == -1)
return -1;
// Fail if name is not nul terminated.
return strnchr((const char *)kp->public_key, kp->public_key_len, '\0') == NULL ? -1 : 0;
@ -761,6 +788,16 @@ const struct keytype keytypes[] = {
.unpacker = unpack_did_name,
.dumper = dump_did_name,
.loader = load_did_name
},
[KEYTYPE_PUBLIC_TAG] = {
.private_key_size = 0,
.public_key_size = 0, // size is derived from the stored key length
.packed_size = 0,
.creator = NULL, // not included in a newly created identity
.packer = pack_public_only,
.unpacker = unpack_public_only,
.dumper = dump_private_public,
.loader = load_unknown
}
// ADD MORE KEY TYPES HERE
};
@ -835,13 +872,16 @@ static int keyring_pack_identity(const keyring_identity *id, unsigned char packe
unsigned ktype = id->keypairs[kp]->type;
const char *kts = keytype_str(ktype, "unknown");
int (*packer)(const keypair *, struct rotbuf *) = NULL;
size_t keypair_len;
size_t keypair_len=0;
const struct keytype *kt = &keytypes[ktype];
if (ktype == 0x00)
FATALF("ktype=0 in keypair kp=%u", kp);
if (ktype < NELS(keytypes)) {
packer = kt->packer;
keypair_len = kt->packed_size;
if (keypair_len==0){
keypair_len = id->keypairs[kp]->private_key_len + id->keypairs[kp]->public_key_len;
}
} else {
packer = pack_private_only;
keypair_len = id->keypairs[kp]->private_key_len;
@ -914,16 +954,24 @@ static int cmp_keypair(const keypair *a, const keypair *b)
{
int c = a->type < b->type ? -1 : a->type > b->type ? 1 : 0;
if (c == 0 && a->public_key_len) {
assert(a->public_key_len == b->public_key_len);
assert(a->public_key != NULL);
assert(b->public_key != NULL);
c = memcmp(a->public_key, b->public_key, a->public_key_len);
int len=a->public_key_len;
if (len>b->public_key_len)
len=b->public_key_len;
c = memcmp(a->public_key, b->public_key, len);
if (c==0 && a->public_key_len!=b->public_key_len)
c = a->public_key_len - b->public_key_len;
}
if (c == 0 && a->private_key_len) {
assert(a->private_key_len == b->private_key_len);
assert(a->private_key != NULL);
assert(b->private_key != NULL);
c = memcmp(a->private_key, b->private_key, a->private_key_len);
int len=a->private_key_len;
if (len>b->private_key_len)
len=b->private_key_len;
c = memcmp(a->private_key, b->private_key, len);
if (c==0 && a->private_key_len!=b->private_key_len)
c = a->private_key_len - b->private_key_len;
}
return c;
}
@ -1007,7 +1055,7 @@ static keyring_identity *keyring_unpack_identity(unsigned char *slot, const char
if (ktype < NELS(keytypes) && kt->unpacker) {
if (config.debug.keyring)
DEBUGF("unpack key type = 0x%02x(%s) at offset %u", ktype, keytype_str(ktype, "unknown"), (int)rotbuf_position(&rbo));
if (kt->unpacker(kp, &rbuf) != 0) {
if (kt->unpacker(kp, &rbuf, keypair_len) != 0) {
// If there is an error, it is probably an empty slot.
if (config.debug.keyring)
DEBUGF("key type 0x%02x does not unpack", ktype);
@ -1413,10 +1461,14 @@ int keyring_set_did(keyring_identity *id, const char *did, const char *name)
}
/* Store DID unpacked for ease of searching */
int len=strlen(did); if (len>31) len=31;
int len=strlen(did);
if (len>31)
len=31;
bcopy(did,&id->keypairs[i]->private_key[0],len);
bzero(&id->keypairs[i]->private_key[len],32-len);
len=strlen(name); if (len>63) len=63;
len=strlen(name);
if (len>63)
len=63;
bcopy(name,&id->keypairs[i]->public_key[0],len);
bzero(&id->keypairs[i]->public_key[len],64-len);
@ -1427,23 +1479,121 @@ int keyring_set_did(keyring_identity *id, const char *did, const char *name)
return 0;
}
int keyring_find_did(const keyring_file *k,int *cn,int *in,int *kp,char *did)
int keyring_find_did(const keyring_file *k, int *cn, int *in, int *kp, const char *did)
{
for (; keyring_sanitise_position(k,cn,in,kp) == 0; ++*kp) {
if (k->contexts[*cn]->identities[*in]->keypairs[*kp]->type==KEYTYPE_DID) {
/* Compare DIDs */
if ((!did[0])
||(did[0]=='*'&&did[1]==0)
||(!strcasecmp(did,(char *)k->contexts[*cn]->identities[*in]
->keypairs[*kp]->private_key))
) {
return 1; // match
}
for(;keyring_next_keytype(k,cn,in,kp,KEYTYPE_DID);++(*kp)) {
/* Compare DIDs */
if ((!did[0])
||(did[0]=='*'&&did[1]==0)
||(!strcasecmp(did,(char *)k->contexts[*cn]->identities[*in]
->keypairs[*kp]->private_key))
) {
return 1; // match
}
}
return 0;
}
int keyring_unpack_tag(const unsigned char *packed, size_t packed_len, const char **name, const unsigned char **value, size_t *length)
{
size_t i;
for (i=0;i<packed_len;i++){
if (packed[i]==0){
*name = (const char*)packed;
if (value)
*value = &packed[i+1];
if (length)
*length = packed_len - (i+1);
return 0;
}
}
return WHY("Did not find NULL values in tag");
}
int keyring_pack_tag(unsigned char *packed, size_t *packed_len, const char *name, const unsigned char *value, size_t length)
{
size_t name_len=strlen(name)+1;
if (packed && *packed_len <name_len+length)
return -1;
*packed_len=name_len+length;
if (packed){
bcopy(name, packed, name_len);
bcopy(value, &packed[name_len], length);
}
return 0;
}
int keyring_set_public_tag(keyring_identity *id, const char *name, const unsigned char *value, size_t length)
{
int i;
for(i=0;i<id->keypair_count;i++){
const char *tag_name;
const unsigned char *tag_value;
size_t tag_length;
if (id->keypairs[i]->type==KEYTYPE_PUBLIC_TAG &&
keyring_unpack_tag(id->keypairs[i]->public_key, id->keypairs[i]->public_key_len,
&tag_name, &tag_value, &tag_length)==0 &&
strcmp(tag_name, name)==0) {
if (config.debug.keyring)
DEBUG("Found existing public tag");
break;
}
}
if (i >= PKR_MAX_KEYPAIRS)
return WHY("Too many key pairs");
/* allocate if needed */
if (i >= id->keypair_count) {
if (config.debug.keyring)
DEBUGF("Creating new public tag @%d", i);
if ((id->keypairs[i] = keyring_alloc_keypair(KEYTYPE_PUBLIC_TAG, 0)) == NULL)
return -1;
++id->keypair_count;
}
if (id->keypairs[i]->public_key)
free(id->keypairs[i]->public_key);
if (keyring_pack_tag(NULL, &id->keypairs[i]->public_key_len, name, value, length))
return -1;
id->keypairs[i]->public_key = emalloc(id->keypairs[i]->public_key_len);
if (!id->keypairs[i]->public_key)
return -1;
if (keyring_pack_tag(id->keypairs[i]->public_key, &id->keypairs[i]->public_key_len, name, value, length))
return -1;
if (config.debug.keyring)
dump("New tag", id->keypairs[i]->public_key, id->keypairs[i]->public_key_len);
return 0;
}
int keyring_find_public_tag(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char **value, size_t *length)
{
for(;keyring_next_keytype(k,cn,in,kp,KEYTYPE_PUBLIC_TAG);++(*kp)) {
keypair *keypair=k->contexts[*cn]->identities[*in]->keypairs[*kp];
const char *tag_name;
if (!keyring_unpack_tag(keypair->public_key, keypair->public_key_len, &tag_name, value, length) &&
strcmp(name, tag_name)==0){
return 1;
}
}
if (value)
*value=NULL;
return 0;
}
int keyring_find_public_tag_value(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char *value, size_t length)
{
const unsigned char *stored_value;
size_t stored_length;
for(;keyring_find_public_tag(k, cn, in, kp, name, &stored_value, &stored_length);++(*kp)) {
if (stored_length == length && memcmp(value, stored_value, length)==0)
return 1;
}
return 0;
}
int keyring_identity_find_keytype(const keyring_file *k, int cn, int in, int keytype)
{
int kp;
@ -1468,24 +1618,27 @@ int keyring_next_identity(const keyring_file *k, int *cn, int *in, int *kp)
int keyring_sanitise_position(const keyring_file *k,int *cn,int *in,int *kp)
{
if (!k) return 1;
if (!k)
return 1;
/* Sanity check passed in position */
if ((*cn)>=k->context_count) return 1;
if ((*in)>=k->contexts[*cn]->identity_count)
{
(*in)=0; (*cn)++;
if ((*cn)>=k->context_count) return 1;
while(1){
if ((*cn)>=k->context_count)
return 1;
if ((*in)>=k->contexts[*cn]->identity_count){
(*in)=(*kp)=0;
(*cn)++;
continue;
}
if ((*kp)>=k->contexts[*cn]->identities[*in]->keypair_count)
{
*kp=0; (*in)++;
if ((*in)>=k->contexts[*cn]->identity_count)
{
(*in)=0; (*cn)++;
if ((*cn)>=k->context_count) return 1;
}
if ((*kp)>=k->contexts[*cn]->identities[*in]->keypair_count){
*kp=0;
(*in)++;
continue;
}
return 0;
return 0;
}
}
unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsigned char **sas_public_out)
@ -1493,32 +1646,28 @@ unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsi
IN();
int cn=0,in=0,kp=0;
if (!keyring_find_sid(k,&cn,&in,&kp,sidp)) {
if (!keyring_find_sid(k,&cn,&in,&kp,sidp))
RETURNNULL(WHYNULL("Could not find SID in keyring, so can't find SAS"));
kp = keyring_identity_find_keytype(k, cn, in, KEYTYPE_CRYPTOSIGN);
if (kp==-1)
RETURNNULL(WHYNULL("Identity lacks SAS"));
unsigned char *sas_private=
k->contexts[cn]->identities[in]->keypairs[kp]->private_key;
unsigned char *sas_public=
k->contexts[cn]->identities[in]->keypairs[kp]->public_key;
if (!rhizome_verify_bundle_privatekey(sas_private,sas_public)){
/* SAS key is invalid (perhaps because it was a pre 0.90 format one),
so replace it */
WARN("SAS key is invalid -- regenerating.");
crypto_sign_edwards25519sha512batch_keypair(sas_public, sas_private);
keyring_commit(k);
}
for(kp=0;kp<k->contexts[cn]->identities[in]->keypair_count;kp++)
if (k->contexts[cn]->identities[in]->keypairs[kp]->type==KEYTYPE_CRYPTOSIGN)
{
unsigned char *sas_private=
k->contexts[cn]->identities[in]->keypairs[kp]->private_key;
unsigned char *sas_public=
k->contexts[cn]->identities[in]->keypairs[kp]->public_key;
if (rhizome_verify_bundle_privatekey(NULL,sas_private,sas_public))
{
/* SAS key is invalid (perhaps because it was a pre 0.90 format one),
so replace it */
WARN("SAS key is invalid -- regenerating.");
crypto_sign_edwards25519sha512batch_keypair(sas_public, sas_private);
keyring_commit(k);
}
if (config.debug.keyring)
DEBUGF("Found SAS entry for %s*", alloca_tohex(sidp->binary, 7));
if (sas_public_out) *sas_public_out=sas_public;
RETURN(sas_private);
}
RETURNNULL(WHYNULL("Identity lacks SAS"));
if (config.debug.keyring)
DEBUGF("Found SAS entry for %s*", alloca_tohex(sidp->binary, 7));
if (sas_public_out) *sas_public_out=sas_public;
RETURN(sas_private);
OUT();
}
@ -1616,7 +1765,7 @@ static int keyring_respond_sas(keyring_file *k, overlay_mdp_frame *req)
alloca_tohex_sid_t(req->out.src.sid), req->out.src.port,
alloca_tohex_sid_t(req->out.dst.sid), req->out.dst.port
);
return overlay_mdp_dispatch(req,0,NULL,0);
return overlay_mdp_dispatch(req, NULL);
}
// someone else is claiming to be me on this network
@ -1642,7 +1791,7 @@ int keyring_send_unlock(struct subscriber *subscriber)
if (crypto_sign_message(subscriber, mdp.out.payload, sizeof(mdp.out.payload), &len))
return -1;
mdp.out.payload_length=len;
return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0);
return overlay_mdp_dispatch(&mdp, NULL);
}
static int keyring_send_challenge(struct subscriber *source, struct subscriber *dest)
@ -1667,7 +1816,7 @@ static int keyring_send_challenge(struct subscriber *source, struct subscriber *
bcopy(source->identity->challenge, &mdp.out.payload[1], sizeof(source->identity->challenge));
mdp.out.payload_length+=sizeof(source->identity->challenge);
return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0);
return overlay_mdp_dispatch(&mdp, NULL);
}
static int keyring_respond_challenge(struct subscriber *subscriber, overlay_mdp_frame *req)
@ -1691,7 +1840,7 @@ static int keyring_respond_challenge(struct subscriber *subscriber, overlay_mdp_
if (crypto_sign_message(subscriber, mdp.out.payload, sizeof(mdp.out.payload), &len))
return -1;
mdp.out.payload_length=len;
return overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0);
return overlay_mdp_dispatch(&mdp, NULL);
}
static int keyring_process_challenge(keyring_file *k, struct subscriber *subscriber, overlay_mdp_frame *req)
@ -1777,7 +1926,7 @@ int keyring_send_sas_request(struct subscriber *subscriber){
mdp.out.payload_length=1;
mdp.out.payload[0]=KEYTYPE_CRYPTOSIGN;
if (overlay_mdp_dispatch(&mdp, 0 /* system generated */, NULL, 0))
if (overlay_mdp_dispatch(&mdp, NULL))
return WHY("Failed to send SAS resolution request");
if (config.debug.keyring)
DEBUGF("Dispatched SAS resolution request");
@ -1788,10 +1937,10 @@ int keyring_send_sas_request(struct subscriber *subscriber){
int keyring_find_sid(const keyring_file *k, int *cn, int *in, int *kp, const sid_t *sidp)
{
for (; keyring_sanitise_position(k, cn, in, kp) == 0; ++*kp)
if (k->contexts[*cn]->identities[*in]->keypairs[*kp]->type == KEYTYPE_CRYPTOBOX
&& memcmp(sidp->binary, k->contexts[*cn]->identities[*in]->keypairs[*kp]->public_key, SID_SIZE) == 0)
for(; keyring_next_keytype(k,cn,in,kp,KEYTYPE_CRYPTOBOX); ++(*kp)) {
if (memcmp(sidp->binary, k->contexts[*cn]->identities[*in]->keypairs[*kp]->public_key, SID_SIZE) == 0)
return 1;
}
return 0;
}

109
keyring.h Normal file
View File

@ -0,0 +1,109 @@
#ifndef __SERVALD_KEYRING_H
#define __SERVALD_KEYRING_H
typedef struct keypair {
int type;
unsigned char *private_key;
size_t private_key_len;
unsigned char *public_key;
size_t public_key_len;
} keypair;
/* Contains just the list of private:public key pairs and types,
the pin used to extract them, and the slot in the keyring file
(so that it can be replaced/rewritten as required). */
#define PKR_MAX_KEYPAIRS 64
#define PKR_SALT_BYTES 32
#define PKR_MAC_BYTES 64
typedef struct keyring_identity {
char *PKRPin;
struct subscriber *subscriber;
time_ms_t challenge_expires;
unsigned char challenge[24];
unsigned int slot;
unsigned int keypair_count;
keypair *keypairs[PKR_MAX_KEYPAIRS];
} keyring_identity;
/* 64K identities, can easily be increased should the need arise,
but keep it low-ish for now so that the 64K pointers don't eat too
much ram on a small device. Should probably think about having
small and large device settings for some of these things */
#define KEYRING_MAX_IDENTITIES 65536
typedef struct keyring_context {
char *KeyRingPin;
unsigned char *KeyRingSalt;
int KeyRingSaltLen;
unsigned int identity_count;
keyring_identity *identities[KEYRING_MAX_IDENTITIES];
} keyring_context;
#define KEYRING_PAGE_SIZE 4096LL
#define KEYRING_BAM_BYTES 2048LL
#define KEYRING_BAM_BITS (KEYRING_BAM_BYTES<<3)
#define KEYRING_SLAB_SIZE (KEYRING_PAGE_SIZE*KEYRING_BAM_BITS)
typedef struct keyring_bam {
off_t file_offset;
unsigned char bitmap[KEYRING_BAM_BYTES];
struct keyring_bam *next;
} keyring_bam;
#define KEYRING_MAX_CONTEXTS 256
typedef struct keyring_file {
int context_count;
keyring_bam *bam;
keyring_context *contexts[KEYRING_MAX_CONTEXTS];
FILE *file;
off_t file_size;
} keyring_file;
void keyring_free(keyring_file *k);
void keyring_release_identity(keyring_file *k, int cn, int id);
#define KEYTYPE_CRYPTOBOX 0x01 // must be lowest
#define KEYTYPE_CRYPTOSIGN 0x02
#define KEYTYPE_RHIZOME 0x03
/* DIDs aren't really keys, but the keyring is a real handy place to keep them,
and keep them private if people so desire */
#define KEYTYPE_DID 0x04
/* Arbitrary name / value pairs */
#define KEYTYPE_PUBLIC_TAG 0x05
/* handle to keyring file for use in running instance */
extern keyring_file *keyring;
/* Public calls to keyring management */
keyring_file *keyring_open(const char *path, int writeable);
keyring_file *keyring_open_instance();
keyring_file *keyring_open_instance_cli(const struct cli_parsed *parsed);
int keyring_enter_pin(keyring_file *k, const char *pin);
int keyring_set_did(keyring_identity *id, const char *did, const char *name);
int keyring_sanitise_position(const keyring_file *k,int *cn,int *in,int *kp);
int keyring_next_keytype(const keyring_file *k, int *cn, int *in, int *kp, int keytype);
int keyring_next_identity(const keyring_file *k,int *cn,int *in,int *kp);
int keyring_identity_find_keytype(const keyring_file *k, int cn, int in, int keytype);
int keyring_find_did(const keyring_file *k,int *cn,int *in,int *kp, const char *did);
int keyring_find_sid(const keyring_file *k,int *cn,int *in,int *kp, const sid_t *sidp);
unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsigned char **sas_public);
int keyring_send_sas_request(struct subscriber *subscriber);
int keyring_commit(keyring_file *k);
keyring_identity *keyring_create_identity(keyring_file *k,keyring_context *c, const char *pin);
int keyring_seed(keyring_file *k);
void keyring_identity_extract(const keyring_identity *id, const sid_t **sidp, const char **didp, const char **namep);
int keyring_load(keyring_file *k, const char *keyring_pin, unsigned entry_pinc, const char **entry_pinv, FILE *input);
int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret);
unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp);
int keyring_mapping_request(keyring_file *k, struct overlay_frame *frame, overlay_mdp_frame *req);
int keyring_send_unlock(struct subscriber *subscriber);
void keyring_release_subscriber(keyring_file *k, const sid_t *sid);
int keyring_set_public_tag(keyring_identity *id, const char *name, const unsigned char *value, size_t length);
int keyring_find_public_tag(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char **value, size_t *length);
int keyring_find_public_tag_value(const keyring_file *k, int *cn, int *in, int *kp, const char *name, const unsigned char *value, size_t length);
int keyring_unpack_tag(const unsigned char *packed, size_t packed_len, const char **name, const unsigned char **value, size_t *length);
int keyring_pack_tag(unsigned char *packed, size_t *packed_len, const char *name, const unsigned char *value, size_t length);
#endif // __SERVALD_KEYRING_H

1
lsif.c
View File

@ -46,7 +46,6 @@
#ifndef SIOCGIFCONF
#include <sys/sockio.h>
#endif
#include <netinet/in.h>
#include <netinet/if_ether.h>
#if __MACH__
#include <net/if_dl.h>

407
mavlink.c
View File

@ -1,407 +0,0 @@
// -*- Mode: C; c-basic-offset: 2; -*-
//
// Copyright (c) 2012 Andrew Tridgell, All Rights Reserved
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// o Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// o Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in
// the documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
// OF THE POSSIBILITY OF SUCH DAMAGE.
//
/*
Portions Copyright (C) 2013 Paul Gardner-Stephen
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "serval.h"
#include "conf.h"
#include "overlay_buffer.h"
#include "golay.h"
#define MAVLINK_MSG_ID_RADIO 166
#define MAVLINK_MSG_ID_DATASTREAM 67
int MAVLINK_MESSAGE_CRCS[]={72, 39, 190, 92, 191, 217, 104, 119, 0, 219, 60, 186, 10, 0, 0, 0, 0, 0, 0, 0, 89, 159, 162, 121, 0, 149, 222, 110, 179, 136, 66, 126, 185, 147, 112, 252, 162, 215, 229, 128, 9, 106, 101, 213, 4, 229, 21, 214, 215, 14, 206, 50, 157, 126, 108, 213, 95, 5, 127, 0, 0, 0, 57, 126, 130, 119, 193, 191, 236, 158, 143, 0, 0, 104, 123, 131, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 143, 29, 208, 188, 118, 242, 19, 97, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 224, 60, 106, 7};
// use '3D' for 3DRadio
#define RADIO_SOURCE_SYSTEM '3'
#define RADIO_SOURCE_COMPONENT 'D'
uint16_t mavlink_crc(unsigned char *buf,int length)
{
uint16_t sum = 0xFFFF;
uint8_t i, stoplen;
stoplen = length + 6;
// MAVLink 1.0 has an extra CRC seed
buf[length+6] = MAVLINK_MESSAGE_CRCS[buf[5]];
stoplen++;
i = 1;
while (i<stoplen) {
uint8_t tmp;
tmp = buf[i] ^ (uint8_t)(sum&0xff);
tmp ^= (tmp<<4);
sum = (sum>>8) ^ (tmp<<8) ^ (tmp<<3) ^ (tmp>>4);
i++;
}
buf[length+6]=sum&0xff;
buf[length+7]=sum>>8;
return sum;
}
/*
we use a hand-crafted MAVLink packet based on the following
message definition
<message name="RADIO" id="166">
<description>Status generated by radio</description>
<field type="uint8_t" name="rssi">local signal strength</field>
<field type="uint8_t" name="remrssi">remote signal strength</field>
<field type="uint8_t" name="txbuf">percentage free space in transmit buffer</field>
<field type="uint8_t" name="noise">background noise level</field>
<field type="uint8_t" name="remnoise">remote background noise level</field>
<field type="uint16_t" name="rxerrors">receive errors</field>
<field type="uint16_t" name="fixed">count of error corrected packets</field>
</message>
*/
struct mavlink_RADIO_v09 {
uint8_t rssi;
uint8_t remrssi;
uint8_t txbuf;
uint8_t noise;
uint8_t remnoise;
uint16_t rxerrors;
uint16_t fixed;
};
struct mavlink_RADIO_v10 {
uint16_t rxerrors;
uint16_t fixed;
uint8_t rssi;
uint8_t remrssi;
uint8_t txbuf;
uint8_t noise;
uint8_t remnoise;
};
/*
Each mavlink frame consists of 0xfe followed by a standard 6 byte header.
Normally the payload plus a 2-byte CRC follows.
We are replacing the CRC check with a Reed-Solomon code to correct as well
as detect upto 16 bytes with errors, in return for a 32-byte overhead.
The nature of the particular library we are using is that the overhead is
basically fixed, but we can shorten the data section.
Note that the mavlink headers are not protected against errors. This is a
limitation of the radio firmware at present. One day we will re-write the
radio firmware so that we can send and receive raw radio frames, and get
rid of the mavlink framing altogether, and just send R-S protected payloads.
Not ideal, but will be fine for now.
*/
#include "fec-3.0.1/fixed.h"
void encode_rs_8(data_t *data, data_t *parity,int pad);
int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad);
int mavlink_encode_packet(struct overlay_interface *interface)
{
int count = ob_remaining(interface->tx_packet);
int startP = !ob_position(interface->tx_packet);
int endP = 1;
if (count+6+32 > 255){
count = 255-6-32;
endP = 0;
}
interface->txbuffer[0]=0xfe; // mavlink v1.0 frame
/* payload len, excluding 6 byte header and 2 byte CRC.
But we use a 4-byte CRC, so need to add two to count to make packet lengths
be as expected.
Note that this construction will result in CRC errors by non-servald
programmes, which is probably more helpful than otherwise.
*/
// we need 32 bytes for the parity, but this field assumes
// that there is a 2 byte CRC, so we can save two bytes
int len = count+32 - 2;
interface->txbuffer[1]=len;
interface->txbuffer[2]=(len & 0xF);
interface->txbuffer[3]=0;
golay_encode(&interface->txbuffer[1]);
interface->txbuffer[4]=(interface->mavlink_seq++) & 0x3f;
if (startP) interface->txbuffer[4]|=0x40;
if (endP) interface->txbuffer[4]|=0x80;
interface->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM;
ob_get_bytes(interface->tx_packet, &interface->txbuffer[6], count);
encode_rs_8(&interface->txbuffer[4], &interface->txbuffer[6+count], 223 - (count+2));
interface->tx_bytes_pending=len + 8;
if (endP){
ob_free(interface->tx_packet);
interface->tx_packet=NULL;
overlay_queue_schedule_next(gettime_ms());
}
return 0;
}
int mavlink_heartbeat(unsigned char *frame,int *outlen)
{
int count=9;
bzero(frame, count+8);
frame[0]=0xfe; // mavlink v1.0 frame
// Must be 9 to indicate heartbeat
frame[1]=count; // payload len, excluding 6 byte header and 2 byte CRC
frame[2]=(count & 0xF); // packet sequence
frame[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC)
golay_encode(&frame[1]);
frame[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE)
// Must be zero to indicate heartbeat
frame[5]=0; // message ID type of this frame: DATA_STREAM
// extra magic number to detect remote heartbeat requests
frame[14]=0x55;
frame[15]=0x05;
golay_encode(&frame[14]);
*outlen=count+8;
return 0;
}
static int parse_heartbeat(struct overlay_interface *interface, const unsigned char *payload)
{
if (payload[0]==0xFE
&& payload[1]==9
&& payload[3]==RADIO_SOURCE_SYSTEM
&& payload[4]==RADIO_SOURCE_COMPONENT
&& payload[5]==MAVLINK_MSG_ID_RADIO){
// we can assume that radio status packets arrive without corruption
interface->radio_rssi=(1.0*payload[10]-payload[13])/1.9;
interface->remote_rssi=(1.0*payload[11] - payload[14])/1.9;
int free_space = payload[12];
int free_bytes = (free_space * 1280) / 100 - 30;
interface->remaining_space = free_bytes;
if (free_bytes>0)
interface->next_tx_allowed = gettime_ms();
if (free_bytes>720)
interface->next_heartbeat=gettime_ms()+1000;
if (config.debug.packetradio) {
INFOF("Link budget = %+ddB, remote link budget = %+ddB, buffer space = %d%% (approx %d)",
interface->radio_rssi,
interface->remote_rssi,
free_space, free_bytes);
}
return 1;
}
return 0;
}
static int mavlink_parse(struct overlay_interface *interface, struct slip_decode_state *state,
int packet_length, unsigned char *payload, int *backtrack)
{
*backtrack=0;
if (packet_length==9){
// make sure we've heard the start and end of a remote heartbeat request
int errs=0;
int tail = golay_decode(&errs, &payload[14]);
if (tail == 0x555){
return 1;
}
return 0;
}
int data_bytes = packet_length - (32 - 2);
// preserve the last 16 bytes of data
unsigned char old_footer[32];
unsigned char *payload_footer=&payload[packet_length+8-sizeof(old_footer)];
bcopy(payload_footer, old_footer, sizeof(old_footer));
int pad=223 - (data_bytes + 2);
int errors=decode_rs_8(&payload[4], NULL, 0, pad);
if (errors==-1){
if (config.debug.mavlink)
DEBUGF("Reed-Solomon error correction failed");
return 0;
}
*backtrack=errors;
int seq=payload[4]&0x3f;
if (config.debug.mavlink){
DEBUGF("Received RS protected message, len: %d, errors: %d, seq: %d, flags:%s%s",
data_bytes,
errors,
seq,
payload[4]&0x40?" start":"",
payload[4]&0x80?" end":"");
}
if (seq != ((state->mavlink_seq+1)&0x3f)){
// reject partial packet if we missed a sequence number
if (config.debug.mavlink)
DEBUGF("Rejecting packet, sequence jumped from %d to %d", state->mavlink_seq, seq);
state->packet_length=sizeof(state->dst)+1;
}
if (payload[4]&0x40){
// start a new packet
state->packet_length=0;
}
state->mavlink_seq=payload[4]&0x3f;
if (state->packet_length + data_bytes > sizeof(state->dst)){
if (config.debug.mavlink)
DEBUG("Fragmented packet is too long or a previous piece was missed - discarding");
state->packet_length=sizeof(state->dst)+1;
return 1;
}
bcopy(&payload[6], &state->dst[state->packet_length], data_bytes);
state->packet_length+=data_bytes;
if (payload[4]&0x80) {
if (config.debug.mavlink)
DEBUGF("PDU Complete (length=%d)",state->packet_length);
state->dst_offset=0;
packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL, 0);
state->packet_length=sizeof(state->dst)+1;
}
return 1;
}
static int decode_length(struct slip_decode_state *state, unsigned char *p)
{
// look for a valid golay encoded length
int errs=0;
int length = golay_decode(&errs, p);
if (length<0 || ((length >>8) & 0xF) != (length&0xF))
return -1;
length=length&0xFF;
if (length!=9 && (length<31 || length+8>255))
return -1;
if (config.debug.mavlink && (errs || state->mavlink_payload_length!=*p))
DEBUGF("Decoded length %d to %d with %d errs", *p, length, errs);
state->mavlink_payload_length=length;
return 0;
}
int mavlink_decode(struct overlay_interface *interface, struct slip_decode_state *state, uint8_t c)
{
if (state->mavlink_payload_start + state->mavlink_payload_offset >= sizeof(state->mavlink_payload)){
// drop one byte if we run out of space
if (config.debug.mavlink)
DEBUGF("Dropped %02x, buffer full", state->mavlink_payload[0]);
bcopy(state->mavlink_payload+1, state->mavlink_payload, sizeof(state->mavlink_payload) -1);
state->mavlink_payload_start--;
}
unsigned char *p = &state->mavlink_payload[state->mavlink_payload_start];
p[state->mavlink_payload_offset++]=c;
while(1){
// look for packet length headers
p = &state->mavlink_payload[state->mavlink_payload_start];
while(state->mavlink_payload_length==0 && state->mavlink_payload_offset>=6){
if (p[0]==0xFE
&& p[1]==9
&& p[3]==RADIO_SOURCE_SYSTEM
&& p[4]==RADIO_SOURCE_COMPONENT
&& p[5]==MAVLINK_MSG_ID_RADIO){
//looks like a valid heartbeat response header, read the rest and process it
state->mavlink_payload_length=9;
break;
}
if (decode_length(state, &p[1])==0)
break;
state->mavlink_payload_start++;
state->mavlink_payload_offset--;
p++;
}
// wait for a whole packet
if (!state->mavlink_payload_length || state->mavlink_payload_offset < state->mavlink_payload_length+8)
return 0;
if (parse_heartbeat(interface, p)){
// cut the bytes of the heartbeat out of the buffer
state->mavlink_payload_offset -= state->mavlink_payload_length+8;
if (state->mavlink_payload_offset){
// shuffle bytes backwards
bcopy(&p[state->mavlink_payload_length+8], p, state->mavlink_payload_offset);
}
// restart parsing for a valid header from the beginning of out buffer
state->mavlink_payload_offset+=state->mavlink_payload_start;
state->mavlink_payload_start=0;
state->mavlink_payload_length=0;
continue;
}
// is this a well formed packet?
int backtrack=0;
if (mavlink_parse(interface, state, state->mavlink_payload_length, p, &backtrack)==1){
// Since we know we've synced with the remote party,
// and there's nothing we can do about any earlier data
// throw away everything before the end of this packet
if (state->mavlink_payload_start && config.debug.mavlink)
dump("Skipped", state->mavlink_payload, state->mavlink_payload_start);
// If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet,
// but we may need to examine the last few bytes to find the start of the next packet.
state->mavlink_payload_offset -= state->mavlink_payload_length+8-backtrack;
if (state->mavlink_payload_offset){
// shuffle all remaining bytes back to the start of the buffer
bcopy(&state->mavlink_payload[state->mavlink_payload_start + state->mavlink_payload_length+8-backtrack],
state->mavlink_payload, state->mavlink_payload_offset);
}
state->mavlink_payload_start=0;
}else{
// ignore the first byte for now and start looking for another packet header
// we may find a heartbeat in the middle that we need to cut out first
state->mavlink_payload_start++;
state->mavlink_payload_offset--;
}
state->mavlink_payload_length=0;
};
}

163
mdp_client.c
View File

@ -27,6 +27,7 @@
#include "overlay_address.h"
#include "overlay_packet.h"
#include "mdp_client.h"
#include "socket.h"
int mdp_socket(void)
{
@ -34,49 +35,70 @@ int mdp_socket(void)
return overlay_mdp_client_socket();
}
int mdp_close(int socket)
static void mdp_unlink(int mdp_sock)
{
// use the same process for closing sockets, though this will need to change once bind is implemented
return overlay_mdp_client_close(socket);
// get the socket name and unlink it from the filesystem if not abstract
struct socket_address addr;
addr.addrlen = sizeof addr.store;
if (getsockname(mdp_sock, &addr.addr, &addr.addrlen))
WHYF_perror("getsockname(%d)", mdp_sock);
else if (addr.addr.sa_family==AF_UNIX
&& addr.addrlen > sizeof addr.local.sun_family
&& addr.addrlen <= sizeof addr.local && addr.local.sun_path[0] != '\0') {
if (unlink(addr.local.sun_path) == -1)
WARNF_perror("unlink(%s)", alloca_str_toprint(addr.local.sun_path));
}
close(mdp_sock);
}
int mdp_send(int socket, const struct mdp_header *header, const unsigned char *payload, ssize_t len)
int mdp_close(int socket)
{
struct sockaddr_un addr;
socklen_t addrlen;
if (make_local_sockaddr(&addr, &addrlen, "mdp.2.socket") == -1)
// tell the daemon to drop all bindings
struct mdp_header header={
.flags = MDP_FLAG_CLOSE,
.local.port = 0,
};
mdp_send(socket, &header, NULL, 0);
// remove socket
mdp_unlink(socket);
return 0;
}
int mdp_send(int socket, const struct mdp_header *header, const uint8_t *payload, size_t len)
{
struct socket_address addr;
if (make_local_sockaddr(&addr, "mdp.2.socket") == -1)
return -1;
struct iovec iov[]={
{
.iov_base = (void *)header,
.iov_len = sizeof(struct mdp_header)
},
{
.iov_base = (void *)payload,
.iov_len = len
struct fragmented_data data={
.fragment_count=2,
.iov={
{
.iov_base = (void*)header,
.iov_len = sizeof(struct mdp_header)
},
{
.iov_base = (void*)payload,
.iov_len = len
}
}
};
struct msghdr hdr={
.msg_name=&addr,
.msg_namelen=addrlen,
.msg_iov=iov,
.msg_iovlen=2,
};
return sendmsg(socket, &hdr, 0);
return send_message(socket, &addr, &data);
}
ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, ssize_t max_len)
ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_t max_len)
{
/* Construct name of socket to receive from. */
struct sockaddr_un mdp_addr;
socklen_t mdp_addrlen;
if (make_local_sockaddr(&mdp_addr, &mdp_addrlen, "mdp.2.socket") == -1)
return -1;
errno=0;
struct socket_address mdp_addr;
if (make_local_sockaddr(&mdp_addr, "mdp.2.socket") == -1)
return WHY("Failed to build socket address");
struct sockaddr_un addr;
struct socket_address addr;
struct iovec iov[]={
{
.iov_base = (void *)header,
@ -89,25 +111,26 @@ ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload,
};
struct msghdr hdr={
.msg_name=&addr,
.msg_namelen=sizeof(struct sockaddr_un),
.msg_name=&addr.addr,
.msg_namelen=sizeof(addr.store),
.msg_iov=iov,
.msg_iovlen=2,
};
ssize_t len = recvmsg(socket, &hdr, 0);
if (len<sizeof(struct mdp_header))
return -1;
return WHYF("Received message is too short (%d)", (int)len);
addr.addrlen=hdr.msg_namelen;
// double check that the incoming address matches the servald daemon
if (cmp_sockaddr((struct sockaddr *)&addr, hdr.msg_namelen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
&& ( addr.sun_family != AF_UNIX
|| real_sockaddr(&addr, hdr.msg_namelen, &addr, &hdr.msg_namelen) <= 0
|| cmp_sockaddr((struct sockaddr *)&addr, hdr.msg_namelen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
if (cmp_sockaddr(&addr, &mdp_addr) != 0
&& ( addr.local.sun_family != AF_UNIX
|| real_sockaddr(&addr, &addr) <= 0
|| cmp_sockaddr(&addr, &mdp_addr) != 0
)
)
return -1;
return WHYF("Received message came from %s instead of %s?",
alloca_socket_address(&addr),
alloca_socket_address(&mdp_addr));
return len - sizeof(struct mdp_header);
}
@ -122,22 +145,25 @@ int overlay_mdp_send(int mdp_sockfd, overlay_mdp_frame *mdp, int flags, int time
return -1;
// Minimise frame length to save work and prevent accidental disclosure of memory contents.
ssize_t len = overlay_mdp_relevant_bytes(mdp);
if (len < 0)
if (len == -1)
return WHY("MDP frame invalid (could not compute length)");
/* Construct name of socket to send to. */
struct sockaddr_un addr;
socklen_t addrlen;
if (make_local_sockaddr(&addr, &addrlen, "mdp.socket") == -1)
struct socket_address addr;
if (make_local_sockaddr(&addr, "mdp.socket") == -1)
return -1;
// Send to that socket
set_nonblock(mdp_sockfd);
int result = sendto(mdp_sockfd, mdp, len, 0, (struct sockaddr *)&addr, addrlen);
ssize_t result = sendto(mdp_sockfd, mdp, (size_t)len, 0, &addr.addr, addr.addrlen);
set_block(mdp_sockfd);
if (result == -1) {
if ((size_t)result != (size_t)len) {
if (result == -1)
WHYF_perror("sendto(fd=%d,len=%zu,addr=%s)", mdp_sockfd, (size_t)len, alloca_socket_address(&addr));
else
WHYF("sendto() sent %zu bytes of MDP reply (%zu) to %s", (size_t)result, (size_t)len, alloca_socket_address(&addr));
mdp->packetTypeAndFlags=MDP_ERROR;
mdp->error.error=1;
snprintf(mdp->error.message,128,"Error sending frame to MDP server.");
return WHY_perror("sendto(f)");
return -1;
} else {
if (!(flags&MDP_AWAITREPLY)) {
return 0;
@ -177,16 +203,15 @@ int overlay_mdp_client_socket(void)
{
/* Create local per-client socket to MDP server (connection is always local) */
int mdp_sockfd;
struct sockaddr_un addr;
socklen_t addrlen;
struct socket_address addr;
uint32_t random_value;
if (urandombytes((unsigned char *)&random_value, sizeof random_value) == -1)
return WHY("urandombytes() failed");
if (make_local_sockaddr(&addr, &addrlen, "mdp.client.%u.%08lx.socket", getpid(), (unsigned long)random_value) == -1)
if (make_local_sockaddr(&addr, "mdp.client.%u.%08lx.socket", getpid(), (unsigned long)random_value) == -1)
return -1;
if ((mdp_sockfd = esocket(AF_UNIX, SOCK_DGRAM, 0)) == -1)
return -1;
if (socket_bind(mdp_sockfd, (struct sockaddr *)&addr, addrlen) == -1) {
if (socket_bind(mdp_sockfd, &addr.addr, addr.addrlen) == -1) {
close(mdp_sockfd);
return -1;
}
@ -200,16 +225,8 @@ int overlay_mdp_client_close(int mdp_sockfd)
overlay_mdp_frame mdp;
mdp.packetTypeAndFlags = MDP_GOODBYE;
overlay_mdp_send(mdp_sockfd, &mdp, 0, 0);
// get the socket name and unlink it from the filesystem if not abstract
struct sockaddr_un addr;
socklen_t addrlen = sizeof addr;
if (getsockname(mdp_sockfd, (struct sockaddr *)&addr, &addrlen))
WHYF_perror("getsockname(%d)", mdp_sockfd);
else if (addrlen > sizeof addr.sun_family && addrlen <= sizeof addr && addr.sun_path[0] != '\0') {
if (unlink(addr.sun_path) == -1)
WARNF_perror("unlink(%s)", alloca_str_toprint(addr.sun_path));
}
close(mdp_sockfd);
mdp_unlink(mdp_sockfd);
return 0;
}
@ -232,36 +249,40 @@ int overlay_mdp_client_poll(int mdp_sockfd, time_ms_t timeout_ms)
int overlay_mdp_recv(int mdp_sockfd, overlay_mdp_frame *mdp, mdp_port_t port, int *ttl)
{
/* Construct name of socket to receive from. */
struct sockaddr_un mdp_addr;
socklen_t mdp_addrlen;
if (make_local_sockaddr(&mdp_addr, &mdp_addrlen, "mdp.socket") == -1)
struct socket_address mdp_addr;
if (make_local_sockaddr(&mdp_addr, "mdp.socket") == -1)
return -1;
/* Check if reply available */
struct sockaddr_un recvaddr;
socklen_t recvaddrlen = sizeof recvaddr;
struct socket_address recvaddr;
recvaddr.addrlen = sizeof recvaddr.store;
ssize_t len;
mdp->packetTypeAndFlags = 0;
set_nonblock(mdp_sockfd);
len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, (struct sockaddr *)&recvaddr, &recvaddrlen);
len = recvwithttl(mdp_sockfd, (unsigned char *)mdp, sizeof(overlay_mdp_frame), ttl, &recvaddr);
if (len == -1)
WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))",
mdp_sockfd, mdp, sizeof(overlay_mdp_frame), *ttl,
&recvaddr, alloca_socket_address(&recvaddr)
);
set_block(mdp_sockfd);
if (len <= 0)
return -1; // no packet received
// If the received address overflowed the buffer, then it cannot have come from the server, whose
// address must always fit within a struct sockaddr_un.
if (recvaddrlen > sizeof recvaddr)
if (recvaddr.addrlen > sizeof recvaddr.store)
return WHY("reply did not come from server: address overrun");
// Compare the address of the sender with the address of our server, to ensure they are the same.
// If the comparison fails, then try using realpath(3) on the sender address and compare again.
if ( cmp_sockaddr((struct sockaddr *)&recvaddr, recvaddrlen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
&& ( recvaddr.sun_family != AF_UNIX
|| real_sockaddr(&recvaddr, recvaddrlen, &recvaddr, &recvaddrlen) <= 0
|| cmp_sockaddr((struct sockaddr *)&recvaddr, recvaddrlen, (struct sockaddr *)&mdp_addr, mdp_addrlen) != 0
if ( cmp_sockaddr(&recvaddr, &mdp_addr) != 0
&& ( recvaddr.local.sun_family != AF_UNIX
|| real_sockaddr(&recvaddr, &recvaddr) <= 0
|| cmp_sockaddr(&recvaddr, &mdp_addr) != 0
)
)
return WHYF("reply did not come from server: %s", alloca_sockaddr(&recvaddr, recvaddrlen));
return WHYF("reply did not come from server: %s", alloca_socket_address(&recvaddr));
// silently drop incoming packets for the wrong port number
if (port>0 && port != mdp->in.dst.port){

36
mdp_client.h
View File

@ -31,8 +31,9 @@ struct mdp_sockaddr {
#define MDP_FLAG_NO_CRYPT (1<<0)
#define MDP_FLAG_NO_SIGN (1<<1)
#define MDP_FLAG_BIND_ALL (1<<2)
#define MDP_FLAG_OK (1<<3)
#define MDP_FLAG_BIND (1<<2)
#define MDP_FLAG_CLOSE (1<<3)
#define MDP_FLAG_ERROR (1<<4)
struct mdp_header {
@ -43,19 +44,36 @@ struct mdp_header {
uint8_t ttl;
};
#define BIND_PRIMARY SID_ANY
#define BIND_ALL SID_BROADCAST
#define TYPE_SID 1
#define TYPE_PIN 2
#define ACTION_LOCK 1
#define ACTION_UNLOCK 2
/* Port numbers for commands sent to the local daemon*/
#define MDP_LISTEN 0
/* lock and unlock identities from the local keyring
* Requests start with an mdp_identity_request structure followed by a list of pins or SIDs
*/
#define MDP_IDENTITY 1
/* Search unlocked identities from the running daemon
* If the request is empty, all identities will be returned
* if the request contains a packed tag / value, identities with a matching tag will be returned
* if the value passed in is zero length, all identities with that tag and any value will be returned
*/
#define MDP_SEARCH_IDS 2
// an identity request is sent to port MDP_IDENTITY, sid ANY
struct mdp_identity_request{
uint8_t action;
uint8_t type;
// followed by a list of SID's or NULL terminated entry pins for the remainder of the payload
// the request is followed by a list of SID's or NULL terminated entry pins for the remainder of the payload
};
#define MDP_IDENTITY 1
#pragma pack(pop)
struct overlay_route_record{
@ -69,13 +87,15 @@ struct overlay_mdp_scan{
struct in_addr addr;
};
/* V2 interface */
/* low level V2 mdp interface */
int mdp_socket(void);
int mdp_close(int socket);
int mdp_send(int socket, const struct mdp_header *header, const unsigned char *payload, ssize_t len);
ssize_t mdp_recv(int socket, struct mdp_header *header, unsigned char *payload, ssize_t max_len);
int mdp_send(int socket, const struct mdp_header *header, const uint8_t *payload, size_t len);
ssize_t mdp_recv(int socket, struct mdp_header *header, uint8_t *payload, ssize_t max_len);
int mdp_poll(int socket, time_ms_t timeout_ms);
/* Client-side MDP function */
int overlay_mdp_client_socket(void);
int overlay_mdp_client_close(int mdp_sockfd);

6
mem.c
View File

@ -51,10 +51,8 @@ void *_emalloc_zero(struct __sourceloc __whence, size_t bytes)
char *_strn_edup(struct __sourceloc __whence, const char *str, size_t len)
{
char *new = _emalloc(__whence, len + 1);
if (new) {
strncpy(new, str, len);
new[len] = '\0';
}
if (new)
strncpy(new, str, len)[len] = '\0';
return new;
}

227
meshms.c
View File

@ -1,9 +1,12 @@
#include <assert.h>
#include "serval.h"
#include "rhizome.h"
#include "log.h"
#include "conf.h"
#include "crypto.h"
#include "strlcpy.h"
#include "keyring.h"
#include "dataformats.h"
#define MESHMS_BLOCK_TYPE_ACK 0x01
#define MESHMS_BLOCK_TYPE_MESSAGE 0x02
@ -82,15 +85,24 @@ static int get_my_conversation_bundle(const sid_t *my_sidp, rhizome_manifest *m)
return -1;
// always consider the content encrypted, we don't need to rely on the manifest itself.
m->payloadEncryption = 1;
rhizome_manifest_set_crypt(m, PAYLOAD_ENCRYPTED);
assert(m->haveSecret);
if (m->haveSecret == NEW_BUNDLE_ID) {
rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
if (rhizome_fill_manifest(m, NULL, NULL, NULL) == -1)
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
if (rhizome_fill_manifest(m, NULL, my_sidp) == -1)
return WHY("Invalid manifest");
if (config.debug.meshms) {
char secret[RHIZOME_BUNDLE_KEY_STRLEN + 1];
rhizome_bytes_to_hex_upper(m->cryptoSignSecret, secret, RHIZOME_BUNDLE_KEY_BYTES);
// The 'meshms' automated test depends on this message; do not alter.
DEBUGF("MESHMS CONVERSATION BUNDLE bid=%s secret=%s",
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
secret
);
}
} else {
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (strcmp(service, RHIZOME_SERVICE_FILE) != 0)
return WHYF("Invalid manifest, service=%s but should be %s", service, RHIZOME_SERVICE_MESHMS2);
if (strcmp(m->service, RHIZOME_SERVICE_FILE) != 0)
return WHYF("Invalid manifest, service=%s but should be %s", m->service, RHIZOME_SERVICE_MESHMS2);
}
return 0;
}
@ -185,33 +197,40 @@ static int get_database_conversations(const sid_t *my_sid, const sid_t *their_si
return 0;
}
static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid){
static struct conversations * find_or_create_conv(const sid_t *my_sid, const sid_t *their_sid)
{
struct conversations *conv=NULL;
if (meshms_conversations_list(my_sid, their_sid, &conv))
return NULL;
if (!conv){
conv = emalloc_zero(sizeof(struct conversations));
bcopy(their_sid->binary, conv->them.binary, sizeof(sid_t));
conv->them = *their_sid;
}
return conv;
}
static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m){
m->journalTail = 0;
const char *my_sidhex = alloca_tohex_sid_t(*my_sid);
const char *their_sidhex = alloca_tohex_sid_t(conv->them);
rhizome_manifest_set(m, "service", RHIZOME_SERVICE_MESHMS2);
rhizome_manifest_set(m, "sender", my_sidhex);
rhizome_manifest_set(m, "recipient", their_sidhex);
rhizome_manifest_set_ll(m, "tail", m->journalTail);
if (rhizome_fill_manifest(m, NULL, my_sid, NULL))
static int create_ply(const sid_t *my_sid, struct conversations *conv, rhizome_manifest *m)
{
if (config.debug.meshms)
DEBUGF("Creating ply for my_sid=%s them=%s",
alloca_tohex_sid_t(conv->them),
alloca_tohex_sid_t(*my_sid));
rhizome_manifest_set_service(m, RHIZOME_SERVICE_MESHMS2);
rhizome_manifest_set_sender(m, my_sid);
rhizome_manifest_set_recipient(m, &conv->them);
rhizome_manifest_set_filesize(m, 0);
rhizome_manifest_set_tail(m, 0);
if (rhizome_fill_manifest(m, NULL, my_sid))
return -1;
assert(m->haveSecret);
assert(m->payloadEncryption == PAYLOAD_ENCRYPTED);
conv->my_ply.bundle_id = m->cryptoSignPublic;
conv->found_my_ply = 1;
return 0;
}
static int append_footer(unsigned char *buffer, char type, int payload_len){
static int append_footer(unsigned char *buffer, char type, int payload_len)
{
payload_len = (payload_len << 4) | (type&0xF);
write_uint16(buffer, payload_len);
return 2;
@ -223,16 +242,18 @@ static int ply_read_open(struct ply_read *ply, const rhizome_bid_t *bid, rhizome
DEBUGF("Opening ply %s", alloca_tohex_rhizome_bid_t(*bid));
if (rhizome_retrieve_manifest(bid, m))
return -1;
int ret = rhizome_open_decrypt_read(m, NULL, &ply->read);
int ret = rhizome_open_decrypt_read(m, &ply->read);
if (ret == 1)
WARNF("Payload was not found for manifest %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (ret != 0)
return ret;
ply->read.offset = ply->read.length = m->fileLength;
assert(m->filesize != RHIZOME_SIZE_UNSET);
ply->read.offset = ply->read.length = m->filesize;
return 0;
}
static int ply_read_close(struct ply_read *ply){
static int ply_read_close(struct ply_read *ply)
{
if (ply->buffer){
free(ply->buffer);
ply->buffer=NULL;
@ -244,17 +265,22 @@ static int ply_read_close(struct ply_read *ply){
// read the next record from the ply (backwards)
// returns 1 on EOF, -1 on failure
static int ply_read_next(struct ply_read *ply){
ply->record_end_offset=ply->read.offset;
static int ply_read_next(struct ply_read *ply)
{
ply->record_end_offset = ply->read.offset;
unsigned char footer[2];
ply->read.offset-=sizeof(footer);
if (ply->read.offset<=0){
if (ply->read.offset <= sizeof footer) {
if (config.debug.meshms)
DEBUGF("EOF");
return 1;
}
if (rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof(footer)) < sizeof(footer))
return -1;
ply->read.offset -= sizeof footer;
ssize_t read;
read = rhizome_read_buffered(&ply->read, &ply->buff, footer, sizeof footer);
if (read == -1)
return WHYF("rhizome_read_buffered() failed");
if ((size_t) read != sizeof footer)
return WHYF("Expected %zu bytes read, got %zu", (size_t) sizeof footer, (size_t) read);
// (rhizome_read automatically advances the offset by the number of bytes read)
ply->record_length=read_uint16(footer);
ply->type = ply->record_length & 0xF;
@ -264,7 +290,7 @@ static int ply_read_next(struct ply_read *ply){
DEBUGF("Found record %d, length %d @%"PRId64, ply->type, ply->record_length, ply->record_end_offset);
// need to allow for advancing the tail and cutting a message in half.
if (ply->record_length + sizeof(footer) > ply->read.offset){
if (ply->record_length + sizeof footer > ply->read.offset){
if (config.debug.meshms)
DEBUGF("EOF");
return 1;
@ -281,9 +307,11 @@ static int ply_read_next(struct ply_read *ply){
ply->buffer = b;
}
int read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length);
if (read!=ply->record_length)
return WHYF("Expected %d bytes read, got %d", ply->record_length, read);
read = rhizome_read_buffered(&ply->read, &ply->buff, ply->buffer, ply->record_length);
if (read == -1)
return WHYF("rhizome_read_buffered() failed");
if ((size_t) read != ply->record_length)
return WHYF("Expected %u bytes read, got %zu", ply->record_length, (size_t) read);
ply->read.offset = record_start;
return 0;
@ -298,7 +326,8 @@ static int ply_find_next(struct ply_read *ply, char type){
}
}
static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, unsigned char *buffer, int len){
static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv, unsigned char *buffer, int len)
{
int ret=-1;
rhizome_manifest *mout = NULL;
rhizome_manifest *m = rhizome_new_manifest();
@ -308,16 +337,17 @@ static int append_meshms_buffer(const sid_t *my_sid, struct conversations *conv,
if (conv->found_my_ply){
if (rhizome_retrieve_manifest(&conv->my_ply.bundle_id, m))
goto end;
// set the author of the manifest as we should already know that
m->author = *my_sid;
if (rhizome_find_bundle_author(m))
rhizome_authenticate_author(m);
if (!m->haveSecret || m->authorship != AUTHOR_AUTHENTIC)
goto end;
}else{
if (create_ply(my_sid, conv, m))
goto end;
}
assert(m->haveSecret);
assert(m->authorship == AUTHOR_AUTHENTIC);
if (rhizome_append_journal_buffer(m, NULL, 0, buffer, len))
if (rhizome_append_journal_buffer(m, 0, buffer, len))
goto end;
if (rhizome_manifest_finalise(m, &mout, 1))
@ -476,21 +506,24 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
struct rhizome_read_buffer buff;
bzero(&buff, sizeof(buff));
int ret = rhizome_open_decrypt_read(m, NULL, &read);
int ret = rhizome_open_decrypt_read(m, &read);
if (ret == -1)
goto end;
unsigned char version=0xFF;
ret=rhizome_read_buffered(&read, &buff, &version, 1);
if (version!=1){
WARN("Expected version 1");
ssize_t r = rhizome_read_buffered(&read, &buff, &version, 1);
ret = -1;
if (r == -1)
goto end;
if (version != 1) {
WARNF("Expected version 1 (got 0x%02x)", version);
goto end;
}
while (1){
while (1) {
sid_t sid;
ret=rhizome_read_buffered(&read, &buff, sid.binary, sizeof(sid));
if (ret<sizeof(sid))
r = rhizome_read_buffered(&read, &buff, sid.binary, sizeof sid.binary);
if (r != sizeof sid.binary)
break;
if (config.debug.meshms)
DEBUGF("Reading existing conversation for %s", alloca_tohex_sid_t(sid));
@ -500,30 +533,29 @@ static int read_known_conversations(rhizome_manifest *m, const sid_t *their_sid,
if (!ptr)
goto end;
unsigned char details[8*3];
ret = rhizome_read_buffered(&read, &buff, details, sizeof(details));
if (ret<0)
r = rhizome_read_buffered(&read, &buff, details, sizeof details);
if (r == -1)
break;
int bytes=ret;
int ofs=0;
ret=unpack_uint(details, bytes, &ptr->their_last_message);
if (ret<0)
int bytes = r;
int ofs = 0;
int unpacked = unpack_uint(details, bytes, &ptr->their_last_message);
if (unpacked == -1)
break;
ofs+=ret;
ret=unpack_uint(details+ofs,bytes-ofs, &ptr->read_offset);
if (ret<0)
ofs += unpacked;
unpacked = unpack_uint(details+ofs, bytes-ofs, &ptr->read_offset);
if (unpacked == -1)
break;
ofs+=ret;
ret=unpack_uint(details+ofs,bytes-ofs, &ptr->their_size);
if (ret<0)
ofs += unpacked;
unpacked = unpack_uint(details+ofs, bytes-ofs, &ptr->their_size);
if (unpacked == -1)
break;
ofs+=ret;
ofs += unpacked;
read.offset += ofs - bytes;
}
ret = 0;
end:
rhizome_read_close(&read);
return 0;
return ret;
}
static ssize_t write_conversation(struct rhizome_write *write, struct conversations *conv)
@ -583,10 +615,8 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
goto end;
// then write it
m->version++;
rhizome_manifest_set_ll(m,"version",m->version);
m->fileLength = (size_t) len + 1;
rhizome_manifest_set_ll(m,"filesize",m->fileLength);
rhizome_manifest_set_version(m, m->version + 1);
rhizome_manifest_set_filesize(m, (size_t)len + 1);
if (rhizome_write_open_manifest(&write, m) == -1)
goto end;
@ -597,8 +627,7 @@ static int write_known_conversations(rhizome_manifest *m, struct conversations *
goto end;
if (rhizome_finish_write(&write))
goto end;
m->filehash = write.id;
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
rhizome_manifest_set_filehash(m, &write.id);
if (rhizome_manifest_finalise(m, &mout, 1))
goto end;
@ -612,7 +641,8 @@ end:
}
// read information about existing conversations from a rhizome payload
static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv){
static int meshms_conversations_list(const sid_t *my_sid, const sid_t *their_sid, struct conversations **conv)
{
int ret=-1;
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
@ -677,13 +707,16 @@ int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
if (rhizome_opendb() == -1)
if (rhizome_opendb() == -1){
keyring_free(keyring);
return -1;
}
struct conversations *conv=NULL;
if (meshms_conversations_list(&sid, NULL, &conv))
if (meshms_conversations_list(&sid, NULL, &conv)){
keyring_free(keyring);
return -1;
}
const char *names[]={
"_id","recipient","read", "last_message", "read_offset"
};
@ -693,10 +726,12 @@ int app_meshms_conversations(const struct cli_parsed *parsed, struct cli_context
cli_row_count(context, rows);
free_conversations(conv);
keyring_free(keyring);
return 0;
}
int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context *context){
int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context *context)
{
const char *my_sidhex, *their_sidhex, *message;
if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1
|| cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, "") == -1
@ -707,16 +742,21 @@ int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
if (rhizome_opendb() == -1)
if (rhizome_opendb() == -1){
keyring_free(keyring);
return -1;
}
sid_t my_sid, their_sid;
fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary));
fromhex(their_sid.binary, their_sidhex, sizeof(their_sid.binary));
struct conversations *conv=find_or_create_conv(&my_sid, &their_sid);
if (!conv)
if (str_to_sid_t(&my_sid, my_sidhex) == -1)
return WHY("invalid sender SID");
if (str_to_sid_t(&their_sid, their_sidhex) == -1)
return WHY("invalid recipient SID");
struct conversations *conv = find_or_create_conv(&my_sid, &their_sid);
if (!conv) {
keyring_free(keyring);
return -1;
}
// construct a message payload
int message_len = strlen(message)+1;
@ -727,10 +767,12 @@ int app_meshms_send_message(const struct cli_parsed *parsed, struct cli_context
int ret = append_meshms_buffer(&my_sid, conv, buffer, message_len);
free_conversations(conv);
keyring_free(keyring);
return ret;
}
int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context){
int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context *context)
{
const char *my_sidhex, *their_sidhex;
if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1
|| cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, "") == -1)
@ -740,17 +782,24 @@ int app_meshms_list_messages(const struct cli_parsed *parsed, struct cli_context
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
if (rhizome_opendb() == -1)
if (rhizome_opendb() == -1){
keyring_free(keyring);
return -1;
}
sid_t my_sid, their_sid;
fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary));
fromhex(their_sid.binary, their_sidhex, sizeof(their_sid.binary));
if (str_to_sid_t(&my_sid, my_sidhex) == -1){
keyring_free(keyring);
return WHY("invalid sender SID");
}
if (str_to_sid_t(&their_sid, their_sidhex) == -1){
keyring_free(keyring);
return WHY("invalid recipient SID");
}
struct conversations *conv=find_or_create_conv(&my_sid, &their_sid);
if (!conv)
if (!conv){
keyring_free(keyring);
return -1;
}
int ret=-1;
const char *names[]={
@ -880,6 +929,7 @@ end:
ply_read_close(&read_theirs);
}
free_conversations(conv);
keyring_free(keyring);
return ret;
}
@ -915,7 +965,8 @@ static int mark_read(struct conversations *conv, const sid_t *their_sid, const c
return ret;
}
int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *context){
int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *context)
{
const char *my_sidhex, *their_sidhex, *offset_str;
if (cli_arg(parsed, "sender_sid", &my_sidhex, str_is_subscriber_id, "") == -1
|| cli_arg(parsed, "recipient_sid", &their_sidhex, str_is_subscriber_id, NULL) == -1
@ -926,9 +977,10 @@ int app_meshms_mark_read(const struct cli_parsed *parsed, struct cli_context *co
return -1;
if (!(keyring = keyring_open_instance_cli(parsed)))
return -1;
if (rhizome_opendb() == -1)
if (rhizome_opendb() == -1){
keyring_free(keyring);
return -1;
}
sid_t my_sid, their_sid;
fromhex(my_sid.binary, my_sidhex, sizeof(my_sid.binary));
if (their_sidhex)
@ -966,5 +1018,6 @@ end:
if (m)
rhizome_manifest_free(m);
free_conversations(conv);
keyring_free(keyring);
return ret;
}

13
monitor-client.c
View File

@ -36,15 +36,15 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#endif
#include <sys/un.h>
#include <fcntl.h>
#include <ctype.h>
#include "constants.h"
#include "conf.h"
#include "log.h"
#include "str.h"
#include "strbuf_helpers.h"
#include "socket.h"
#include "monitor-client.h"
#include <ctype.h>
#define STATE_INIT 0
#define STATE_DATA 1
@ -72,12 +72,11 @@ int monitor_client_open(struct monitor_state **res)
int fd;
if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1)
return WHYF_perror("socket(AF_UNIX, SOCK_STREAM, 0)");
struct sockaddr_un addr;
socklen_t addrlen;
if (make_local_sockaddr(&addr, &addrlen, "monitor.socket") == -1)
struct socket_address addr;
if (make_local_sockaddr(&addr, "monitor.socket") == -1)
return -1;
INFOF("Attempting to connect to %s", alloca_sockaddr(&addr, addrlen));
if (socket_connect(fd, (struct sockaddr*)&addr, addrlen) == -1) {
INFOF("Attempting to connect to %s", alloca_socket_address(&addr));
if (socket_connect(fd, &addr.addr, addr.addrlen) == -1) {
close(fd);
return -1;
}

11
monitor.c
View File

@ -32,6 +32,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "strbuf_helpers.h"
#include "overlay_address.h"
#include "monitor-client.h"
#include "socket.h"
#include "dataformats.h"
#ifdef HAVE_UCRED_H
#include <ucred.h>
@ -80,11 +82,10 @@ int monitor_setup_sockets()
int sock = -1;
if ((sock = esocket(AF_UNIX, SOCK_STREAM, 0)) == -1)
goto error;
struct sockaddr_un addr;
socklen_t addrlen;
if (make_local_sockaddr(&addr, &addrlen, "monitor.socket") == -1)
struct socket_address addr;
if (make_local_sockaddr(&addr, "monitor.socket") == -1)
goto error;
if (socket_bind(sock, (struct sockaddr*)&addr, addrlen) == -1)
if (socket_bind(sock, &addr.addr, addr.addrlen) == -1)
goto error;
if (socket_listen(sock, MAX_MONITOR_SOCKETS) == -1)
goto error;
@ -97,7 +98,7 @@ int monitor_setup_sockets()
named_socket.poll.fd=sock;
named_socket.poll.events=POLLIN;
watch(&named_socket);
INFOF("Monitor socket: fd=%d %s", sock, alloca_sockaddr(&addr, addrlen));
INFOF("Monitor socket: fd=%d %s", sock, alloca_socket_address(&addr));
return 0;
error:

24
net.c
View File

@ -22,12 +22,12 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <fcntl.h>
#include <sys/uio.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <time.h>
#include "serval.h"
#include "conf.h"
#include "net.h"
#include "socket.h"
#include "str.h"
#include "strbuf_helpers.h"
@ -139,31 +139,25 @@ ssize_t _write_str_nonblock(int fd, const char *str, struct __sourceloc __whence
return _write_all_nonblock(fd, str, strlen(str), __whence);
}
ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
struct sockaddr *recvaddr, socklen_t *recvaddrlen)
ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl, struct socket_address *recvaddr)
{
struct msghdr msg;
struct iovec iov[1];
struct cmsghdr cmsgcmsg[16];
iov[0].iov_base=buffer;
iov[0].iov_len=bufferlen;
bzero(&msg,sizeof(msg));
msg.msg_name = recvaddr;
msg.msg_namelen = *recvaddrlen;
msg.msg_name = &recvaddr->store;
msg.msg_namelen = recvaddr->addrlen;
msg.msg_iov = &iov[0];
msg.msg_iovlen = 1;
// setting the following makes the data end up in the wrong place
// msg.msg_iov->iov_base=iov_buffer;
// msg.msg_iov->iov_len=sizeof(iov_buffer);
struct cmsghdr cmsgcmsg[16];
msg.msg_control = &cmsgcmsg[0];
msg.msg_controllen = sizeof(struct cmsghdr)*16;
msg.msg_control = cmsgcmsg;
msg.msg_controllen = sizeof cmsgcmsg;
msg.msg_flags = 0;
ssize_t len = recvmsg(sock,&msg,0);
if (len == -1 && errno != EAGAIN && errno != EWOULDBLOCK)
return WHY_perror("recvmsg");
return WHYF_perror("recvmsg(%d,%p,0)", sock, &msg);
#if 0
if (config.debug.packetrx) {
@ -193,7 +187,7 @@ ssize_t recvwithttl(int sock,unsigned char *buffer, size_t bufferlen,int *ttl,
}
}
}
*recvaddrlen=msg.msg_namelen;
recvaddr->addrlen = msg.msg_namelen;
return len;
}

6
net.h
View File

@ -21,7 +21,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/types.h> // for size_t, ssize_t
#include <sys/socket.h> // for struct sockaddr, socklen_t
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h> // for struct in_addr
#endif
#include <arpa/inet.h> // for in_addr_t
#include "log.h" // for __WHENCE__ and struct __sourceloc
@ -50,6 +52,8 @@ ssize_t _write_all_nonblock(int fd, const void *buf, size_t len, struct __source
ssize_t _writev_all(int fd, const struct iovec *iov, int iovcnt, struct __sourceloc __whence);
ssize_t _write_str(int fd, const char *str, struct __sourceloc __whence);
ssize_t _write_str_nonblock(int fd, const char *str, struct __sourceloc __whence);
ssize_t recvwithttl(int sock, unsigned char *buffer, size_t bufferlen, int *ttl, struct sockaddr *recvaddr, socklen_t *recvaddrlen);
struct socket_address;
ssize_t recvwithttl(int sock, unsigned char *buffer, size_t bufferlen, int *ttl, struct socket_address *);
#endif // __SERVALD_NET_H

3
os.h
View File

@ -54,6 +54,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* }
*/
typedef int64_t time_ms_t;
#define PRItime_ms_t PRId64
time_ms_t gettime_ms();
time_ms_t sleep_ms(time_ms_t milliseconds);
@ -65,7 +66,7 @@ __SERVALDNA_OS_INLINE void bzero(void *buf, size_t len) {
#endif
#ifndef HAVE_BCOPY
__SERVALDNA_OS_INLINE void bcopy(void *src, void *dst, size_t len) {
__SERVALDNA_OS_INLINE void bcopy(const void *src, void *dst, size_t len) {
memcpy(dst, src, len);
}
#endif

1
overlay.c
View File

@ -72,6 +72,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "conf.h"
#include "rhizome.h"
#include "strbuf.h"
#include "keyring.h"
int overlayMode=0;

213
overlay_address.c
View File

@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
since for things like number resolution we are happy to send repeat requests.
*/
#include <assert.h>
#include "serval.h"
#include "conf.h"
#include "str.h"
@ -46,7 +47,7 @@ static struct broadcast bpilist[MAX_BPIS];
// each slot either points to another tree node or a struct subscriber.
struct tree_node{
// bit flags for the type of object each element points to
int is_tree;
uint16_t is_tree;
union{
struct tree_node *tree_nodes[16];
@ -66,58 +67,106 @@ static unsigned char get_nibble(const unsigned char *sidp, int pos)
return byte&0xF;
}
// find a subscriber struct from a whole or abbreviated subscriber id
struct subscriber *find_subscriber(const unsigned char *sidp, int len, int create)
static void free_subscriber(struct subscriber *subscriber)
{
if (subscriber->link_state || subscriber->destination)
FATAL("Can't free a subscriber that is being used in routing");
if (subscriber->sync_state)
FATAL("Can't free a subscriber that is being used by rhizome");
if (subscriber->identity)
FATAL("Can't free a subscriber that is unlocked in the keyring");
free(subscriber);
}
static void free_children(struct tree_node *parent)
{
int i;
for (i=0;i<16;i++){
if (parent->is_tree & (1<<i)){
free_children(parent->tree_nodes[i]);
free(parent->tree_nodes[i]);
parent->tree_nodes[i]=NULL;
}else if(parent->subscribers[i]){
free_subscriber(parent->subscribers[i]);
parent->subscribers[i]=NULL;
}
}
parent->is_tree=0;
}
void free_subscribers()
{
// don't attempt to free anything if we're running as a server
// who knows where subscriber ptr's may have leaked to.
if (serverMode)
FATAL("Freeing subscribers from a running daemon is not supported");
free_children(&root);
}
// find a subscriber struct from a whole or abbreviated subscriber id
struct subscriber *_find_subscriber(struct __sourceloc __whence, const unsigned char *sidp, int len, int create)
{
IN();
if (config.debug.subscriber)
DEBUGF("find_subscriber(sid=%s, create=%d)", alloca_tohex(sidp, len), create);
struct tree_node *ptr = &root;
int pos=0;
if (len!=SID_SIZE)
create =0;
do{
struct subscriber *ret = NULL;
do {
unsigned char nibble = get_nibble(sidp, pos++);
if (ptr->is_tree & (1<<nibble)){
ptr = ptr->tree_nodes[nibble];
}else if(!ptr->subscribers[nibble]){
// subscriber is not yet known
if (create){
struct subscriber *ret=(struct subscriber *)malloc(sizeof(struct subscriber));
memset(ret,0,sizeof(struct subscriber));
ptr->subscribers[nibble]=ret;
if (create && (ret = (struct subscriber *) emalloc_zero(sizeof(struct subscriber)))) {
ptr->subscribers[nibble] = ret;
ret->sid = *(const sid_t *)sidp;
ret->abbreviate_len=pos;
ret->abbreviate_len = pos;
if (config.debug.subscriber)
DEBUGF("set node[%.*s].subscribers[%c]=%p (sid=%s, abbrev_len=%d)",
pos - 1, alloca_tohex(sidp, len), hexdigit_upper[nibble],
ret, alloca_tohex_sid_t(ret->sid), ret->abbreviate_len
);
}
return ptr->subscribers[nibble];
goto done;
}else{
// there's a subscriber in this slot, does it match the rest of the sid we've been given?
struct subscriber *ret = ptr->subscribers[nibble];
ret = ptr->subscribers[nibble];
if (memcmp(ret->sid.binary, sidp, len) == 0)
return ret;
goto done;
// if we need to insert this subscriber, we have to make a new tree node first
if (!create)
return NULL;
if (!create) {
ret = NULL;
goto done;
}
// create a new tree node and move the existing subscriber into it
struct tree_node *new=(struct tree_node *)malloc(sizeof(struct tree_node));
memset(new,0,sizeof(struct tree_node));
ptr->tree_nodes[nibble]=new;
struct tree_node *new = (struct tree_node *) emalloc_zero(sizeof(struct tree_node));
if (new == NULL) {
ret = NULL;
goto done;
}
if (config.debug.subscriber)
DEBUGF("create node[%.*s]", pos, alloca_tohex(sidp, len));
ptr->tree_nodes[nibble] = new;
ptr->is_tree |= (1<<nibble);
ptr=new;
nibble=get_nibble(ret->sid.binary, pos);
ptr->subscribers[nibble]=ret;
ret->abbreviate_len=pos+1;
ptr = new;
nibble = get_nibble(ret->sid.binary, pos);
ptr->subscribers[nibble] = ret;
ret->abbreviate_len = pos + 1;
if (config.debug.subscriber)
DEBUGF("set node[%.*s].subscribers[%c]=%p(sid=%s, abbrev_len=%d)",
pos, alloca_tohex(sidp, len), hexdigit_upper[nibble],
ret, alloca_tohex_sid_t(ret->sid), ret->abbreviate_len
);
// then go around the loop again to compare the next nibble against the sid until we find an empty slot.
}
}while(pos < len*2);
// abbreviation is not unique
return NULL;
} while(pos < len*2);
done:
if (config.debug.subscriber)
DEBUGF("find_subscriber() return %p", ret);
RETURN(ret);
}
/*
@ -198,35 +247,28 @@ int overlay_broadcast_drop_check(struct broadcast *addr)
}
}
int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast)
void overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast)
{
return ob_append_bytes(b, broadcast->id, BROADCAST_LEN);
ob_append_bytes(b, broadcast->id, BROADCAST_LEN);
}
// append an appropriate abbreviation into the address
int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber)
void overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber)
{
if (!subscriber)
return WHY("No address supplied");
if(context
&& subscriber == context->point_to_point_device){
if (ob_append_byte(b, OA_CODE_P2P_YOU))
return -1;
}else if(context
assert(subscriber != NULL);
if (context && subscriber == context->point_to_point_device)
ob_append_byte(b, OA_CODE_P2P_YOU);
else if(context
&& !subscriber->send_full
&& subscriber == my_subscriber
&& context->point_to_point_device
&& (context->encoding_header==0 || !context->interface->local_echo)){
if (ob_append_byte(b, OA_CODE_P2P_ME))
return -1;
}else if (context && subscriber==context->sender){
if (ob_append_byte(b, OA_CODE_SELF))
return -1;
}else if(context && subscriber==context->previous){
if (ob_append_byte(b, OA_CODE_PREVIOUS))
return -1;
}else{
&& (context->encoding_header==0 || !context->interface->local_echo))
ob_append_byte(b, OA_CODE_P2P_ME);
else if (context && subscriber==context->sender)
ob_append_byte(b, OA_CODE_SELF);
else if (context && subscriber==context->previous)
ob_append_byte(b, OA_CODE_PREVIOUS);
else {
int len=SID_SIZE;
if (subscriber->send_full){
subscriber->send_full=0;
@ -237,17 +279,15 @@ int overlay_address_append(struct decode_context *context, struct overlay_buffer
if (len>SID_SIZE)
len=SID_SIZE;
}
if (ob_append_byte(b, len))
return -1;
if (ob_append_bytes(b, subscriber->sid.binary, len))
return -1;
ob_append_byte(b, len);
ob_append_bytes(b, subscriber->sid.binary, len);
}
if (context)
context->previous = subscriber;
return 0;
}
static int add_explain_response(struct subscriber *subscriber, void *context){
static int add_explain_response(struct subscriber *subscriber, void *context)
{
struct decode_context *response = context;
// only explain a SID once every half second.
time_ms_t now = gettime_ms();
@ -256,8 +296,13 @@ static int add_explain_response(struct subscriber *subscriber, void *context){
subscriber->last_explained = now;
if (!response->please_explain){
response->please_explain = calloc(sizeof(struct overlay_frame),1);
response->please_explain->payload=ob_new();
if ((response->please_explain = emalloc_zero(sizeof(struct overlay_frame))) == NULL)
return 1; // stop walking
if ((response->please_explain->payload = ob_new()) == NULL) {
free(response->please_explain);
response->please_explain = NULL;
return 1; // stop walking
}
ob_limitsize(response->please_explain->payload, 1024);
}
@ -265,6 +310,8 @@ static int add_explain_response(struct subscriber *subscriber, void *context){
// the header of this packet must include our full sid.
if (subscriber->reachable==REACHABLE_SELF){
if (subscriber==my_subscriber){
if (config.debug.subscriber)
DEBUGF("Explaining SELF sid=%s", alloca_tohex_sid_t(subscriber->sid));
response->please_explain->source_full=1;
return 0;
}
@ -272,18 +319,22 @@ static int add_explain_response(struct subscriber *subscriber, void *context){
}
// add the whole subscriber id to the payload, stop if we run out of space
DEBUGF("Adding full sid by way of explanation %s", alloca_tohex_sid_t(subscriber->sid));
if (ob_append_byte(response->please_explain->payload, SID_SIZE))
if (config.debug.subscriber)
DEBUGF("Explaining sid=%s", alloca_tohex_sid_t(subscriber->sid));
ob_checkpoint(response->please_explain->payload);
ob_append_byte(response->please_explain->payload, SID_SIZE);
ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE);
if (ob_overrun(response->please_explain->payload)) {
ob_rewind(response->please_explain->payload);
return 1;
if (ob_append_bytes(response->please_explain->payload, subscriber->sid.binary, SID_SIZE))
return 1;
}
// let the routing engine know that we had to explain this sid, we probably need to re-send routing info
link_explained(subscriber);
return 0;
}
static int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int len, struct subscriber **subscriber){
static int find_subscr_buffer(struct decode_context *context, struct overlay_buffer *b, int len, struct subscriber **subscriber)
{
if (len<=0 || len>SID_SIZE){
return WHYF("Invalid abbreviation length %d", len);
}
@ -309,7 +360,8 @@ static int find_subscr_buffer(struct decode_context *context, struct overlay_buf
// add the abbreviation you told me about
if (!context->please_explain){
context->please_explain = calloc(sizeof(struct overlay_frame),1);
context->please_explain->payload=ob_new();
if ((context->please_explain->payload = ob_new()) == NULL)
return -1;
ob_limitsize(context->please_explain->payload, MDP_MTU);
}
@ -360,7 +412,8 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer
// add the abbreviation you told me about
if (!context->please_explain){
context->please_explain = calloc(sizeof(struct overlay_frame),1);
context->please_explain->payload=ob_new();
if ((context->please_explain->payload = ob_new()) == NULL)
return -1;
ob_limitsize(context->please_explain->payload, MDP_MTU);
}
@ -394,11 +447,13 @@ int overlay_address_parse(struct decode_context *context, struct overlay_buffer
}
// once we've finished parsing a packet, complete and send a please explain if required.
int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination){
int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination)
{
IN();
struct overlay_frame *frame=context->please_explain;
if (!frame)
if (frame == NULL)
RETURN(0);
assert(frame->payload != NULL);
frame->type = OF_TYPE_PLEASEEXPLAIN;
if (source)
@ -430,7 +485,7 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc
}
frame->queue=OQ_MESH_MANAGEMENT;
if (!overlay_payload_enqueue(frame))
if (overlay_payload_enqueue(frame) != -1)
RETURN(0);
op_free(frame);
RETURN(-1);
@ -438,7 +493,8 @@ int send_please_explain(struct decode_context *context, struct subscriber *sourc
}
// process an incoming request for explanation of subscriber abbreviations
int process_explain(struct overlay_frame *frame){
int process_explain(struct overlay_frame *frame)
{
struct overlay_buffer *b=frame->payload;
struct decode_context context;
@ -463,14 +519,17 @@ int process_explain(struct overlay_frame *frame){
if (len==SID_SIZE){
// This message is also used to inform people of previously unknown subscribers
// make sure we know this one
INFOF("Storing explain response for %s", alloca_tohex(sid, len));
find_subscriber(sid,len,1);
}else{
// reply to the sender with all subscribers that match this abbreviation
INFOF("Sending responses for %s", alloca_tohex(sid, len));
INFOF("Sending explain responses for %s", alloca_tohex(sid, len));
walk_tree(&root, 0, sid, len, sid, len, add_explain_response, &context);
}
}
send_please_explain(&context, frame->destination, frame->source);
if (context.please_explain)
send_please_explain(&context, frame->destination, frame->source);
else if (config.debug.subscriber)
DEBUG("No explain responses");
return 0;
}

15
overlay_address.h
View File

@ -57,9 +57,6 @@ struct subscriber{
int max_packet_version;
// overlay routing information
struct overlay_node *node;
// link state routing information
struct link_state *link_state;
@ -85,7 +82,7 @@ struct subscriber{
unsigned char sas_valid;
// private keys for local identities
keyring_identity *identity;
struct keyring_identity *identity;
};
struct broadcast{
@ -113,7 +110,9 @@ struct decode_context{
extern struct subscriber *my_subscriber;
extern struct subscriber *directory_service;
struct subscriber *find_subscriber(const unsigned char *sid, int len, int create);
struct subscriber *_find_subscriber(struct __sourceloc, const unsigned char *sid, int len, int create);
#define find_subscriber(sid, len, create) _find_subscriber(__WHENCE__, sid, len, create)
void enum_subscribers(struct subscriber *start, int(*callback)(struct subscriber *, void *), void *context);
int set_reachable(struct subscriber *subscriber, struct network_destination *destination, struct subscriber *next_hop);
int load_subscriber_address(struct subscriber *subscriber);
@ -122,11 +121,13 @@ int process_explain(struct overlay_frame *frame);
int overlay_broadcast_drop_check(struct broadcast *addr);
int overlay_broadcast_generate_address(struct broadcast *addr);
int overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast);
int overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber);
void overlay_broadcast_append(struct overlay_buffer *b, struct broadcast *broadcast);
void overlay_address_append(struct decode_context *context, struct overlay_buffer *b, struct subscriber *subscriber);
int overlay_broadcast_parse(struct overlay_buffer *b, struct broadcast *broadcast);
int overlay_address_parse(struct decode_context *context, struct overlay_buffer *b, struct subscriber **subscriber);
int send_please_explain(struct decode_context *context, struct subscriber *source, struct subscriber *destination);
void free_subscribers();
#endif

443
overlay_buffer.c
View File

@ -17,6 +17,7 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <assert.h>
#include "serval.h"
#include "conf.h"
#include "mem.h"
@ -30,57 +31,64 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
In either case, functions that don't take an offset use and advance the position.
*/
struct overlay_buffer *ob_new(void)
struct overlay_buffer *_ob_new(struct __sourceloc __whence)
{
struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
if (!ret) return NULL;
struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
if (config.debug.overlaybuffer)
DEBUGF("ob_new() return %p", ret);
if (ret == NULL)
return NULL;
ob_unlimitsize(ret);
return ret;
}
// index an existing static buffer.
// and allow other callers to use the ob_ convenience methods for reading and writing up to size bytes.
struct overlay_buffer *ob_static(unsigned char *bytes, int size){
struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
if (!ret) return NULL;
struct overlay_buffer *_ob_static(struct __sourceloc __whence, unsigned char *bytes, int size)
{
struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
if (config.debug.overlaybuffer)
DEBUGF("ob_static(bytes=%p, size=%d) return %p", bytes, size, ret);
if (ret == NULL)
return NULL;
ret->bytes = bytes;
ret->allocSize = size;
ret->allocated = NULL;
ob_unlimitsize(ret);
return ret;
}
// create a new overlay buffer from an existing piece of another buffer.
// Both buffers will point to the same memory region.
// It is up to the caller to ensure this buffer is not used after the parent buffer is freed.
struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length){
struct overlay_buffer *_ob_slice(struct __sourceloc __whence, struct overlay_buffer *b, int offset, int length)
{
if (offset+length > b->allocSize) {
WHY("Buffer isn't long enough to slice");
return NULL;
return NULL;
}
struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
if (!ret)
struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
if (config.debug.overlaybuffer)
DEBUGF("ob_slice(b=%p, offset=%d, length=%d) return %p", b, offset, length, ret);
if (ret == NULL)
return NULL;
ret->bytes = b->bytes+offset;
ret->allocSize = length;
ret->allocated = NULL;
ob_unlimitsize(ret);
return ret;
}
struct overlay_buffer *ob_dup(struct overlay_buffer *b){
struct overlay_buffer *ret=calloc(sizeof(struct overlay_buffer),1);
struct overlay_buffer *_ob_dup(struct __sourceloc __whence, struct overlay_buffer *b)
{
struct overlay_buffer *ret = emalloc_zero(sizeof(struct overlay_buffer));
if (config.debug.overlaybuffer)
DEBUGF("ob_dup(b=%p) return %p", b, ret);
if (ret == NULL)
return NULL;
ret->sizeLimit = b->sizeLimit;
ret->position = b->position;
ret->checkpointLength = b->checkpointLength;
if (b->bytes && b->allocSize){
// duplicate any bytes that might be relevant
int byteCount = b->sizeLimit;
@ -88,97 +96,109 @@ struct overlay_buffer *ob_dup(struct overlay_buffer *b){
byteCount = b->position;
if (byteCount > b->allocSize)
byteCount = b->allocSize;
ob_append_bytes(ret, b->bytes, byteCount);
if (byteCount)
ob_append_bytes(ret, b->bytes, byteCount);
}
return ret;
}
int ob_free(struct overlay_buffer *b)
void _ob_free(struct __sourceloc __whence, struct overlay_buffer *b)
{
if (!b) return WHY("Asked to free NULL");
if (b->bytes && b->allocated) free(b->allocated);
// we're about to free this anyway, why are we clearing it?
b->bytes=NULL;
b->allocated=NULL;
b->allocSize=0;
b->sizeLimit=0;
assert(b != NULL);
if (config.debug.overlaybuffer)
DEBUGF("ob_free(b=%p)", b);
if (b->allocated)
free(b->allocated);
free(b);
}
int _ob_checkpoint(struct __sourceloc __whence, struct overlay_buffer *b)
{
assert(b != NULL);
b->checkpointLength = b->position;
if (config.debug.overlaybuffer)
DEBUGF("ob_checkpoint(b=%p) checkpointLength=%d", b, b->checkpointLength);
return 0;
}
int ob_checkpoint(struct overlay_buffer *b)
int _ob_rewind(struct __sourceloc __whence, struct overlay_buffer *b)
{
if (!b) return WHY("Asked to checkpoint NULL");
b->checkpointLength=b->position;
assert(b != NULL);
b->position = b->checkpointLength;
if (config.debug.overlaybuffer)
DEBUGF("ob_rewind(b=%p) position=%d", b, b->position);
return 0;
}
int ob_rewind(struct overlay_buffer *b)
void _ob_limitsize(struct __sourceloc __whence, struct overlay_buffer *b, int bytes)
{
if (!b) return WHY("Asked to rewind NULL");
b->position=b->checkpointLength;
return 0;
assert(b != NULL);
assert(bytes >= 0);
assert(b->position >= 0);
assert(b->position <= bytes);
assert(b->checkpointLength <= bytes);
if (b->bytes && b->allocated == NULL)
assert(bytes <= b->allocSize);
b->sizeLimit = bytes;
if (config.debug.overlaybuffer)
DEBUGF("ob_limitsize(b=%p, bytes=%d) sizeLimit=%d", b, bytes, b->sizeLimit);
}
int ob_limitsize(struct overlay_buffer *b,int bytes)
void _ob_unlimitsize(struct __sourceloc __whence, struct overlay_buffer *b)
{
if (!b) return WHY("Asked to limit size of NULL");
if (b->position>bytes) return WHY("Length of data in buffer already exceeds size limit");
if (b->checkpointLength>bytes) return WHY("Checkpointed length of data in buffer already exceeds size limit");
if (b->bytes && (!b->allocated) && bytes > b->allocSize) return WHY("Size limit exceeds buffer size");
if (bytes<0) return WHY("Can't limit buffer to a negative size");
b->sizeLimit=bytes;
return 0;
assert(b != NULL);
b->sizeLimit = -1;
if (config.debug.overlaybuffer)
DEBUGF("ob_unlimitsize(b=%p) sizeLimit=%d", b, b->sizeLimit);
}
int ob_unlimitsize(struct overlay_buffer *b)
void _ob_flip(struct __sourceloc __whence, struct overlay_buffer *b)
{
if (!b) return WHY("b is NULL");
b->sizeLimit=-1;
return 0;
if (config.debug.overlaybuffer)
DEBUGF("ob_flip(b=%p) checkpointLength=0 position=0", b);
b->checkpointLength = 0;
ob_limitsize(b, b->position);
b->position = 0;
}
int ob_flip(struct overlay_buffer *b)
{
b->checkpointLength=0;
if (ob_limitsize(b, b->position))
return -1;
b->position=0;
return 0;
}
int _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b,int bytes)
/* Return 1 if space is available, 0 if not.
*/
ssize_t _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b, size_t bytes)
{
assert(b != NULL);
if (config.debug.overlaybuffer)
DEBUGF("ob_makespace(b=%p, bytes=%zd) b->bytes=%p b->position=%d b->allocSize=%d",
b, bytes, b->bytes, b->position, b->allocSize);
assert(b->position >= 0);
if (b->sizeLimit != -1)
assert(b->sizeLimit >= 0);
assert(b->allocSize >= 0);
if (b->position)
assert(b->bytes != NULL);
if (b->sizeLimit != -1 && b->position + bytes > b->sizeLimit) {
if (config.debug.packetformats)
DEBUGF("asked for space to %u, beyond size limit of %u", b->position + bytes, b->sizeLimit);
return -1;
}
// already enough space?
if (b->position + bytes <= b->allocSize)
DEBUGF("ob_makespace(): asked for space to %zu, beyond size limit of %u", b->position + bytes, b->sizeLimit);
return 0;
if (b->bytes && !b->allocated)
return WHY("Can't resize a static buffer");
if (0)
DEBUGF("ob_makespace(%p,%d)\n b->bytes=%p,b->position=%d,b->allocSize=%d\n",
b,bytes,b->bytes,b->position,b->allocSize);
}
if (b->position + bytes <= b->allocSize)
return 1;
// Don't realloc a static buffer.
if (b->bytes && b->allocated == NULL) {
if (config.debug.packetformats)
DEBUGF("ob_makespace(): asked for space to %zu, beyond static buffer size of %u", b->position + bytes, b->allocSize);
return 0;
}
int newSize=b->position+bytes;
if (newSize<64) newSize=64;
if (newSize&63) newSize+=64-(newSize&63);
if (newSize>1024) {
if (newSize&1023) newSize+=1024-(newSize&1023);
}
if (newSize>65536) {
if (newSize&65535) newSize+=65536-(newSize&65535);
}
if (0) DEBUGF("realloc(b->bytes=%p,newSize=%d)", b->bytes,newSize);
if (newSize>1024 && (newSize&1023))
newSize+=1024-(newSize&1023);
if (newSize>65536 && (newSize&65535))
newSize+=65536-(newSize&65535);
if (config.debug.overlaybuffer)
DEBUGF("realloc(b->bytes=%p,newSize=%d)", b->bytes,newSize);
/* XXX OSX realloc() seems to be able to corrupt things if the heap is not happy when calling realloc(), making debugging memory corruption much harder.
So will do a three-stage malloc,bcopy,free to see if we can tease bugs out that way. */
So will do a three-stage malloc,bcopy,free to see if we can tease bugs out that way. */
/*
unsigned char *r=realloc(b->bytes,newSize);
if (!r) return WHY("realloc() failed");
@ -196,94 +216,130 @@ int _ob_makespace(struct __sourceloc __whence, struct overlay_buffer *b,int byte
sleep_ms(36000000);
}
}
unsigned char *new=malloc(newSize+4096);
if (!new) return WHY("realloc() failed");
unsigned char *new = emalloc(newSize+4096);
{
int i;
for(i=0;i<4096;i++) new[newSize+i]=0xbd;
}
#else
unsigned char *new=malloc(newSize);
unsigned char *new = emalloc(newSize);
#endif
if (!new)
return 0;
bcopy(b->bytes,new,b->position);
if (b->allocated) free(b->allocated);
if (b->allocated) {
assert(b->allocated == b->bytes);
free(b->allocated);
}
b->bytes=new;
b->allocated=new;
b->allocSize=newSize;
return 0;
return 1;
}
/*
Functions that append data and increase the size of the buffer if possible / required
*/
int _ob_append_byte(struct __sourceloc __whence, struct overlay_buffer *b,unsigned char byte)
void _ob_append_byte(struct __sourceloc __whence, struct overlay_buffer *b, unsigned char byte)
{
if (_ob_makespace(__whence, b,1)) return WHY("ob_makespace() failed");
b->bytes[b->position++] = byte;
return 0;
const int bytes = 1;
if (ob_makespace(b, bytes)) {
b->bytes[b->position] = byte;
if (config.debug.overlaybuffer)
DEBUGF("ob_append_byte(b=%p, byte=0x%02x) %p[%d]=%02x position=%d", b, byte, b->bytes, b->position, byte, b->position + bytes);
} else {
if (config.debug.overlaybuffer)
DEBUGF("ob_append_byte(b=%p, byte=0x%02x) OVERRUN position=%d", b, byte, b->position + bytes);
}
b->position += bytes;
}
unsigned char *_ob_append_space(struct __sourceloc __whence, struct overlay_buffer *b,int count)
unsigned char *_ob_append_space(struct __sourceloc __whence, struct overlay_buffer *b, int count)
{
if (_ob_makespace(__whence, b,count)) {
WHY("ob_makespace() failed");
return NULL;
}
unsigned char *r=&b->bytes[b->position];
b->position+=count;
assert(count > 0);
unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL;
b->position += count;
if (config.debug.overlaybuffer)
DEBUGF("ob_append_space(b=%p, count=%d) position=%d return %p", b, count, b->position, r);
return r;
}
int _ob_append_bytes(struct __sourceloc __whence, struct overlay_buffer *b, const unsigned char *bytes, int count)
void _ob_append_bytes(struct __sourceloc __whence, struct overlay_buffer *b, const unsigned char *bytes, int count)
{
if (_ob_makespace(__whence, b,count)) return WHY("ob_makespace() failed");
bcopy(bytes,&b->bytes[b->position],count);
b->position+=count;
return 0;
assert(count > 0);
unsigned char *r = ob_makespace(b, count) ? &b->bytes[b->position] : NULL;
if (r) {
bcopy(bytes, r, count);
if (config.debug.overlaybuffer)
DEBUGF("ob_append_bytes(b=%p, bytes=%p, count=%d) position=%d return %p", b, bytes, count, b->position + count, r);
} else {
if (config.debug.overlaybuffer)
DEBUGF("ob_append_bytes(b=%p, bytes=%p, count=%d) OVERRUN position=%d return NULL", b, bytes, count, b->position + count);
}
if (config.debug.overlaybuffer)
dump("ob_append_bytes", bytes, count);
b->position += count;
}
int _ob_append_buffer(struct __sourceloc __whence, struct overlay_buffer *b, struct overlay_buffer *s){
return _ob_append_bytes(__whence, b, s->bytes, s->position);
void _ob_append_buffer(struct __sourceloc __whence, struct overlay_buffer *b, struct overlay_buffer *s)
{
ob_append_bytes(b, s->bytes, s->position);
}
int _ob_append_ui16(struct __sourceloc __whence, struct overlay_buffer *b, uint16_t v)
void _ob_append_ui16(struct __sourceloc __whence, struct overlay_buffer *b, uint16_t v)
{
if (_ob_makespace(__whence, b, 2)) return WHY("ob_makespace() failed");
b->bytes[b->position] = (v >> 8) & 0xFF;
b->bytes[b->position+1] = v & 0xFF;
b->position+=2;
return 0;
const int bytes = 2;
if (ob_makespace(b, bytes)) {
b->bytes[b->position] = (v >> 8) & 0xFF;
b->bytes[b->position+1] = v & 0xFF;
if (config.debug.overlaybuffer)
DEBUGF("ob_append_ui16(b=%p, v=%u) %p[%d]=%s position=%d", b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes);
} else {
if (config.debug.overlaybuffer)
DEBUGF("ob_append_ui16(b=%p, v=%u) OVERRUN position=%d", b, v, b->position + bytes);
}
b->position += bytes;
}
int _ob_append_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
void _ob_append_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
{
if (_ob_makespace(__whence, b, 4)) return WHY("ob_makespace() failed");
b->bytes[b->position] = (v >> 24) & 0xFF;
b->bytes[b->position+1] = (v >> 16) & 0xFF;
b->bytes[b->position+2] = (v >> 8) & 0xFF;
b->bytes[b->position+3] = v & 0xFF;
b->position+=4;
return 0;
const int bytes = 4;
if (ob_makespace(b, bytes)) {
b->bytes[b->position] = (v >> 24) & 0xFF;
b->bytes[b->position+1] = (v >> 16) & 0xFF;
b->bytes[b->position+2] = (v >> 8) & 0xFF;
b->bytes[b->position+3] = v & 0xFF;
if (config.debug.overlaybuffer)
DEBUGF("ob_append_ui32(b=%p, v=%"PRIu32") %p[%d]=%s position=%d",
b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes);
} else {
if (config.debug.overlaybuffer)
DEBUGF("ob_append_ui32(b=%p, v=%"PRIu32") OVERRUN position=%d", b, v, b->position + bytes);
}
b->position += bytes;
}
int _ob_append_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
void _ob_append_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
{
if (_ob_makespace(__whence, b, 8)) return WHY("ob_makespace() failed");
b->bytes[b->position] = (v >> 56) & 0xFF;
b->bytes[b->position+1] = (v >> 48) & 0xFF;
b->bytes[b->position+2] = (v >> 40) & 0xFF;
b->bytes[b->position+3] = (v >> 32) & 0xFF;
b->bytes[b->position+4] = (v >> 24) & 0xFF;
b->bytes[b->position+5] = (v >> 16) & 0xFF;
b->bytes[b->position+6] = (v >> 8) & 0xFF;
b->bytes[b->position+7] = v & 0xFF;
b->position+=8;
return 0;
const int bytes = 8;
if (ob_makespace(b, bytes)) {
b->bytes[b->position] = (v >> 56) & 0xFF;
b->bytes[b->position+1] = (v >> 48) & 0xFF;
b->bytes[b->position+2] = (v >> 40) & 0xFF;
b->bytes[b->position+3] = (v >> 32) & 0xFF;
b->bytes[b->position+4] = (v >> 24) & 0xFF;
b->bytes[b->position+5] = (v >> 16) & 0xFF;
b->bytes[b->position+6] = (v >> 8) & 0xFF;
b->bytes[b->position+7] = v & 0xFF;
if (config.debug.overlaybuffer)
DEBUGF("ob_append_ui64(b=%p, v=%"PRIu64") %p[%d]=%s position=%d",
b, v, b->bytes, b->position, alloca_tohex(&b->bytes[b->position], bytes), b->position + bytes);
} else {
if (config.debug.overlaybuffer)
DEBUGF("ob_append_ui64(b=%p, v=%"PRIu64") OVERRUN position=%d", b, v, b->position + bytes);
}
b->position += bytes;
}
int measure_packed_uint(uint64_t v){
@ -320,36 +376,28 @@ int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v){
return i;
}
int _ob_append_packed_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
void _ob_append_packed_ui32(struct __sourceloc __whence, struct overlay_buffer *b, uint32_t v)
{
do{
if (_ob_append_byte(__whence, b, (v&0x7f) | (v>0x7f?0x80:0)))
return -1;
v = v>>7;
}while(v!=0);
return 0;
do {
ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0));
v = v >> 7;
} while (v != 0);
}
int _ob_append_packed_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
void _ob_append_packed_ui64(struct __sourceloc __whence, struct overlay_buffer *b, uint64_t v)
{
do{
if (ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0)))
return -1;
v = v>>7;
}while(v!=0);
return 0;
do {
ob_append_byte(b, (v&0x7f) | (v>0x7f?0x80:0));
v = v >> 7;
} while (v != 0);
}
int _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l)
void _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l)
{
if (l<0||l>0xffff) return -1;
b->var_length_offset=b->position;
return _ob_append_ui16(__whence, b,l);
assert(l >= 0);
assert(l <= 0xffff);
b->var_length_offset = b->position;
ob_append_ui16(b, l);
}
@ -359,7 +407,8 @@ int _ob_append_rfs(struct __sourceloc __whence, struct overlay_buffer *b, int l)
// make sure a range of bytes is valid for reading
int test_offset(struct overlay_buffer *b,int start,int length){
int test_offset(struct overlay_buffer *b,int start,int length)
{
if (!b) return -1;
if (start<0) return -1;
if (b->sizeLimit>=0 && start+length>b->sizeLimit) return -1;
@ -367,12 +416,17 @@ int test_offset(struct overlay_buffer *b,int start,int length){
return 0;
}
int ob_getbyte(struct overlay_buffer *b, int ofs)
// next byte without advancing
int ob_peek(struct overlay_buffer *b)
{
if (test_offset(b, ofs, 1))
if (test_offset(b, b->position, 1))
return -1;
return b->bytes[ofs];
return b->bytes[b->position];
}
void ob_skip(struct overlay_buffer *b, unsigned n)
{
b->position += n;
}
int ob_get_bytes(struct overlay_buffer *b, unsigned char *buff, int len){
@ -464,46 +518,75 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b)
return ret;
}
int ob_get(struct overlay_buffer *b){
int ob_get(struct overlay_buffer *b)
{
if (test_offset(b, b->position, 1))
return -1;
return b->bytes[b->position++];
}
int ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v)
void _ob_set_ui16(struct __sourceloc __whence, struct overlay_buffer *b, int offset, uint16_t v)
{
if (test_offset(b, offset, 2))
return -1;
const int bytes = 2;
assert(b != NULL);
assert(offset >= 0);
if (b->sizeLimit != -1)
assert(offset + bytes <= b->sizeLimit);
assert(offset + bytes <= b->allocSize);
b->bytes[offset] = (v >> 8) & 0xFF;
b->bytes[offset+1] = v & 0xFF;
return 0;
if (config.debug.overlaybuffer)
DEBUGF("ob_set_ui16(b=%p, offset=%d, v=%u) %p[%d]=%s", b, offset, v, b->bytes, offset, alloca_tohex(&b->bytes[offset], bytes));
}
int ob_set(struct overlay_buffer *b, int ofs, unsigned char byte)
void _ob_set(struct __sourceloc __whence, struct overlay_buffer *b, int offset, unsigned char byte)
{
if (test_offset(b, ofs, 1))
return -1;
b->bytes[ofs] = byte;
return 0;
const int bytes = 1;
assert(b != NULL);
assert(offset >= 0);
if (b->sizeLimit != -1)
assert(offset + bytes <= b->sizeLimit);
assert(offset + bytes <= b->allocSize);
b->bytes[offset] = byte;
if (config.debug.overlaybuffer)
DEBUGF("ob_set(b=%p, offset=%d, byte=0x%02x) %p[%d]=%s", b, offset, byte, b->bytes, offset, alloca_tohex(&b->bytes[offset], bytes));
}
int ob_patch_rfs(struct overlay_buffer *b){
return ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2));
void _ob_patch_rfs(struct __sourceloc __whence, struct overlay_buffer *b)
{
ob_set_ui16(b,b->var_length_offset,b->position - (b->var_length_offset + 2));
}
int ob_position(struct overlay_buffer *b){
int ob_position(struct overlay_buffer *b)
{
return b->position;
}
int ob_limit(struct overlay_buffer *b){
int ob_limit(struct overlay_buffer *b)
{
return b->sizeLimit;
}
int ob_remaining(struct overlay_buffer *b){
int ob_remaining(struct overlay_buffer *b)
{
assert(b->sizeLimit != -1);
return b->sizeLimit - b->position;
}
unsigned char *ob_ptr(struct overlay_buffer *b){
int _ob_overrun(struct __sourceloc __whence, struct overlay_buffer *b)
{
assert(b->allocSize >= 0);
if (b->sizeLimit != -1)
assert(b->sizeLimit >= 0);
int ret = b->position > (b->sizeLimit != -1 && b->sizeLimit < b->allocSize ? b->sizeLimit : b->allocSize);
if (config.debug.overlaybuffer)
DEBUGF("ob_overrun(b=%p) return %d", b, ret);
return ret;
}
unsigned char *ob_ptr(struct overlay_buffer *b)
{
return b->bytes;
}

73
overlay_buffer.h
View File

@ -17,8 +17,8 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef _SERVALD_OVERLAY_BUFFER_H
#define _SERVALD_OVERLAY_BUFFER_H
#ifndef __SERVALD__OVERLAY_BUFFER_H
#define __SERVALD__OVERLAY_BUFFER_H
struct overlay_buffer {
unsigned char *bytes;
@ -42,31 +42,47 @@ struct overlay_buffer {
int var_length_offset;
};
struct overlay_buffer *ob_new(void);
struct overlay_buffer *ob_static(unsigned char *bytes, int size);
struct overlay_buffer *ob_slice(struct overlay_buffer *b, int offset, int length);
struct overlay_buffer *ob_dup(struct overlay_buffer *b);
int ob_free(struct overlay_buffer *b);
int ob_checkpoint(struct overlay_buffer *b);
int ob_rewind(struct overlay_buffer *b);
int ob_limitsize(struct overlay_buffer *b,int bytes);
int ob_flip(struct overlay_buffer *b);
int ob_unlimitsize(struct overlay_buffer *b);
int _ob_makespace(struct __sourceloc whence, struct overlay_buffer *b,int bytes);
int ob_set(struct overlay_buffer *b, int ofs, unsigned char byte);
struct overlay_buffer *_ob_new(struct __sourceloc __whence);
struct overlay_buffer *_ob_static(struct __sourceloc __whence, unsigned char *bytes, int size);
struct overlay_buffer *_ob_slice(struct __sourceloc __whence, struct overlay_buffer *b, int offset, int length);
struct overlay_buffer *_ob_dup(struct __sourceloc __whence, struct overlay_buffer *b);
void _ob_free(struct __sourceloc __whence, struct overlay_buffer *b);
int _ob_checkpoint(struct __sourceloc __whence, struct overlay_buffer *b);
int _ob_rewind(struct __sourceloc __whence, struct overlay_buffer *b);
void _ob_limitsize(struct __sourceloc __whence, struct overlay_buffer *b,int bytes);
void _ob_flip(struct __sourceloc __whence, struct overlay_buffer *b);
void _ob_unlimitsize(struct __sourceloc __whence, struct overlay_buffer *b);
ssize_t _ob_makespace(struct __sourceloc whence, struct overlay_buffer *b, size_t bytes);
void _ob_set(struct __sourceloc __whence, struct overlay_buffer *b, int ofs, unsigned char byte);
void _ob_set_ui16(struct __sourceloc __whence, struct overlay_buffer *b, int offset, uint16_t v);
void _ob_patch_rfs(struct __sourceloc __whence, struct overlay_buffer *b);
int _ob_append_byte(struct __sourceloc whence, struct overlay_buffer *b,unsigned char byte);
int _ob_append_bytes(struct __sourceloc whence, struct overlay_buffer *b,const unsigned char *bytes,int count);
int _ob_append_buffer(struct __sourceloc whence, struct overlay_buffer *b,struct overlay_buffer *s);
void _ob_append_byte(struct __sourceloc whence, struct overlay_buffer *b,unsigned char byte);
void _ob_append_bytes(struct __sourceloc whence, struct overlay_buffer *b,const unsigned char *bytes,int count);
void _ob_append_buffer(struct __sourceloc whence, struct overlay_buffer *b,struct overlay_buffer *s);
unsigned char *_ob_append_space(struct __sourceloc whence, struct overlay_buffer *b,int count);
int _ob_append_ui16(struct __sourceloc whence, struct overlay_buffer *b, uint16_t v);
int _ob_append_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
int _ob_append_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
int _ob_append_packed_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
int _ob_append_packed_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
int _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l);
void _ob_append_ui16(struct __sourceloc whence, struct overlay_buffer *b, uint16_t v);
void _ob_append_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
void _ob_append_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
void _ob_append_packed_ui32(struct __sourceloc whence, struct overlay_buffer *b, uint32_t v);
void _ob_append_packed_ui64(struct __sourceloc whence, struct overlay_buffer *b, uint64_t v);
void _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l);
#define ob_new() _ob_new(__WHENCE__)
#define ob_static(bytes, size) _ob_static(__WHENCE__, bytes, size)
#define ob_slice(b, off, len) _ob_slice(__WHENCE__, b, off, len)
#define ob_dup(b) _ob_dup(__WHENCE__, b)
#define ob_free(b) _ob_free(__WHENCE__, b)
#define ob_checkpoint(b) _ob_checkpoint(__WHENCE__, b)
#define ob_rewind(b) _ob_rewind(__WHENCE__, b)
#define ob_limitsize(b, size) _ob_limitsize(__WHENCE__, b, size)
#define ob_flip(b) _ob_flip(__WHENCE__, b)
#define ob_unlimitsize(b) _ob_unlimitsize(__WHENCE__, b)
#define ob_makespace(b, bytes) _ob_makespace(__WHENCE__, b, bytes)
#define ob_set(b, off, byte) _ob_set(__WHENCE__, b, off, byte)
#define ob_set_ui16(b, off, v) _ob_set_ui16(__WHENCE__, b, off, v)
#define ob_patch_rfs(b) _ob_patch_rfs(__WHENCE__, b)
#define ob_append_byte(b, byte) _ob_append_byte(__WHENCE__, b, byte)
#define ob_append_bytes(b, bytes, count) _ob_append_bytes(__WHENCE__, b, bytes, count)
#define ob_append_buffer(b, s) _ob_append_buffer(__WHENCE__, b, s)
@ -78,9 +94,9 @@ int _ob_append_rfs(struct __sourceloc whence, struct overlay_buffer *b,int l);
#define ob_append_packed_ui64(b, v) _ob_append_packed_ui64(__WHENCE__, b, v)
#define ob_append_rfs(b, l) _ob_append_rfs(__WHENCE__, b, l)
int ob_patch_rfs(struct overlay_buffer *b);
// get one byte, -ve number indicates failure
int ob_getbyte(struct overlay_buffer *b,int ofs);
int ob_peek(struct overlay_buffer *b);
void ob_skip(struct overlay_buffer *b, unsigned n);
// get one byte from the current position, -ve number indicates failure
int ob_get(struct overlay_buffer *b);
int ob_get_bytes(struct overlay_buffer *b, unsigned char *buff, int len);
@ -89,7 +105,6 @@ uint64_t ob_get_ui64(struct overlay_buffer *b);
uint32_t ob_get_ui32(struct overlay_buffer *b);
uint16_t ob_get_ui16(struct overlay_buffer *b);
int ob_dump(struct overlay_buffer *b,char *desc);
int ob_set_ui16(struct overlay_buffer *b, int offset, uint16_t v);
uint32_t ob_get_packed_ui32(struct overlay_buffer *b);
uint64_t ob_get_packed_ui64(struct overlay_buffer *b);
@ -98,5 +113,9 @@ uint64_t ob_get_packed_ui64(struct overlay_buffer *b);
int ob_position(struct overlay_buffer *b);
int ob_limit(struct overlay_buffer *b);
int ob_remaining(struct overlay_buffer *b);
int _ob_overrun(struct __sourceloc, struct overlay_buffer *b);
unsigned char* ob_ptr(struct overlay_buffer *b);
#endif
#define ob_overrun(b) _ob_overrun(__WHENCE__, b)
#endif //__SERVALD__OVERLAY_BUFFER_H

254
overlay_interface.c
View File

@ -19,18 +19,20 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <time.h>
#include <fnmatch.h>
#include "serval.h"
#include "conf.h"
#include "net.h"
#include "socket.h"
#include "strbuf.h"
#include "strbuf_helpers.h"
#include "overlay_buffer.h"
#include "overlay_packet.h"
#include "str.h"
#include "radio_link.h"
#ifdef HAVE_IFADDRS_H
#include <ifaddrs.h>
@ -49,7 +51,6 @@ struct profile_total sock_any_stats;
static void overlay_interface_poll(struct sched_ent *alarm);
static int re_init_socket(int interface_index);
static void write_stream_buffer(overlay_interface *interface);
static void
overlay_interface_close(overlay_interface *interface){
@ -59,6 +60,8 @@ overlay_interface_close(overlay_interface *interface){
unschedule(&interface->alarm);
unwatch(&interface->alarm);
close(interface->alarm.poll.fd);
if (interface->radio_link_state)
radio_link_free(interface);
interface->alarm.poll.fd=-1;
interface->state=INTERFACE_STATE_DOWN;
}
@ -76,8 +79,7 @@ void interface_state_html(struct strbuf *b, struct overlay_interface *interface)
switch(interface->type){
case OVERLAY_INTERFACE_PACKETRADIO:
strbuf_puts(b, "Type: Packet Radio<br>");
strbuf_sprintf(b, "RSSI: %ddB<br>",interface->radio_rssi);
strbuf_sprintf(b, "Remote RSSI: %ddB<br>",interface->remote_rssi);
radio_link_state_html(b, interface);
break;
case OVERLAY_INTERFACE_ETHERNET:
strbuf_puts(b, "Type: Ethernet<br>");
@ -248,37 +250,39 @@ int overlay_interface_compare(overlay_interface *one, overlay_interface *two)
// OSX doesn't recieve broadcast packets on sockets bound to an interface's address
// So we have to bind a socket to INADDR_ANY to receive these packets.
static void
overlay_interface_read_any(struct sched_ent *alarm){
overlay_interface_read_any(struct sched_ent *alarm)
{
if (alarm->poll.revents & POLLIN) {
int plen=0;
int recvttl=1;
unsigned char packet[16384];
overlay_interface *interface=NULL;
struct sockaddr src_addr;
socklen_t addrlen = sizeof(src_addr);
struct socket_address recvaddr;
recvaddr.addrlen = sizeof recvaddr.store;
/* Read only one UDP packet per call to share resources more fairly, and also
enable stats to accurately count packets received */
plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &src_addr, &addrlen);
plen = recvwithttl(alarm->poll.fd, packet, sizeof(packet), &recvttl, &recvaddr);
if (plen == -1) {
WHY_perror("recvwithttl(c)");
WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))",
alarm->poll.fd, packet, sizeof packet, recvttl,
&recvaddr, alloca_socket_address(&recvaddr)
);
unwatch(alarm);
close(alarm->poll.fd);
return;
}
struct in_addr src = ((struct sockaddr_in *)&src_addr)->sin_addr;
/* Try to identify the real interface that the packet arrived on */
interface = overlay_interface_find(src, 0);
interface = overlay_interface_find(recvaddr.inet.sin_addr, 0);
/* Drop the packet if we don't find a match */
if (!interface){
if (config.debug.overlayinterfaces)
DEBUGF("Could not find matching interface for packet received from %s", inet_ntoa(src));
DEBUGF("Could not find matching interface for packet received from %s", inet_ntoa(recvaddr.inet.sin_addr));
return;
}
packetOkOverlay(interface, packet, plen, recvttl, &src_addr, addrlen);
packetOkOverlay(interface, packet, plen, recvttl, &recvaddr);
}
if (alarm->poll.revents & (POLLHUP | POLLERR)) {
INFO("Closing broadcast socket due to error");
@ -411,8 +415,6 @@ overlay_interface_init(const char *name, struct in_addr src_addr, struct in_addr
set_destination_ref(&interface->destination, NULL);
interface->destination = new_destination(interface, ifconfig->encapsulation);
interface->throttle_bytes_per_second = ifconfig->throttle;
interface->throttle_burst_write_size = ifconfig->burst_size;
/* Pick a reasonable default MTU.
This will ultimately get tuned by the bandwidth and other properties of the interface */
interface->mtu = 1200;
@ -532,10 +534,7 @@ overlay_interface_init(const char *name, struct in_addr src_addr, struct in_addr
switch (ifconfig->socket_type) {
case SOCK_STREAM:
interface->slip_decode_state.dst_offset=0;
/* The encapsulation type should be configurable, but for now default to the one that should
be safe on the RFD900 radios, and that also allows us to receive RSSI reports inline */
interface->slip_decode_state.encapsulator=SLIP_FORMAT_MAVLINK;
radio_link_init(interface);
interface->alarm.poll.events=POLLIN|POLLOUT;
watch(&interface->alarm);
@ -574,25 +573,27 @@ cleanup:
return cleanup_ret;
}
static void interface_read_dgram(struct overlay_interface *interface){
static void interface_read_dgram(struct overlay_interface *interface)
{
int plen=0;
unsigned char packet[8096];
struct sockaddr src_addr;
socklen_t addrlen = sizeof(src_addr);
struct socket_address recvaddr;
recvaddr.addrlen = sizeof recvaddr.store;
/* Read only one UDP packet per call to share resources more fairly, and also
enable stats to accurately count packets received */
int recvttl=1;
plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &src_addr, &addrlen);
plen = recvwithttl(interface->alarm.poll.fd,packet, sizeof(packet), &recvttl, &recvaddr);
if (plen == -1) {
WHY_perror("recvwithttl(c)");
WHYF_perror("recvwithttl(%d,%p,%zu,&%d,%p(%s))",
interface->alarm.poll.fd, packet, sizeof packet, recvttl,
&recvaddr, alloca_socket_address(&recvaddr)
);
overlay_interface_close(interface);
return;
}
packetOkOverlay(interface, packet, plen, recvttl, &src_addr, addrlen);
packetOkOverlay(interface, packet, plen, recvttl, &recvaddr);
}
struct file_packet{
@ -658,9 +659,6 @@ static void interface_read_file(struct overlay_interface *interface)
return;
}
if (config.debug.overlayinterfaces)
DEBUGF("Read interface %s (size=%"PRId64") at offset=%d",interface->name, (int64_t)length, interface->recv_offset);
ssize_t nread = read(interface->alarm.poll.fd, &packet, sizeof packet);
if (nread == -1){
WHY_perror("read");
@ -669,14 +667,27 @@ static void interface_read_file(struct overlay_interface *interface)
}
if (nread == sizeof packet) {
if (config.debug.overlayinterfaces)
DEBUGF("Read from interface %s (filesize=%"PRId64") at offset=%d: src_addr=%s dst_addr=%s pid=%d length=%d",
interface->name, (int64_t)length, interface->recv_offset,
alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr),
alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr),
packet.pid,
packet.payload_length
);
interface->recv_offset += nread;
if (should_drop(interface, packet.dst_addr) || (packet.pid == getpid() && !interface->local_echo)){
if (config.debug.packetrx)
DEBUGF("Ignoring packet from %d, addressed to %s:%d", packet.pid,
inet_ntoa(packet.dst_addr.sin_addr), ntohs(packet.dst_addr.sin_port));
DEBUGF("Ignoring packet from pid=%d src_addr=%s dst_addr=%s",
packet.pid,
alloca_sockaddr_in(&packet.src_addr),
alloca_sockaddr_in(&packet.dst_addr)
);
}else{
packetOkOverlay(interface, packet.payload, packet.payload_length, -1,
(struct sockaddr*)&packet.src_addr, (socklen_t) sizeof(packet.src_addr));
struct socket_address srcaddr;
srcaddr.addrlen = sizeof packet.src_addr;
srcaddr.inet = packet.src_addr;
packetOkOverlay(interface, packet.payload, packet.payload_length, -1, &srcaddr);
}
}
}
@ -710,101 +721,15 @@ static void interface_read_stream(struct overlay_interface *interface){
OUT();
return;
}
struct slip_decode_state *state=&interface->slip_decode_state;
int i;
for (i=0;i<nread;i++)
mavlink_decode(interface, state, buffer[i]);
radio_link_decode(interface, buffer[i]);
OUT();
}
static void write_stream_buffer(overlay_interface *interface){
time_ms_t now = gettime_ms();
// Throttle output to a prescribed bit-rate
// first, reduce the number of bytes based on the configured burst size
int bytes_allowed=interface->throttle_burst_write_size;
int total_written=0;
while (interface->tx_bytes_pending>0 || interface->tx_packet || interface->next_heartbeat <= now) {
if (interface->tx_bytes_pending==0){
if (interface->next_heartbeat <= now){
// Queue a hearbeat now
mavlink_heartbeat(interface->txbuffer,&interface->tx_bytes_pending);
if (config.debug.packetradio)
DEBUGF("Sending heartbeat");
interface->next_heartbeat = now+1000;
}else if(interface->tx_packet && interface->remaining_space >= 256 + 8+9){
// prepare a new link layer packet in txbuffer
if (mavlink_encode_packet(interface))
break;
if (interface->remaining_space - interface->tx_bytes_pending < 256 + 8+9)
interface->next_heartbeat = now;
}
}
if (interface->next_tx_allowed > now)
break;
int bytes = interface->tx_bytes_pending;
if (interface->throttle_burst_write_size && bytes>bytes_allowed)
bytes=bytes_allowed;
if (bytes<=0)
break;
int written=write(interface->alarm.poll.fd, interface->txbuffer, bytes);
if (written<=0){
DEBUGF("Blocking for POLLOUT");
break;
}
interface->remaining_space-=written;
interface->tx_bytes_pending-=written;
total_written+=written;
bytes_allowed-=written;
if (interface->tx_bytes_pending){
bcopy(&interface->txbuffer[written],&interface->txbuffer[0],
interface->tx_bytes_pending);
DEBUGF("Partial write, %d left", interface->tx_bytes_pending);
}
}
if (total_written>0){
// Now when are we allowed to send more?
int rate = interface->throttle_bytes_per_second;
if (interface->remaining_space<=0)
rate = 600;
if (rate){
int delay = total_written*1000/rate;
if (config.debug.throttling)
DEBUGF("Throttling for %dms (%d).", delay, interface->remaining_space);
interface->next_tx_allowed = now + delay;
}
}
time_ms_t next_write = interface->next_tx_allowed;
if (interface->tx_bytes_pending<=0){
next_write = interface->next_heartbeat;
}
if (interface->alarm.alarm==-1 || next_write < interface->alarm.alarm){
interface->alarm.alarm = next_write;
interface->alarm.deadline = interface->alarm.alarm+10;
}
if (interface->tx_bytes_pending>0 && next_write <= now){
// more to write, so set the POLLOUT flag
interface->alarm.poll.events|=POLLOUT;
} else {
// Nothing to write, so clear POLLOUT flag
interface->alarm.poll.events&=~POLLOUT;
}
watch(&interface->alarm);
}
static void overlay_interface_poll(struct sched_ent *alarm)
{
struct overlay_interface *interface = (overlay_interface *)alarm;
@ -816,7 +741,7 @@ static void overlay_interface_poll(struct sched_ent *alarm)
if (interface->state==INTERFACE_STATE_UP
&& interface->destination->tick_ms>0
&& interface->send_broadcasts
&& !interface->tx_packet){
&& !radio_link_is_busy(interface)){
if (now >= interface->destination->last_tx+interface->destination->tick_ms)
overlay_send_tick_packet(interface->destination);
@ -827,8 +752,8 @@ static void overlay_interface_poll(struct sched_ent *alarm)
switch(interface->socket_type){
case SOCK_STREAM:
write_stream_buffer(interface);
break;
radio_link_tx(interface);
return;
case SOCK_DGRAM:
break;
case SOCK_FILE:
@ -848,14 +773,8 @@ static void overlay_interface_poll(struct sched_ent *alarm)
if (alarm->poll.revents & POLLOUT){
switch(interface->socket_type){
case SOCK_STREAM:
write_stream_buffer(interface);
if (alarm->alarm!=-1 && interface->state==INTERFACE_STATE_UP) {
if (alarm->alarm < now)
alarm->alarm = now;
unschedule(alarm);
schedule(alarm);
}
break;
radio_link_tx(interface);
return;
case SOCK_DGRAM:
case SOCK_FILE:
//XXX error? fatal?
@ -871,14 +790,9 @@ static void overlay_interface_poll(struct sched_ent *alarm)
case SOCK_STREAM:
interface_read_stream(interface);
// if we read a valid heartbeat packet, we may be able to write more bytes now.
if (interface->state==INTERFACE_STATE_UP && interface->remaining_space>0){
write_stream_buffer(interface);
if (alarm->alarm!=-1 && interface->state==INTERFACE_STATE_UP) {
if (alarm->alarm < now)
alarm->alarm = now;
unschedule(alarm);
schedule(alarm);
}
if (interface->state==INTERFACE_STATE_UP){
radio_link_tx(interface);
return;
}
break;
case SOCK_FILE:
@ -918,24 +832,7 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
switch(interface->socket_type){
case SOCK_STREAM:
{
if (interface->tx_packet){
ob_free(buffer);
return WHYF("Cannot send two packets to a stream at the same time");
}
// prepare the buffer for reading
ob_flip(buffer);
interface->tx_packet = buffer;
write_stream_buffer(interface);
if (interface->alarm.alarm!=-1){
unschedule(&interface->alarm);
schedule(&interface->alarm);
}
return 0;
}
return radio_link_queue_packet(interface, buffer);
case SOCK_FILE:
{
@ -962,9 +859,21 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
not support seeking. */
if (errno != ESPIPE)
return WHY_perror("lseek");
DEBUGF("Write to interface %s at unknown offset", interface->name);
DEBUGF("Write to interface %s at offset unknown: src_addr=%s dst_addr=%s pid=%d length=%d",
interface->name,
alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr),
alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr),
packet.pid,
packet.payload_length
);
} else
DEBUGF("Write to interface %s at offset=%"PRId64, interface->name, (int64_t)fsize);
DEBUGF("Write to interface %s at offset=%"PRId64": src_addr=%s dst_addr=%s pid=%d length=%d",
interface->name, (int64_t)fsize,
alloca_sockaddr(&packet.src_addr, sizeof packet.src_addr),
alloca_sockaddr(&packet.dst_addr, sizeof packet.dst_addr),
packet.pid,
packet.payload_length
);
}
ssize_t nwrite = write(interface->alarm.poll.fd, &packet, sizeof(packet));
if (nwrite == -1)
@ -977,13 +886,22 @@ int overlay_broadcast_ensemble(struct network_destination *destination, struct o
case SOCK_DGRAM:
{
if (config.debug.overlayinterfaces)
DEBUGF("Sending %d byte overlay frame on %s to %s",len,interface->name,inet_ntoa(destination->address.sin_addr));
int sent=sendto(interface->alarm.poll.fd,
bytes, len, 0,
DEBUGF("Sending %zu byte overlay frame on %s to %s", (size_t)len, interface->name, inet_ntoa(destination->address.sin_addr));
ssize_t sent = sendto(interface->alarm.poll.fd,
bytes, (size_t)len, 0,
(struct sockaddr *)&destination->address, sizeof(destination->address));
ob_free(buffer);
if (sent!= len){
WHY_perror("sendto(c)");
if (sent == -1 || (size_t)sent != (size_t)len) {
if (sent == -1)
WHYF_perror("sendto(fd=%d,len=%zu,addr=%s) on interface %s",
interface->alarm.poll.fd,
(size_t)len,
alloca_sockaddr((struct sockaddr *)&destination->address, sizeof destination->address),
interface->name
);
else
WHYF("sendto() sent %zu bytes of overlay frame (%zu) to interface %s (socket=%d)",
(size_t)sent, (size_t)len, interface->name, interface->alarm.poll.fd);
// close the interface if we had any error while sending broadcast packets,
// unicast packets should not bring the interface down
if (destination == interface->destination)

55
overlay_link.c
View File

@ -1,9 +1,11 @@
#include <assert.h>
#include "serval.h"
#include "conf.h"
#include "str.h"
#include "overlay_address.h"
#include "overlay_buffer.h"
#include "overlay_packet.h"
#include "keyring.h"
#define MIN_BURST_LENGTH 5000
@ -221,14 +223,14 @@ int overlay_send_probe(struct subscriber *peer, struct network_destination *dest
frame->ttl=1;
frame->queue=queue;
frame->destinations[frame->destination_count++].destination=add_destination_ref(destination);
frame->payload = ob_new();
frame->source_full = 1;
// TODO call mdp payload encryption / signing without calling overlay_mdp_dispatch...
if (overlay_mdp_encode_ports(frame->payload, MDP_PORT_ECHO, MDP_PORT_PROBE)){
if ((frame->payload = ob_new()) == NULL) {
op_free(frame);
return -1;
}
frame->source_full = 1;
// TODO call mdp payload encryption / signing without calling overlay_mdp_dispatch...
overlay_mdp_encode_ports(frame->payload, MDP_PORT_ECHO, MDP_PORT_PROBE);
// not worried about byte order here as we are the only node that should be parsing the contents.
unsigned char *dst=ob_append_space(frame->payload, sizeof(struct probe_contents));
if (!dst){
@ -253,25 +255,21 @@ int overlay_send_probe(struct subscriber *peer, struct network_destination *dest
}
// append the address of a unicast link into a packet buffer
static int overlay_append_unicast_address(struct subscriber *subscriber, struct overlay_buffer *buff)
static void overlay_append_unicast_address(struct subscriber *subscriber, struct overlay_buffer *buff)
{
if (subscriber->destination
&& subscriber->destination->unicast
&& subscriber->destination->address.sin_family==AF_INET){
if (overlay_address_append(NULL, buff, subscriber))
return -1;
if (ob_append_ui32(buff, subscriber->destination->address.sin_addr.s_addr))
return -1;
if (ob_append_ui16(buff, subscriber->destination->address.sin_port))
return -1;
ob_checkpoint(buff);
if ( subscriber->destination
&& subscriber->destination->unicast
&& subscriber->destination->address.sin_family==AF_INET
) {
overlay_address_append(NULL, buff, subscriber);
ob_append_ui32(buff, subscriber->destination->address.sin_addr.s_addr);
ob_append_ui16(buff, subscriber->destination->address.sin_port);
if (config.debug.overlayrouting)
DEBUGF("Added STUN info for %s", alloca_tohex_sid_t(subscriber->sid));
}else{
if (config.debug.overlayrouting)
DEBUGF("Unable to give address of %s, %d", alloca_tohex_sid_t(subscriber->sid),subscriber->reachable);
}
return 0;
}
int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp)
@ -295,29 +293,27 @@ int overlay_mdp_service_stun_req(overlay_mdp_frame *mdp)
struct overlay_buffer *replypayload = ob_static(reply.out.payload, sizeof(reply.out.payload));
ob_checkpoint(replypayload);
while(ob_remaining(payload)>0){
while (ob_remaining(payload) > 0) {
struct subscriber *subscriber=NULL;
if (overlay_address_parse(NULL, payload, &subscriber))
break;
if (!subscriber){
if (config.debug.overlayrouting)
DEBUGF("Unknown subscriber");
continue;
}
if (overlay_append_unicast_address(subscriber, replypayload))
overlay_append_unicast_address(subscriber, replypayload);
if (ob_overrun(payload))
break;
ob_checkpoint(replypayload);
}
ob_rewind(replypayload);
reply.out.payload_length=ob_position(replypayload);
if (reply.out.payload_length){
if (config.debug.overlayrouting)
DEBUGF("Sending reply");
overlay_mdp_dispatch(&reply,0 /* system generated */, NULL,0);
overlay_mdp_dispatch(&reply, NULL);
}
ob_free(replypayload);
ob_free(payload);
@ -387,11 +383,12 @@ int overlay_send_stun_request(struct subscriber *server, struct subscriber *requ
struct overlay_buffer *payload = ob_static(mdp.out.payload, sizeof(mdp.out.payload));
overlay_address_append(NULL, payload, request);
mdp.out.payload_length=ob_position(payload);
if (config.debug.overlayrouting)
DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid));
overlay_mdp_dispatch(&mdp,0 /* system generated */,
NULL,0);
if (!ob_overrun(payload)) {
mdp.out.payload_length=ob_position(payload);
if (config.debug.overlayrouting)
DEBUGF("Sending STUN request to %s", alloca_tohex_sid_t(server->sid));
overlay_mdp_dispatch(&mdp, NULL);
}
ob_free(payload);
return 0;
}

950
overlay_mdp.c
View File

File diff suppressed because it is too large Load Diff

34
overlay_mdp_services.c
View File

@ -28,6 +28,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "rhizome.h"
#include "crypto.h"
#include "log.h"
#include "keyring.h"
#include "dataformats.h"
int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, uint64_t version, uint64_t fileOffset, uint32_t bitmap, uint16_t blockLength)
{
@ -98,7 +100,7 @@ int rhizome_mdp_send_block(struct subscriber *dest, const rhizome_bid_t *bid, ui
reply.out.payload[0]='T';
// send packet
if (overlay_mdp_dispatch(&reply,0 /* system generated */, NULL,0))
if (overlay_mdp_dispatch(&reply, NULL))
break;
}
@ -137,11 +139,11 @@ int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp)
unsigned char *bidprefix=&mdp->out.payload[1];
uint64_t version=read_uint64(&mdp->out.payload[1+16]);
uint64_t offset=read_uint64(&mdp->out.payload[1+16+8]);
int count=mdp->out.payload_length-(1+16+8+8);
size_t count = mdp->out.payload_length-(1+16+8+8);
unsigned char *bytes=&mdp->out.payload[1+16+8+8];
if (config.debug.rhizome_mdp_rx)
DEBUGF("bidprefix=%02x%02x%02x%02x*, offset=%"PRId64", count=%d",
DEBUGF("bidprefix=%02x%02x%02x%02x*, offset=%"PRId64", count=%zu",
bidprefix[0],bidprefix[1],bidprefix[2],bidprefix[3],offset,count);
/* Now see if there is a slot that matches. If so, then
@ -151,7 +153,7 @@ int overlay_mdp_service_rhizomeresponse(overlay_mdp_frame *mdp)
a slot to capture this files as it is being requested
by someone else.
*/
rhizome_received_content(bidprefix,version,offset,count,bytes,type);
rhizome_received_content(bidprefix,version,offset, count, bytes, type);
RETURN(0);
}
@ -251,8 +253,7 @@ int overlay_mdp_service_echo(overlay_mdp_frame *mdp)
mdp->packetTypeAndFlags&=~(MDP_NOCRYPT|MDP_NOSIGN);
/* queue frame for delivery */
overlay_mdp_dispatch(mdp,0 /* system generated */,
NULL,0);
overlay_mdp_dispatch(mdp, NULL);
mdp->packetTypeAndFlags=preserved;
/* and switch addresses back around in case the caller was planning on
@ -333,34 +334,33 @@ static int overlay_mdp_service_trace(overlay_mdp_frame *mdp){
ob_unlimitsize(b);
// always write a full sid into the payload
my_subscriber->send_full=1;
if (overlay_address_append(&context, b, my_subscriber)){
overlay_address_append(&context, b, my_subscriber);
if (ob_overrun(b)) {
ret = WHYF("Unable to append my address to the trace");
goto end;
}
mdp->out.payload_length = ob_position(b);
mdp->out.src.sid = my_subscriber->sid;
mdp->out.dst.sid = next->sid;
ret = overlay_mdp_dispatch(mdp, 0, NULL, 0);
ret = overlay_mdp_dispatch(mdp, NULL);
end:
ob_free(b);
RETURN(ret);
}
static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, overlay_mdp_frame *mdp)
static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, const uint8_t *payload, size_t len)
{
int offset=0;
while (offset<mdp->out.payload_length) {
while (offset<len) {
rhizome_manifest *m = rhizome_new_manifest();
if (!m)
return WHY("Unable to allocate manifest");
unsigned char *bar = &mdp->out.payload[offset];
const unsigned char *bar = &payload[offset];
if (!rhizome_retrieve_manifest_by_prefix(&bar[RHIZOME_BAR_PREFIX_OFFSET], RHIZOME_BAR_PREFIX_BYTES, m)){
rhizome_advertise_manifest(frame->source, m);
// pre-emptively send the payload if it will fit in a single packet
if (m->fileLength > 0 && m->fileLength <= 1024)
rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->fileLength);
if (m->filesize > 0 && m->filesize <= 1024)
rhizome_mdp_send_block(frame->source, &m->cryptoSignPublic, m->version, 0, 0, m->filesize);
}
rhizome_manifest_free(m);
offset+=RHIZOME_BAR_BYTES;
@ -368,7 +368,7 @@ static int overlay_mdp_service_manifest_requests(struct overlay_frame *frame, ov
return 0;
}
int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp)
int overlay_mdp_try_internal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp)
{
IN();
switch(mdp->out.dst.port) {
@ -383,7 +383,7 @@ int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_fr
case MDP_PORT_STUN: RETURN(overlay_mdp_service_stun(mdp));
case MDP_PORT_RHIZOME_REQUEST: RETURN(overlay_mdp_service_rhizomerequest(frame, mdp));
case MDP_PORT_RHIZOME_RESPONSE: RETURN(overlay_mdp_service_rhizomeresponse(mdp));
case MDP_PORT_RHIZOME_MANIFEST_REQUEST: RETURN(overlay_mdp_service_manifest_requests(frame, mdp));
case MDP_PORT_RHIZOME_MANIFEST_REQUEST: RETURN(overlay_mdp_service_manifest_requests(frame, mdp->out.payload, mdp->out.payload_length));
case MDP_PORT_RHIZOME_SYNC: RETURN(overlay_mdp_service_rhizome_sync(frame, mdp));
}

3
overlay_olsr.c
View File

@ -252,6 +252,8 @@ int olsr_send(struct overlay_frame *frame){
struct decode_context context;
bzero(&context, sizeof context);
struct overlay_buffer *b=ob_new();
if (b == NULL)
return 0;
// build olsr specific frame header
ob_append_byte(b, PACKET_FORMAT_NUMBER);
@ -259,7 +261,6 @@ int olsr_send(struct overlay_frame *frame){
// address the packet as transmitted by me
overlay_address_append(&context, b, my_subscriber);
overlay_address_append(&context, b, frame->source);
overlay_broadcast_append(b, &frame->broadcast_id);
ob_append_byte(b, frame->modifiers);

29
overlay_packetformats.c
View File

@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "serval.h"
#include "conf.h"
#include "socket.h"
#include "str.h"
#include "strbuf.h"
#include "overlay_buffer.h"
@ -42,20 +43,16 @@ int overlay_packet_init_header(int packet_version, int encapsulation,
if (encapsulation !=ENCAP_OVERLAY && encapsulation !=ENCAP_SINGLE)
return WHY("Invalid packet encapsulation");
if (ob_append_byte(buff, packet_version))
return -1;
if (ob_append_byte(buff, encapsulation))
return -1;
ob_append_byte(buff, packet_version);
ob_append_byte(buff, encapsulation);
if (context->interface->point_to_point
&& context->interface->other_device
&& packet_version>=1)
if ( context->interface->point_to_point
&& context->interface->other_device
&& packet_version>=1
)
context->point_to_point_device = context->interface->other_device;
context->encoding_header=1;
if (overlay_address_append(context, buff, my_subscriber))
return -1;
overlay_address_append(context, buff, my_subscriber);
context->encoding_header=0;
context->sender = my_subscriber;
@ -324,7 +321,7 @@ int parseEnvelopeHeader(struct decode_context *context, struct overlay_interface
}
int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, size_t len,
int recvttl, struct sockaddr *recvaddr, socklen_t recvaddrlen)
int recvttl, struct socket_address *recvaddr)
{
IN();
/*
@ -389,8 +386,8 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
}
}
if (recvaddr&&recvaddr->sa_family!=AF_INET)
RETURN(WHYF("Unexpected protocol family %d",recvaddr->sa_family));
if (recvaddr && recvaddr->addr.sa_family != AF_INET)
RETURN(WHYF("Unexpected protocol family %d", recvaddr->addr.sa_family));
struct overlay_frame f;
struct decode_context context;
@ -403,11 +400,11 @@ int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, s
f.interface = interface;
if (recvaddr)
f.recvaddr = *((struct sockaddr_in *)recvaddr);
f.recvaddr = recvaddr->inet;
else
bzero(&f.recvaddr, sizeof f.recvaddr);
int ret=parseEnvelopeHeader(&context, interface, (struct sockaddr_in *)recvaddr, b);
int ret=parseEnvelopeHeader(&context, interface, recvaddr ? &recvaddr->inet : NULL, b);
if (ret){
ob_free(b);
RETURN(ret);

66
overlay_payload.c
View File

@ -51,34 +51,28 @@ static int overlay_frame_build_header(int packet_version, struct decode_context
if (type!=OF_TYPE_DATA)
flags |= PAYLOAD_FLAG_LEGACY_TYPE;
if (ob_append_byte(buff, flags)) return -1;
ob_append_byte(buff, flags);
if (!(flags & PAYLOAD_FLAG_SENDER_SAME)){
if (overlay_address_append(context, buff, source)) return -1;
}
if (!(flags & PAYLOAD_FLAG_SENDER_SAME))
overlay_address_append(context, buff, source);
if (flags & PAYLOAD_FLAG_TO_BROADCAST){
if (!(flags & PAYLOAD_FLAG_ONE_HOP)){
if (overlay_broadcast_append(buff, broadcast)) return -1;
}
}else{
if (overlay_address_append(context, buff, destination)) return -1;
if (!(flags & PAYLOAD_FLAG_ONE_HOP)){
if (overlay_address_append(context, buff, next_hop)) return -1;
}
if (!(flags & PAYLOAD_FLAG_ONE_HOP))
overlay_broadcast_append(buff, broadcast);
} else {
overlay_address_append(context, buff, destination);
if (!(flags & PAYLOAD_FLAG_ONE_HOP))
overlay_address_append(context, buff, next_hop);
}
if (!(flags & PAYLOAD_FLAG_ONE_HOP)){
if (ob_append_byte(buff, ttl | ((queue&3)<<5))) return -1;
}
if (!(flags & PAYLOAD_FLAG_ONE_HOP))
ob_append_byte(buff, ttl | ((queue&3)<<5));
if (flags & PAYLOAD_FLAG_LEGACY_TYPE){
if (ob_append_byte(buff, type)) return -1;
}
if (flags & PAYLOAD_FLAG_LEGACY_TYPE)
ob_append_byte(buff, type);
if (packet_version >= 1)
if (ob_append_byte(buff, sequence))
return -1;
ob_append_byte(buff, sequence);
return 0;
}
@ -112,20 +106,17 @@ int overlay_frame_append_payload(struct decode_context *context, int encapsulati
p->queue, p->type, p->modifiers, will_retransmit,
p->ttl, p->mdp_sequence&0xFF,
broadcast, p->next_hop,
p->destination, p->source))
p->destination, p->source) == -1)
goto cleanup;
if (encapsulation == ENCAP_OVERLAY){
if (ob_append_ui16(b, ob_position(p->payload)))
goto cleanup;
}
if (encapsulation == ENCAP_OVERLAY)
ob_append_ui16(b, ob_position(p->payload));
if (ob_append_bytes(b, ob_ptr(p->payload), ob_position(p->payload))) {
WHYF("could not append payload of %u bytes", ob_position(p->payload));
goto cleanup;
}
return 0;
if (ob_position(p->payload))
ob_append_bytes(b, ob_ptr(p->payload), ob_position(p->payload));
if (!ob_overrun(b))
return 0;
cleanup:
ob_rewind(b);
@ -150,13 +141,18 @@ struct overlay_frame *op_dup(struct overlay_frame *in)
if (!in) return NULL;
/* clone the frame */
struct overlay_frame *out=malloc(sizeof(struct overlay_frame));
if (!out) { WHY("malloc() failed"); return NULL; }
struct overlay_frame *out = emalloc(sizeof(struct overlay_frame));
if (out == NULL)
return NULL;
/* copy main data structure */
bcopy(in,out,sizeof(struct overlay_frame));
if (in->payload)
out->payload=ob_dup(in->payload);
if (in->payload) {
if ((out->payload = ob_dup(in->payload)) == NULL) {
free(out);
return NULL;
}
}
return out;
}

79
overlay_queue.c
View File

@ -17,10 +17,12 @@
*/
#include <assert.h>
#include "serval.h"
#include "conf.h"
#include "overlay_buffer.h"
#include "overlay_packet.h"
#include "radio_link.h"
#include "str.h"
#include "strbuf.h"
@ -150,30 +152,25 @@ int overlay_payload_enqueue(struct overlay_frame *p)
Complain if there are too many frames in the queue.
*/
if (!p) return WHY("Cannot queue NULL");
if (p->queue>=OQ_MAX)
return WHY("Invalid queue specified");
assert(p != NULL);
assert(p->queue < OQ_MAX);
assert(p->payload != NULL);
overlay_txqueue *queue = &overlay_tx[p->queue];
if (config.debug.packettx)
DEBUGF("Enqueuing packet for %s* (q[%d]length = %d)",
DEBUGF("Enqueuing packet for %s* (q[%d].length = %d)",
p->destination?alloca_tohex_sid_t_trunc(p->destination->sid, 14): alloca_tohex(p->broadcast_id.id, BROADCAST_LEN),
p->queue, queue->length);
if (p->payload && ob_remaining(p->payload)<0){
// HACK, maybe should be done in each caller
// set the size of the payload based on the position written
ob_limitsize(p->payload,ob_position(p->payload));
}
if (ob_overrun(p->payload))
return WHY("Packet content overrun -- not queueing");
if (ob_position(p->payload) >= MDP_MTU)
FATAL("Queued packet is too big");
if (queue->length>=queue->maxLength)
return WHYF("Queue #%d congested (size = %d)",p->queue,queue->maxLength);
if (ob_position(p->payload)>=MDP_MTU)
FATAL("Queued packet is too big");
// it should be safe to try sending all packets with an mdp sequence
if (p->packet_version<=0)
p->packet_version=1;
@ -226,11 +223,13 @@ int overlay_payload_enqueue(struct overlay_frame *p)
return 0;
}
static void
static int
overlay_init_packet(struct outgoing_packet *packet, int packet_version,
struct network_destination *destination){
struct network_destination *destination)
{
packet->context.interface = destination->interface;
packet->buffer=ob_new();
if ((packet->buffer = ob_new()) == NULL)
return -1;
packet->packet_version = packet_version;
packet->context.packet_version = packet_version;
packet->destination = add_destination_ref(destination);
@ -238,20 +237,24 @@ overlay_init_packet(struct outgoing_packet *packet, int packet_version,
packet->seq=-1;
else
packet->seq = destination->sequence_number = (destination->sequence_number + 1) & 0xFFFF;
ob_limitsize(packet->buffer, destination->interface->mtu);
int i=destination->interface - overlay_interfaces;
overlay_packet_init_header(packet_version, destination->encapsulation,
&packet->context, packet->buffer,
destination->unicast,
i, packet->seq);
int i = destination->interface - overlay_interfaces;
if (overlay_packet_init_header(packet_version, destination->encapsulation,
&packet->context, packet->buffer,
destination->unicast,
i, packet->seq) == -1
) {
ob_free(packet->buffer);
packet->buffer = NULL;
return -1;
}
packet->header_length = ob_position(packet->buffer);
if (config.debug.overlayframes)
DEBUGF("Creating %d packet for interface %s, seq %d, %s",
packet_version,
destination->interface->name, destination->sequence_number,
destination->unicast?"unicast":"broadcast");
return 0;
}
int overlay_queue_schedule_next(time_ms_t next_allowed_packet){
@ -288,7 +291,7 @@ overlay_calc_queue_time(overlay_txqueue *queue, struct overlay_frame *frame){
int i;
for(i=0;i<frame->destination_count;i++)
{
if (frame->destinations[i].destination->interface->tx_packet)
if (radio_link_is_busy(frame->destinations[i].destination->interface))
continue;
time_ms_t next_packet = limit_next_allowed(&frame->destinations[i].destination->transfer_limit);
if (frame->destinations[i].transmit_time){
@ -331,8 +334,9 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
while(frame){
if (frame->enqueued_at + queue->latencyTarget < now){
if (config.debug.overlayframes)
DEBUGF("Dropping frame type %x for %s due to expiry timeout",
frame->type, frame->destination?alloca_tohex_sid_t(frame->destination->sid):"All");
DEBUGF("Dropping frame type %x (length %d) for %s due to expiry timeout",
frame->type, frame->payload->checkpointLength,
frame->destination?alloca_tohex_sid_t(frame->destination->sid):"All");
frame = overlay_queue_remove(queue, frame);
continue;
}
@ -400,8 +404,7 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
}
}else{
// skip this interface if the stream tx buffer has data
if (dest->interface->socket_type==SOCK_STREAM
&& dest->interface->tx_packet)
if (radio_link_is_busy(dest->interface))
continue;
// can we send a packet on this interface now?
@ -411,11 +414,11 @@ overlay_stuff_packet(struct outgoing_packet *packet, overlay_txqueue *queue, tim
// send a packet to this destination
if (frame->source_full)
my_subscriber->send_full=1;
overlay_init_packet(packet, frame->packet_version, dest);
destination_index=i;
frame->destinations[i].sent_sequence = dest->sequence_number;
break;
if (overlay_init_packet(packet, frame->packet_version, dest) != -1) {
destination_index=i;
frame->destinations[i].sent_sequence = dest->sequence_number;
break;
}
}
}
}
@ -532,12 +535,12 @@ static void overlay_send_packet(struct sched_ent *alarm){
overlay_fill_send_packet(&packet, gettime_ms());
}
int overlay_send_tick_packet(struct network_destination *destination){
int overlay_send_tick_packet(struct network_destination *destination)
{
struct outgoing_packet packet;
bzero(&packet, sizeof(struct outgoing_packet));
overlay_init_packet(&packet, 0, destination);
overlay_fill_send_packet(&packet, gettime_ms());
if (overlay_init_packet(&packet, 0, destination) != -1)
overlay_fill_send_packet(&packet, gettime_ms());
return 0;
}

26
performance_timing.c
View File

@ -149,17 +149,8 @@ int fd_showstats()
stats = stats->_next;
}
// Show periodic rhizome transfer information, but only
// if there are some active rhizome transfers.
if (rhizome_active_fetch_count()!=0)
INFOF("Rhizome transfer progress: %d,%d,%d,%d,%d,%d (remaining %d)",
rhizome_active_fetch_bytes_received(0),
rhizome_active_fetch_bytes_received(1),
rhizome_active_fetch_bytes_received(2),
rhizome_active_fetch_bytes_received(3),
rhizome_active_fetch_bytes_received(4),
rhizome_active_fetch_bytes_received(5),
rhizome_fetch_queue_bytes());
// Show periodic rhizome transfer information
rhizome_fetch_log_short_status();
// Report any functions that take too much time
if (!config.debug.timing)
@ -168,8 +159,8 @@ int fd_showstats()
while(stats!=NULL){
/* If a function spends more than 1 second in any
notionally 3 second period, then dob on it */
if (stats->total_time>1000
&&strcmp(stats->name,"Idle (in poll)"))
if ((stats->total_time>1000 || stats->calls > 10000)
&& strcmp(stats->name,"Idle (in poll)"))
fd_showstat(&total,stats);
stats = stats->_next;
}
@ -208,6 +199,15 @@ void dump_stack(int log_level)
}
}
unsigned fd_depth()
{
unsigned depth = 0;
struct call_stats *call;
for (call = current_call; call; call = call->prev)
++depth;
return depth;
}
int fd_func_enter(struct __sourceloc __whence, struct call_stats *this_call)
{
if (config.debug.profiling)

546
radio_link.c Normal file
View File

@ -0,0 +1,546 @@
// -*- Mode: C; c-basic-offset: 2; -*-
//
// Copyright (c) 2012 Andrew Tridgell, All Rights Reserved
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
// o Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// o Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in
// the documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
// INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
// OF THE POSSIBILITY OF SUCH DAMAGE.
//
/*
Portions Copyright (C) 2013 Paul Gardner-Stephen
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "serval.h"
#include "conf.h"
#include "overlay_buffer.h"
#include "golay.h"
#include "radio_link.h"
#define MAVLINK_MSG_ID_RADIO 166
#define MAVLINK_MSG_ID_DATASTREAM 67
// use '3D' for 3DRadio
#define RADIO_SOURCE_SYSTEM '3'
#define RADIO_SOURCE_COMPONENT 'D'
/*
we use a hand-crafted MAVLink packet based on the following
message definition
struct mavlink_RADIO_v10 {
uint16_t rxerrors; // receive errors
uint16_t fixed; // count of error corrected packets
uint8_t rssi; // local signal strength
uint8_t remrssi; // remote signal strength
uint8_t txbuf; // percentage free space in transmit buffer
uint8_t noise; // background noise level
uint8_t remnoise; // remote background noise level
};
*/
#define FEC_LENGTH 32
#define FEC_MAX_BYTES 223
#define RADIO_HEADER_LENGTH 6
#define RADIO_USED_HEADER_LENGTH 4
#define RADIO_CRC_LENGTH 2
#define LINK_PAYLOAD_MTU (LINK_MTU - FEC_LENGTH - RADIO_HEADER_LENGTH - RADIO_CRC_LENGTH)
struct radio_link_state{
// next seq for transmission
int tx_seq;
// small buffer for parsing incoming bytes from the serial interface,
// looking for recoverable link layer packets
// should be large enough to hold at least one packet from the remote end
// plus one heartbeat packet from the local firmware
uint8_t payload[LINK_MTU*3];
// decoded length of next link layer packet
// including all header and footer bytes
int payload_length;
// last rx seq for reassembly
int seq;
// offset within payload that we have found a valid looking header
int payload_start;
// offset after payload_start for incoming bytes
int payload_offset;
// small buffer for assembling mdp payloads.
uint8_t dst[MDP_MTU];
// length of recovered packet
int packet_length;
// next firmware heartbeat
time_ms_t next_heartbeat;
time_ms_t last_packet;
// parsed rssi
int radio_rssi;
int remote_rssi;
// estimated firmware buffer space
int32_t remaining_space;
// next serial write
uint64_t next_tx_allowed;
// partially sent packet
struct overlay_buffer *tx_packet;
// serial write buffer
uint8_t txbuffer[LINK_MTU];
int tx_bytes;
int tx_pos;
};
/*
Each mavlink frame consists of 0xfe followed by a standard 6 byte header.
Normally the payload plus a 2-byte CRC follows.
We are replacing the CRC check with a Reed-Solomon code to correct as well
as detect upto 16 bytes with errors, in return for a 32-byte overhead.
The nature of the particular library we are using is that the overhead is
basically fixed, but we can shorten the data section.
Note that the mavlink headers are not protected against errors. This is a
limitation of the radio firmware at present. One day we will re-write the
radio firmware so that we can send and receive raw radio frames, and get
rid of the mavlink framing altogether, and just send R-S protected payloads.
Not ideal, but will be fine for now.
*/
#include "fec-3.0.1/fixed.h"
void encode_rs_8(data_t *data, data_t *parity,int pad);
int decode_rs_8(data_t *data, int *eras_pos, int no_eras, int pad);
int radio_link_free(struct overlay_interface *interface)
{
if (interface->radio_link_state){
free(interface->radio_link_state);
interface->radio_link_state=NULL;
}
return 0;
}
int radio_link_init(struct overlay_interface *interface)
{
interface->radio_link_state = emalloc_zero(sizeof(struct radio_link_state));
return 0;
}
void radio_link_state_html(struct strbuf *b, struct overlay_interface *interface)
{
struct radio_link_state *state = interface->radio_link_state;
strbuf_sprintf(b, "RSSI: %ddB<br>", state->radio_rssi);
strbuf_sprintf(b, "Remote RSSI: %ddB<br>", state->remote_rssi);
}
// write a new link layer packet to interface->txbuffer
// consuming more bytes from the next interface->tx_packet if required
static int radio_link_encode_packet(struct radio_link_state *link_state)
{
// if we have nothing interesting left to send, don't create a packet at all
if (!link_state->tx_packet)
return 0;
int count = ob_remaining(link_state->tx_packet);
int startP = (ob_position(link_state->tx_packet) == 0);
int endP = 1;
if (count > LINK_PAYLOAD_MTU){
count = LINK_PAYLOAD_MTU;
endP = 0;
}
link_state->txbuffer[0]=0xfe; // mavlink v1.0 magic header
// we need to add FEC_LENGTH for FEC, but the length field doesn't include the expected headers or CRC
int len = count + FEC_LENGTH - RADIO_CRC_LENGTH;
link_state->txbuffer[1]=len; // mavlink payload length
link_state->txbuffer[2]=(len & 0xF);
link_state->txbuffer[3]=0;
// add golay encoding so that decoding the actual length is more reliable
golay_encode(&link_state->txbuffer[1]);
link_state->txbuffer[4]=(link_state->tx_seq++) & 0x3f;
if (startP) link_state->txbuffer[4]|=0x40;
if (endP) link_state->txbuffer[4]|=0x80;
link_state->txbuffer[5]=MAVLINK_MSG_ID_DATASTREAM;
ob_get_bytes(link_state->tx_packet, &link_state->txbuffer[6], count);
encode_rs_8(&link_state->txbuffer[4], &link_state->txbuffer[6+count], FEC_MAX_BYTES - (count+2));
link_state->tx_bytes=len + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
if (endP){
ob_free(link_state->tx_packet);
link_state->tx_packet=NULL;
overlay_queue_schedule_next(gettime_ms());
}
return 0;
}
int radio_link_is_busy(struct overlay_interface *interface)
{
if (interface->radio_link_state && interface->radio_link_state->tx_packet)
return 1;
return 0;
}
int radio_link_queue_packet(struct overlay_interface *interface, struct overlay_buffer *buffer)
{
struct radio_link_state *link_state = interface->radio_link_state;
if (link_state->tx_packet){
ob_free(buffer);
return WHYF("Cannot send two packets to a stream at the same time");
}
// prepare the buffer for reading
ob_flip(buffer);
link_state->tx_packet = buffer;
radio_link_tx(interface);
return 0;
}
static int build_heartbeat(struct radio_link_state *link_state)
{
int count=9;
bzero(link_state->txbuffer, count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH);
link_state->txbuffer[0]=0xfe; // mavlink v1.0 link_state->txbuffer
// Must be 9 to indicate heartbeat
link_state->txbuffer[1]=count; // payload len, excluding 6 byte header and 2 byte CRC
link_state->txbuffer[2]=(count & 0xF); // packet sequence
link_state->txbuffer[3]=0x00; // system ID of sender (MAV_TYPE_GENERIC)
// we're golay encoding the length to improve the probability of skipping it correctly
golay_encode(&link_state->txbuffer[1]);
link_state->txbuffer[4]=0xf1; // component ID of sender (MAV_COMP_ID_UART_BRIDGE)
// Must be zero to indicate heartbeat
link_state->txbuffer[5]=0; // message ID type of this link_state->txbuffer: DATA_STREAM
// extra magic number to help correctly detect remote heartbeat requests
link_state->txbuffer[14]=0x55;
link_state->txbuffer[15]=0x05;
golay_encode(&link_state->txbuffer[14]);
link_state->tx_bytes = count + RADIO_CRC_LENGTH + RADIO_HEADER_LENGTH;
if (config.debug.radio_link)
DEBUGF("Produced heartbeat");
return 0;
}
// write a new link layer packet to interface->txbuffer
// consuming more bytes from the next interface->tx_packet if required
int radio_link_tx(struct overlay_interface *interface)
{
struct radio_link_state *link_state = interface->radio_link_state;
unschedule(&interface->alarm);
interface->alarm.alarm = 0;
time_ms_t next_tick = interface->destination->last_tx+interface->destination->tick_ms;
time_ms_t now = gettime_ms();
while(1){
if (link_state->tx_bytes){
if (link_state->next_tx_allowed > now){
interface->alarm.alarm = link_state->next_tx_allowed;
break;
}
int written=write(interface->alarm.poll.fd, &link_state->txbuffer[link_state->tx_pos], link_state->tx_bytes);
if (written<=0){
interface->alarm.poll.events|=POLLOUT;
break;
}
link_state->remaining_space-=written;
link_state->tx_bytes-=written;
if (link_state->tx_bytes)
link_state->tx_pos+=written;
else
link_state->tx_pos=0;
continue;
}
interface->alarm.poll.events&=~POLLOUT;
if (link_state->next_heartbeat<=now){
build_heartbeat(link_state);
link_state->next_heartbeat = now + 1000;
continue;
}
// out of space? Don't bother to send anything interesting
// until we hear the next heartbeat response
if (link_state->remaining_space < LINK_MTU + HEARTBEAT_SIZE){
interface->alarm.alarm = link_state->next_heartbeat;
break;
}
if (link_state->remaining_space < LINK_MTU + HEARTBEAT_SIZE)
link_state->next_heartbeat = now;
if (!link_state->tx_packet){
// finished current packet, wait for more.
interface->alarm.alarm = next_tick;
break;
}
// encode another packet fragment
radio_link_encode_packet(link_state);
link_state->last_packet = now;
}
watch(&interface->alarm);
if (interface->alarm.alarm<now)
interface->alarm.alarm=now;
if (interface->alarm.alarm){
interface->alarm.deadline = interface->alarm.alarm+100;
schedule(&interface->alarm);
}
return 0;
}
static int parse_heartbeat(struct radio_link_state *state, const unsigned char *payload)
{
if (payload[0]==0xFE
&& payload[1]==9
&& payload[3]==RADIO_SOURCE_SYSTEM
&& payload[4]==RADIO_SOURCE_COMPONENT
&& payload[5]==MAVLINK_MSG_ID_RADIO){
// we can assume that radio status packets arrive without corruption
state->radio_rssi=(1.0*payload[10]-payload[13])/1.9;
state->remote_rssi=(1.0*payload[11] - payload[14])/1.9;
int free_space = payload[12];
int free_bytes = (free_space * 1280) / 100 - 30;
state->remaining_space = free_bytes;
if (free_bytes>0)
state->next_tx_allowed = gettime_ms();
if (free_bytes>720)
state->next_heartbeat=gettime_ms()+1000;
if (config.debug.packetradio)
INFOF("Link budget = %+ddB, remote link budget = %+ddB, buffer space = %d%% (approx %d)",
state->radio_rssi,
state->remote_rssi,
free_space, free_bytes);
return 1;
}
return 0;
}
static int radio_link_parse(struct overlay_interface *interface, struct radio_link_state *state,
size_t packet_length, uint8_t *payload, int *backtrack)
{
*backtrack=0;
if (packet_length==17){
// if we've heard the start and end of a remote heartbeat request
// we can skip it without checking anything else
int errs=0;
int tail = golay_decode(&errs, &payload[14]);
if (tail == 0x555){
if (config.debug.radio_link)
DEBUGF("Decoded remote heartbeat request");
return 1;
}
return 0;
}
size_t data_bytes = packet_length - (RADIO_USED_HEADER_LENGTH + FEC_LENGTH);
int errors=decode_rs_8(&payload[4], NULL, 0, FEC_MAX_BYTES - data_bytes);
if (errors==-1){
if (config.debug.radio_link)
DEBUGF("Reed-Solomon error correction failed");
return 0;
}
*backtrack=errors;
data_bytes -= 2;
int seq=payload[4]&0x3f;
if (config.debug.radio_link){
DEBUGF("Received RS protected message, len: %zd, errors: %d, seq: %d, flags:%s%s",
data_bytes,
errors,
seq,
payload[4]&0x40?" start":"",
payload[4]&0x80?" end":"");
}
if (seq != ((state->seq+1)&0x3f)){
// reject partial packet if we missed a sequence number
if (config.debug.radio_link)
DEBUGF("Rejecting packet, sequence jumped from %d to %d", state->seq, seq);
state->packet_length=sizeof(state->dst)+1;
}
if (payload[4]&0x40){
// start a new packet
state->packet_length=0;
}
state->seq=payload[4]&0x3f;
if (state->packet_length + data_bytes > sizeof(state->dst)){
if (config.debug.radio_link)
DEBUG("Fragmented packet is too long or a previous piece was missed - discarding");
state->packet_length=sizeof(state->dst)+1;
return 1;
}
bcopy(&payload[RADIO_HEADER_LENGTH], &state->dst[state->packet_length], data_bytes);
state->packet_length+=data_bytes;
if (payload[4]&0x80) {
if (config.debug.radio_link)
DEBUGF("PDU Complete (length=%d)",state->packet_length);
packetOkOverlay(interface, state->dst, state->packet_length, -1, NULL);
state->packet_length=sizeof(state->dst)+1;
}
return 1;
}
static int decode_length(struct radio_link_state *state, unsigned char *p)
{
// look for a valid golay encoded length
int errs=0;
int length = golay_decode(&errs, p);
if (length<0 || ((length >>8) & 0xF) != (length&0xF))
return -1;
length=length&0xFF;
length += RADIO_HEADER_LENGTH + RADIO_CRC_LENGTH;
if (length!=17 && (length <= FEC_LENGTH || length > LINK_MTU))
return -1;
if (config.debug.radio_link && (errs || state->payload_length!=*p))
DEBUGF("Decoded length %d to %d with %d errs", *p, length, errs);
state->payload_length=length;
return 0;
}
// add one byte at a time from the serial link, and attempt to decode packets
int radio_link_decode(struct overlay_interface *interface, uint8_t c)
{
IN();
struct radio_link_state *state=interface->radio_link_state;
if (state->payload_start + state->payload_offset >= sizeof(state->payload)){
// drop one byte if we run out of space
if (config.debug.radio_link)
DEBUGF("Dropped %02x, buffer full", state->payload[0]);
bcopy(state->payload+1, state->payload, sizeof(state->payload) -1);
state->payload_start--;
}
unsigned char *p = &state->payload[state->payload_start];
p[state->payload_offset++]=c;
while(1){
// look for packet length headers
p = &state->payload[state->payload_start];
while(state->payload_length==0 && state->payload_offset>=6){
if (p[0]==0xFE
&& p[1]==9
&& p[3]==RADIO_SOURCE_SYSTEM
&& p[4]==RADIO_SOURCE_COMPONENT
&& p[5]==MAVLINK_MSG_ID_RADIO){
//looks like a valid heartbeat response header, read the rest and process it
state->payload_length=17;
break;
}
if (decode_length(state, &p[1])==0)
break;
state->payload_start++;
state->payload_offset--;
p++;
}
// wait for a whole packet
if (!state->payload_length || state->payload_offset < state->payload_length)
RETURN(0);
if (parse_heartbeat(state, p)){
// cut the bytes of the heartbeat out of the buffer
state->payload_offset -= state->payload_length;
if (state->payload_offset){
// shuffle bytes backwards
bcopy(&p[state->payload_length], p, state->payload_offset);
}
// restart parsing for a valid header from the beginning of out buffer
state->payload_offset+=state->payload_start;
state->payload_start=0;
state->payload_length=0;
continue;
}
// is this a well formed packet?
int backtrack=0;
if (radio_link_parse(interface, state, state->payload_length, p, &backtrack)==1){
// Since we know we've synced with the remote party,
// and there's nothing we can do about any earlier data
// throw away everything before the end of this packet
if (state->payload_start && config.debug.radio_link)
dump("Skipped", state->payload, state->payload_start);
// If the packet is truncated by less than 16 bytes, RS protection should be enough to recover the packet,
// but we may need to examine the last few bytes to find the start of the next packet.
state->payload_offset -= state->payload_length - backtrack;
if (state->payload_offset){
// shuffle all remaining bytes back to the start of the buffer
bcopy(&state->payload[state->payload_start + state->payload_length - backtrack],
state->payload, state->payload_offset);
}
state->payload_start=0;
}else{
// ignore the first byte for now and start looking for another packet header
// we may find a heartbeat in the middle that we need to cut out first
state->payload_start++;
state->payload_offset--;
}
state->payload_length=0;
};
RETURN(0);
}

15
radio_link.h Normal file
View File

@ -0,0 +1,15 @@
#ifndef __SERVALD_RADIO_LINK_H
#define __SERVALD_RADIO_LINK_H
#define HEARTBEAT_SIZE (8+9)
#define LINK_MTU 255
int radio_link_free(struct overlay_interface *interface);
int radio_link_init(struct overlay_interface *interface);
int radio_link_decode(struct overlay_interface *interface, uint8_t c);
int radio_link_tx(struct overlay_interface *interface);
void radio_link_state_html(struct strbuf *b, struct overlay_interface *interface);
int radio_link_is_busy(struct overlay_interface *interface);
int radio_link_queue_packet(struct overlay_interface *interface, struct overlay_buffer *buffer);
#endif

206
rhizome.c
View File

@ -17,11 +17,13 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <stdlib.h>
#include <assert.h>
#include "serval.h"
#include "conf.h"
#include "str.h"
#include "rhizome.h"
#include <stdlib.h>
#include "dataformats.h"
int is_rhizome_enabled()
{
@ -129,14 +131,11 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
if (rhizome_read_manifest_file(m, manifest_path, buffer_len) == -1)
return WHY("could not read manifest file");
if (rhizome_manifest_verify(m))
if (!rhizome_manifest_validate(m))
return WHY("manifest is invalid");
if (!rhizome_manifest_verify(m))
return WHY("could not verify manifest");
/* Make sure we store signatures */
// TODO, why do we need this? Why isn't the state correct from rhizome_read_manifest_file?
// This feels like a hack...
m->manifest_bytes=m->manifest_all_bytes;
/* Do we already have this manifest or newer? */
int64_t dbVersion = -1;
if (sqlite_exec_int64(&dbVersion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END) == -1)
@ -152,115 +151,130 @@ int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path,
return rhizome_add_manifest(m, 1);
}
int rhizome_manifest_check_sanity(rhizome_manifest *m_in)
int rhizome_manifest_check_sanity(rhizome_manifest *m)
{
/* Ensure manifest meets basic sanity checks. */
const char *service = rhizome_manifest_get(m_in, "service", NULL, 0);
const char *sender = rhizome_manifest_get(m_in, "sender", NULL, 0);
const char *recipient = rhizome_manifest_get(m_in, "recipient", NULL, 0);
if (service == NULL || !service[0])
return WHY("Manifest missing 'service' field");
if (rhizome_manifest_get_ll(m_in, "date") == -1)
return WHY("Manifest missing 'date' field");
/* Get manifest version number. */
m_in->version = rhizome_manifest_get_ll(m_in, "version");
if (m_in->version==-1)
return WHY("Manifest must have a version number");
if (strcasecmp(service, RHIZOME_SERVICE_FILE) == 0) {
const char *name = rhizome_manifest_get(m_in, "name", NULL, 0);
if (name == NULL)
return WHY("Manifest missing 'name' field");
} else if (strcasecmp(service, RHIZOME_SERVICE_MESHMS) == 0
|| strcasecmp(service, RHIZOME_SERVICE_MESHMS2) == 0) {
if (sender == NULL || !sender[0])
return WHY("MeshMS Manifest missing 'sender' field");
if (!str_is_subscriber_id(sender))
return WHYF("MeshMS Manifest contains invalid 'sender' field: %s", sender);
if (recipient == NULL || !recipient[0])
return WHY("MeshMS Manifest missing 'recipient' field");
if (!str_is_subscriber_id(recipient))
return WHYF("MeshMS Manifest contains invalid 'recipient' field: %s", recipient);
} else {
return WHY("Invalid service type");
int ret = 0;
if (m->version == 0)
ret = WHY("Manifest must have a version number");
if (m->filesize == RHIZOME_SIZE_UNSET)
ret = WHY("Manifest missing 'filesize' field");
else if (m->filesize && rhizome_filehash_t_is_zero(m->filehash))
ret = WHY("Manifest 'filehash' field has not been set");
if (m->service == NULL)
ret = WHY("Manifest missing 'service' field");
else if (strcasecmp(m->service, RHIZOME_SERVICE_FILE) == 0) {
if (m->name == NULL)
ret = WHY("Manifest with service='" RHIZOME_SERVICE_FILE "' missing 'name' field");
} else if (strcasecmp(m->service, RHIZOME_SERVICE_MESHMS) == 0
|| strcasecmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) {
if (!m->has_sender)
ret = WHYF("Manifest with service='%s' missing 'sender' field", m->service);
if (!m->has_recipient)
ret = WHYF("Manifest with service='%s' missing 'recipient' field", m->service);
}
if (config.debug.rhizome)
DEBUGF("sender='%s'", sender ? sender : "(null)");
/* passes all sanity checks */
return 0;
else if (!rhizome_str_is_manifest_service(m->service))
ret = WHYF("Manifest invalid 'service' field %s", alloca_str_toprint(m->service));
if (!m->has_date)
ret = WHY("Manifest missing 'date' field");
return ret;
}
/*
A bundle can either be an ordinary manifest-payload pair, or a group description.
- Group descriptions are manifests with no payload that have the "isagroup" variable set. They
get stored in the manifests table AND a reference is added to the grouplist table. Any
manifest, including any group manifest, may be a member of zero or one group. This allows a
nested, i.e., multi-level group hierarchy where sub-groups will only typically be discovered
by joining the parent group.
*/
int rhizome_manifest_bind_id(rhizome_manifest *m_in)
/* Sets the bundle key "BK" field of a manifest. Returns 1 if the field was set, 0 if not.
*
* This function must not be called unless the bundle secret is known.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_manifest_add_bundle_key(rhizome_manifest *m)
{
if (rhizome_manifest_createid(m_in) == -1)
return -1;
/* The ID is implicit in transit, but we need to store it in the file, so that reimporting
manifests on receiver nodes works easily. We might implement something that strips the id
variable out of the manifest when sending it, or some other scheme to avoid sending all the
extra bytes. */
if (!is_sid_t_any(m_in->author)) {
/* Set the BK using the provided authorship information.
Serval Security Framework defines BK as being:
BK = privateKey XOR sha512(RS##BID), where BID = cryptoSignPublic,
and RS is the rhizome secret for the specified author.
The nice thing about this specification is that:
privateKey = BK XOR sha512(RS##BID), so the same function can be used
to encrypt and decrypt the BK field. */
const unsigned char *rs;
int rs_len=0;
unsigned char bkbytes[RHIZOME_BUNDLE_KEY_BYTES];
if (rhizome_find_secret(&m_in->author, &rs_len, &rs))
return WHYF("Failed to obtain RS for %s to calculate BK", alloca_tohex_sid_t(m_in->author));
if (!rhizome_secret2bk(&m_in->cryptoSignPublic, rs, rs_len, bkbytes, m_in->cryptoSignSecret)) {
char bkhex[RHIZOME_BUNDLE_KEY_STRLEN + 1];
(void) tohex(bkhex, RHIZOME_BUNDLE_KEY_STRLEN, bkbytes);
if (config.debug.rhizome) DEBUGF("set BK=%s", bkhex);
rhizome_manifest_set(m_in, "BK", bkhex);
} else
return WHY("Failed to set BK");
IN();
assert(m->haveSecret);
switch (m->authorship) {
case ANONYMOUS: // there can be no BK field without an author
case AUTHOR_UNKNOWN: // we already know the author is not in the keyring
case AUTHENTICATION_ERROR: // already tried and failed to get Rhizome Secret
break;
case AUTHOR_NOT_CHECKED:
case AUTHOR_LOCAL:
case AUTHOR_AUTHENTIC:
case AUTHOR_IMPOSTOR: {
/* Set the BK using the provided author. Serval Security Framework defines BK as being:
* BK = privateKey XOR sha512(RS##BID)
* where BID = cryptoSignPublic,
* RS is the rhizome secret for the specified author.
* The nice thing about this specification is that:
* privateKey = BK XOR sha512(RS##BID)
* so the same function can be used to encrypt and decrypt the BK field.
*/
const unsigned char *rs;
size_t rs_len = 0;
enum rhizome_secret_disposition d = find_rhizome_secret(&m->author, &rs_len, &rs);
switch (d) {
case FOUND_RHIZOME_SECRET: {
rhizome_bk_t bkey;
if (rhizome_secret2bk(&m->cryptoSignPublic, rs, rs_len, bkey.binary, m->cryptoSignSecret) == 0) {
rhizome_manifest_set_bundle_key(m, &bkey);
m->authorship = AUTHOR_AUTHENTIC;
RETURN(1);
} else
m->authorship = AUTHENTICATION_ERROR;
}
break;
case IDENTITY_NOT_FOUND:
m->authorship = AUTHOR_UNKNOWN;
break;
case IDENTITY_HAS_NO_RHIZOME_SECRET:
m->authorship = AUTHENTICATION_ERROR;
break;
default:
FATALF("find_rhizome_secret() returned unknown code %d", (int)d);
break;
}
}
break;
default:
FATALF("m->authorship = %d", (int)m->authorship);
}
return 0;
rhizome_manifest_del_bundle_key(m);
switch (m->authorship) {
case AUTHOR_UNKNOWN:
WHYF("Cannot set BK because author=%s is not in keyring", alloca_tohex_sid_t(m->author));
break;
case AUTHENTICATION_ERROR:
WHY("Cannot set BK due to error");
break;
default:
break;
}
RETURN(0);
}
int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
int rhizome_add_manifest(rhizome_manifest *m, int ttl)
{
if (config.debug.rhizome)
DEBUGF("rhizome_add_manifest(m_in=%p, ttl=%d)",m_in, ttl);
DEBUGF("rhizome_add_manifest(m=%p, ttl=%d)",m, ttl);
if (m_in->finalised==0)
if (m->finalised==0)
return WHY("Manifest must be finalised before being stored");
/* Store time to live, clamped to within legal range */
m_in->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
m->ttl = ttl < 0 ? 0 : ttl > 254 ? 254 : ttl;
if (rhizome_manifest_check_sanity(m_in))
if (rhizome_manifest_check_sanity(m))
return -1;
if (m_in->fileLength && !rhizome_exists(&m_in->filehash))
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0 && !rhizome_exists(&m->filehash))
return WHY("File has not been imported");
/* If the manifest already has an ID */
if (rhizome_bid_t_is_zero(m_in->cryptoSignPublic))
if (rhizome_bid_t_is_zero(m->cryptoSignPublic))
return WHY("Manifest does not have an ID");
/* Discard the new manifest unless it is newer than the most recent known version with the same ID */
int64_t storedversion = -1;
switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m_in->cryptoSignPublic, END)) {
switch (sqlite_exec_int64(&storedversion, "SELECT version FROM MANIFESTS WHERE id = ?;", RHIZOME_BID_T, &m->cryptoSignPublic, END)) {
case -1:
return WHY("Select failed");
case 0:
@ -268,18 +282,18 @@ int rhizome_add_manifest(rhizome_manifest *m_in,int ttl)
break;
case 1:
if (config.debug.rhizome)
DEBUGF("Found existing version=%"PRId64", new version=%"PRId64, storedversion, m_in->version);
if (m_in->version < storedversion)
DEBUGF("Found existing version=%"PRId64", new version=%"PRId64, storedversion, m->version);
if (m->version < storedversion)
return WHY("Newer version exists");
if (m_in->version == storedversion)
return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m_in->cryptoSignPublic), m_in->version);
if (m->version == storedversion)
return WHYF("Already have %s:%"PRId64", not adding", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
break;
default:
return WHY("Select found too many rows!");
}
/* Okay, it is written, and can be put directly into the rhizome database now */
return rhizome_store_bundle(m_in);
return rhizome_store_bundle(m);
}
/* When voice traffic is being carried, we need to throttle Rhizome down

456
rhizome.h
View File

@ -21,12 +21,13 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#define __SERVALDNA__RHIZOME_H
#include <sqlite3.h>
#include <limits.h>
#include "sha2.h"
#include "uuid.h"
#include "str.h"
#include "strbuf.h"
#include "http_server.h"
#include "nacl.h"
#include <sys/stat.h>
#ifndef __RHIZOME_INLINE
# if __GNUC__ && !__GNUC_STDC_INLINE__
@ -104,6 +105,8 @@ __RHIZOME_INLINE int rhizome_is_bk_none(const rhizome_bk_t *bk) {
}
#define alloca_tohex_rhizome_bk_t(bk) alloca_tohex((bk).binary, sizeof (*(rhizome_bk_t*)0).binary)
int cmp_rhizome_bk_t(const rhizome_bk_t *a, const rhizome_bk_t *b);
int str_to_rhizome_bk_t(rhizome_bk_t *bk, const char *hex);
extern time_ms_t rhizome_voice_timeout;
@ -118,13 +121,10 @@ extern time_ms_t rhizome_voice_timeout;
#define RHIZOME_IDLE_TIMEOUT 20000
#define EXISTING_BUNDLE_ID 1
#define NEW_BUNDLE_ID 2
typedef struct rhizome_signature {
unsigned char signature[crypto_sign_edwards25519sha512batch_BYTES
+crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES+1];
int signatureLength;
size_t signatureLength;
} rhizome_signature;
#define RHIZOME_BAR_BYTES 32
@ -138,85 +138,246 @@ typedef struct rhizome_signature {
#define MAX_MANIFEST_VARS 256
#define MAX_MANIFEST_BYTES 8192
typedef struct rhizome_manifest {
int manifest_record_number;
int manifest_bytes;
int manifest_all_bytes;
unsigned char manifestdata[MAX_MANIFEST_BYTES];
unsigned char manifesthash[crypto_hash_sha512_BYTES];
/* CryptoSign key pair for this manifest.
The filename as distributed on Rhizome will be the public key
of this pair, thus ensuring that noone can tamper with a bundle
except the creator. */
#define RHIZOME_SIZE_UNSET UINT64_MAX
typedef struct rhizome_manifest
{
int manifest_record_number;
/* CryptoSign key pair for this manifest. The public key is the Bundle ID
* (aka Manifest ID).
*/
rhizome_bid_t cryptoSignPublic;
unsigned char cryptoSignSecret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES];
/* Whether we have the secret for this manifest on hand */
int haveSecret;
int var_count;
char *vars[MAX_MANIFEST_VARS];
char *values[MAX_MANIFEST_VARS];
/* Whether cryptoSignSecret is correct (ie, bundle secret is known)
*/
enum { SECRET_UNKNOWN = 0, EXISTING_BUNDLE_ID, NEW_BUNDLE_ID } haveSecret;
int sig_count;
/* Parties who have signed this manifest (raw byte format) */
/* Version of the manifest. Typically the number of milliseconds since 1970.
* A value of zero (0) means it has not been set yet.
* TODO: change this to uint64_t.
*/
int64_t version;
/* Payload is described by the offset of its tail (number of missing bytes
* before the first byte in the payload), its size (number of bytes) and the
* hash of its content. Bundle size = tail + filesize.
*/
uint64_t tail;
uint64_t filesize;
rhizome_filehash_t filehash;
/* All the manifest fields in original order (the order affects the manifest
* hash which was used to sign the manifest, so the signature can only be
* checked if order is preserved).
*
* TODO: reduce to only unknown fields.
*
* TODO: store all vars and values as NUL terminated strings within
* manifestdata[], not malloc()/free() heap, to reduce memory fragmentation
* and allow manifest struct copying without string lifetime issues.
*/
unsigned short var_count;
const char *vars[MAX_MANIFEST_VARS];
const char *values[MAX_MANIFEST_VARS];
/* Parties who have signed this manifest (binary format, malloc(3)).
* Recognised signature types:
* 0x17 = crypto_sign_edwards25519sha512batch()
*/
unsigned short sig_count;
unsigned char *signatories[MAX_MANIFEST_VARS];
/*
0x17 = crypto_sign_edwards25519sha512batch()
*/
unsigned char signatureTypes[MAX_MANIFEST_VARS];
uint8_t signatureTypes[MAX_MANIFEST_VARS];
// errors only involve the correctness of fields that are mandatory for
// proper operation of the transport and storage layer
int errors;
// a warning indicates that the manifest cannot be perfectly understood by this version of rhizome
// during add, the manifest should not be finalised and imported
// during extract an error should be displayed.
int warnings;
time_ms_t inserttime;
/* Set non-zero after variables have been packed and
signature blocks appended.
All fields below may not be valid until the manifest has been finalised */
int finalised;
/* Set to non-zero if a manifest has been parsed that cannot be fully
* understood by this version of Rhizome (probably from a future or a very
* old past version of Rhizome). During add (local injection), the manifest
* should not be imported. During extract (local decode) a warning or error
* message should be logged. Manifests marked as malformed are still
* transported, imported and exported normally, as long as their signature is
* valid.
*/
unsigned short malformed;
/* Set non-zero after variables have been packed and signature blocks
* appended. All fields below may not be valid until the manifest has been
* finalised.
*/
bool_t finalised;
/* Whether the manifest contains a signature that corresponds to the manifest
* id (ie public key).
*/
bool_t selfSigned;
/* If set, unlink(2) the associated file when freeing the manifest.
*/
bool_t dataFileUnlinkOnFree;
/* Set if the ID field (cryptoSignPublic) contains a bundle ID.
*/
bool_t has_id;
/* Set if the tail field is valid, ie, the bundle is a journal.
*/
bool_t is_journal;
/* Set if the date field is valid, ie, the manifest contains a valid "date"
* field.
*/
bool_t has_date;
/* Set if the bundle_key field is valid, ie, the manifest contains a valid
* "BK" field.
*/
bool_t has_bundle_key;
/* Set if the sender and recipient fields are valid, ie, the manifest
* contains a valid "sender"/"recipient" field.
*/
bool_t has_sender;
bool_t has_recipient;
/* Local authorship. Useful for dividing bundle lists between "sent" and
* "inbox" views.
*/
enum rhizome_bundle_authorship {
ANONYMOUS = 0, // 'author' element is not valid
AUTHOR_NOT_CHECKED, // 'author' element is valid but not checked
AUTHENTICATION_ERROR, // author check failed, don't try again
AUTHOR_UNKNOWN, // author is not a local identity
AUTHOR_LOCAL, // author is in keyring (unlocked) but not verified
AUTHOR_IMPOSTOR, // author is a local identity but fails verification
AUTHOR_AUTHENTIC // a local identity is the verified author
} authorship;
/* time-to-live in hops of this manifest. */
int ttl;
/* When finalised, we keep the filehash and maximum priority due to any
group membership handy */
int64_t fileLength;
int64_t journalTail;
rhizome_filehash_t filehash;
int fileHighestPriority;
/* Absolute path of the file associated with the manifest */
char *dataFileName;
/* If set, unlink(2) the associated file when freeing the manifest */
int dataFileUnlinkOnFree;
const char *dataFileName;
/* Whether the paylaod is encrypted or not */
int payloadEncryption;
enum rhizome_manifest_crypt {
PAYLOAD_CRYPT_UNKNOWN = 0,
PAYLOAD_CLEAR,
PAYLOAD_ENCRYPTED
} payloadEncryption;
unsigned char payloadKey[RHIZOME_CRYPT_KEY_BYTES];
unsigned char payloadNonce[crypto_stream_xsalsa20_NONCEBYTES];
/* Whether the manifest contains a signature that corresponds to the
manifest id (ie public key) */
int selfSigned;
/* From the "date" field, if present. The number of milliseconds since 1970
* when the bundle was last modified.
*/
time_ms_t date;
/* Version of the manifest. Typically the number of milliseconds since 1970. */
int64_t version;
int group_count;
char *groups[MAX_MANIFEST_VARS];
/* From the "service" field, which should always be present.
*/
const char *service;
/* Author of the manifest. A reference to a local keyring entry. Manifests
* not authored locally will have the ANY author (all zeros).
/* From the optional "name" field. NULL if there is no "name" field in the
* manifest.
*/
const char *name;
/* Bundle Key "BK" field from the manifest.
*/
rhizome_bk_t bundle_key;
/* Sender and recipient fields, if present in the manifest.
*/
sid_t sender;
sid_t recipient;
/* Local data, not encapsulated in the bundle. The ROWID of the SQLite
* MANIFESTS table row in which this manifest is stored. Zero if the
* manifest has not been stored yet.
*/
uint64_t rowid;
/* Local data, not encapsulated in the bundle. The system time of the most
* recent INSERT or UPDATE of the manifest into the store. Zero if the manifest
* has not been stored yet.
*/
time_ms_t inserttime;
/* Local data, not encapsulated in the bundle. The author of the manifest.
* A reference to a local keyring entry. Manifests not authored locally will
* have an ANY author (all zeros).
*/
sid_t author;
/* Unused. SHOULD BE DELETED.
*/
unsigned group_count;
char *groups[MAX_MANIFEST_VARS];
size_t manifest_body_bytes;
size_t manifest_all_bytes;
unsigned char manifestdata[MAX_MANIFEST_BYTES];
unsigned char manifesthash[crypto_hash_sha512_BYTES];
} rhizome_manifest;
/* These setter functions (methods) are needed because the relevant attributes
* are stored in two places: in the vars[] array and in a dedicated struct
* element.
*
* TODO: refactor to remove the redundancy, possibly removing these setter
* functions as well.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
#define rhizome_manifest_set_id(m,v) _rhizome_manifest_set_id(__WHENCE__,(m),(v))
#define rhizome_manifest_set_version(m,v) _rhizome_manifest_set_version(__WHENCE__,(m),(v))
#define rhizome_manifest_set_filesize(m,v) _rhizome_manifest_set_filesize(__WHENCE__,(m),(v))
#define rhizome_manifest_set_filehash(m,v) _rhizome_manifest_set_filehash(__WHENCE__,(m),(v))
#define rhizome_manifest_set_tail(m,v) _rhizome_manifest_set_tail(__WHENCE__,(m),(v))
#define rhizome_manifest_set_bundle_key(m,v) _rhizome_manifest_set_bundle_key(__WHENCE__,(m),(v))
#define rhizome_manifest_del_bundle_key(m) _rhizome_manifest_del_bundle_key(__WHENCE__,(m))
#define rhizome_manifest_set_service(m,v) _rhizome_manifest_set_service(__WHENCE__,(m),(v))
#define rhizome_manifest_del_service(m) _rhizome_manifest_del_service(__WHENCE__,(m))
#define rhizome_manifest_set_name(m,v) _rhizome_manifest_set_name(__WHENCE__,(m),(v))
#define rhizome_manifest_del_name(m) _rhizome_manifest_del_name(__WHENCE__,(m))
#define rhizome_manifest_set_date(m,v) _rhizome_manifest_set_date(__WHENCE__,(m),(v))
#define rhizome_manifest_del_date(m) _rhizome_manifest_del_date(__WHENCE__,(m))
#define rhizome_manifest_set_sender(m,v) _rhizome_manifest_set_sender(__WHENCE__,(m),(v))
#define rhizome_manifest_del_sender(m) _rhizome_manifest_del_sender(__WHENCE__,(m))
#define rhizome_manifest_set_recipient(m,v) _rhizome_manifest_set_recipient(__WHENCE__,(m),(v))
#define rhizome_manifest_del_recipient(m) _rhizome_manifest_del_recipient(__WHENCE__,(m))
#define rhizome_manifest_set_crypt(m,v) _rhizome_manifest_set_crypt(__WHENCE__,(m),(v))
#define rhizome_manifest_set_rowid(m,v) _rhizome_manifest_set_rowid(__WHENCE__,(m),(v))
#define rhizome_manifest_set_inserttime(m,v) _rhizome_manifest_set_inserttime(__WHENCE__,(m),(v))
#define rhizome_manifest_set_author(m,v) _rhizome_manifest_set_author(__WHENCE__,(m),(v))
#define rhizome_manifest_del_author(m) _rhizome_manifest_del_author(__WHENCE__,(m))
void _rhizome_manifest_set_id(struct __sourceloc, rhizome_manifest *, const rhizome_bid_t *);
void _rhizome_manifest_set_version(struct __sourceloc, rhizome_manifest *, int64_t); // TODO change to uint64_t
void _rhizome_manifest_set_filesize(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_filehash(struct __sourceloc, rhizome_manifest *, const rhizome_filehash_t *);
void _rhizome_manifest_set_tail(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_bundle_key(struct __sourceloc, rhizome_manifest *, const rhizome_bk_t *);
void _rhizome_manifest_del_bundle_key(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_service(struct __sourceloc, rhizome_manifest *, const char *);
void _rhizome_manifest_del_service(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_name(struct __sourceloc, rhizome_manifest *, const char *);
void _rhizome_manifest_del_name(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_date(struct __sourceloc, rhizome_manifest *, time_ms_t);
void _rhizome_manifest_del_date(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_sender(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_sender(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_recipient(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_recipient(struct __sourceloc, rhizome_manifest *);
void _rhizome_manifest_set_crypt(struct __sourceloc, rhizome_manifest *, enum rhizome_manifest_crypt);
void _rhizome_manifest_set_rowid(struct __sourceloc, rhizome_manifest *, uint64_t);
void _rhizome_manifest_set_inserttime(struct __sourceloc, rhizome_manifest *, time_ms_t);
void _rhizome_manifest_set_author(struct __sourceloc, rhizome_manifest *, const sid_t *);
void _rhizome_manifest_del_author(struct __sourceloc, rhizome_manifest *);
/* Supported service identifiers. These go in the 'service' field of every
* manifest, and indicate which application must be used to process the bundle
* after it is received by Rhizome.
@ -246,6 +407,7 @@ int create_rhizome_datastore_dir();
#define FORM_RHIZOME_IMPORT_PATH(buf,fmt,...) (form_rhizome_import_path((buf), sizeof(buf), (fmt), ##__VA_ARGS__))
extern sqlite3 *rhizome_db;
uuid_t rhizome_db_uuid;
int rhizome_opendb();
int rhizome_close_db();
@ -261,14 +423,6 @@ int rhizome_cleanup(struct rhizome_cleanup_report *report);
int rhizome_manifest_createid(rhizome_manifest *m);
int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed);
int rhizome_strn_is_manifest_id(const char *text);
int rhizome_str_is_manifest_id(const char *text);
int rhizome_strn_is_bundle_key(const char *text);
int rhizome_str_is_bundle_key(const char *text);
int rhizome_strn_is_bundle_crypt_key(const char *text);
int rhizome_str_is_bundle_crypt_key(const char *text);
int rhizome_strn_is_file_hash(const char *text);
int rhizome_str_is_file_hash(const char *text);
int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call);
@ -285,35 +439,45 @@ sqlite_retry_state sqlite_retry_state_init(int serverLimit, int serverSleep, int
#define SQLITE_RETRY_STATE_DEFAULT sqlite_retry_state_init(-1,-1,-1,-1)
struct rhizome_manifest_summary {
rhizome_bid_t bid;
int64_t version;
size_t body_len;
};
int rhizome_manifest_inspect(const char *buf, size_t len, struct rhizome_manifest_summary *summ);
int rhizome_write_manifest_file(rhizome_manifest *m, const char *filename, char append);
int rhizome_manifest_selfsign(rhizome_manifest *m);
int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priority);
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp);
int rhizome_read_manifest_file(rhizome_manifest *m, const char *filename, size_t bufferPAndSize);
int rhizome_manifest_validate(rhizome_manifest *m);
int rhizome_hash_file(rhizome_manifest *m, const char *path, rhizome_filehash_t *hash_out, uint64_t *size_out);
char *rhizome_manifest_get(const rhizome_manifest *m, const char *var, char *out, int maxlen);
int64_t rhizome_manifest_get_ll(rhizome_manifest *m, const char *var);
int rhizome_manifest_set_ll(rhizome_manifest *m,char *var, int64_t value);
int rhizome_manifest_set(rhizome_manifest *m, const char *var, const char *value);
int rhizome_manifest_del(rhizome_manifest *m, const char *var);
int64_t rhizome_file_size(char *filename);
void _rhizome_manifest_free(struct __sourceloc __whence, rhizome_manifest *m);
#define rhizome_manifest_free(m) _rhizome_manifest_free(__WHENCE__,m)
rhizome_manifest *_rhizome_new_manifest(struct __sourceloc __whence);
#define rhizome_new_manifest() _rhizome_new_manifest(__WHENCE__)
int rhizome_manifest_pack_variables(rhizome_manifest *m);
int rhizome_store_bundle(rhizome_manifest *m);
int rhizome_remove_file_datainvalid(sqlite_retry_state *retry, const rhizome_filehash_t *hashp);
int rhizome_manifest_add_group(rhizome_manifest *m,char *groupid);
int rhizome_clean_payload(const char *fileidhex);
int rhizome_store_file(rhizome_manifest *m,const unsigned char *key);
int rhizome_bundle_import_files(rhizome_manifest *m, const char *manifest_path, const char *filepath);
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp, rhizome_bk_t *bsk);
int rhizome_fill_manifest(rhizome_manifest *m, const char *filepath, const sid_t *authorSidp);
int rhizome_apply_bundle_secret(rhizome_manifest *, const rhizome_bk_t *);
int rhizome_manifest_add_bundle_key(rhizome_manifest *);
void rhizome_find_bundle_author_and_secret(rhizome_manifest *m);
int rhizome_lookup_author(rhizome_manifest *m);
void rhizome_authenticate_author(rhizome_manifest *m);
int rhizome_manifest_verify(rhizome_manifest *m);
int rhizome_manifest_check_sanity(rhizome_manifest *m_in);
int rhizome_manifest_bind_id(rhizome_manifest *m_in);
int rhizome_manifest_finalise(rhizome_manifest *m, rhizome_manifest **mout, int deduplicate);
int rhizome_add_manifest(rhizome_manifest *m_in,int ttl);
@ -355,6 +519,7 @@ enum sqlbind_type {
TOHEX, // const unsigned char *binary, unsigned bytes
TEXT_TOUPPER, // const char *text,
TEXT_LEN_TOUPPER, // const char *text, unsigned bytes
UUID_T, // const uuid_t *uuidp
NUL = 1 << 15, // NUL (no arg) ; NUL|INT, ...
INDEX = 0xfade0000, // INDEX|INT, int index, ...
NAMED = 0xdead0000 // NAMED|INT, const char *label, ...
@ -402,7 +567,7 @@ int _sqlite_vexec_strbuf_retry(struct __sourceloc, sqlite_retry_state *retry, st
#define sqlite_exec_strbuf_retry(rs,sb,sql,arg,...) _sqlite_exec_strbuf_retry(__WHENCE__, (rs), (sb), (sql), arg, ##__VA_ARGS__)
double rhizome_manifest_get_double(rhizome_manifest *m,char *var,double default_value);
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs);
int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs);
int rhizome_update_file_priority(const char *fileid);
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found);
int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar);
@ -410,9 +575,6 @@ int64_t rhizome_bar_version(const unsigned char *bar);
uint64_t rhizome_bar_bidprefix_ll(unsigned char *bar);
int rhizome_is_bar_interesting(unsigned char *bar);
int rhizome_is_manifest_interesting(rhizome_manifest *m);
int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
const char *sender_sid, const char *recipient_sid,
int limit, int offset, char count_rows);
int rhizome_retrieve_manifest(const rhizome_bid_t *bid, rhizome_manifest *m);
int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned prefix_len, rhizome_manifest *m);
int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m);
@ -426,7 +588,12 @@ int rhizome_delete_file(const rhizome_filehash_t *hashp);
int rhizome_fetching_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int monitor_announce_bundle(rhizome_manifest *m);
int rhizome_find_secret(const sid_t *authorSidp, int *rs_len, const unsigned char **rs);
enum rhizome_secret_disposition {
FOUND_RHIZOME_SECRET = 0,
IDENTITY_NOT_FOUND,
IDENTITY_HAS_NO_RHIZOME_SECRET,
};
enum rhizome_secret_disposition find_rhizome_secret(const sid_t *authorSidp, size_t *rs_len, const unsigned char **rs);
int rhizome_bk_xor_stream(
const rhizome_bid_t *bidp,
const unsigned char *rs,
@ -448,16 +615,40 @@ int rhizome_secret2bk(
const unsigned char secret[crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]
);
unsigned char *rhizome_bundle_shared_secret(rhizome_manifest *m);
int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk,
const unsigned char *pk,rhizome_signature *out);
int rhizome_verify_bundle_privatekey(rhizome_manifest *m, const unsigned char *sk,
const unsigned char *pk);
int rhizome_find_bundle_author(rhizome_manifest *m);
int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned char *pk);
int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int timeout);
int rhizome_ignore_manifest_check(unsigned char *bid_prefix, int prefix_len);
/* Rhizome list cursor for iterating over all or a subset of manifests in the store.
*/
struct rhizome_list_cursor {
// Query parameters that narrow the set of listed bundles.
const char *service;
const char *name;
bool_t is_sender_set;
bool_t is_recipient_set;
sid_t sender;
sid_t recipient;
// If set, then the cursor moves in ascending (chronological) order starting
// from the first bundle with rowid > rowid_since. If zero, then the cursor
// moves in descending (reverse chronological) order starting from the most
// recent bundle.
uint64_t rowid_since;
// Set by calling the next() function.
rhizome_manifest *manifest;
// Private state.
sqlite3_stmt *_statement;
uint64_t _rowid_current;
uint64_t _rowid_last; // for re-opening query
};
int rhizome_list_open(sqlite_retry_state *, struct rhizome_list_cursor *);
int rhizome_list_next(sqlite_retry_state *, struct rhizome_list_cursor *);
void rhizome_list_commit(struct rhizome_list_cursor *);
void rhizome_list_release(struct rhizome_list_cursor *);
/* one manifest is required per candidate, plus a few spare.
so MAX_RHIZOME_MANIFESTS must be > MAX_CANDIDATES.
*/
@ -471,7 +662,7 @@ rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_leng
struct rhizome_write_buffer
{
struct rhizome_write_buffer *_next;
int64_t offset;
uint64_t offset;
size_t buffer_size;
size_t data_size;
unsigned char data[0];
@ -483,10 +674,10 @@ struct rhizome_write
uint64_t temp_id;
char id_known;
int64_t tail;
int64_t file_offset;
int64_t written_offset;
int64_t file_length;
uint64_t tail;
uint64_t file_offset;
uint64_t written_offset;
uint64_t file_length;
struct rhizome_write_buffer *buffer_list;
size_t buffer_size;
@ -521,9 +712,9 @@ struct rhizome_read
int64_t blob_rowid;
int blob_fd;
int64_t tail;
int64_t offset;
int64_t length;
uint64_t tail;
uint64_t offset;
uint64_t length;
};
/* Rhizome-specific HTTP request handling.
@ -532,40 +723,45 @@ typedef struct rhizome_http_request
{
struct http_request http; // MUST BE FIRST ELEMENT
/* Identify request from others being run.
Monotonic counter feeds it. Only used for debugging when we write
post-<uuid>.log files for multi-part form requests. */
/* Identify request from others being run. Monotonic counter feeds it. Only
* used for debugging when we write post-<uuid>.log files for multi-part form
* requests.
*/
unsigned int uuid;
struct rhizome_read read_state;
/* File currently being written to while decoding POST multipart form */
/* For receiving a POST multipart form:
*/
// Which part is currently being received
enum rhizome_direct_mime_part { NONE = 0, MANIFEST, DATA } current_part;
// Temporary file currently current part is being written to
int part_fd;
/* Which parts have been received in POST multipart form */
// Which parts have already been received
bool_t received_manifest;
bool_t received_data;
/* Name of data file supplied */
// Name of data file supplied in part's Content-Disposition header, filename
// parameter (if any)
char data_file_name[MIME_FILENAME_MAXLEN + 1];
/* The source specification data which are used in different ways by different
request types */
char source[1024];
int64_t source_index;
int64_t source_count;
int source_record_size;
unsigned int source_flags;
const char *sql_table;
const char *sql_row;
int64_t rowid;
/* source_index used for offset in blob */
int64_t blob_end;
union {
/* For responses that send part or all of a payload.
*/
struct rhizome_read read_state;
/* For responses that list manifests.
*/
struct {
enum { LIST_HEADER = 0, LIST_ROWS, LIST_DONE } phase;
uint64_t rowid_highest;
size_t rowcount;
time_ms_t end_time;
struct rhizome_list_cursor cursor;
} list;
} u;
} rhizome_http_request;
int rhizome_received_content(const unsigned char *bidprefix,uint64_t version,
uint64_t offset,int count,unsigned char *bytes,
uint64_t offset, size_t count,unsigned char *bytes,
int type);
int64_t rhizome_database_create_blob_for(const char *filehashhex_or_tempid,
int64_t fileLength,int priority);
@ -702,15 +898,14 @@ enum rhizome_start_fetch_result {
enum rhizome_start_fetch_result rhizome_fetch_request_manifest_by_prefix(const struct sockaddr_in *peerip, const sid_t *sidp, const unsigned char *prefix, size_t prefix_length);
int rhizome_any_fetch_active();
int rhizome_any_fetch_queued();
int rhizome_fetch_queue_bytes();
int rhizome_fetch_status_html(struct strbuf *b);
int rhizome_fetch_has_queue_space(unsigned char log2_size);
struct http_response_parts {
uint16_t code;
char *reason;
int64_t range_start;
int64_t content_length;
uint64_t range_start;
uint64_t content_length;
char *content_start;
};
@ -719,9 +914,9 @@ int unpack_http_response(char *response, struct http_response_parts *parts);
/* rhizome storage methods */
int rhizome_exists(const rhizome_filehash_t *hashp);
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority);
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority);
int rhizome_write_buffer(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size);
int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size);
int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m);
int rhizome_write_file(struct rhizome_write *write, const char *filename);
int rhizome_fail_write(struct rhizome_write *write);
@ -730,21 +925,20 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath);
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length);
int rhizome_stat_file(rhizome_manifest *m, const char *filepath);
int rhizome_add_file(rhizome_manifest *m, const char *filepath);
int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk);
int rhizome_derive_payload_key(rhizome_manifest *m);
int rhizome_open_write_journal(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size);
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len);
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename);
int rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, unsigned char *buffer, size_t len);
int rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename);
int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *hashp, uint64_t start_offset, uint64_t length);
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset,
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset,
const unsigned char *key, const unsigned char *nonce);
int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp);
ssize_t rhizome_read(struct rhizome_read *read, unsigned char *buffer, size_t buffer_length);
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len);
int rhizome_read_close(struct rhizome_read *read);
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk);
int rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state);
int rhizome_extract_file(rhizome_manifest *m, const char *filepath);
int rhizome_dump_file(const rhizome_filehash_t *hashp, const char *filepath, int64_t *length);
int rhizome_read_cached(const rhizome_bid_t *bid, uint64_t version, time_ms_t timeout,
uint64_t fileOffset, unsigned char *buffer, size_t length);

1451
rhizome_bundle.c
View File

File diff suppressed because it is too large Load Diff

581
rhizome_crypto.c
View File

@ -17,16 +17,19 @@ along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <stdlib.h>
#include <ctype.h>
#include <assert.h>
#include "crypto_sign_edwards25519sha512batch.h"
#include "nacl/src/crypto_sign_edwards25519sha512batch_ref/ge.h"
#include "serval.h"
#include "conf.h"
#include "str.h"
#include "rhizome.h"
#include "crypto.h"
#include <stdlib.h>
#include <ctype.h>
#include "keyring.h"
#include "dataformats.h"
/* Work out the encrypt/decrypt key for the supplied manifest.
If the manifest is not encrypted, then return NULL.
@ -40,8 +43,10 @@ int rhizome_manifest_createid(rhizome_manifest *m)
{
if (crypto_sign_edwards25519sha512batch_keypair(m->cryptoSignPublic.binary, m->cryptoSignSecret))
return WHY("Failed to create keypair for manifest ID.");
rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
rhizome_manifest_set_id(m, &m->cryptoSignPublic);
m->haveSecret = NEW_BUNDLE_ID;
// A new Bundle ID and secret invalidates any existing BK field.
rhizome_manifest_del_bundle_key(m);
return 0;
}
@ -56,10 +61,14 @@ static int generate_keypair(const char *seed, struct signing_key *key)
unsigned char hash[crypto_hash_sha512_BYTES];
crypto_hash_sha512(hash, (unsigned char *)seed, strlen(seed));
// The first 256 bits of the hash will be used as the private key of the BID.
bcopy(hash, key->Private, sizeof(key->Private));
if (crypto_sign_compute_public_key(key->Private, key->Public.binary))
// The first 256 bits (32 bytes) of the hash will be used as the private key of the BID.
bcopy(hash, key->Private, sizeof key->Private);
if (crypto_sign_compute_public_key(key->Private, key->Public.binary) == -1)
return WHY("Could not generate public key");
// The last 32 bytes of the private key should be identical to the public key. This is what
// crypto_sign_edwards25519sha512batch_keypair() returns, and there is code that depends on it.
// TODO: Refactor the Rhizome private/public keypair to eliminate this duplication.
bcopy(key->Public.binary, key->Private + RHIZOME_BUNDLE_KEY_BYTES, sizeof key->Public.binary);
return 0;
}
@ -70,16 +79,20 @@ int rhizome_get_bundle_from_seed(rhizome_manifest *m, const char *seed)
struct signing_key key;
if (generate_keypair(seed, &key))
return -1;
int ret=rhizome_retrieve_manifest(&key.Public, m);
int ret = rhizome_retrieve_manifest(&key.Public, m);
if (ret == -1)
return -1;
m->haveSecret=(ret==0)?EXISTING_BUNDLE_ID:NEW_BUNDLE_ID;
m->cryptoSignPublic = key.Public;
if (ret == 1) {
// manifest not retrieved
rhizome_manifest_set_id(m, &key.Public); // zerofills m->cryptoSignSecret
m->haveSecret = NEW_BUNDLE_ID;
} else {
m->haveSecret = EXISTING_BUNDLE_ID;
}
bcopy(key.Private, m->cryptoSignSecret, sizeof m->cryptoSignSecret);
if (ret == 1)
rhizome_manifest_set(m, "id", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
// Disabled for performance, these asserts should nevertheless always hold.
//assert(cmp_rhizome_bid_t(&m->cryptoSignPublic, &key.Public) == 0);
//assert(memcmp(m->cryptoSignPublic.binary, m->cryptoSignSecret + RHIZOME_BUNDLE_KEY_BYTES, sizeof m->cryptoSignPublic.binary) == 0);
return ret;
}
@ -114,10 +127,12 @@ int rhizome_bk_xor_stream(
return 0;
}
/*
CryptoSign Secret Keys in cupercop-20120525 onwards have the public key as the
second half of the secret key. The public key is the BID, so this simplifies
the BK<-->SECRET conversion processes. */
/* CryptoSign Secret Keys in cupercop-20120525 onwards have the public key as the second half of the
* secret key. The public key is the BID, so this simplifies the BK<-->SECRET conversion processes.
*
* Returns 0 if the BK decodes correctly to the bundle secret, 1 if not. Returns -1 if there is an
* error.
*/
int rhizome_bk2secret(rhizome_manifest *m,
const rhizome_bid_t *bidp,
const unsigned char *rs, const size_t rs_len,
@ -128,21 +143,16 @@ int rhizome_bk2secret(rhizome_manifest *m,
{
IN();
unsigned char xor_stream[RHIZOME_BUNDLE_KEY_BYTES];
if (rhizome_bk_xor_stream(bidp,rs,rs_len,xor_stream,RHIZOME_BUNDLE_KEY_BYTES))
if (rhizome_bk_xor_stream(bidp, rs, rs_len, xor_stream, RHIZOME_BUNDLE_KEY_BYTES))
RETURN(WHY("rhizome_bk_xor_stream() failed"));
int i;
/* XOR and store secret part of secret key */
for(i = 0; i != RHIZOME_BUNDLE_KEY_BYTES; i++)
unsigned i;
for (i = 0; i != RHIZOME_BUNDLE_KEY_BYTES; ++i)
secret[i] = bkin[i] ^ xor_stream[i];
/* Copy BID as public-key part of secret key */
for(;i!=crypto_sign_edwards25519sha512batch_SECRETKEYBYTES;++i)
secret[i] = bidp->binary[i - RHIZOME_BUNDLE_KEY_BYTES];
bzero(xor_stream, sizeof xor_stream);
RETURN(rhizome_verify_bundle_privatekey(m, secret, bidp->binary));
/* Copy BID as public-key part of secret key */
bcopy(bidp->binary, secret + RHIZOME_BUNDLE_KEY_BYTES, sizeof bidp->binary);
RETURN(rhizome_verify_bundle_privatekey(secret, bidp->binary) ? 0 : 1);
OUT();
}
@ -171,257 +181,245 @@ int rhizome_secret2bk(
}
/* Given the SID of a bundle's author, search for an identity in the keyring and return its
* Rhizome secret if found.
/* Given a SID, search the keyring for an identity with the same SID and return its Rhizome secret
* if found.
*
* Returns -1 if an error occurs.
* Returns 0 if the author's rhizome secret is found; '*rs' is set to point to the secret key in the
* keyring, and '*rs_len' is set to the key length.
* Returns 2 if the author's identity is not in the keyring.
* Returns 3 if the author's identity is in the keyring but has no rhizome secret.
* Returns FOUND_RHIZOME_SECRET if the author's rhizome secret is found; '*rs' is set to point to
* the secret key in the keyring, and '*rs_len' is set to the key length.
*
* Returns IDENTITY_NOT_FOUND if the SID is not in the keyring.
*
* Returns IDENTITY_HAS_NO_RHIZOME_SECRET if the SID is in the keyring but has no Rhizome Secret.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_find_secret(const sid_t *authorSidp, int *rs_len, const unsigned char **rs)
enum rhizome_secret_disposition find_rhizome_secret(const sid_t *authorSidp, size_t *rs_len, const unsigned char **rs)
{
IN();
int cn=0, in=0, kp=0;
if (!keyring_find_sid(keyring,&cn,&in,&kp, authorSidp)) {
if (config.debug.rhizome)
DEBUGF("identity sid=%s is not in keyring", alloca_tohex_sid_t(*authorSidp));
return 2;
RETURN(IDENTITY_NOT_FOUND);
}
kp = keyring_identity_find_keytype(keyring, cn, in, KEYTYPE_RHIZOME);
if (kp == -1) {
if (config.debug.rhizome)
DEBUGF("identity sid=%s has no Rhizome Secret", alloca_tohex_sid_t(*authorSidp));
return 3;
WARNF("Identity sid=%s has no Rhizome Secret", alloca_tohex_sid_t(*authorSidp));
RETURN(IDENTITY_HAS_NO_RHIZOME_SECRET);
}
int rslen = keyring->contexts[cn]->identities[in]->keypairs[kp]->private_key_len;
if (rslen < 16 || rslen > 1024)
return WHYF("identity sid=%s has invalid Rhizome Secret: length=%d", alloca_tohex_sid_t(*authorSidp), rslen);
assert(rslen >= 16);
assert(rslen <= 1024);
if (rs_len)
*rs_len = rslen;
if (rs)
*rs = keyring->contexts[cn]->identities[in]->keypairs[kp]->private_key;
return 0;
RETURN(FOUND_RHIZOME_SECRET);
}
/* Given the SID of a bundle's author and the bundle ID, XOR a bundle key (private or public) with
* RS##BID where RS is the rhizome secret of the bundle's author, and BID is the bundle's public key
* (aka the Bundle ID).
*
* This will convert a manifest BK field into the bundle's private key, or vice versa.
*
* Returns -1 if an error occurs.
* Returns 0 if the author's private key is located and the XOR is performed successfully.
* Returns 2 if the author's identity is not in the keyring (this return code from
* rhizome_find_secret()).
* Returns 3 if the author's identity is in the keyring but has no rhizome secret (this return code
* from rhizome_find_secret()).
*
* Looks up the SID in the keyring, and if it is present and has a valid-looking RS, calls
* rhizome_bk_xor_rs() to perform the XOR.
/* Attempt to authenticate the authorship of the given bundle, and set the 'authorship' element
* accordingly. If the manifest has nk BK field, then no authentication can be performed.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
/* See if the manifest has a BK entry, and if so, use it to obtain the private key for the BID. The
* manifest's 'author' field must contain the (binary) SID of the purported author of the bundle,
* which is used to look up the author's rhizome secret in the keyring.
*
* Returns 0 if a valid private key was extracted, with the private key in the manifest
* 'cryptoSignSecret' field and the 'haveSecret' field set to 1.
*
* Returns 1 if the manifest does not have a BK field.
*
* Returns 2 if the author is not found in the keyring (not unlocked?) -- this return code from
* rhizome_bk_xor().
*
* Returns 3 if the author is found in the keyring but has no rhizome secret -- this return code
* from rhizome_bk_xor().
*
* Returns 4 if the author is found in the keyring and has a rhizome secret but the private bundle
* key formed using it does not verify.
*
* Returns -1 on error.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_extract_privatekey(rhizome_manifest *m, rhizome_bk_t *bsk)
void rhizome_authenticate_author(rhizome_manifest *m)
{
IN();
unsigned char bkBytes[RHIZOME_BUNDLE_KEY_BYTES];
char *bk = rhizome_manifest_get(m, "BK", NULL, 0);
int result;
if (bk){
if (fromhexstr(bkBytes, bk, RHIZOME_BUNDLE_KEY_BYTES) == -1)
RETURN(WHYF("invalid BK field: %s", bk));
if (is_sid_t_any(m->author)) {
result=rhizome_find_bundle_author(m);
}else{
int rs_len;
const unsigned char *rs;
result = rhizome_find_secret(&m->author, &rs_len, &rs);
if (result==0)
result = rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret);
}
if (result == 0 && bsk && !rhizome_is_bk_none(bsk)){
// If a bundle secret key was supplied that does not match the secret key derived from the
// author, then warn but carry on using the author's.
if (memcmp(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES) != 0)
WARNF("Supplied bundle secret key is invalid -- ignoring");
}
}else if(bsk && !rhizome_is_bk_none(bsk)){
bcopy(m->cryptoSignPublic.binary, &m->cryptoSignSecret[RHIZOME_BUNDLE_KEY_BYTES], sizeof m->cryptoSignPublic.binary);
bcopy(bsk, m->cryptoSignSecret, RHIZOME_BUNDLE_KEY_BYTES);
if (rhizome_verify_bundle_privatekey(m, m->cryptoSignSecret, m->cryptoSignPublic.binary))
result=5;
else
result=0;
}else{
result=1;
if (!m->has_bundle_key)
RETURNVOID;
switch (m->authorship) {
case ANONYMOUS:
rhizome_find_bundle_author_and_secret(m);
break;
case AUTHOR_NOT_CHECKED:
case AUTHOR_LOCAL: {
if (config.debug.rhizome)
DEBUGF("manifest[%d] authenticate author=%s", m->manifest_record_number, alloca_tohex_sid_t(m->author));
size_t rs_len;
const unsigned char *rs;
enum rhizome_secret_disposition d = find_rhizome_secret(&m->author, &rs_len, &rs);
switch (d) {
case FOUND_RHIZOME_SECRET:
if (config.debug.rhizome)
DEBUGF("author has Rhizome secret");
switch (rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, m->cryptoSignSecret)) {
case 0:
if (config.debug.rhizome)
DEBUGF("authentic");
m->authorship = AUTHOR_AUTHENTIC;
if (!m->haveSecret)
m->haveSecret = EXISTING_BUNDLE_ID;
break;
case -1:
if (config.debug.rhizome)
DEBUGF("error");
m->authorship = AUTHENTICATION_ERROR;
break;
default:
if (config.debug.rhizome)
DEBUGF("impostor");
m->authorship = AUTHOR_IMPOSTOR;
break;
}
break;
case IDENTITY_NOT_FOUND:
if (config.debug.rhizome)
DEBUGF("author not found");
m->authorship = AUTHOR_UNKNOWN;
break;
case IDENTITY_HAS_NO_RHIZOME_SECRET:
if (config.debug.rhizome)
DEBUGF("author has no Rhizome secret");
m->authorship = AUTHENTICATION_ERROR;
break;
default:
FATALF("find_rhizome_secret() returned unknown code %d", (int)d);
break;
}
}
break;
case AUTHENTICATION_ERROR:
case AUTHOR_UNKNOWN:
case AUTHOR_IMPOSTOR:
case AUTHOR_AUTHENTIC:
// work has already been done, don't repeat it
break;
default:
FATALF("m->authorship = %d", (int)m->authorship);
break;
}
if (result == 0){
m->haveSecret=EXISTING_BUNDLE_ID;
}else{
memset(m->cryptoSignSecret, 0, sizeof m->cryptoSignSecret);
m->haveSecret=0;
}
RETURN(result);
OUT();
}
/* Same as rhizome_extract_privatekey, except warnings become errors and are logged */
int rhizome_extract_privatekey_required(rhizome_manifest *m, rhizome_bk_t *bsk)
/* If the given bundle secret key corresponds to the bundle's ID (public key) then store it in the
* manifest structure and mark the secret key as known. Return 1 if the secret key was assigned,
* 0 if not.
*
* This function should only be called on a manifest that already has a public key (ID) and does
* not have a known secret key.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_apply_bundle_secret(rhizome_manifest *m, const rhizome_bk_t *bsk)
{
int result = rhizome_extract_privatekey(m, bsk);
switch (result) {
case -1:
case 0:
return result;
case 1:
return WHY("Bundle contains no BK field, and no bundle secret supplied");
case 2:
return WHY("Author unknown");
case 3:
return WHY("Author does not have a Rhizome Secret");
case 4:
return WHY("Author does not have permission to modify manifest");
case 5:
return WHY("Bundle secret is not valid for this manifest");
default:
return WHYF("Unknown result from rhizome_extract_privatekey(): %d", result);
IN();
if (config.debug.rhizome)
DEBUGF("manifest[%d] bsk=%s", m->manifest_record_number, bsk ? alloca_tohex_rhizome_bk_t(*bsk) : "NULL");
assert(m->haveSecret == SECRET_UNKNOWN);
assert(is_all_matching(m->cryptoSignSecret, sizeof m->cryptoSignSecret, 0));
assert(!rhizome_bid_t_is_zero(m->cryptoSignPublic));
assert(bsk != NULL);
assert(!rhizome_is_bk_none(bsk));
if (rhizome_verify_bundle_privatekey(bsk->binary, m->cryptoSignPublic.binary)) {
if (config.debug.rhizome)
DEBUG("bundle secret verifies ok");
bcopy(bsk->binary, m->cryptoSignSecret, sizeof bsk->binary);
bcopy(m->cryptoSignPublic.binary, m->cryptoSignSecret + sizeof bsk->binary, sizeof m->cryptoSignPublic.binary);
m->haveSecret = EXISTING_BUNDLE_ID;
RETURN(1);
}
RETURN(0);
OUT();
}
/* Discover if the given manifest was created (signed) by any unlocked identity currently in the
* keyring.
*
* Returns 0 if an identity is found with permission to alter the bundle, after setting the manifest
* 'author' field to the SID of the identity and the manifest 'cryptoSignSecret' field to the bundle
* secret key and the 'haveSecret' field to 1.
* This function must only be called if the bundle secret is not known. If it is known, then
* use
*
* Returns 1 if no identity in the keyring is the author of this bundle.
* If the authorship is already known (ie, not ANONYMOUS) then returns without changing anything.
* That means this function can be called several times on the same manifest, but will only perform
* any work the first time.
*
* Returns 4 if the manifest has no BK field.
* If the manifest has no bundle key (BK) field, then it is anonymous, so leaves 'authorship'
* unchanged and returns.
*
* Returns -1 if an error occurs, eg, the manifest contains an invalid BK field.
* If an identity is found in the keyring with permission to alter the bundle, then sets the
* manifest 'authorship' field to AUTHOR_AUTHENTIC, the 'author' field to the SID of the identity,
* the manifest 'cryptoSignSecret' field to the bundle secret key and the 'haveSecret' field to
* EXISTING_BUNDLE_ID.
*
* If no identity is found in the keyring that combines with the bundle key (BK) field to yield
* the bundle's secret key, then leaves the manifest 'authorship' field as ANONYMOUS.
*
* If an error occurs, eg, the keyring contains an invalid Rhizome Secret or a cryptographic
* operation fails, then sets the 'authorship' field to AUTHENTICATION_ERROR and leaves the
* 'author', 'haveSecret' and 'cryptoSignSecret' fields unchanged.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_find_bundle_author(rhizome_manifest *m)
void rhizome_find_bundle_author_and_secret(rhizome_manifest *m)
{
IN();
char *bk = rhizome_manifest_get(m, "BK", NULL, 0);
if (!bk) {
if (config.debug.rhizome)
DEBUGF("missing BK field");
RETURN(4);
}
unsigned char bkBytes[RHIZOME_BUNDLE_KEY_BYTES];
if (fromhexstr(bkBytes, bk, RHIZOME_BUNDLE_KEY_BYTES) == -1)
RETURN(WHYF("invalid BK field: %s", bk));
if (m->authorship != ANONYMOUS)
RETURNVOID;
assert(is_sid_t_any(m->author));
if (!m->has_bundle_key)
RETURNVOID;
int cn = 0, in = 0, kp = 0;
for (; keyring_next_identity(keyring, &cn, &in, &kp); ++kp) {
const sid_t *authorSidp = (const sid_t *) keyring->contexts[cn]->identities[in]->keypairs[kp]->public_key;
//if (config.debug.rhizome) DEBUGF("try author identity sid=%s", alloca_tohex_sid_t(*authorSidp));
int rkp = keyring_identity_find_keytype(keyring, cn, in, KEYTYPE_RHIZOME);
if (rkp != -1) {
int rs_len = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key_len;
if (rs_len < 16 || rs_len > 1024)
RETURN(WHYF("invalid Rhizome Secret: length=%d", rs_len));
size_t rs_len = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key_len;
if (rs_len < 16 || rs_len > 1024) {
WHYF("invalid Rhizome Secret: length=%zu", rs_len);
m->authorship = AUTHENTICATION_ERROR;
RETURNVOID;
}
const unsigned char *rs = keyring->contexts[cn]->identities[in]->keypairs[rkp]->private_key;
if (!rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, bkBytes, m->cryptoSignSecret)) {
m->haveSecret=EXISTING_BUNDLE_ID;
if (cmp_sid_t(&m->author, authorSidp) != 0){
m->author = *authorSidp;
if (config.debug.rhizome)
DEBUGF("found bundle author sid=%s", alloca_tohex_sid_t(m->author));
// if this bundle is already in the database, update the author.
if (m->inserttime)
sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
"UPDATE MANIFESTS SET author = ? WHERE id = ?;",
SID_T, &m->author,
RHIZOME_BID_T, &m->cryptoSignPublic,
END);
}
RETURN(0); // bingo
unsigned char *secretp = m->cryptoSignSecret;
if (m->haveSecret)
secretp = alloca(sizeof m->cryptoSignSecret);
if (rhizome_bk2secret(m, &m->cryptoSignPublic, rs, rs_len, m->bundle_key.binary, secretp) == 0) {
if (m->haveSecret) {
if (memcmp(secretp, m->cryptoSignSecret, sizeof m->cryptoSignSecret) != 0)
FATALF("Bundle secret does not match derived secret");
} else
m->haveSecret = EXISTING_BUNDLE_ID;
if (config.debug.rhizome)
DEBUGF("found bundle author sid=%s", alloca_tohex_sid_t(*authorSidp));
rhizome_manifest_set_author(m, authorSidp);
m->authorship = AUTHOR_AUTHENTIC;
// if this bundle is already in the database, update the author.
if (m->rowid)
sqlite_exec_void_loglevel(LOG_LEVEL_WARN,
"UPDATE MANIFESTS SET author = ? WHERE rowid = ?;",
SID_T, &m->author,
INT64, m->rowid,
END);
RETURNVOID; // bingo
}
}
}
assert(m->authorship == ANONYMOUS);
if (config.debug.rhizome)
DEBUG("bundle author not found");
RETURN(1);
OUT();
}
/* Verify the validity of the manifest's secret key, ie, is the given manifest's 'cryptoSignSecret'
* field actually the secret key corresponding to the public key in 'cryptoSignPublic'?
* Return 0 if valid, 1 if not. Return -1 if an error occurs.
/* Verify the validity of a given secret manifest key. Return 1 if valid, 0 if not.
*
* There is no NaCl API to efficiently test this. We use a modified version of
* crypto_sign_keypair() to accomplish this task.
*/
int rhizome_verify_bundle_privatekey(rhizome_manifest *m,
const unsigned char *sk,
const unsigned char *pkin)
int rhizome_verify_bundle_privatekey(const unsigned char *sk, const unsigned char *pkin)
{
IN();
unsigned char pk[32];
int i;
crypto_sign_compute_public_key(sk,pk);
for (i = 0;i < 32;++i)
if (pkin[i] != pk[i]) {
if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary)
m->haveSecret=0;
RETURN(-1);
}
if (m&&sk==m->cryptoSignSecret&&pkin==m->cryptoSignPublic.binary) {
if (config.debug.rhizome)
DEBUGF("We have the private key for this bundle.");
m->haveSecret=EXISTING_BUNDLE_ID;
}
RETURN(0);
OUT();
rhizome_bid_t pk;
if (crypto_sign_compute_public_key(sk, pk.binary) == -1)
RETURN(0);
int ret = bcmp(pkin, pk.binary, sizeof pk.binary) == 0;
RETURN(ret);
}
int rhizome_sign_hash(rhizome_manifest *m,
rhizome_signature *out)
int rhizome_sign_hash(rhizome_manifest *m, rhizome_signature *out)
{
IN();
if (!m->haveSecret && rhizome_extract_privatekey_required(m, NULL))
RETURN(-1);
int ret=rhizome_sign_hash_with_key(m, m->cryptoSignSecret, m->cryptoSignPublic.binary, out);
assert(m->haveSecret);
int ret = rhizome_sign_hash_with_key(m, m->cryptoSignSecret, m->cryptoSignPublic.binary, out);
RETURN(ret);
OUT();
}
@ -436,7 +434,7 @@ int rhizome_sign_hash_with_key(rhizome_manifest *m,const unsigned char *sk,
int mLen = crypto_hash_sha512_BYTES;
int r = crypto_sign_edwards25519sha512batch(signatureBuffer, &sigLen, &hash[0], mLen, sk);
if (r)
RETURN(WHY("crypto_sign_edwards25519sha512batch() failed."));
RETURN(WHY("crypto_sign_edwards25519sha512batch() failed"));
/* Here we use knowledge of the internal structure of the signature block
to remove the hash, since that is implicitly transported, thus reducing the
actual signature size down to 64 bytes.
@ -459,7 +457,7 @@ typedef struct manifest_signature_block_cache {
#define SIG_CACHE_SIZE 1024
manifest_signature_block_cache sig_cache[SIG_CACHE_SIZE];
int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char *sig,int sig_len)
int rhizome_manifest_lookup_signature_validity(const unsigned char *hash, const unsigned char *sig, int sig_len)
{
IN();
unsigned int slot=0;
@ -504,80 +502,54 @@ int rhizome_manifest_lookup_signature_validity(unsigned char *hash,unsigned char
OUT();
}
int rhizome_manifest_extract_signature(rhizome_manifest *m,int *ofs)
int rhizome_manifest_extract_signature(rhizome_manifest *m, unsigned *ofs)
{
IN();
if (!m)
RETURN(WHY("NULL pointer passed in as manifest"));
if (config.debug.rhizome)
DEBUGF("m->manifest_all_bytes=%d m->manifest_bytes=%d *ofs=%d", m->manifest_all_bytes, m->manifest_bytes, *ofs);
if ((*ofs)>=m->manifest_all_bytes) { RETURN(0); }
int sigType=m->manifestdata[*ofs];
int len=(sigType&0x3f)*4+4+1;
/* Each signature type is required to have a different length to detect it.
At present only crypto_sign_edwards25519sha512batch() signatures are
supported. */
int r;
if (m->sig_count<MAX_MANIFEST_VARS)
switch(sigType)
{
case 0x17: /* crypto_sign_edwards25519sha512batch() */
/* Reconstitute signature block */
r=rhizome_manifest_lookup_signature_validity
(m->manifesthash,&m->manifestdata[(*ofs)+1],96);
#ifdef DEPRECATED
unsigned char sigBuf[256];
unsigned char verifyBuf[256];
unsigned char publicKey[256];
bcopy(&m->manifestdata[(*ofs)+1],&sigBuf[0],64);
bcopy(&m->manifesthash[0],&sigBuf[64],crypto_hash_sha512_BYTES);
/* Get public key of signatory */
bcopy(&m->manifestdata[(*ofs)+1+64],&publicKey[0],crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
unsigned long long mlen=0;
int r=crypto_sign_edwards25519sha512batch_open(verifyBuf,&mlen,&sigBuf[0],128, publicKey);
#endif
if (r) {
(*ofs)+=len;
m->errors++;
RETURN(WHY("Error in signature block (verification failed)."));
} else {
/* Signature block passes, so add to list of signatures */
m->signatureTypes[m->sig_count]=len;
m->signatories[m->sig_count]
=malloc(crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
if(!m->signatories[m->sig_count]) {
(*ofs)+=len;
RETURN(WHY("malloc() failed when reading signature block"));
}
bcopy(&m->manifestdata[(*ofs)+1+64],m->signatories[m->sig_count],
crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
m->sig_count++;
if (config.debug.rhizome) DEBUG("Signature passed.");
}
break;
default:
(*ofs)+=len;
m->errors++;
RETURN(WHYF("Encountered illegal or malformed signature block (unknown type=0x%02x @ offset 0x%x)",sigType,(*ofs)-len));
}
else
if (config.debug.rhizome_manifest)
DEBUGF("*ofs=%u m->manifest_all_bytes=%zu", *ofs, m->manifest_all_bytes);
assert((*ofs) < m->manifest_all_bytes);
const unsigned char *sig = m->manifestdata + *ofs;
uint8_t sigType = m->manifestdata[*ofs];
uint8_t len = (sigType << 2) + 4 + 1;
if (*ofs + len > m->manifest_all_bytes) {
WARNF("Invalid signature at offset %u: type=%#02x gives len=%u that overruns manifest size",
*ofs, sigType, len);
RETURN(1);
}
*ofs += len;
assert (m->sig_count <= NELS(m->signatories));
if (m->sig_count == NELS(m->signatories)) {
WARN("Too many signature blocks in manifest");
RETURN(2);
}
switch (sigType) {
case 0x17: // crypto_sign_edwards25519sha512batch()
{
(*ofs)+=len;
WHY("Too many signature blocks in manifest.");
m->errors++;
assert(len == 97);
/* Reconstitute signature block */
int r = rhizome_manifest_lookup_signature_validity(m->manifesthash, sig + 1, 96);
if (r) {
WARN("Signature verification failed");
RETURN(4);
}
m->signatureTypes[m->sig_count] = len;
if ((m->signatories[m->sig_count] = emalloc(crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES)) == NULL)
RETURN(-1);
bcopy(sig + 1 + 64, m->signatories[m->sig_count], crypto_sign_edwards25519sha512batch_PUBLICKEYBYTES);
m->sig_count++;
if (config.debug.rhizome)
DEBUG("Signature verified");
RETURN(0);
}
(*ofs)+=len;
RETURN(0);
OUT();
}
WARNF("Unsupported signature at ofs=%u: type=%#02x", sig - m->manifestdata, sigType);
RETURN(3);
}
// add value to nonce, with the same result regardless of CPU endian order
// allowing for any carry value up to the size of the whole nonce
static void add_nonce(unsigned char *nonce, int64_t value){
static void add_nonce(unsigned char *nonce, uint64_t value)
{
int i=crypto_stream_xsalsa20_NONCEBYTES -1;
while(i>=0 && value>0){
int x = nonce[i]+(value & 0xFF);
@ -590,13 +562,10 @@ static void add_nonce(unsigned char *nonce, int64_t value){
/* crypt a block of a stream, allowing for offsets that don't align perfectly to block boundaries
* for efficiency the caller should use a buffer size of (n*RHIZOME_CRYPT_PAGE_SIZE)
*/
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t stream_offset,
const unsigned char *key, const unsigned char *nonce){
if (stream_offset<0)
return WHY("Invalid stream offset");
int64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, uint64_t stream_offset,
const unsigned char *key, const unsigned char *nonce)
{
uint64_t nonce_offset = stream_offset & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
size_t offset=0;
unsigned char block_nonce[crypto_stream_xsalsa20_NONCEBYTES];
@ -632,34 +601,24 @@ int rhizome_crypt_xor_block(unsigned char *buffer, size_t buffer_size, int64_t s
return 0;
}
int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
int rhizome_derive_payload_key(rhizome_manifest *m)
{
// don't do anything if the manifest isn't flagged as being encrypted
if (!m->payloadEncryption)
if (m->payloadEncryption != PAYLOAD_ENCRYPTED)
return 0;
if (m->payloadEncryption!=1)
return WHYF("Unsupported encryption scheme %d", m->payloadEncryption);
char *sender = rhizome_manifest_get(m, "sender", NULL, 0);
char *recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
if (sender && recipient){
sid_t sender_sid, recipient_sid;
if (cf_opt_sid(&sender_sid, sender)!=CFOK)
return WHYF("Unable to parse sender sid");
if (cf_opt_sid(&recipient_sid, recipient)!=CFOK)
return WHYF("Unable to parse recipient sid");
if (m->has_sender && m->has_recipient){
unsigned char *nm_bytes=NULL;
int cn=0,in=0,kp=0;
if (!keyring_find_sid(keyring, &cn, &in, &kp, &sender_sid)){
if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->sender)){
cn=in=kp=0;
if (!keyring_find_sid(keyring, &cn, &in, &kp, &recipient_sid)){
return WHYF("Neither the sender %s nor the recipient %s appears in our keyring", sender, recipient);
if (!keyring_find_sid(keyring, &cn, &in, &kp, &m->recipient)){
return WHYF("Neither the sender %s nor the recipient %s appears in our keyring",
alloca_tohex_sid_t(m->sender),
alloca_tohex_sid_t(m->recipient));
}
nm_bytes=keyring_get_nm_bytes(&recipient_sid, &sender_sid);
nm_bytes=keyring_get_nm_bytes(&m->recipient, &m->sender);
}else{
nm_bytes=keyring_get_nm_bytes(&sender_sid, &recipient_sid);
nm_bytes=keyring_get_nm_bytes(&m->sender, &m->recipient);
}
if (!nm_bytes)
@ -670,10 +629,8 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
bcopy(hash, m->payloadKey, RHIZOME_CRYPT_KEY_BYTES);
}else{
if(!m->haveSecret){
if (rhizome_extract_privatekey_required(m, bsk))
return -1;
}
if (!m->haveSecret)
return WHY("Cannot derive payload key because bundle secret is unknown");
unsigned char raw_key[9+crypto_sign_edwards25519sha512batch_SECRETKEYBYTES]="sasquatch";
bcopy(m->cryptoSignSecret, &raw_key[9], crypto_sign_edwards25519sha512batch_SECRETKEYBYTES);
@ -687,9 +644,9 @@ int rhizome_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk)
// journal bundles must always have the same nonce, regardless of version.
// otherwise, generate nonce from version#bundle id#version;
unsigned char raw_nonce[8 + 8 + sizeof m->cryptoSignPublic.binary];
write_uint64(&raw_nonce[0], m->journalTail>=0?0:m->version);
write_uint64(&raw_nonce[0], m->is_journal ? 0 : m->version);
bcopy(m->cryptoSignPublic.binary, &raw_nonce[8], sizeof m->cryptoSignPublic.binary);
write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->journalTail>=0?0:m->version);
write_uint64(&raw_nonce[8 + sizeof m->cryptoSignPublic.binary], m->is_journal ? 0 : m->version);
unsigned char hash[crypto_hash_sha512_BYTES];

671
rhizome_database.c
View File

@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "strbuf.h"
#include "strbuf_helpers.h"
#include "str.h"
#include "keyring.h"
static char rhizome_thisdatastore_path[256];
@ -75,7 +76,8 @@ int create_rhizome_datastore_dir()
return emkdirs(rhizome_datastore_path(), 0700);
}
sqlite3 *rhizome_db=NULL;
sqlite3 *rhizome_db = NULL;
uuid_t rhizome_db_uuid;
/* XXX Requires a messy join that might be slow. */
int rhizome_manifest_priority(sqlite_retry_state *retry, const rhizome_bid_t *bidp)
@ -106,11 +108,32 @@ int is_debug_rhizome_ads()
static int (*sqlite_trace_func)() = is_debug_rhizome;
const struct __sourceloc *sqlite_trace_whence = NULL;
static int sqlite_trace_done;
/* This is called by SQLite when executing a statement using sqlite3_step(). Unfortunately, it is
* not called on PRAGMA statements, and possibly others. Hence the use of the profile callback (see
* below).
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static void sqlite_trace_callback(void *context, const char *rendered_sql)
{
if (sqlite_trace_func())
logMessage(LOG_LEVEL_DEBUG, sqlite_trace_whence ? *sqlite_trace_whence : __HERE__, "%s", rendered_sql);
++sqlite_trace_done;
}
/* This is called by SQLite when an executed statement finishes. We use it to log rendered SQL
* statements when the trace callback (above) has not been called, eg, for PRAGMA statements. This
* requires that the 'sqlite_trace_done' static be reset to zero whenever a new prepared statement
* is about to be stepped.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static void sqlite_profile_callback(void *context, const char *rendered_sql, sqlite_uint64 nanosec)
{
if (!sqlite_trace_done)
sqlite_trace_callback(context, rendered_sql);
}
/* This function allows code like:
@ -141,33 +164,31 @@ void sqlite_log(void *ignored, int result, const char *msg)
WARNF("Sqlite: %d %s", result, msg);
}
void verify_bundles(){
void verify_bundles()
{
// assume that only the manifest itself can be trusted
// fetch all manifests and reinsert them.
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// This cursor must be ordered descending as re-inserting the manifests will give them a new higher manifest id.
// If we didn't, we'd get stuck in an infinite loop.
sqlite3_stmt *statement = sqlite_prepare(&retry, "SELECT ROWID, MANIFEST FROM MANIFESTS ORDER BY ROWID DESC;");
while(sqlite_step_retry(&retry, statement)==SQLITE_ROW){
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
sqlite3_int64 rowid = sqlite3_column_int64(statement, 0);
const void *manifest = sqlite3_column_blob(statement, 1);
size_t manifest_length = sqlite3_column_bytes(statement, 1);
rhizome_manifest *m=rhizome_new_manifest();
int ret=0;
ret = rhizome_read_manifest_file(m, manifest, manifest_length);
if (ret==0 && m->errors)
ret=-1;
if (ret==0)
ret=rhizome_manifest_verify(m);
if (ret==0){
m->finalised=1;
m->manifest_bytes=m->manifest_all_bytes;
// store it again, to ensure it is valid and stored correctly with matching file content.
ret=rhizome_store_bundle(m);
int ret = -1;
if ( rhizome_read_manifest_file(m, manifest, manifest_length) == 0
&& rhizome_manifest_validate(m)
&& rhizome_manifest_verify(m)
) {
assert(m->finalised);
// Store it again, to ensure that MANIFESTS columns are up to date.
ret = rhizome_store_bundle(m);
}
if (ret!=0){
DEBUGF("Removing invalid manifest entry @%lld", rowid);
if (ret) {
if (config.debug.rhizome)
DEBUGF("Removing invalid manifest entry @%lld", rowid);
sqlite_exec_void_retry(&retry, "DELETE FROM MANIFESTS WHERE ROWID = ?;", INT64, rowid, END);
}
rhizome_manifest_free(m);
@ -176,20 +197,25 @@ void verify_bundles(){
}
/*
* The MANIFESTS table 'author' column records the cryptographically verified SID of the author
* that has write permission on the bundle, ie, possesses the Rhizome secret key that generated the
* BID, and hence can derive the Bundle Secret from the bundle's BK field:
* - The MANIFESTS table 'author' column is set to the author SID when a bundle is created
* locally by a non-secret identity, so no verification need be performed for one's own
* bundles while they remain in the Rhizome store.
* - When a bundle is imported, the 'author' column is set to NULL to indicate that no
* verification has passed yet. This includes one's own bundles that have been purged from
* the local Rhizome store then recovered from a remote Rhizome node.
* - When a manifest with NULL 'author' is examined closely, ie extracted, not merely
* listed, the keyring is searched for an identity that is the author. If an author is
* found, the MANIFESTS table 'author' column is updated. This allows one to regain the
* ability to overwrite one's own bundles that have been lost but recovered from an exterior
* Rhizome node.
* The MANIFESTS table 'author' column records the cryptographically verified SID of the author that
* has write permission on the bundle, ie, possesses the Rhizome secret key that generated the BID,
* and hence can derive the Bundle Secret from the bundle's BK field:
*
* - The MANIFESTS table 'author' column is set to the author SID when a bundle is created locally
* by a non-secret identity, so no verification need be performed for one's own bundles while they
* remain in the local Rhizome store.
*
* - When a bundle is imported, the 'author' column is set to NULL to indicate that no verification
* has passed yet. This includes one's own bundles that have been purged from the local Rhizome
* store then recovered from a remote Rhizome node.
*
* - When a manifest with NULL 'author' is examined closely, ie extracted, not merely listed, the
* keyring is searched for an identity that is the author. If the identity is found and its
* Rhizome Secret unlocks the Bundle Key (ie, reveals a Bundle Secret that yields the Bundle's ID
* as its public key), the MANIFESTS table 'author' column is updated. This allows one to regain
* the ability to overwrite one's own bundles that have been lost but
* recovered from an exterior Rhizome node.
*
* - The above check automates the "own bundle recovery" mechanism at the expense of a CPU-heavy
* cryptographic check every time a foreign bundle is examined, but at least listing is fast.
* This will not scale as many identities are added to the keyring. It will eventually have to be
@ -201,7 +227,10 @@ void verify_bundles(){
int rhizome_opendb()
{
if (rhizome_db) return 0;
if (rhizome_db) {
assert(uuid_is_valid(&rhizome_db_uuid));
return 0;
}
IN();
@ -226,6 +255,7 @@ int rhizome_opendb()
RETURN(WHYF("SQLite could not open database %s: %s", dbpath, sqlite3_errmsg(rhizome_db)));
}
sqlite3_trace(rhizome_db, sqlite_trace_callback, NULL);
sqlite3_profile(rhizome_db, sqlite_profile_callback, NULL);
int loglevel = (config.debug.rhizome) ? LOG_LEVEL_DEBUG : LOG_LEVEL_SILENT;
/* Read Rhizome configuration */
@ -251,11 +281,9 @@ int rhizome_opendb()
) {
RETURN(WHY("Failed to create schema"));
}
/* Create indexes if they don't already exist */
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS bundlesizeindex ON manifests (filesize);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_HASH ON MANIFESTS(filehash);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=1;", END);
}
if (version<2){
@ -267,26 +295,52 @@ int rhizome_opendb()
verify_bundles();
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=2;", END);
}
if (version<3){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE INDEX IF NOT EXISTS IDX_MANIFESTS_ID_VERSION ON MANIFESTS(id, version);", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=3;", END);
}
if (version<4){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "ALTER TABLE MANIFESTS ADD COLUMN tail integer;", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=4;", END);
}
if (version<5){
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "CREATE TABLE IF NOT EXISTS IDENTITY(uuid text not null); ", END);
sqlite_exec_void_loglevel(LOG_LEVEL_WARN, "PRAGMA user_version=5;", END);
}
char buf[UUID_STRLEN + 1];
int r = sqlite_exec_strbuf_retry(&retry, strbuf_local(buf, sizeof buf), "SELECT uuid from IDENTITY LIMIT 1;", END);
if (r == -1)
RETURN(-1);
if (r) {
if (!str_to_uuid(buf, &rhizome_db_uuid, NULL)) {
WHYF("IDENTITY table contains malformed UUID %s -- overwriting", alloca_str_toprint(buf));
if (uuid_generate_random(&rhizome_db_uuid) == -1)
RETURN(WHY("Cannot generate new UUID for Rhizome database"));
if (sqlite_exec_void_retry(&retry, "UPDATE IDENTITY SET uuid = ? LIMIT 1;", UUID_T, &rhizome_db_uuid, END) == -1)
RETURN(WHY("Failed to update new UUID in Rhizome database"));
if (config.debug.rhizome)
DEBUGF("Updated Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
}
} else if (r == 0) {
if (uuid_generate_random(&rhizome_db_uuid) == -1)
RETURN(WHY("Cannot generate UUID for Rhizome database"));
if (sqlite_exec_void_retry(&retry, "INSERT INTO IDENTITY (uuid) VALUES (?);", UUID_T, &rhizome_db_uuid, END) == -1)
RETURN(WHY("Failed to insert UUID into Rhizome database"));
if (config.debug.rhizome)
DEBUGF("Set Rhizome database UUID to %s", alloca_uuid_str(rhizome_db_uuid));
}
// TODO recreate tables with collate nocase on hex columns
/* Future schema updates should be performed here.
The above schema can be assumed to exist.
All changes should attempt to preserve any existing data */
// We can't delete a file that is being transferred in another process at this very moment...
if (config.rhizome.clean_on_open)
rhizome_cleanup(NULL);
INFOF("Opened Rhizome database %s, UUID=%s", dbpath, alloca_uuid_str(rhizome_db_uuid));
RETURN(0);
OUT();
}
@ -437,6 +491,7 @@ sqlite3_stmt *_sqlite_prepare(struct __sourceloc __whence, int log_level, sqlite
while (1) {
switch (sqlite3_prepare_v2(rhizome_db, sqltext, -1, &statement, NULL)) {
case SQLITE_OK:
sqlite_trace_done = 0;
RETURN(statement);
case SQLITE_BUSY:
case SQLITE_LOCKED:
@ -504,7 +559,7 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
return -1;
}
#define BIND_DEBUG(TYP,FUNC,ARGFMT,...) \
if (config.debug.rhizome_bind) \
if (config.debug.rhizome_sql_bind) \
DEBUGF("%s%s %s(%d," ARGFMT ") %s", #TYP, strbuf_str(ext), #FUNC, index, ##__VA_ARGS__, sqlite3_sql(statement))
#define BIND_RETRY(FUNC, ...) \
do { \
@ -684,10 +739,10 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
if (hashp == NULL) {
BIND_NULL(RHIZOME_FILEHASH_T);
} else {
char hash_hex[RHIZOME_FILEHASH_STRLEN];
tohex(hash_hex, sizeof hash_hex, hashp->binary);
BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%zu,SQLITE_TRANSIENT", hash_hex, sizeof hash_hex);
BIND_RETRY(sqlite3_bind_text, hash_hex, sizeof hash_hex, SQLITE_TRANSIENT);
char hash_hex[RHIZOME_FILEHASH_STRLEN + 1];
tohex(hash_hex, RHIZOME_FILEHASH_STRLEN, hashp->binary);
BIND_DEBUG(RHIZOME_FILEHASH_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", hash_hex, RHIZOME_FILEHASH_STRLEN);
BIND_RETRY(sqlite3_bind_text, hash_hex, RHIZOME_FILEHASH_STRLEN, SQLITE_TRANSIENT);
}
}
break;
@ -734,6 +789,19 @@ int _sqlite_vbind(struct __sourceloc __whence, int log_level, sqlite_retry_state
}
}
break;
case UUID_T: {
const uuid_t *uuidp = va_arg(ap, const uuid_t *);
++argnum;
if (uuidp == NULL) {
BIND_NULL(UUID_T);
} else {
char uuid_str[UUID_STRLEN + 1];
uuid_to_str(uuidp, uuid_str);
BIND_DEBUG(UUID_T, sqlite3_bind_text, "%s,%u,SQLITE_TRANSIENT", uuid_str, UUID_STRLEN);
BIND_RETRY(sqlite3_bind_text, uuid_str, UUID_STRLEN, SQLITE_TRANSIENT);
}
}
break;
#undef BIND_RETRY
default:
FATALF("at bind arg %u, unsupported bind code typ=0x%08x: %s", argnum, typ, sqlite3_sql(statement));
@ -1270,49 +1338,32 @@ int rhizome_drop_stored_file(const rhizome_filehash_t *hashp, int maximum_priori
*/
int rhizome_store_bundle(rhizome_manifest *m)
{
if (!m->finalised) return WHY("Manifest was not finalised");
if (!m->finalised)
return WHY("Manifest was not finalised");
if (m->haveSecret) {
/* We used to store the secret in the database, but we don't anymore, as we use
the BK field in the manifest. So nothing to do here. */
} else {
/* We don't have the secret for this manifest, so only allow updates if
the self-signature is valid */
if (!m->selfSigned)
return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt.");
}
// If we don't have the secret for this manifest, only store it if its self-signature is valid
if (!m->haveSecret && !m->selfSigned)
return WHY("Manifest is not signed, and I don't have the key. Manifest might be forged or corrupt.");
/* Bind BAR to data field */
unsigned char bar[RHIZOME_BAR_BYTES];
rhizome_manifest_to_bar(m,bar);
/* Store the file (but not if it is already in the database) */
if (m->fileLength > 0 && !rhizome_exists(&m->filehash))
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0 && !rhizome_exists(&m->filehash))
return WHY("File should already be stored by now");
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
sid_t *sender = NULL;
const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0);
if (sender_field) {
sender = (sid_t *) alloca(sizeof *sender);
if (str_to_sid_t(sender, sender_field) == -1)
return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field));
}
sid_t *recipient = NULL;
const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0);
if (recipient_field) {
recipient = (sid_t *) alloca(sizeof *recipient);
if (str_to_sid_t(recipient, recipient_field) == -1)
return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field));
}
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
if (sqlite_exec_void_retry(&retry, "BEGIN TRANSACTION;", END) == -1)
return WHY("Failed to begin transaction");
time_ms_t now = gettime_ms();
// The INSERT OR REPLACE statement will delete a row with the same ID (primary key) if it exists,
// so a new autoincremented ROWID will be allocated whether or not the manifest with this ID is
// already in the table. Other code depends on this property: that ROWID is monotonically
// increasing with time and unique.
sqlite3_stmt *stmt;
if ((stmt = sqlite_prepare_bind(&retry,
"INSERT OR REPLACE INTO MANIFESTS("
@ -1333,18 +1384,19 @@ int rhizome_store_bundle(rhizome_manifest *m)
"?,?,?,?,?,?,?,?,?,?,?,?,?"
");",
RHIZOME_BID_T, &m->cryptoSignPublic,
STATIC_BLOB, m->manifestdata, m->manifest_bytes,
STATIC_BLOB, m->manifestdata, m->manifest_all_bytes,
INT64, m->version,
INT64, (int64_t) gettime_ms(),
INT64, (int64_t) now,
STATIC_BLOB, bar, RHIZOME_BAR_BYTES,
INT64, m->fileLength,
RHIZOME_FILEHASH_T|NUL, m->fileLength > 0 ? &m->filehash : NULL,
SID_T|NUL, is_sid_t_any(m->author) ? NULL : &m->author,
STATIC_TEXT, service,
STATIC_TEXT|NUL, name,
SID_T|NUL, sender,
SID_T|NUL, recipient,
INT64, m->journalTail,
INT64, m->filesize,
RHIZOME_FILEHASH_T|NUL, m->filesize > 0 ? &m->filehash : NULL,
// Only store the author if it is known to be authentic.
SID_T|NUL, m->authorship == AUTHOR_AUTHENTIC ? &m->author : NULL,
STATIC_TEXT, m->service,
STATIC_TEXT|NUL, m->name,
SID_T|NUL, m->has_sender ? &m->sender : NULL,
SID_T|NUL, m->has_recipient ? &m->recipient : NULL,
INT64, m->tail,
END
)
) == NULL)
@ -1353,12 +1405,15 @@ int rhizome_store_bundle(rhizome_manifest *m)
goto rollback;
sqlite3_finalize(stmt);
stmt = NULL;
rhizome_manifest_set_rowid(m, sqlite3_last_insert_rowid(rhizome_db));
rhizome_manifest_set_inserttime(m, now);
// if (serverMode)
// rhizome_sync_bundle_inserted(bar);
// TODO remove old payload?
#if 0
if (rhizome_manifest_get(m,"isagroup",NULL,0)!=NULL) {
int closed=rhizome_manifest_get_ll(m,"closedgroup");
if (closed<1) closed=0;
@ -1380,11 +1435,13 @@ int rhizome_store_bundle(rhizome_manifest *m)
sqlite3_finalize(stmt);
stmt = NULL;
}
#endif
#if 0
if (m->group_count > 0) {
if ((stmt = sqlite_prepare(&retry, "INSERT OR REPLACE INTO GROUPMEMBERSHIPS (manifestid, groupid) VALUES (?, ?);")) == NULL)
goto rollback;
int i;
unsigned i;
for (i=0;i<m->group_count;i++){
if (sqlite_bind(&retry, stmt, RHIZOME_BID_T, &m->cryptoSignPublic, TEXT, m->groups[i]) == -1)
goto rollback;
@ -1395,14 +1452,15 @@ int rhizome_store_bundle(rhizome_manifest *m)
sqlite3_finalize(stmt);
stmt = NULL;
}
#endif
if (sqlite_exec_void_retry(&retry, "COMMIT;", END) != -1){
// This message used in tests; do not modify or remove.
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
INFOF("RHIZOME ADD MANIFEST service=%s bid=%s version=%"PRId64,
service ? service : "NULL",
m->service ? m->service : "NULL",
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
m->version
);
);
monitor_announce_bundle(m);
if (serverMode)
rhizome_sync_announce();
@ -1416,183 +1474,188 @@ rollback:
return -1;
}
int rhizome_list_manifests(struct cli_context *context, const char *service, const char *name,
const char *sender_sid, const char *recipient_sid,
int limit, int offset, char count_rows)
/* The cursor struct must be zerofilled and the query parameters optionally filled in prior to
* calling this function.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_list_open(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
{
if (config.debug.rhizome)
DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64,
c,
alloca_str_toprint(c->service),
alloca_str_toprint(c->name),
c->is_sender_set ? alloca_tohex_sid_t(c->sender) : "UNSET",
c->is_recipient_set ? alloca_tohex_sid_t(c->recipient) : "UNSET",
c->rowid_since,
c->_rowid_last
);
IN();
strbuf b = strbuf_alloca(1024);
strbuf_sprintf(b, "SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE 1=1");
if (service && *service)
strbuf_sprintf(b, " AND service = ?1");
if (name && *name)
strbuf_sprintf(b, " AND name like ?2");
if (sender_sid && *sender_sid)
strbuf_sprintf(b, " AND sender = ?3");
if (recipient_sid && *recipient_sid)
strbuf_sprintf(b, " AND recipient = ?4");
strbuf_sprintf(b, " ORDER BY inserttime DESC");
if (offset)
strbuf_sprintf(b, " OFFSET %u", offset);
if (c->service)
strbuf_puts(b, " AND service = @service");
if (c->name)
strbuf_puts(b, " AND name like @name");
if (c->is_sender_set)
strbuf_puts(b, " AND sender = @sender");
if (c->is_recipient_set)
strbuf_puts(b, " AND recipient = @recipient");
if (c->rowid_since) {
strbuf_puts(b, " AND rowid > @last ORDER BY rowid ASC"); // oldest first
if (c->_rowid_last < c->rowid_since)
c->_rowid_last = c->rowid_since;
} else {
if (c->_rowid_last)
strbuf_puts(b, " AND rowid < @last");
strbuf_puts(b, " ORDER BY rowid DESC"); // most recent first
}
if (strbuf_overrun(b))
RETURN(WHYF("SQL command too long: %s", strbuf_str(b)));
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare(&retry, strbuf_str(b));
if (!statement)
c->_statement = sqlite_prepare(retry, strbuf_str(b));
if (c->_statement == NULL)
RETURN(-1);
int ret = 0;
if (service && *service)
ret = sqlite3_bind_text(statement, 1, service, -1, SQLITE_STATIC);
if (ret==SQLITE_OK && name && *name)
ret = sqlite3_bind_text(statement, 2, name, -1, SQLITE_STATIC);
if (ret==SQLITE_OK && sender_sid && *sender_sid)
ret = sqlite3_bind_text(statement, 3, sender_sid, -1, SQLITE_STATIC);
if (ret==SQLITE_OK && recipient_sid && *recipient_sid)
ret = sqlite3_bind_text(statement, 4, recipient_sid, -1, SQLITE_STATIC);
if (ret!=SQLITE_OK){
ret = WHYF("Failed to bind parameters: %s", sqlite3_errmsg(rhizome_db));
goto cleanup;
}
ret=0;
size_t rows = 0;
const char *names[]={
"_id",
"service",
"id",
"version",
"date",
".inserttime",
".author",
".fromhere",
"filesize",
"filehash",
"sender",
"recipient",
"name"
};
cli_columns(context, 13, names);
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
++rows;
if (limit>0 && rows>limit)
break;
if (!( sqlite3_column_count(statement) == 6
&& sqlite3_column_type(statement, 0) == SQLITE_TEXT
&& sqlite3_column_type(statement, 1) == SQLITE_BLOB
&& sqlite3_column_type(statement, 2) == SQLITE_INTEGER
&& sqlite3_column_type(statement, 3) == SQLITE_INTEGER
&& ( sqlite3_column_type(statement, 4) == SQLITE_TEXT
|| sqlite3_column_type(statement, 4) == SQLITE_NULL
)
)) {
ret = WHY("Incorrect statement column");
break;
}
rhizome_manifest *m = rhizome_new_manifest();
if (m == NULL) {
ret = WHY("Out of manifests");
break;
}
const char *q_manifestid = (const char *) sqlite3_column_text(statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
int64_t q_version = sqlite3_column_int64(statement, 2);
int64_t q_inserttime = sqlite3_column_int64(statement, 3);
const char *q_author = (const char *) sqlite3_column_text(statement, 4);
int64_t rowid = sqlite3_column_int64(statement, 5);
if (rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1) {
WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
} else {
int64_t blob_version = rhizome_manifest_get_ll(m, "version");
if (blob_version != q_version)
WARNF("MANIFESTS row id=%s version=%"PRId64" does not match manifest blob.version=%"PRId64,
q_manifestid, q_version, blob_version);
int match = 1;
const char *blob_service = rhizome_manifest_get(m, "service", NULL, 0);
if (service[0] && !(blob_service && strcasecmp(service, blob_service) == 0))
match = 0;
const char *blob_sender = rhizome_manifest_get(m, "sender", NULL, 0);
const char *blob_recipient = rhizome_manifest_get(m, "recipient", NULL, 0);
if (match && sender_sid[0]) {
if (!(blob_sender && strcasecmp(sender_sid, blob_sender) == 0))
match = 0;
}
if (match && recipient_sid[0]) {
if (!(blob_recipient && strcasecmp(recipient_sid, blob_recipient) == 0))
match = 0;
}
if (match) {
const char *blob_name = rhizome_manifest_get(m, "name", NULL, 0);
int64_t blob_date = rhizome_manifest_get_ll(m, "date");
const char *blob_filehash = rhizome_manifest_get(m, "filehash", NULL, 0);
int from_here = 0;
sid_t senderSid;
sid_t recipientSid;
if (blob_sender)
str_to_sid_t(&senderSid, blob_sender);
if (blob_recipient)
str_to_sid_t(&recipientSid, blob_recipient);
if (q_author) {
if (config.debug.rhizome) DEBUGF("q_author=%s", alloca_str_toprint(q_author));
str_to_sid_t(&m->author, q_author);
int cn = 0, in = 0, kp = 0;
from_here = keyring_find_sid(keyring, &cn, &in, &kp, &m->author);
}
if (!from_here && blob_sender) {
if (config.debug.rhizome) DEBUGF("blob_sender=%s", alloca_str_toprint(blob_sender));
int cn = 0, in = 0, kp = 0;
from_here = keyring_find_sid(keyring, &cn, &in, &kp, &senderSid);
}
cli_put_long(context, rowid, ":");
cli_put_string(context, blob_service, ":");
cli_put_hexvalue(context, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, ":");
cli_put_long(context, blob_version, ":");
cli_put_long(context, blob_date, ":");
cli_put_long(context, q_inserttime, ":");
cli_put_hexvalue(context, q_author ? m->author.binary : NULL, sizeof m->author.binary, ":");
cli_put_long(context, from_here, ":");
cli_put_long(context, m->fileLength, ":");
unsigned char filehash[SHA512_DIGEST_LENGTH];
if (m->fileLength)
fromhex(filehash, blob_filehash, SHA512_DIGEST_LENGTH);
cli_put_hexvalue(context, m->fileLength?filehash:NULL, SHA512_DIGEST_LENGTH, ":");
cli_put_hexvalue(context, blob_sender ? senderSid.binary : NULL, sizeof senderSid.binary, ":");
cli_put_hexvalue(context, blob_recipient ? recipientSid.binary : NULL, sizeof recipientSid.binary, ":");
cli_put_string(context, blob_name, "\n");
}
}
if (m) rhizome_manifest_free(m);
}
if (ret==0 && count_rows){
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW)
++rows;
}
cli_row_count(context, rows);
cleanup:
sqlite3_finalize(statement);
RETURN(ret);
if (c->service && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@service", c->service, END) == -1)
goto failure;
if (c->name && sqlite_bind(retry, c->_statement, NAMED|STATIC_TEXT, "@name", c->name, END) == -1)
goto failure;
if (c->is_sender_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@sender", &c->sender, END) == -1)
goto failure;
if (c->is_recipient_set && sqlite_bind(retry, c->_statement, NAMED|SID_T, "@recipient", &c->recipient, END) == -1)
goto failure;
if (c->_rowid_last && sqlite_bind(retry, c->_statement, NAMED|INT64, "@last", c->_rowid_last, END) == -1)
goto failure;
c->manifest = NULL;
c->_rowid_current = 0;
RETURN(0);
OUT();
failure:
sqlite3_finalize(c->_statement);
c->_statement = NULL;
RETURN(-1);
OUT();
}
/* Guaranteed to return manifests with monotonically descending rowid. The first manifest will have
* the greatest rowid.
*
* Returns 1 if a new manifest has been fetched from the list, in which case the cursor 'manifest'
* field points to the fetched manifest. Returns 0 if there are no more manifests in the list.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int rhizome_list_next(sqlite_retry_state *retry, struct rhizome_list_cursor *c)
{
if (config.debug.rhizome)
DEBUGF("c=%p c->service=%s c->name=%s c->sender=%s c->recipient=%s c->rowid_since=%"PRIu64" c->_rowid_last=%"PRIu64,
c,
alloca_str_toprint(c->service),
alloca_str_toprint(c->name),
c->is_sender_set ? alloca_tohex_sid_t(c->sender) : "UNSET",
c->is_recipient_set ? alloca_tohex_sid_t(c->recipient) : "UNSET",
c->rowid_since,
c->_rowid_last
);
IN();
if (c->_statement == NULL && rhizome_list_open(retry, c) == -1)
RETURN(-1);
while (1) {
if (c->manifest) {
rhizome_manifest_free(c->manifest);
c->_rowid_current = 0;
c->manifest = NULL;
}
if (sqlite_step_retry(retry, c->_statement) != SQLITE_ROW)
break;
assert(sqlite3_column_count(c->_statement) == 6);
assert(sqlite3_column_type(c->_statement, 0) == SQLITE_TEXT);
assert(sqlite3_column_type(c->_statement, 1) == SQLITE_BLOB);
assert(sqlite3_column_type(c->_statement, 2) == SQLITE_INTEGER);
assert(sqlite3_column_type(c->_statement, 3) == SQLITE_INTEGER);
assert(sqlite3_column_type(c->_statement, 4) == SQLITE_TEXT || sqlite3_column_type(c->_statement, 4) == SQLITE_NULL);
assert(sqlite3_column_type(c->_statement, 5) == SQLITE_INTEGER);
uint64_t q_rowid = sqlite3_column_int64(c->_statement, 5);
if (c->_rowid_current && (c->rowid_since ? q_rowid >= c->_rowid_current : q_rowid <= c->_rowid_current)) {
WHYF("Query returned rowid=%"PRIu64" out of order (last was %"PRIu64") -- skipped", q_rowid, c->_rowid_current);
continue;
}
c->_rowid_current = q_rowid;
if (q_rowid <= c->rowid_since) {
WHYF("Query returned rowid=%"PRIu64" <= rowid_since=%"PRIu64" -- skipped", q_rowid, c->rowid_since);
continue;
}
const char *q_manifestid = (const char *) sqlite3_column_text(c->_statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(c->_statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(c->_statement, 1); // must call after sqlite3_column_blob()
int64_t q_version = sqlite3_column_int64(c->_statement, 2);
int64_t q_inserttime = sqlite3_column_int64(c->_statement, 3);
const char *q_author = (const char *) sqlite3_column_text(c->_statement, 4);
sid_t *author = NULL;
if (q_author) {
author = alloca(sizeof *author);
if (str_to_sid_t(author, q_author) == -1) {
WHYF("MANIFESTS row id=%s has invalid author column %s -- skipped", q_manifestid, alloca_str_toprint(q_author));
continue;
}
}
rhizome_manifest *m = c->manifest = rhizome_new_manifest();
if (m == NULL)
RETURN(-1);
if ( rhizome_read_manifest_file(m, manifestblob, manifestblobsize) == -1
|| !rhizome_manifest_validate(m)
) {
WHYF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
continue;
}
if (m->version != q_version) {
WHYF("MANIFESTS row id=%s version=%"PRId64" does not match manifest blob version=%"PRId64" -- skipped",
q_manifestid, q_version, m->version);
continue;
}
if (author)
rhizome_manifest_set_author(m, author);
rhizome_manifest_set_rowid(m, q_rowid);
rhizome_manifest_set_inserttime(m, q_inserttime);
if (c->service && !(m->service && strcasecmp(c->service, m->service) == 0))
continue;
if (c->is_sender_set && !(m->has_sender && cmp_sid_t(&c->sender, &m->sender) == 0))
continue;
if (c->is_recipient_set && !(m->has_recipient && cmp_sid_t(&c->recipient, &m->recipient) == 0))
continue;
assert(c->_rowid_current != 0);
// Don't do rhizome_verify_author(m); too CPU expensive for a listing. Save that for when
// the bundle is extracted or exported.
RETURN(1);
}
assert(c->_rowid_current == 0);
RETURN(0);
OUT();
}
void rhizome_list_commit(struct rhizome_list_cursor *c)
{
if (config.debug.rhizome)
DEBUGF("c=%p c->rowid_since=%"PRIu64" c->_rowid_current=%"PRIu64" c->_rowid_last=%"PRIu64,
c, c->rowid_since, c->_rowid_current, c->_rowid_last);
assert(c->_rowid_current != 0);
if (c->_rowid_last == 0 || (c->rowid_since ? c->_rowid_current > c->_rowid_last : c->_rowid_current < c->_rowid_last))
c->_rowid_last = c->_rowid_current;
}
void rhizome_list_release(struct rhizome_list_cursor *c)
{
if (config.debug.rhizome)
DEBUGF("c=%p", c);
if (c->manifest) {
rhizome_manifest_free(c->manifest);
c->_rowid_current = 0;
c->manifest = NULL;
}
if (c->_statement) {
sqlite3_finalize(c->_statement);
c->_statement = NULL;
}
}
void rhizome_bytes_to_hex_upper(unsigned const char *in, char *out, int byteCount)
@ -1631,56 +1694,36 @@ int rhizome_update_file_priority(const char *fileid)
*/
int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
{
const char *service = rhizome_manifest_get(m, "service", NULL, 0);
if (service == NULL)
if (m->service == NULL)
return WHY("Manifest has no service");
const char *name = rhizome_manifest_get(m, "name", NULL, 0);
sid_t *sender = NULL;
const char *sender_field = rhizome_manifest_get(m, "sender", NULL, 0);
if (sender_field) {
sender = (sid_t *) alloca(sizeof *sender);
if (str_to_sid_t(sender, sender_field) == -1)
return WHYF("invalid field in manifest bid=%s: sender=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(sender_field));
}
sid_t *recipient = NULL;
const char *recipient_field = rhizome_manifest_get(m, "recipient", NULL, 0);
if (recipient_field) {
recipient = (sid_t *) alloca(sizeof *recipient);
if (str_to_sid_t(recipient, recipient_field) == -1)
return WHYF("invalid field in manifest bid=%s: recipient=%s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), alloca_str_toprint(recipient_field));
}
char sqlcmd[1024];
strbuf b = strbuf_local(sqlcmd, sizeof sqlcmd);
strbuf_puts(b, "SELECT id, manifest, author FROM manifests WHERE filesize = ? AND service = ?");
if (m->fileLength != 0)
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0)
strbuf_puts(b, " AND filehash = ?");
if (name)
if (m->name)
strbuf_puts(b, " AND name = ?");
if (sender)
if (m->has_sender)
strbuf_puts(b, " AND sender = ?");
if (recipient)
if (m->has_recipient)
strbuf_puts(b, " AND recipient = ?");
if (strbuf_overrun(b))
return WHYF("SQL command too long: %s", strbuf_str(b));
int ret = 0;
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->fileLength, STATIC_TEXT, service, END);
sqlite3_stmt *statement = sqlite_prepare_bind(&retry, strbuf_str(b), INT64, m->filesize, STATIC_TEXT, m->service, END);
if (!statement)
return -1;
int field = 2;
if (m->fileLength != 0)
if (m->filesize > 0)
sqlite_bind(&retry, statement, INDEX|RHIZOME_FILEHASH_T, ++field, &m->filehash, END);
if (name)
sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, name, END);
if (sender)
sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, sender, END);
if (recipient)
sqlite_bind(&retry, statement, INDEX|SID_T|NUL, ++field, recipient, END);
if (m->name)
sqlite_bind(&retry, statement, INDEX|STATIC_TEXT, ++field, m->name, END);
if (m->has_sender)
sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->sender, END);
if (m->has_recipient)
sqlite_bind(&retry, statement, INDEX|SID_T, ++field, &m->recipient, END);
int rows = 0;
while (sqlite_step_retry(&retry, statement) == SQLITE_ROW) {
@ -1695,24 +1738,28 @@ int rhizome_find_duplicate(const rhizome_manifest *m, rhizome_manifest **found)
const unsigned char *q_manifestid = sqlite3_column_text(statement, 0);
const char *manifestblob = (char *) sqlite3_column_blob(statement, 1);
size_t manifestblobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
if (rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1) {
if ( rhizome_read_manifest_file(blob_m, manifestblob, manifestblobsize) == -1
|| !rhizome_manifest_validate(blob_m)
) {
WARNF("MANIFESTS row id=%s has invalid manifest blob -- skipped", q_manifestid);
goto next;
}
if (rhizome_manifest_verify(blob_m)) {
if (!rhizome_manifest_verify(blob_m)) {
WARNF("MANIFESTS row id=%s fails verification -- skipped", q_manifestid);
goto next;
}
const char *q_author = (const char *) sqlite3_column_text(statement, 2);
if (q_author) {
if (config.debug.rhizome)
strbuf_sprintf(b, " .author=%s", q_author);
str_to_sid_t(&blob_m->author, q_author);
sid_t author;
if (str_to_sid_t(&author, q_author) == -1)
WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_manifestid, alloca_str_toprint(q_author));
else
rhizome_manifest_set_author(blob_m, &author);
}
// check that we can re-author this manifest
if (rhizome_extract_privatekey(blob_m, NULL)){
rhizome_authenticate_author(blob_m);
if (m->authorship != AUTHOR_AUTHENTIC)
goto next;
}
*found = blob_m;
if (config.debug.rhizome)
DEBUGF("Found duplicate payload, %s", q_manifestid);
@ -1734,15 +1781,20 @@ static int unpack_manifest_row(rhizome_manifest *m, sqlite3_stmt *statement)
int64_t q_inserttime = sqlite3_column_int64(statement, 3);
const char *q_author = (const char *) sqlite3_column_text(statement, 4);
size_t q_blobsize = sqlite3_column_bytes(statement, 1); // must call after sqlite3_column_blob()
if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1)
return WHYF("Manifest %s exists but is invalid", q_id);
uint64_t q_rowid = sqlite3_column_int64(statement, 5);
if (rhizome_read_manifest_file(m, q_blob, q_blobsize) == -1 || !rhizome_manifest_validate(m))
return WHYF("Manifest bid=%s in database but invalid", q_id);
if (q_author) {
if (str_to_sid_t(&m->author, q_author) == -1)
WARNF("manifest id=%s contains invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author));
sid_t author;
if (str_to_sid_t(&author, q_author) == -1)
WARNF("MANIFESTS row id=%s has invalid author=%s -- ignored", q_id, alloca_str_toprint(q_author));
else
rhizome_manifest_set_author(m, &author);
}
if (m->version != q_version)
WARNF("Version mismatch, manifest is %"PRId64", database is %"PRId64, m->version, q_version);
m->inserttime = q_inserttime;
rhizome_manifest_set_rowid(m, q_rowid);
rhizome_manifest_set_inserttime(m, q_inserttime);
return 0;
}
@ -1757,7 +1809,7 @@ int rhizome_retrieve_manifest(const rhizome_bid_t *bidp, rhizome_manifest *m)
{
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, manifest, version, inserttime, author FROM manifests WHERE id = ?",
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id = ?",
RHIZOME_BID_T, bidp,
END);
if (!statement)
@ -1787,7 +1839,7 @@ int rhizome_retrieve_manifest_by_prefix(const unsigned char *prefix, unsigned pr
like[prefix_strlen] = '%';
like[prefix_strlen + 1] = '\0';
sqlite3_stmt *statement = sqlite_prepare_bind(&retry,
"SELECT id, manifest, version, inserttime, author FROM manifests WHERE id like ?",
"SELECT id, manifest, version, inserttime, author, rowid FROM manifests WHERE id like ?",
TEXT, like,
END);
if (!statement)
@ -1937,7 +1989,7 @@ static int is_interesting(const char *id_hex, int64_t version)
int rhizome_is_bar_interesting(unsigned char *bar)
{
int64_t version = rhizome_bar_version(bar);
char id_hex[RHIZOME_MANIFEST_ID_STRLEN];
char id_hex[RHIZOME_BAR_PREFIX_BYTES *2 + 2];
tohex(id_hex, RHIZOME_BAR_PREFIX_BYTES * 2, &bar[RHIZOME_BAR_PREFIX_OFFSET]);
strcat(id_hex, "%");
return is_interesting(id_hex, version);
@ -1945,10 +1997,5 @@ int rhizome_is_bar_interesting(unsigned char *bar)
int rhizome_is_manifest_interesting(rhizome_manifest *m)
{
char id[RHIZOME_MANIFEST_ID_STRLEN + 1];
if (!rhizome_manifest_get(m, "id", id, sizeof id))
// dodgy manifest, we don't want to receive it
return WHY("Ignoring bad manifest (no ID field)");
str_toupper_inplace(id);
return is_interesting(id, m->version);
return is_interesting(alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
}

11
rhizome_direct.c
View File

@ -442,11 +442,12 @@ rhizome_manifest *rhizome_direct_get_manifest(unsigned char *bid_prefix,int pref
if (!manifestblob) goto error;
rhizome_manifest *m=rhizome_new_manifest();
if (rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1)
{
rhizome_manifest_free(m);
goto error;
}
if ( rhizome_read_manifest_file(m,manifestblob,manifestblobsize)==-1
|| !rhizome_manifest_validate(m)
) {
rhizome_manifest_free(m);
goto error;
}
DEBUGF("Read manifest");
sqlite3_blob_close(blob);

98
rhizome_direct_http.c
View File

@ -54,7 +54,7 @@ static void rhizome_direct_clear_temporary_files(rhizome_http_request *r)
}
}
int rhizome_direct_import_end(struct http_request *hr)
static int rhizome_direct_import_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (!r->received_manifest) {
@ -168,7 +168,7 @@ int rhizome_direct_enquiry_end(struct http_request *hr)
return 0;
}
int rhizome_direct_addfile_end(struct http_request *hr)
static int rhizome_direct_addfile_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
// If given a file without a manifest, we should only accept if it we are configured to do so, and
@ -222,23 +222,22 @@ int rhizome_direct_addfile_end(struct http_request *hr)
return 0;
}
// If manifest template did not specify a service field, then by default it is "file".
if (rhizome_manifest_get(m, "service", NULL, 0) == NULL)
rhizome_manifest_set(m, "service", RHIZOME_SERVICE_FILE);
sid_t *author = NULL;
if (!is_sid_t_any(config.rhizome.api.addfile.default_author))
author = &config.rhizome.api.addfile.default_author;
rhizome_bk_t bsk = config.rhizome.api.addfile.bundle_secret_key;
if (rhizome_fill_manifest(m, r->data_file_name, author, &bsk)) {
if (!rhizome_is_bk_none(&config.rhizome.api.addfile.bundle_secret_key))
rhizome_apply_bundle_secret(m, &config.rhizome.api.addfile.bundle_secret_key);
if (m->service == NULL)
rhizome_manifest_set_service(m, RHIZOME_SERVICE_FILE);
const sid_t *author = is_sid_t_any(config.rhizome.api.addfile.default_author) ? NULL : &config.rhizome.api.addfile.default_author;
if (rhizome_fill_manifest(m, r->data_file_name, author)) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
http_request_simple_response(&r->http, 500, "Internal Error: Could not fill manifest");
return 0;
}
m->payloadEncryption=0;
rhizome_manifest_set_ll(m,"crypt",m->payloadEncryption?1:0);
rhizome_manifest_set_crypt(m, PAYLOAD_CLEAR);
// import file contents
// TODO, stream file into database
if (m->fileLength) {
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize > 0) {
if (rhizome_add_file(m, payload_path)) {
rhizome_manifest_free(m);
rhizome_direct_clear_temporary_files(r);
@ -258,7 +257,7 @@ int rhizome_direct_addfile_end(struct http_request *hr)
if (config.debug.rhizome)
DEBUGF("Import sans-manifest appeared to succeed");
/* Respond with the manifest that was added. */
http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_bytes);
http_request_response_static(&r->http, 200, "text/plain", (const char *)m->manifestdata, m->manifest_all_bytes);
/* clean up after ourselves */
if (mout && mout != m)
rhizome_manifest_free(mout);
@ -271,14 +270,14 @@ int rhizome_direct_addfile_end(struct http_request *hr)
}
}
void rhizome_direct_process_mime_start(struct http_request *hr)
static void rhizome_direct_process_mime_start(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(r->current_part == NONE);
assert(r->part_fd == -1);
}
void rhizome_direct_process_mime_end(struct http_request *hr)
static void rhizome_direct_process_mime_end(struct http_request *hr)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->part_fd != -1) {
@ -302,19 +301,19 @@ void rhizome_direct_process_mime_end(struct http_request *hr)
r->current_part = NONE;
}
void rhizome_direct_process_mime_content_disposition(struct http_request *hr, const struct mime_content_disposition *cd)
static void rhizome_direct_process_mime_part_header(struct http_request *hr, const struct mime_part_headers *h)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (strcmp(cd->name, "data") == 0) {
if (strcmp(h->content_disposition.name, "data") == 0) {
r->current_part = DATA;
strncpy(r->data_file_name, cd->filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0';
strncpy(r->data_file_name, h->content_disposition.filename, sizeof r->data_file_name)[sizeof r->data_file_name - 1] = '\0';
}
else if (strcmp(cd->name, "manifest") == 0) {
else if (strcmp(h->content_disposition.name, "manifest") == 0) {
r->current_part = MANIFEST;
} else
return;
char path[512];
if (form_temporary_file_path(r, path, cd->name) == -1) {
if (form_temporary_file_path(r, path, h->content_disposition.name) == -1) {
http_request_simple_response(&r->http, 500, "Internal Error: Buffer overrun");
return;
}
@ -325,7 +324,7 @@ void rhizome_direct_process_mime_content_disposition(struct http_request *hr, co
}
}
void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len)
static void rhizome_direct_process_mime_body(struct http_request *hr, const char *buf, size_t len)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
if (r->part_fd != -1) {
@ -344,7 +343,7 @@ int rhizome_direct_import(rhizome_http_request *r, const char *remainder)
}
r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition;
r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header;
r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
r->http.handle_content_end = rhizome_direct_import_end;
r->current_part = NONE;
@ -361,7 +360,7 @@ int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder)
}
r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition;
r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header;
r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
r->http.handle_content_end = rhizome_direct_enquiry_end;
r->current_part = NONE;
@ -394,7 +393,7 @@ int rhizome_direct_addfile(rhizome_http_request *r, const char *remainder)
}
r->http.form_data.handle_mime_part_start = rhizome_direct_process_mime_start;
r->http.form_data.handle_mime_part_end = rhizome_direct_process_mime_end;
r->http.form_data.handle_mime_content_disposition = rhizome_direct_process_mime_content_disposition;
r->http.form_data.handle_mime_part_header = rhizome_direct_process_mime_part_header;
r->http.form_data.handle_mime_body = rhizome_direct_process_mime_body;
r->http.handle_content_end = rhizome_direct_addfile_end;
r->current_part = NONE;
@ -637,12 +636,10 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* Get filehash and size from manifest if present */
if (config.debug.rhizome_tx) {
DEBUGF("bundle id = %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
DEBUGF("bundle filehash = '%s'", alloca_tohex_rhizome_filehash_t(m->filehash));
DEBUGF("file size = %"PRId64, m->fileLength);
DEBUGF("bundle filehash = %s", alloca_tohex_rhizome_filehash_t(m->filehash));
DEBUGF("file size = %"PRId64, m->filesize);
DEBUGF("version = %"PRId64, m->version);
}
int64_t version = rhizome_manifest_get_ll(m, "version");
if (config.debug.rhizome_tx)
DEBUGF("version = %"PRId64,version);
/* We now have everything we need to compose the POST request and send it.
*/
@ -661,15 +658,16 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
"\r\n";
/* Work out what the content length should be */
if (config.debug.rhizome_tx)
DEBUGF("manifest_all_bytes=%d, manifest_bytes=%d", m->manifest_all_bytes,m->manifest_bytes);
int content_length
=strlen(template2)-2 /* minus 2 for the "%s" that gets replaced */
+strlen(boundary)
+m->manifest_all_bytes
+strlen(template3)-2 /* minus 2 for the "%s" that gets replaced */
+strlen(boundary)
+m->fileLength
+strlen("\r\n--")+strlen(boundary)+strlen("--\r\n");
DEBUGF("manifest_all_bytes=%u, manifest_body_bytes=%u", m->manifest_all_bytes, m->manifest_body_bytes);
assert(m->filesize != RHIZOME_SIZE_UNSET);
size_t content_length =
strlen(template2) - 2 /* minus 2 for the "%s" that gets replaced */
+ strlen(boundary)
+ m->manifest_all_bytes
+ strlen(template3) - 2 /* minus 2 for the "%s" that gets replaced */
+ strlen(boundary)
+ m->filesize
+ strlen("\r\n--") + strlen(boundary) + strlen("--\r\n");
int len=snprintf(buffer,8192,template,content_length,boundary);
len+=snprintf(&buffer[len],8192-len,template2,boundary);
@ -705,7 +703,7 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
/* send file contents */
{
rhizome_filehash_t filehash;
if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, version, &filehash) == -1)
if (rhizome_database_filehash_from_id(&m->cryptoSignPublic, m->version, &filehash) == -1)
goto closeit;
struct rhizome_read read;
@ -713,28 +711,26 @@ void rhizome_direct_http_dispatch(rhizome_direct_sync_request *r)
if (rhizome_open_read(&read, &filehash))
goto closeit;
int64_t read_ofs;
for(read_ofs=0;read_ofs<m->fileLength;){
uint64_t read_ofs;
for(read_ofs=0;read_ofs<m->filesize;){
unsigned char buffer[4096];
read.offset=read_ofs;
int bytes_read = rhizome_read(&read, buffer, sizeof buffer);
if (bytes_read<0){
ssize_t bytes_read = rhizome_read(&read, buffer, sizeof buffer);
if (bytes_read == -1) {
rhizome_read_close(&read);
goto closeit;
}
int write_ofs=0;
while(write_ofs < bytes_read){
int written = write(sock, buffer + write_ofs, bytes_read - write_ofs);
if (written<0){
size_t write_ofs = 0;
while (write_ofs < (size_t) bytes_read){
ssize_t written = write(sock, buffer + write_ofs, (size_t) bytes_read - write_ofs);
if (written == -1){
WHY_perror("write");
rhizome_read_close(&read);
goto closeit;
}
write_ofs+=written;
write_ofs += (size_t) written;
}
read_ofs+=bytes_read;
read_ofs += (size_t) bytes_read;
}
rhizome_read_close(&read);
}

260
rhizome_fetch.c
View File

@ -26,6 +26,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "str.h"
#include "strbuf_helpers.h"
#include "overlay_address.h"
#include "socket.h"
#include "dataformats.h"
/* Represents a queued fetch of a bundle payload, for which the manifest is already known.
*/
@ -73,7 +75,7 @@ struct rhizome_fetch_slot {
/* HTTP streaming reception of manifests */
char manifest_buffer[1024];
int manifest_bytes;
unsigned manifest_bytes;
/* MDP transport specific elements */
rhizome_bid_t bid;
@ -100,7 +102,7 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot);
*/
struct rhizome_fetch_queue {
struct rhizome_fetch_slot active; // must be first element in struct
int candidate_queue_size;
unsigned candidate_queue_size;
struct rhizome_fetch_candidate *candidate_queue;
unsigned char log_size_threshold; // will only queue payloads smaller than this.
};
@ -114,7 +116,6 @@ struct rhizome_fetch_candidate queue3[4];
struct rhizome_fetch_candidate queue4[2];
struct rhizome_fetch_candidate queue5[2];
#define NELS(a) (sizeof (a) / sizeof *(a))
#define slotno(slot) (int)((struct rhizome_fetch_queue *)(slot) - &rhizome_fetch_queues[0])
/* Static allocation of the queue structures. Must be in order of ascending log_size_threshold.
@ -150,53 +151,69 @@ static const char * fetch_state(int state)
}
}
int rhizome_active_fetch_count()
{
int i,active=0;
for(i=0;i<NQUEUES;i++)
if (rhizome_fetch_queues[i].active.state!=RHIZOME_FETCH_FREE)
active++;
return active;
}
int rhizome_active_fetch_bytes_received(int q)
static uint64_t rhizome_active_fetch_bytes_received(int q)
{
if (q<0 || q>=NQUEUES) return -1;
if (rhizome_fetch_queues[q].active.state==RHIZOME_FETCH_FREE) return -1;
return (int)rhizome_fetch_queues[q].active.write_state.file_offset;
return rhizome_fetch_queues[q].active.write_state.file_offset;
}
int rhizome_fetch_queue_bytes(){
int i,j,bytes=0;
static uint64_t rhizome_fetch_queue_bytes()
{
uint64_t bytes = 0;
unsigned i;
for(i=0;i<NQUEUES;i++){
if (rhizome_fetch_queues[i].active.state!=RHIZOME_FETCH_FREE){
int received=rhizome_fetch_queues[i].active.write_state.file_offset;
bytes+=rhizome_fetch_queues[i].active.manifest->fileLength - received;
assert(rhizome_fetch_queues[i].active.manifest->filesize != RHIZOME_SIZE_UNSET);
bytes += rhizome_fetch_queues[i].active.manifest->filesize - rhizome_fetch_queues[i].active.write_state.file_offset;
}
unsigned j;
for (j=0;j<rhizome_fetch_queues[i].candidate_queue_size;j++){
if (rhizome_fetch_queues[i].candidate_queue[j].manifest)
bytes+=rhizome_fetch_queues[i].candidate_queue[j].manifest->fileLength;
if (rhizome_fetch_queues[i].candidate_queue[j].manifest) {
assert(rhizome_fetch_queues[i].candidate_queue[j].manifest->filesize != RHIZOME_SIZE_UNSET);
bytes += rhizome_fetch_queues[i].candidate_queue[j].manifest->filesize;
}
}
}
return bytes;
}
void rhizome_fetch_log_short_status()
{
int i,active=0;
for(i=0;i<NQUEUES;i++)
if (rhizome_fetch_queues[i].active.state!=RHIZOME_FETCH_FREE)
active++;
if (!active)
return;
INFOF("Rhizome transfer progress: %"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64" (remaining %"PRIu64")",
rhizome_active_fetch_bytes_received(0),
rhizome_active_fetch_bytes_received(1),
rhizome_active_fetch_bytes_received(2),
rhizome_active_fetch_bytes_received(3),
rhizome_active_fetch_bytes_received(4),
rhizome_active_fetch_bytes_received(5),
rhizome_fetch_queue_bytes());
}
int rhizome_fetch_status_html(strbuf b)
{
int i,j;
unsigned i;
for(i=0;i<NQUEUES;i++){
struct rhizome_fetch_queue *q=&rhizome_fetch_queues[i];
int used=0;
unsigned used=0;
unsigned j;
for (j=0;j<q->candidate_queue_size;j++){
if (q->candidate_queue[j].manifest)
used++;
}
strbuf_sprintf(b, "<p>Slot %d, (%d of %d): ", i, used, q->candidate_queue_size);
strbuf_sprintf(b, "<p>Slot %u, (%u of %u): ", i, used, q->candidate_queue_size);
if (q->active.state!=RHIZOME_FETCH_FREE){
strbuf_sprintf(b, "%s %"PRId64" of %"PRId64" from %s*",
strbuf_sprintf(b, "%s %"PRIu64" of %"PRIu64" from %s*",
fetch_state(q->active.state),
q->active.write_state.file_offset,
q->active.manifest->fileLength,
q->active.manifest->filesize,
alloca_tohex_sid_t_trunc(q->active.peer_sid, 16));
}else{
strbuf_puts(b, "inactive");
@ -206,7 +223,8 @@ int rhizome_fetch_status_html(strbuf b)
for (j=0; j< q->candidate_queue_size;j++){
if (q->candidate_queue[j].manifest){
candidates++;
candidate_size += q->candidate_queue[j].manifest->fileLength;
assert(q->candidate_queue[j].manifest->filesize != RHIZOME_SIZE_UNSET);
candidate_size += q->candidate_queue[j].manifest->filesize;
}
}
if (candidates)
@ -224,10 +242,9 @@ static struct profile_total fetch_stats = { .name="rhizome_fetch_poll" };
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static struct rhizome_fetch_queue *rhizome_find_queue(uint64_t size)
static struct rhizome_fetch_queue *rhizome_find_queue(unsigned char log_size)
{
int i;
unsigned char log_size = log2ll(size);
for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
if (log_size < q->log_size_threshold)
@ -272,9 +289,10 @@ static struct rhizome_fetch_slot *fetch_search_slot(const unsigned char *id, int
// find the first matching candidate for this bundle
static struct rhizome_fetch_candidate *fetch_search_candidate(const unsigned char *id, int prefix_length)
{
int i, j;
unsigned i;
for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
unsigned j;
for (j = 0; j < q->candidate_queue_size; j++) {
struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
if (!c->manifest)
@ -304,12 +322,12 @@ rhizome_manifest * rhizome_fetch_search(const unsigned char *id, int prefix_leng
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch_queue *q, int i)
static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch_queue *q, unsigned i)
{
struct rhizome_fetch_candidate * const c = &q->candidate_queue[i];
struct rhizome_fetch_candidate * e = &q->candidate_queue[q->candidate_queue_size - 1];
if (config.debug.rhizome_rx)
DEBUGF("insert queue[%d] candidate[%d]", (int)(q - rhizome_fetch_queues), i);
DEBUGF("insert queue[%d] candidate[%u]", (int)(q - rhizome_fetch_queues), i);
assert(i >= 0 && i < q->candidate_queue_size);
assert(i == 0 || c[-1].manifest);
if (e->manifest) // queue is full
@ -330,7 +348,7 @@ static struct rhizome_fetch_candidate *rhizome_fetch_insert(struct rhizome_fetch
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, int i)
static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, unsigned i)
{
assert(i >= 0 && i < q->candidate_queue_size);
struct rhizome_fetch_candidate *c = &q->candidate_queue[i];
@ -348,10 +366,10 @@ static void rhizome_fetch_unqueue(struct rhizome_fetch_queue *q, int i)
static void candidate_unqueue(struct rhizome_fetch_candidate *c)
{
int i, index;
unsigned i;
for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
index = c - q->candidate_queue;
unsigned index = c - q->candidate_queue;
if (index>=0 && index < q->candidate_queue_size){
rhizome_fetch_unqueue(q, index);
return;
@ -457,10 +475,9 @@ int rhizome_queue_ignore_manifest(unsigned char *bid_prefix, int prefix_len, int
static int rhizome_import_received_bundle(struct rhizome_manifest *m)
{
m->finalised = 1;
m->manifest_bytes = m->manifest_all_bytes; // store the signatures too
if (config.debug.rhizome_rx) {
DEBUGF("manifest len=%d has %d signatories. Associated file = %"PRId64" bytes",
m->manifest_bytes, m->sig_count, m->fileLength);
DEBUGF("manifest len=%u has %u signatories. Associated filesize=%"PRIu64" bytes",
m->manifest_all_bytes, m->sig_count, m->filesize);
dump("manifest", m->manifestdata, m->manifest_all_bytes);
}
return rhizome_add_manifest(m, m->ttl - 1 /* TTL */);
@ -489,24 +506,27 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
strbuf r = strbuf_local(slot->request, sizeof slot->request);
strbuf_sprintf(r, "GET /rhizome/file/%s HTTP/1.0\r\n", alloca_tohex_rhizome_filehash_t(slot->manifest->filehash));
if (slot->manifest->journalTail>=0){
if (slot->manifest->is_journal){
// if we're fetching a journal bundle, work out how many bytes we have of a previous version
// and therefore what range of bytes we should ask for
slot->previous = rhizome_new_manifest();
if (rhizome_retrieve_manifest(&slot->manifest->cryptoSignPublic, slot->previous)){
rhizome_manifest_free(slot->previous);
slot->previous=NULL;
// check that the new journal is valid and has some overlapping bytes
}else if (slot->previous->journalTail > slot->manifest->journalTail
|| slot->previous->fileLength + slot->previous->journalTail < slot->manifest->journalTail){
// check that the new journal is valid and has some overlapping bytes
}else if ( !slot->previous->is_journal
|| slot->previous->tail > slot->manifest->tail
|| slot->previous->filesize + slot->previous->tail < slot->manifest->tail
){
rhizome_manifest_free(slot->previous);
slot->previous=NULL;
}else{
assert(slot->previous->fileLength >= slot->manifest->journalTail);
assert(slot->manifest->fileLength > 0);
strbuf_sprintf(r, "Range: bytes=%"PRId64"-%"PRId64"\r\n",
slot->previous->fileLength - slot->manifest->journalTail, slot->manifest->fileLength - 1);
assert(slot->previous->filesize >= slot->manifest->tail);
assert(slot->manifest->filesize > 0);
strbuf_sprintf(r, "Range: bytes=%"PRIu64"-%"PRIu64"\r\n",
slot->previous->filesize - slot->manifest->tail,
slot->manifest->filesize - 1
);
}
}
@ -516,7 +536,7 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
RETURN(WHY("request overrun"));
slot->request_len = strbuf_len(r);
if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->fileLength, RHIZOME_PRIORITY_DEFAULT))
if (rhizome_open_write(&slot->write_state, &slot->manifest->filehash, slot->manifest->filesize, RHIZOME_PRIORITY_DEFAULT))
RETURN(-1);
} else {
strbuf r = strbuf_local(slot->request, sizeof slot->request);
@ -526,8 +546,8 @@ static int schedule_fetch(struct rhizome_fetch_slot *slot)
slot->request_len = strbuf_len(r);
slot->manifest_bytes=0;
slot->write_state.file_offset=0;
slot->write_state.file_length=-1;
slot->write_state.file_offset = 0;
slot->write_state.file_length = RHIZOME_SIZE_UNSET;
}
slot->request_ofs = 0;
@ -647,16 +667,16 @@ rhizome_fetch(struct rhizome_fetch_slot *slot, rhizome_manifest *m, const struct
*/
if (config.debug.rhizome_rx)
DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRId64" peerip=%s",
DEBUGF("Fetching bundle slot=%d bid=%s version=%"PRId64" size=%"PRIu64" peerip=%s",
slotno(slot),
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic),
m->version,
m->fileLength,
m->filesize,
alloca_sockaddr(peerip, sizeof(struct sockaddr_in))
);
// If the payload is empty, no need to fetch, so import now.
if (m->fileLength == 0) {
if (m->filesize == 0) {
if (config.debug.rhizome_rx)
DEBUGF(" manifest fetch not started -- nil payload, so importing instead");
if (rhizome_import_received_bundle(m) == -1)
@ -769,7 +789,7 @@ static void rhizome_start_next_queued_fetch(struct rhizome_fetch_slot *slot)
IN();
struct rhizome_fetch_queue *q;
for (q = (struct rhizome_fetch_queue *) slot; q >= rhizome_fetch_queues; --q) {
int i = 0;
unsigned i = 0;
struct rhizome_fetch_candidate *c;
while (i < q->candidate_queue_size && (c = &q->candidate_queue[i])->manifest) {
int result = rhizome_fetch(slot, c->manifest, &c->peer_ipandport, &c->peer_sid);
@ -815,17 +835,14 @@ static void rhizome_start_next_queued_fetches(struct sched_ent *alarm)
/* Do we have space to add a fetch candidate of this size? */
int rhizome_fetch_has_queue_space(unsigned char log2_size){
int i;
for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
if (log2_size < q->log_size_threshold){
// is there an empty candidate?
int j=0;
for (j=0;j < q->candidate_queue_size;j++)
if (!q->candidate_queue[j].manifest)
return 1;
return 0;
}
struct rhizome_fetch_queue *q = rhizome_find_queue(log2_size);
if (q){
// is there an empty candidate?
unsigned j=0;
for (j=0;j < q->candidate_queue_size;j++)
if (!q->candidate_queue[j].manifest)
return 1;
return 0;
}
return 0;
}
@ -859,8 +876,8 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
int priority=100; /* normal priority */
if (config.debug.rhizome_rx)
DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRId64" priority=%d:",
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->fileLength, priority);
DEBUGF("Considering import bid=%s version=%"PRId64" size=%"PRIu64" priority=%d:",
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, m->filesize, priority);
if (!rhizome_is_manifest_interesting(m)) {
if (config.debug.rhizome_rx)
@ -875,8 +892,9 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
DEBUGF(" is new (have version %"PRId64")", stored_version);
}
if (m->fileLength == 0) {
if (rhizome_manifest_verify(m) != 0) {
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize == 0) {
if (!rhizome_manifest_verify(m)) {
WHY("Error verifying manifest when considering for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
@ -889,9 +907,9 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
}
// Find the proper queue for the payload. If there is none suitable, it is an error.
struct rhizome_fetch_queue *qi = rhizome_find_queue(m->fileLength);
struct rhizome_fetch_queue *qi = rhizome_find_queue(log2ll(m->filesize));
if (!qi) {
WHYF("No suitable fetch queue for bundle size=%"PRId64, m->fileLength);
WHYF("No suitable fetch queue for bundle size=%"PRIu64, m->filesize);
rhizome_manifest_free(m);
RETURN(-1);
}
@ -900,9 +918,10 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
// may have changed between versions.) If a newer or the same version is already queued, then
// ignore this one. Otherwise, unqueue all older candidates.
int ci = -1;
int i, j;
unsigned i;
for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
unsigned j;
for (j = 0; j < q->candidate_queue_size; ) {
struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
if (c->manifest) {
@ -911,7 +930,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
rhizome_manifest_free(m);
RETURN(0);
}
if (!m->selfSigned && rhizome_manifest_verify(m)) {
if (!m->selfSigned && !rhizome_manifest_verify(m)) {
WHY("Error verifying manifest when considering queuing for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
@ -937,7 +956,7 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
RETURN(1);
}
if (!m->selfSigned && rhizome_manifest_verify(m)) {
if (!m->selfSigned && !rhizome_manifest_verify(m)) {
WHY("Error verifying manifest when considering queuing for import");
/* Don't waste time looking at this manifest again for a while */
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
@ -953,18 +972,18 @@ int rhizome_suggest_queue_manifest_import(rhizome_manifest *m, const struct sock
if (config.debug.rhizome_rx) {
DEBUG("Rhizome fetch queues:");
int i, j;
unsigned i, j;
for (i = 0; i < NQUEUES; ++i) {
struct rhizome_fetch_queue *q = &rhizome_fetch_queues[i];
for (j = 0; j < q->candidate_queue_size; ++j) {
struct rhizome_fetch_candidate *c = &q->candidate_queue[j];
if (!c->manifest)
break;
DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%"PRId64, i, j,
DEBUGF("%d:%d manifest=%p bid=%s priority=%d size=%"PRIu64, i, j,
c->manifest,
alloca_tohex_rhizome_bid_t(c->manifest->cryptoSignPublic),
c->priority,
c->manifest->fileLength
c->manifest->filesize
);
}
}
@ -1025,17 +1044,19 @@ static void rhizome_fetch_mdp_slot_callback(struct sched_ent *alarm)
struct rhizome_fetch_slot *slot=(struct rhizome_fetch_slot*)alarm;
time_ms_t now = gettime_ms();
if (now-slot->last_write_time>slot->mdpIdleTimeout) {
if (now - slot->last_write_time > slot->mdpIdleTimeout) {
DEBUGF("MDP connection timed out: last RX %"PRId64"ms ago (read %"PRId64" of %"PRId64" bytes)",
now-slot->last_write_time,
slot->write_state.file_offset, slot->write_state.file_length);
slot->write_state.file_offset,
slot->write_state.file_length);
rhizome_fetch_close(slot);
OUT();
return;
}
if (config.debug.rhizome_rx)
DEBUGF("Timeout: Resending request for slot=0x%p (%"PRId64" of %"PRId64" received)",
slot,slot->write_state.file_offset, slot->write_state.file_length);
DEBUGF("Timeout: Resending request for slot=0x%p (%"PRIu64" of %"PRIu64" received)",
slot, slot->write_state.file_offset,
slot->write_state.file_length);
rhizome_fetch_mdp_requestblocks(slot);
OUT();
}
@ -1050,7 +1071,7 @@ static int rhizome_fetch_mdp_touch_timeout(struct rhizome_fetch_slot *slot)
// For now, we will just make the timeout 1 second from the time of the last
// received block.
unschedule(&slot->alarm);
slot->alarm.alarm=gettime_ms()+1000;
slot->alarm.alarm=gettime_ms()+config.rhizome.mdp_stall_timeout;
slot->alarm.deadline=slot->alarm.alarm+500;
schedule(&slot->alarm);
return 0;
@ -1108,7 +1129,7 @@ static int rhizome_fetch_mdp_requestblocks(struct rhizome_fetch_slot *slot)
slot->write_state.file_offset,
slot->bidVersion);
overlay_mdp_dispatch(&mdp,0 /* system generated */,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
// remember when we sent the request so that we can adjust the inter-request
// interval based on how fast the packets arrive.
@ -1133,11 +1154,14 @@ static int pipe_journal(struct rhizome_fetch_slot *slot){
* [ | written | overlap | new content]
*/
uint64_t start = slot->manifest->journalTail - slot->previous->journalTail + slot->write_state.file_offset;
uint64_t length = slot->previous->fileLength - slot->manifest->journalTail - slot->write_state.file_offset;
assert(slot->manifest->tail != RHIZOME_SIZE_UNSET);
assert(slot->previous->tail != RHIZOME_SIZE_UNSET);
assert(slot->previous->filesize != RHIZOME_SIZE_UNSET);
uint64_t start = slot->manifest->tail - slot->previous->tail + slot->write_state.file_offset;
uint64_t length = slot->previous->filesize - slot->manifest->tail - slot->write_state.file_offset;
// of course there might not be any overlap
if (start < slot->previous->fileLength && length>0){
if (start>=0 && start < slot->previous->filesize && length>0){
if (config.debug.rhizome)
DEBUGF("Copying %"PRId64" bytes from previous journal", length);
rhizome_journal_pipe(&slot->write_state, &slot->previous->filehash, start, length);
@ -1170,8 +1194,8 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
}
if (config.debug.rhizome_rx)
DEBUGF("Trying to switch to MDP for Rhizome fetch: slot=0x%p (%"PRId64" bytes)",
slot,slot->write_state.file_length);
DEBUGF("Trying to switch to MDP for Rhizome fetch: slot=0x%p (%"PRIu64" bytes)",
slot, slot->write_state.file_length);
/* close socket and stop watching it */
if (slot->alarm.poll.fd>=0) {
@ -1205,8 +1229,15 @@ static int rhizome_fetch_switch_to_mdp(struct rhizome_fetch_slot *slot)
down too much. Much careful thought is required to optimise this
transport.
*/
slot->mdpIdleTimeout=config.rhizome.idle_timeout; // give up if nothing received for 5 seconds
slot->mdpRXBlockLength=config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size
slot->mdpIdleTimeout = config.rhizome.idle_timeout; // give up if nothing received for 5 seconds
unsigned char log_size=log2ll(slot->manifest->filesize);
struct rhizome_fetch_queue *q=rhizome_find_queue(log_size);
// increase the timeout based on the queue number
if (q)
slot->mdpIdleTimeout *= 1+(q - rhizome_fetch_queues);
slot->mdpRXBlockLength = config.rhizome.rhizome_mdp_block_size; // Rhizome over MDP block size
rhizome_fetch_mdp_requestblocks(slot);
RETURN(0);
@ -1289,7 +1320,9 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
call schedule queued items. */
rhizome_manifest *m = rhizome_new_manifest();
if (m) {
if (rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1) {
if ( rhizome_read_manifest_file(m, slot->manifest_buffer, (size_t)slot->manifest_bytes) == -1
|| !rhizome_manifest_validate(m)
) {
DEBUGF("Couldn't read manifest");
rhizome_manifest_free(m);
} else {
@ -1308,17 +1341,18 @@ int rhizome_write_complete(struct rhizome_fetch_slot *slot)
time_ms_t interval = now - slot->start_time;
if (interval <= 0)
interval = 1;
DEBUGF("Closing rhizome fetch slot = 0x%p. Received %"PRId64" bytes in %"PRId64"ms (%"PRId64"KB/sec).",
DEBUGF("Closing rhizome fetch slot = 0x%p. Received %"PRIu64" bytes in %"PRIu64"ms (%"PRIu64"KB/sec).",
slot, slot->write_state.file_offset,
(int64_t)interval,
(int64_t)((slot->write_state.file_offset) / interval));
(uint64_t)interval,
slot->write_state.file_offset / (uint64_t)interval
);
}
rhizome_fetch_close(slot);
RETURN(-1);
}
int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer, int bytes)
int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer, size_t bytes)
{
IN();
@ -1327,17 +1361,18 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer
// Truncate to known length of file (handy for reading from journal bundles that
// might grow while we are reading from them).
if (bytes>(slot->write_state.file_length-(slot->write_state.file_offset))){
bytes=slot->write_state.file_length-(slot->write_state.file_offset);
if (bytes > slot->write_state.file_length - slot->write_state.file_offset) {
bytes = slot->write_state.file_length - slot->write_state.file_offset;
}
if (!slot->manifest){
/* We are reading a manifest. Read it into a buffer. */
int count=bytes;
if (count+slot->manifest_bytes>1024) count=1024-slot->manifest_bytes;
unsigned count = bytes;
if (count + slot->manifest_bytes > 1024)
count = 1024 - slot->manifest_bytes;
bcopy(buffer,&slot->manifest_buffer[slot->manifest_bytes],count);
slot->manifest_bytes+=count;
slot->write_state.file_offset+=count;
slot->write_state.file_offset += count;
} else {
/* We are reading a file. Stream it into the database. */
@ -1358,7 +1393,7 @@ int rhizome_write_content(struct rhizome_fetch_slot *slot, unsigned char *buffer
int rhizome_received_content(const unsigned char *bidprefix,
uint64_t version, uint64_t offset,
int count,unsigned char *bytes,int type)
size_t count, unsigned char *bytes, int type)
{
IN();
if (!is_rhizome_mdp_enabled()) {
@ -1368,7 +1403,7 @@ int rhizome_received_content(const unsigned char *bidprefix,
if (slot && slot->bidVersion == version && slot->state == RHIZOME_FETCH_RXFILEMDP){
if (config.debug.rhizome)
DEBUGF("Rhizome over MDP receiving %d bytes.",count);
DEBUGF("Rhizome over MDP receiving %zu bytes.", count);
if (rhizome_random_write(&slot->write_state, offset, bytes, count)){
if (config.debug.rhizome)
DEBUGF("Write failed!");
@ -1398,18 +1433,18 @@ int rhizome_received_content(const unsigned char *bidprefix,
rhizome_manifest *m = NULL;
struct rhizome_fetch_candidate *c = NULL;
if (slot && slot->bidVersion == version && slot->manifest->fileLength==count
&& slot->state!=RHIZOME_FETCH_RXFILEMDP){
if (slot && slot->bidVersion == version && slot->manifest->filesize == count
&& slot->state != RHIZOME_FETCH_RXFILEMDP) {
m=slot->manifest;
}else{
slot = NULL;
c = fetch_search_candidate(bidprefix, 16);
if (c && c->manifest->version==version && c->manifest->fileLength==count)
if (c && c->manifest->version == version && c->manifest->filesize == count)
m=c->manifest;
}
if (m){
if (rhizome_import_buffer(m, bytes, count)>=0 && !rhizome_import_received_bundle(m)){
if (rhizome_import_buffer(m, bytes, count) >= 0 && !rhizome_import_received_bundle(m)){
INFOF("Completed MDP transfer in one hit for file %s",
alloca_tohex_rhizome_filehash_t(m->filehash));
if (c)
@ -1459,8 +1494,9 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
} else {
if (errno!=EAGAIN) {
if (config.debug.rhizome_rx)
DEBUGF("Empty read, closing connection: received %"PRId64" of %"PRId64" bytes",
slot->write_state.file_offset,slot->write_state.file_length);
DEBUGF("Empty read, closing connection: received %"PRIu64" of %"PRIu64" bytes",
slot->write_state.file_offset,
slot->write_state.file_length);
rhizome_fetch_switch_to_mdp(slot);
}
return;
@ -1509,18 +1545,18 @@ void rhizome_fetch_poll(struct sched_ent *alarm)
rhizome_fetch_switch_to_mdp(slot);
return;
}
if (slot->write_state.file_length==-1)
slot->write_state.file_length=parts.content_length;
if (slot->write_state.file_length == RHIZOME_SIZE_UNSET)
slot->write_state.file_length = parts.content_length;
else if (parts.content_length + parts.range_start != slot->write_state.file_length)
WARNF("Expected content length %"PRId64", got %"PRId64" + %"PRId64,
WARNF("Expected content length %"PRIu64", got %"PRIu64" + %"PRIu64,
slot->write_state.file_length, parts.content_length, parts.range_start);
/* We have all we need. The file is already open, so just write out any initial bytes of
the body we read.
*/
slot->state = RHIZOME_FETCH_RXFILE;
if (slot->previous && parts.range_start){
if (parts.range_start != slot->previous->fileLength - slot->manifest->journalTail)
WARNF("Expected Content-Range header to start @%"PRId64, slot->previous->fileLength - slot->manifest->journalTail);
if (parts.range_start != slot->previous->filesize - slot->manifest->tail)
WARNF("Expected Content-Range header to start @%"PRIu64, slot->previous->filesize - slot->manifest->tail);
pipe_journal(slot);
}

350
rhizome_http.c
View File

@ -1,6 +1,6 @@
/*
Serval Distributed Numbering Architecture (DNA)
Copyright (C) 2010 Paul Gardner-Stephen
Serval DNA Rhizome HTTP external interface
Copyright (C) 2013 Serval Project, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -21,37 +21,46 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/socket.h>
#include <signal.h>
#ifdef HAVE_SYS_FILIO_H
#include <sys/filio.h>
# include <sys/filio.h>
#endif
#include <sys/uio.h>
#include <assert.h>
#include "serval.h"
#include "overlay_address.h"
#include "conf.h"
#include "str.h"
#include "strbuf_helpers.h"
#include "rhizome.h"
#include "http_server.h"
#define RHIZOME_SERVER_MAX_LIVE_REQUESTS 32
typedef int HTTP_HANDLER(rhizome_http_request *r, const char *remainder);
struct http_handler{
const char *path;
int (*parser)(rhizome_http_request *r, const char *remainder);
HTTP_HANDLER *parser;
};
static int rhizome_status_page(rhizome_http_request *r, const char *remainder);
static int rhizome_file_page(rhizome_http_request *r, const char *remainder);
static int manifest_by_prefix_page(rhizome_http_request *r, const char *remainder);
static int interface_page(rhizome_http_request *r, const char *remainder);
static int neighbour_page(rhizome_http_request *r, const char *remainder);
static int fav_icon_header(rhizome_http_request *r, const char *remainder);
static int root_page(rhizome_http_request *r, const char *remainder);
static HTTP_HANDLER restful_rhizome_bundlelist_json;
static HTTP_HANDLER restful_rhizome_newsince;
extern int rhizome_direct_import(rhizome_http_request *r, const char *remainder);
extern int rhizome_direct_enquiry(rhizome_http_request *r, const char *remainder);
extern int rhizome_direct_dispatch(rhizome_http_request *r, const char *remainder);
static HTTP_HANDLER rhizome_status_page;
static HTTP_HANDLER rhizome_file_page;
static HTTP_HANDLER manifest_by_prefix_page;
static HTTP_HANDLER interface_page;
static HTTP_HANDLER neighbour_page;
static HTTP_HANDLER fav_icon_header;
static HTTP_HANDLER root_page;
extern HTTP_HANDLER rhizome_direct_import;
extern HTTP_HANDLER rhizome_direct_enquiry;
extern HTTP_HANDLER rhizome_direct_dispatch;
struct http_handler paths[]={
{"/restful/rhizome/bundlelist.json", restful_rhizome_bundlelist_json},
{"/restful/rhizome/newsince/", restful_rhizome_newsince},
{"/rhizome/status", rhizome_status_page},
{"/rhizome/file/", rhizome_file_page},
{"/rhizome/import", rhizome_direct_import},
@ -226,7 +235,7 @@ success:
static void rhizome_server_finalise_http_request(struct http_request *_r)
{
rhizome_http_request *r = (rhizome_http_request *) _r;
rhizome_read_close(&r->read_state);
rhizome_read_close(&r->u.read_state);
request_count--;
}
@ -246,7 +255,7 @@ void rhizome_server_poll(struct sched_ent *alarm)
} else {
struct sockaddr_in *peerip=NULL;
if (addr.sa_family == AF_INET) {
peerip = (struct sockaddr_in *)&addr;
peerip = (struct sockaddr_in *)&addr; // network order
INFOF("RHIZOME HTTP SERVER, ACCEPT addrlen=%u family=%u port=%u addr=%u.%u.%u.%u",
addr_len, peerip->sin_family, peerip->sin_port,
((unsigned char*)&peerip->sin_addr.s_addr)[0],
@ -267,8 +276,8 @@ void rhizome_server_poll(struct sched_ent *alarm)
request_count++;
request->uuid = rhizome_http_request_uuid_counter++;
request->data_file_name[0] = '\0';
request->read_state.blob_fd = -1;
request->read_state.blob_rowid = -1;
request->u.read_state.blob_fd = -1;
request->u.read_state.blob_rowid = -1;
if (peerip)
request->http.client_sockaddr_in = *peerip;
request->http.handle_headers = rhizome_dispatch;
@ -312,6 +321,248 @@ int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_
OUT();
}
static int is_from_loopback(const struct http_request *r)
{
return r->client_sockaddr_in.sin_family == AF_INET
&& ((unsigned char*)&r->client_sockaddr_in.sin_addr.s_addr)[0] == IN_LOOPBACKNET;
}
/* Return 1 if the given authorization credentials are acceptable.
* Return 0 if not.
*/
static int is_authorized(const struct http_client_authorization *auth)
{
if (auth->scheme != BASIC)
return 0;
unsigned i;
for (i = 0; i != config.rhizome.api.restful.users.ac; ++i) {
if ( strcmp(config.rhizome.api.restful.users.av[i].key, auth->credentials.basic.user) == 0
&& strcmp(config.rhizome.api.restful.users.av[i].value.password, auth->credentials.basic.password) == 0
)
return 1;
}
return 0;
}
static int authorize(struct http_request *r)
{
if (!is_from_loopback(r)) {
http_request_simple_response(r, 403, NULL);
return 0;
}
if (!is_authorized(&r->request_header.authorization)) {
r->response.header.www_authenticate.scheme = BASIC;
r->response.header.www_authenticate.realm = "Serval Rhizome";
http_request_simple_response(r, 401, NULL);
return 0;
}
return 1;
}
#define LIST_TOKEN_STRLEN (BASE64_ENCODED_LEN(sizeof(uuid_t) + 8))
#define alloca_list_token(rowid) list_token_to_str(alloca(LIST_TOKEN_STRLEN + 1), (rowid))
static char *list_token_to_str(char *buf, uint64_t rowid)
{
struct iovec iov[2];
iov[0].iov_base = rhizome_db_uuid.u.binary;
iov[0].iov_len = sizeof rhizome_db_uuid.u.binary;
iov[1].iov_base = &rowid;
iov[1].iov_len = sizeof rowid;
size_t n = base64url_encodev(buf, iov, 2);
assert(n == LIST_TOKEN_STRLEN);
buf[n] = '\0';
return buf;
}
static int strn_to_list_token(const char *str, uint64_t *rowidp, const char **afterp)
{
unsigned char token[sizeof rhizome_db_uuid.u.binary + sizeof *rowidp];
if (base64url_decode(token, sizeof token, str, 0, afterp, 0, NULL) != sizeof token)
return 0;
if (cmp_uuid_t(&rhizome_db_uuid, (uuid_t *) &token) != 0)
return 0;
memcpy(rowidp, token + sizeof rhizome_db_uuid.u.binary, sizeof *rowidp);
return 1;
}
static HTTP_CONTENT_GENERATOR restful_rhizome_bundlelist_json_content;
static int restful_rhizome_bundlelist_json(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
if (*remainder)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (!authorize(&r->http))
return 0;
r->u.list.phase = LIST_HEADER;
r->u.list.rowcount = 0;
bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
return 0;
}
static int restful_rhizome_newsince(rhizome_http_request *r, const char *remainder)
{
if (!is_rhizome_http_enabled())
return 1;
uint64_t rowid;
const char *end = NULL;
if (!strn_to_list_token(remainder, &rowid, &end) || strcmp(end, "/bundlelist.json") != 0)
return 1;
if (r->http.verb != HTTP_VERB_GET) {
http_request_simple_response(&r->http, 405, NULL);
return 0;
}
if (!authorize(&r->http))
return 0;
r->u.list.phase = LIST_HEADER;
r->u.list.rowcount = 0;
bzero(&r->u.list.cursor, sizeof r->u.list.cursor);
r->u.list.cursor.rowid_since = rowid;
r->u.list.end_time = gettime_ms() + config.rhizome.api.restful.newsince_timeout * 1000;
http_request_response_generated(&r->http, 200, "application/json", restful_rhizome_bundlelist_json_content);
return 0;
}
static int restful_rhizome_bundlelist_json_content_chunk(sqlite_retry_state *retry, struct rhizome_http_request *r, strbuf b)
{
const char *headers[] = {
".token",
"_id",
"service",
"id",
"version",
"date",
".inserttime",
".author",
".fromhere",
"filesize",
"filehash",
"sender",
"recipient",
"name"
};
switch (r->u.list.phase) {
case LIST_HEADER:
strbuf_puts(b, "{\n\"header\":[");
unsigned i;
for (i = 0; i != NELS(headers); ++i) {
if (i)
strbuf_putc(b, ',');
strbuf_json_string(b, headers[i]);
}
strbuf_puts(b, "],\n\"rows\":[");
if (!strbuf_overrun(b))
r->u.list.phase = LIST_ROWS;
return 1;
case LIST_ROWS:
{
int ret = rhizome_list_next(retry, &r->u.list.cursor);
if (ret == -1)
return -1;
if (ret == 0) {
time_ms_t now;
if (r->u.list.cursor.rowid_since == 0 || (now = gettime_ms()) >= r->u.list.end_time) {
strbuf_puts(b, "\n]\n}\n");
if (!strbuf_overrun(b))
r->u.list.phase = LIST_DONE;
return 0;
}
time_ms_t wake_at = now + config.rhizome.api.restful.newsince_poll_ms;
if (wake_at > r->u.list.end_time)
wake_at = r->u.list.end_time;
http_request_pause_response(&r->http, wake_at);
return 0;
}
rhizome_manifest *m = r->u.list.cursor.manifest;
assert(m->filesize != RHIZOME_SIZE_UNSET);
rhizome_lookup_author(m);
if (r->u.list.rowcount != 0)
strbuf_putc(b, ',');
strbuf_puts(b, "\n[");
if (m->rowid > r->u.list.rowid_highest) {
strbuf_json_string(b, alloca_list_token(m->rowid));
r->u.list.rowid_highest = m->rowid;
} else
strbuf_json_null(b);
strbuf_putc(b, ',');
strbuf_sprintf(b, "%"PRIu64, m->rowid);
strbuf_putc(b, ',');
strbuf_json_string(b, m->service);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
strbuf_putc(b, ',');
strbuf_sprintf(b, "%"PRId64, m->version);
strbuf_putc(b, ',');
if (m->has_date)
strbuf_sprintf(b, "%"PRItime_ms_t, m->date);
else
strbuf_json_null(b);
strbuf_putc(b, ',');
strbuf_sprintf(b, "%"PRItime_ms_t",", m->inserttime);
switch (m->authorship) {
case AUTHOR_LOCAL:
case AUTHOR_AUTHENTIC:
strbuf_json_hex(b, m->author.binary, sizeof m->author.binary);
strbuf_puts(b, ",1,");
break;
default:
strbuf_json_null(b);
strbuf_puts(b, ",1,");
break;
}
strbuf_sprintf(b, "%"PRIu64, m->filesize);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->filesize ? m->filehash.binary : NULL, sizeof m->filehash.binary);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->has_sender ? m->sender.binary : NULL, sizeof m->sender.binary);
strbuf_putc(b, ',');
strbuf_json_hex(b, m->has_recipient ? m->recipient.binary : NULL, sizeof m->recipient.binary);
strbuf_putc(b, ',');
strbuf_json_string(b, m->name);
strbuf_puts(b, "]");
if (!strbuf_overrun(b)) {
rhizome_list_commit(&r->u.list.cursor);
++r->u.list.rowcount;
}
return 1;
}
case LIST_DONE:
return 0;
}
abort();
}
static int restful_rhizome_bundlelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
{
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(bufsz > 0);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
int ret = rhizome_list_open(&retry, &r->u.list.cursor);
if (ret == -1)
return -1;
strbuf b = strbuf_local((char *)buf, bufsz);
while ((ret = restful_rhizome_bundlelist_json_content_chunk(&retry, r, b)) != -1) {
if (strbuf_overrun(b)) {
if (config.debug.rhizome)
DEBUGF("overrun by %zu bytes", strbuf_count(b) - strbuf_len(b));
result->need = strbuf_count(b) + 1 - result->generated;
break;
}
result->generated = strbuf_len(b);
if (ret == 0)
break;
}
rhizome_list_release(&r->u.list.cursor);
return ret;
}
static int neighbour_page(rhizome_http_request *r, const char *remainder)
{
if (r->http.verb != HTTP_VERB_GET) {
@ -378,30 +629,31 @@ static int rhizome_status_page(rhizome_http_request *r, const char *remainder)
return 0;
}
static int rhizome_file_content(struct http_request *hr)
static int rhizome_file_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
{
// Only read multiples of 4k from disk.
const size_t blocksz = 1 << 12;
// Ask for a large buffer for all future reads.
const size_t preferred_bufsz = 16 * blocksz;
// Reads the next part of the payload into the supplied buffer.
rhizome_http_request *r = (rhizome_http_request *) hr;
assert(r->http.response_buffer_sent == 0);
assert(r->http.response_buffer_length == 0);
assert(r->read_state.offset < r->read_state.length);
uint64_t readlen = r->read_state.length - r->read_state.offset;
size_t suggested_size = 64 * 1024;
if (suggested_size > readlen)
suggested_size = readlen;
if (r->http.response_buffer_size < suggested_size)
http_request_set_response_bufsize(&r->http, suggested_size);
if (r->http.response_buffer == NULL)
http_request_set_response_bufsize(&r->http, 1);
if (r->http.response_buffer == NULL)
return -1;
ssize_t len = rhizome_read(&r->read_state,
(unsigned char *)r->http.response_buffer,
r->http.response_buffer_size);
if (len == -1)
return -1;
assert((size_t) len <= r->http.response_buffer_size);
r->http.response_buffer_length += (size_t) len;
return 0;
assert(r->u.read_state.offset < r->u.read_state.length);
uint64_t remain = r->u.read_state.length - r->u.read_state.offset;
size_t readlen = bufsz;
if (remain < bufsz)
readlen = remain;
else
readlen &= ~(blocksz - 1);
if (readlen > 0) {
ssize_t n = rhizome_read(&r->u.read_state, buf, readlen);
if (n == -1)
return -1;
result->generated = (size_t) n;
}
assert(r->u.read_state.offset <= r->u.read_state.length);
remain = r->u.read_state.length - r->u.read_state.offset;
result->need = remain < preferred_bufsz ? remain : preferred_bufsz;
return remain ? 1 : 0;
}
static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
@ -422,35 +674,35 @@ static int rhizome_file_page(rhizome_http_request *r, const char *remainder)
rhizome_filehash_t filehash;
if (str_to_rhizome_filehash_t(&filehash, remainder) == -1)
return 1;
bzero(&r->read_state, sizeof r->read_state);
int n = rhizome_open_read(&r->read_state, &filehash);
bzero(&r->u.read_state, sizeof r->u.read_state);
int n = rhizome_open_read(&r->u.read_state, &filehash);
if (n == -1) {
http_request_simple_response(&r->http, 500, NULL);
return 0;
}
if (n != 0)
return 1;
if (r->read_state.length == -1 && rhizome_read(&r->read_state, NULL, 0)) {
rhizome_read_close(&r->read_state);
if (r->u.read_state.length == -1 && rhizome_read(&r->u.read_state, NULL, 0)) {
rhizome_read_close(&r->u.read_state);
return 1;
}
assert(r->read_state.length != -1);
r->http.response.header.resource_length = r->read_state.length;
assert(r->u.read_state.length != -1);
r->http.response.header.resource_length = r->u.read_state.length;
if (r->http.request_header.content_range_count > 0) {
assert(r->http.request_header.content_range_count == 1);
struct http_range closed;
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->read_state.length);
unsigned n = http_range_close(&closed, r->http.request_header.content_ranges, 1, r->u.read_state.length);
if (n == 0 || http_range_bytes(&closed, 1) == 0) {
http_request_simple_response(&r->http, 416, NULL); // Request Range Not Satisfiable
return 0;
}
r->http.response.header.content_range_start = closed.first;
r->http.response.header.content_length = closed.last - closed.first + 1;
r->read_state.offset = closed.first;
r->u.read_state.offset = closed.first;
} else {
r->http.response.header.content_range_start = 0;
r->http.response.header.content_length = r->http.response.header.resource_length;
r->read_state.offset = 0;
r->u.read_state.offset = 0;
}
http_request_response_generated(&r->http, 200, "application/binary", rhizome_file_content);
return 0;

131
rhizome_packetformats.c
View File

@ -27,7 +27,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
/* Android doesn't have log2(), and we don't really need to do floating point
@ -80,17 +79,18 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
if (!m) { RETURN(WHY("null manifest passed in")); }
int i;
/* Manifest prefix */
unsigned i;
for(i=0;i<RHIZOME_BAR_PREFIX_BYTES;i++)
bar[RHIZOME_BAR_PREFIX_OFFSET+i]=m->cryptoSignPublic.binary[i];
/* file length */
bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->fileLength);
assert(m->filesize != RHIZOME_SIZE_UNSET);
bar[RHIZOME_BAR_FILESIZE_OFFSET]=log2ll(m->filesize);
/* Version */
for(i=0;i<7;i++) bar[RHIZOME_BAR_VERSION_OFFSET+6-i]=(m->version>>(8*i))&0xff;
/* geo bounding box */
#if 0
/* geo bounding box TODO: replace with bounding circle!!! */
double minLat=rhizome_manifest_get_double(m,"min_lat",-90);
if (minLat<-90) minLat=-90; if (minLat>90) minLat=90;
double minLong=rhizome_manifest_get_double(m,"min_long",-180);
@ -99,6 +99,12 @@ int rhizome_manifest_to_bar(rhizome_manifest *m,unsigned char *bar)
if (maxLat<-90) maxLat=-90; if (maxLat>90) maxLat=90;
double maxLong=rhizome_manifest_get_double(m,"max_long",+180);
if (maxLong<-180) maxLong=-180; if (maxLong>180) maxLong=180;
#else
double minLat = -90;
double minLong = -180;
double maxLat = +90;
double maxLong = +180;
#endif
unsigned short v;
int o=RHIZOME_BAR_GEOBOX_OFFSET;
v=(minLat+90)*(65535/180); bar[o++]=(v>>8)&0xff; bar[o++]=(v>>0)&0xff;
@ -162,11 +168,12 @@ static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, cons
DEBUG("Found a BAR that is the wrong size - ignoring");
continue;
}
if (ob_append_bytes(e, (unsigned char *)data, blob_bytes)){
if (ob_remaining(e) < blob_bytes) {
// out of room
count--;
break;
}
ob_append_bytes(e, (unsigned char *)data, blob_bytes);
*last_rowid=rowid;
}
if (statement)
@ -178,7 +185,8 @@ static int append_bars(struct overlay_buffer *e, sqlite_retry_state *retry, cons
Always advertise the most recent 3 manifests in the table, cycle through the rest of the table, adding 17 BAR's at a time
*/
int64_t bundles_available=0;
void overlay_rhizome_advertise(struct sched_ent *alarm){
void overlay_rhizome_advertise(struct sched_ent *alarm)
{
bundles_available=0;
static int64_t bundle_last_rowid=INT64_MAX;
@ -191,7 +199,7 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){
int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);
sqlite_retry_state retry = SQLITE_RETRY_STATE_DEFAULT;
// DEPRECATE REST OF THIS CODE WHICH SEEMS TO BE CAUSING TOO MUCH CHATTER
// TODO: DEPRECATE REST OF THIS CODE WHICH SEEMS TO BE CAUSING TOO MUCH CHATTER
// ESPECIALLY FOR PACKET-RADIO
goto end;
@ -210,31 +218,28 @@ void overlay_rhizome_advertise(struct sched_ent *alarm){
frame->source = my_subscriber;
frame->ttl = 1;
frame->queue = OQ_OPPORTUNISTIC;
frame->payload = ob_new();
if ((frame->payload = ob_new()) == NULL) {
op_free(frame);
goto end;
}
ob_limitsize(frame->payload, 800);
ob_append_byte(frame->payload, 2);
ob_append_ui16(frame->payload, rhizome_http_server_port);
int64_t rowid=0;
int count = append_bars(frame->payload, &retry,
"SELECT BAR,ROWID FROM MANIFESTS ORDER BY ROWID DESC LIMIT 3",
&rowid);
if (count>=3){
if (bundle_last_rowid>rowid || bundle_last_rowid<=0)
bundle_last_rowid=rowid;
count = append_bars(frame->payload, &retry,
"SELECT BAR,ROWID FROM MANIFESTS WHERE ROWID < ? ORDER BY ROWID DESC LIMIT 17",
&bundle_last_rowid);
if (count<17)
bundle_last_rowid=INT64_MAX;
}
if (overlay_payload_enqueue(frame))
if (overlay_payload_enqueue(frame) == -1)
op_free(frame);
end:
sqlite_set_tracefunc(oldfunc);
alarm->alarm = gettime_ms()+config.rhizome.advertise.interval;
@ -256,21 +261,20 @@ int rhizome_advertise_manifest(struct subscriber *dest, rhizome_manifest *m){
else
frame->ttl = 1;
frame->queue = OQ_OPPORTUNISTIC;
frame->payload = ob_new();
if ((frame->payload = ob_new()) == NULL)
goto error;
ob_limitsize(frame->payload, 800);
if (ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS)) goto error;
if (ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0)) goto error;
if (ob_append_ui16(frame->payload, m->manifest_all_bytes)) goto error;
if (ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes)) goto error;
ob_append_byte(frame->payload, HAS_PORT|HAS_MANIFESTS);
ob_append_ui16(frame->payload, is_rhizome_http_enabled()?rhizome_http_server_port:0);
ob_append_ui16(frame->payload, m->manifest_all_bytes);
ob_append_bytes(frame->payload, m->manifestdata, m->manifest_all_bytes);
ob_append_byte(frame->payload, 0xFF);
if (overlay_payload_enqueue(frame)) goto error;
if (overlay_payload_enqueue(frame) == -1)
goto error;
if (config.debug.rhizome_ads)
DEBUGF("Advertising manifest %s %"PRId64" to %s",
alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version, dest?alloca_tohex_sid_t(dest->sid):"broadcast");
return 0;
error:
op_free(frame);
return -1;
@ -290,7 +294,6 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
int ad_frame_type=ob_get(f->payload);
struct sockaddr_in httpaddr = context->addr;
httpaddr.sin_port = htons(RHIZOME_HTTP_PORT);
size_t manifest_length;
rhizome_manifest *m=NULL;
int (*oldfunc)() = sqlite_set_tracefunc(is_debug_rhizome_ads);
@ -301,74 +304,66 @@ int overlay_rhizome_saw_advertisements(int i, struct decode_context *context, st
if (ad_frame_type & HAS_MANIFESTS){
/* Extract whole manifests */
while(f->payload->position < f->payload->sizeLimit) {
if (ob_getbyte(f->payload, f->payload->position)==0xff){
f->payload->position++;
while (ob_remaining(f->payload) > 0) {
if (ob_peek(f->payload) == 0xff) {
ob_skip(f->payload, 1);
break;
}
manifest_length = ob_get_ui16(f->payload);
size_t manifest_length = ob_get_ui16(f->payload);
if (manifest_length==0) continue;
unsigned char *data = ob_get_bytes_ptr(f->payload, manifest_length);
if (!data) {
WHYF("Illegal manifest length field in rhizome advertisement frame %zu vs %d",
manifest_length, f->payload->sizeLimit - f->payload->position);
manifest_length, ob_remaining(f->payload));
break;
}
/* Read manifest without verifying signatures (which would waste lots of
energy, everytime we see a manifest that we already have).
In fact, it would be better here to do a really rough and ready parser
to get the id and version fields out, and avoid the memory copies that
otherwise happen.
But we do need to make sure that at least one signature is there.
*/
m = rhizome_new_manifest();
if (!m) {
WHY("Out of manifests");
// Briefly inspect the manifest to see if it looks interesting.
struct rhizome_manifest_summary summ;
if (!rhizome_manifest_inspect((char *)data, manifest_length, &summ)) {
if (config.debug.rhizome_ads)
DEBUG("Ignoring manifest that looks malformed");
goto next;
}
if (rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1) {
WHY("Error parsing manifest body");
goto next;
}
/* trim manifest ID to a prefix for ease of debugging
(that is the only use of this */
if (config.debug.rhizome_ads){
int64_t version = rhizome_manifest_get_ll(m, "version");
DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), version);
}
if (config.debug.rhizome_ads)
DEBUGF("manifest id=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(summ.bid), summ.version);
/* Crude signature presence test */
if (m->manifest_bytes >= m->manifest_all_bytes){
// no signature was found when parsing?
/* ignore the announcement, but don't ignore other people
offering the same manifest */
// If it looks like there is no signature at all, ignore the announcement but don't brown-list
// the manifest ID, so that we will still process other offers of the same manifest with
// signatures.
if (summ.body_len == manifest_length) {
if (config.debug.rhizome_ads)
DEBUG("Ignoring manifest announcment with no signature");
goto next;
}
if (rhizome_ignore_manifest_check(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary)){
if (rhizome_ignore_manifest_check(summ.bid.binary, sizeof summ.bid.binary)){
/* Ignoring manifest that has caused us problems recently */
if (config.debug.rhizome_ads)
DEBUGF("Ignoring manifest with errors: %s", alloca_tohex_rhizome_bid_t(m->cryptoSignPublic));
DEBUGF("Ignoring manifest with errors bid=%s", alloca_tohex_rhizome_bid_t(summ.bid));
goto next;
}
if (m->errors > 0){
if (config.debug.rhizome_ads)
DEBUG("Unverified manifest has errors - so not processing any further.");
/* Don't waste any time on this manifest in future attempts for at least
a minute. */
// The manifest looks potentially interesting, so now do a full parse and validation.
if ((m = rhizome_new_manifest()) == NULL)
goto next;
if ( rhizome_read_manifest_file(m, (char *)data, manifest_length) == -1
|| !rhizome_manifest_validate(m)
) {
WARN("Malformed manifest");
// Don't attend to this manifest for at least a minute
rhizome_queue_ignore_manifest(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary, 60000);
goto next;
}
/* Manifest is okay, so see if it is worth storing */
assert(m->has_id);
assert(m->version != 0);
assert(cmp_rhizome_bid_t(&m->cryptoSignPublic, &summ.bid) == 0);
assert(m->version == summ.version);
assert(m->manifest_body_bytes == summ.body_len);
// are we already fetching this bundle [or later]?
rhizome_manifest *mf=rhizome_fetch_search(m->cryptoSignPublic.binary, sizeof m->cryptoSignPublic.binary);
if (mf && mf->version >= m->version)
@ -479,7 +474,7 @@ next:
lookup_time = (end_time - start_time);
if (mdp.out.payload_length>0)
overlay_mdp_dispatch(&mdp,0 /* system generated */,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
end:
sqlite_set_tracefunc(oldfunc);

258
rhizome_store.c
View File

@ -14,7 +14,7 @@ int rhizome_exists(const rhizome_filehash_t *hashp)
return gotfile;
}
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, int64_t file_length, int priority)
int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *expectedHashp, uint64_t file_length, int priority)
{
write->blob_fd=-1;
@ -136,13 +136,14 @@ int rhizome_open_write(struct rhizome_write *write, const rhizome_filehash_t *ex
* */
// encrypt and hash data, data buffers must be passed in file order.
static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer, int data_size){
if (data_size<=0)
static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer, size_t data_size)
{
if (data_size <= 0)
return WHY("No content supplied");
/* Make sure we aren't being asked to write more data than we expected */
if (write_state->file_offset + data_size > write_state->file_length)
return WHYF("Too much content supplied, %"PRId64" + %d > %"PRId64,
return WHYF("Too much content supplied, %"PRIu64" + %zu > %"PRIu64,
write_state->file_offset, data_size, write_state->file_length);
if (write_state->crypt){
@ -157,7 +158,7 @@ static int prepare_data(struct rhizome_write *write_state, unsigned char *buffer
write_state->file_offset+=data_size;
if (config.debug.rhizome)
DEBUGF("Processed %"PRId64" of %"PRId64, write_state->file_offset, write_state->file_length);
DEBUGF("Processed %"PRIu64" of %"PRIu64, write_state->file_offset, write_state->file_length);
return 0;
}
@ -189,6 +190,11 @@ static int write_get_lock(struct rhizome_write *write_state){
// write data to disk
static int write_data(struct rhizome_write *write_state, uint64_t file_offset, unsigned char *buffer, size_t data_size)
{
if (config.debug.rhizome) {
DEBUGF("write_state->file_length=%"PRIu64" file_offset=%"PRIu64, write_state->file_length, file_offset);
//dump("buffer", buffer, data_size);
}
if (data_size<=0)
return 0;
@ -198,7 +204,8 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
if (write_state->blob_fd != -1) {
int ofs=0;
// keep trying until all of the data is written.
lseek(write_state->blob_fd, file_offset, SEEK_SET);
if (lseek64(write_state->blob_fd, (off64_t) file_offset, SEEK_SET) == -1)
return WHYF_perror("lseek64(%d,%"PRIu64",SEEK_SET)", write_state->blob_fd, file_offset);
while(ofs < data_size){
int r=write(write_state->blob_fd, buffer + ofs, data_size - ofs);
if (r<0)
@ -225,10 +232,10 @@ static int write_data(struct rhizome_write *write_state, uint64_t file_offset, u
}
}
write_state->written_offset=file_offset + data_size;
write_state->written_offset = file_offset + data_size;
if (config.debug.rhizome)
DEBUGF("Wrote %"PRId64" of %"PRId64, file_offset + data_size, write_state->file_length);
DEBUGF("Wrote %"PRIu64" of %"PRIu64, file_offset + data_size, write_state->file_length);
return 0;
}
@ -254,8 +261,12 @@ static int write_release_lock(struct rhizome_write *write_state){
// Write data buffers in any order, the data will be cached and streamed into the database in file order.
// Though there is an upper bound on the amount of cached data
int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsigned char *buffer, size_t data_size)
int rhizome_random_write(struct rhizome_write *write_state, uint64_t offset, unsigned char *buffer, size_t data_size)
{
if (config.debug.rhizome) {
DEBUGF("write_state->file_length=%"PRIu64" offset=%"PRIu64, write_state->file_length, offset);
//dump("buffer", buffer, data_size);
}
if (offset + data_size > write_state->file_length)
data_size = write_state->file_length - offset;
@ -269,10 +280,10 @@ int rhizome_random_write(struct rhizome_write *write_state, int64_t offset, unsi
// cache up to RHIZOME_BUFFER_MAXIMUM_SIZE or file length before attempting to write everything in one go.
// (Not perfect if the range overlaps)
uint64_t new_size = write_state->written_offset + write_state->buffer_size + data_size;
if (new_size>=write_state->file_length || new_size>=RHIZOME_BUFFER_MAXIMUM_SIZE)
if (new_size >= write_state->file_length || new_size >= RHIZOME_BUFFER_MAXIMUM_SIZE)
should_write = 1;
}
int64_t last_offset = write_state->written_offset;
uint64_t last_offset = write_state->written_offset;
while(1){
@ -396,12 +407,10 @@ int rhizome_write_file(struct rhizome_write *write, const char *filename){
ret = write_get_lock(write);
if (ret)
goto end;
while(write->file_offset < write->file_length){
int size=sizeof(buffer);
while(write->file_offset < write->file_length) {
size_t size = sizeof buffer;
if (write->file_offset + size > write->file_length)
size=write->file_length - write->file_offset;
size = write->file_length - write->file_offset;
size_t r = fread(buffer, 1, size, f);
if (ferror(f)){
ret = WHY_perror("fread");
@ -451,7 +460,7 @@ int rhizome_finish_write(struct rhizome_write *write)
}
if (write->file_offset < write->file_length){
WHYF("Only processed %"PRId64" bytes, expected %"PRId64, write->file_offset, write->file_length);
WHYF("Only processed %"PRIu64" bytes, expected %"PRIu64, write->file_offset, write->file_length);
}
int fd = write->blob_fd;
@ -550,14 +559,15 @@ failure:
// import a file for an existing bundle with a known file hash
int rhizome_import_file(rhizome_manifest *m, const char *filepath)
{
if (m->fileLength<=0)
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize == 0)
return 0;
/* Import the file first, checking the hash as we go */
struct rhizome_write write;
bzero(&write, sizeof(write));
int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
int ret=rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
if (ret!=0)
return ret;
@ -578,16 +588,18 @@ int rhizome_import_file(rhizome_manifest *m, const char *filepath)
// store a whole payload from a single buffer
int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t length)
{
if (m->fileLength<=0)
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (m->filesize == 0)
return 0;
if (length!=m->fileLength)
return WHYF("Expected %"PRId64" bytes, got %zu", m->fileLength, length);
if (length != m->filesize)
return WHYF("Expected %"PRIu64" bytes, got %zu", m->filesize, length);
/* Import the file first, checking the hash as we go */
struct rhizome_write write;
bzero(&write, sizeof(write));
int ret=rhizome_open_write(&write, &m->filehash, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
int ret=rhizome_open_write(&write, &m->filehash, m->filesize, RHIZOME_PRIORITY_DEFAULT);
if (ret!=0)
return ret;
@ -607,48 +619,39 @@ int rhizome_import_buffer(rhizome_manifest *m, unsigned char *buffer, size_t len
int rhizome_stat_file(rhizome_manifest *m, const char *filepath)
{
int64_t existing = rhizome_manifest_get_ll(m, "filesize");
m->fileLength = 0;
uint64_t size = 0;
if (filepath[0]) {
struct stat stat;
if (lstat(filepath,&stat))
return WHYF("Could not stat() payload file '%s'",filepath);
m->fileLength = stat.st_size;
if (lstat(filepath, &stat))
return WHYF_perror("lstat(%s)", alloca_str_toprint(filepath));
size = stat.st_size;
}
// fail if the file is shorter than specified by the manifest
if (existing > m->fileLength)
// Fail if the file is shorter than already specified by the manifest.
if (m->filesize != RHIZOME_SIZE_UNSET && size < m->filesize)
return WHY("Manifest length is longer than the file");
// if the file is longer than specified by the manifest, ignore the end.
if (existing!=-1 && existing < m->fileLength)
m->fileLength = existing;
rhizome_manifest_set_ll(m, "filesize", m->fileLength);
if (m->fileLength == 0){
m->filehash = RHIZOME_FILEHASH_NONE;
rhizome_manifest_del(m, "filehash");
}
// If the file is longer than already specified by the manifest, ignore the end of the file.
if (m->filesize == RHIZOME_SIZE_UNSET || size > m->filesize)
rhizome_manifest_set_filesize(m, size);
return 0;
}
static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_write *write)
static int rhizome_write_derive_key(rhizome_manifest *m, struct rhizome_write *write)
{
if (!m->payloadEncryption)
if (m->payloadEncryption != PAYLOAD_ENCRYPTED)
return 0;
// if the manifest specifies encryption, make sure we can generate the payload key and encrypt the contents as we go
if (rhizome_derive_key(m, bsk))
if (rhizome_derive_payload_key(m))
return -1;
if (config.debug.rhizome)
DEBUGF("Encrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
write->crypt=1;
if (m->journalTail>0)
write->tail = m->journalTail;
if (m->is_journal && m->tail > 0)
write->tail = m->tail;
bcopy(m->payloadKey, write->key, sizeof(write->key));
bcopy(m->payloadNonce, write->nonce, sizeof(write->nonce));
@ -657,10 +660,11 @@ static int rhizome_write_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, stru
int rhizome_write_open_manifest(struct rhizome_write *write, rhizome_manifest *m)
{
if (rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT))
assert(m->filesize != RHIZOME_SIZE_UNSET);
if (rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT))
return -1;
if (rhizome_write_derive_key(m, NULL, write))
if (rhizome_write_derive_key(m, write))
return -1;
return 0;
}
@ -672,18 +676,14 @@ int rhizome_add_file(rhizome_manifest *m, const char *filepath)
// Stream the file directly into the database, encrypting & hashing as we go.
struct rhizome_write write;
bzero(&write, sizeof(write));
if (rhizome_write_open_manifest(&write, m))
goto failure;
if (rhizome_write_file(&write, filepath))
goto failure;
if (rhizome_finish_write(&write))
goto failure;
m->filehash = write.id;
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
rhizome_manifest_set_filehash(m, &write.id);
return 0;
failure:
rhizome_fail_write(&write);
return -1;
@ -704,7 +704,7 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp
" AND FILES.datavalid != 0", RHIZOME_FILEHASH_T, &read->id, END) == -1)
return -1;
if (read->blob_rowid != -1) {
read->length = -1; // discover the length on opening the db BLOB
read->length = RHIZOME_SIZE_UNSET; // discover the length on opening the db BLOB
} else {
// No row in FILEBLOBS, look for an external blob file.
char blob_path[1024];
@ -716,8 +716,10 @@ int rhizome_open_read(struct rhizome_read *read, const rhizome_filehash_t *hashp
return 1; // file not available
return WHYF_perror("open(%s)", alloca_str_toprint(blob_path));
}
if ((read->length = lseek(read->blob_fd, 0, SEEK_END)) == -1)
return WHYF_perror("lseek(%s,0,SEEK_END)", alloca_str_toprint(blob_path));
off64_t pos = lseek64(read->blob_fd, 0, SEEK_END);
if (pos == -1)
return WHYF_perror("lseek64(%s,0,SEEK_END)", alloca_str_toprint(blob_path));
read->length = pos;
if (config.debug.externalblobs)
DEBUGF("Opened stored file %s as fd %d, len %"PRIx64,blob_path, read->blob_fd, read->length);
}
@ -731,8 +733,8 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read
{
IN();
if (read_state->blob_fd != -1) {
if (lseek(read_state->blob_fd, (off_t) read_state->offset, SEEK_SET) == -1)
RETURN(WHYF_perror("lseek(%d,%lu,SEEK_SET)", read_state->blob_fd, (unsigned long)read_state->offset));
if (lseek64(read_state->blob_fd, (off64_t) read_state->offset, SEEK_SET) == -1)
RETURN(WHYF_perror("lseek64(%d,%"PRIu64",SEEK_SET)", read_state->blob_fd, read_state->offset));
if (bufsz == 0)
RETURN(0);
ssize_t rd = read(read_state->blob_fd, buffer, bufsz);
@ -755,7 +757,7 @@ static ssize_t rhizome_read_retry(sqlite_retry_state *retry, struct rhizome_read
RETURN(WHYF("sqlite3_blob_open() failed: %s", sqlite3_errmsg(rhizome_db)));
}
assert(blob != NULL);
if (read_state->length == -1)
if (read_state->length == RHIZOME_SIZE_UNSET)
read_state->length = sqlite3_blob_bytes(blob);
// A NULL buffer skips the actual sqlite3_blob_read() call, which is useful just to work out
// the length.
@ -813,6 +815,7 @@ ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, siz
}
if (read_state->crypt && buffer && bytes_read>0){
dump("before decrypt", buffer, bytes_read);
if(rhizome_crypt_xor_block(
buffer, bytes_read,
read_state->offset + read_state->tail,
@ -820,62 +823,73 @@ ssize_t rhizome_read(struct rhizome_read *read_state, unsigned char *buffer, siz
RETURN(-1);
}
}
read_state->offset+=bytes_read;
read_state->offset += bytes_read;
if (config.debug.rhizome) {
DEBUGF("read %zu bytes, read_state->offset=%"PRIu64, bytes_read, read_state->offset);
//dump("buffer", buffer, bytes_read);
}
RETURN(bytes_read);
OUT();
}
/* Read len bytes from read->offset into data, using *buffer to cache any reads */
int rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len)
ssize_t rhizome_read_buffered(struct rhizome_read *read, struct rhizome_read_buffer *buffer, unsigned char *data, size_t len)
{
size_t bytes_copied=0;
while (len>0){
//DEBUGF("len=%zu read->length=%"PRIu64" read->offset=%"PRIu64" buffer->offset=%"PRIu64"", len, read->length, read->offset, buffer->offset);
// make sure we only attempt to read data that actually exists
if (read->length !=-1 && read->offset + len > read->length)
if (read->length != RHIZOME_SIZE_UNSET && read->offset + len > read->length)
len = read->length - read->offset;
// if we can supply either the beginning or end of the data from cache, do that first.
uint64_t ofs=read->offset - buffer->offset;
if (ofs<=buffer->len){
size_t size=len;
if (size > buffer->len - ofs)
size = buffer->len - ofs;
if (size>0){
// copy into the start of the data buffer
bcopy(buffer->data + ofs, data, size);
data+=size;
len-=size;
read->offset+=size;
bytes_copied+=size;
continue;
if (read->offset >= buffer->offset) {
assert(read->offset - buffer->offset <= SIZE_MAX);
size_t ofs = read->offset - buffer->offset;
if (ofs <= buffer->len){
size_t size = len;
if (size > buffer->len - ofs)
size = buffer->len - ofs;
if (size > 0){
// copy into the start of the data buffer
bcopy(buffer->data + ofs, data, size);
data+=size;
len-=size;
read->offset+=size;
bytes_copied+=size;
continue;
}
}
}
ofs = (read->offset+len) - buffer->offset;
if (ofs>0 && ofs<=buffer->len){
size_t size=len;
if (size > ofs)
size = ofs;
if (size>0){
// copy into the end of the data buffer
bcopy(buffer->data + ofs - size, data + len - size, size);
len-=size;
bytes_copied+=size;
continue;
if (read->offset + len > buffer->offset) {
assert(read->offset + len - buffer->offset <= SIZE_MAX);
size_t ofs = read->offset + len - buffer->offset;
if (ofs <= buffer->len){
size_t size = len;
if (size > ofs)
size = ofs;
if (size>0){
// copy into the end of the data buffer
bcopy(buffer->data + ofs - size, data + len - size, size);
len-=size;
bytes_copied+=size;
continue;
}
}
}
// ok, so we need to read a new buffer to fulfill the request.
// remember the requested read offset so we can put it back
ofs = read->offset;
uint64_t ofs = read->offset;
buffer->offset = read->offset = ofs & ~(RHIZOME_CRYPT_PAGE_SIZE -1);
ssize_t len = rhizome_read(read, buffer->data, sizeof(buffer->data));
ssize_t r = rhizome_read(read, buffer->data, sizeof buffer->data);
read->offset = ofs;
buffer->len = 0;
if (len == -1)
if (r == -1)
return -1;
buffer->len = (size_t) len;
buffer->len = (size_t) r;
}
return bytes_copied;
}
@ -1066,31 +1080,31 @@ static int write_file(struct rhizome_read *read, const char *filepath){
return ret;
}
static int read_derive_key(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state){
read_state->crypt=m->payloadEncryption;
static int read_derive_key(rhizome_manifest *m, struct rhizome_read *read_state)
{
read_state->crypt = m->payloadEncryption == PAYLOAD_ENCRYPTED;
if (read_state->crypt){
// if the manifest specifies encryption, make sure we can generate the payload key and encrypt
// the contents as we go
if (rhizome_derive_key(m, bsk)) {
if (rhizome_derive_payload_key(m)) {
rhizome_read_close(read_state);
return WHY("Unable to decrypt bundle, valid key not found");
}
if (config.debug.rhizome)
DEBUGF("Decrypting payload contents for %s, %"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (m->journalTail>0)
read_state->tail = m->journalTail;
DEBUGF("Decrypting payload contents for bid=%s version=%"PRId64, alloca_tohex_rhizome_bid_t(m->cryptoSignPublic), m->version);
if (m->is_journal && m->tail > 0)
read_state->tail = m->tail;
bcopy(m->payloadKey, read_state->key, sizeof(read_state->key));
bcopy(m->payloadNonce, read_state->nonce, sizeof(read_state->nonce));
}
return 0;
}
int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhizome_read *read_state)
int rhizome_open_decrypt_read(rhizome_manifest *m, struct rhizome_read *read_state)
{
int ret = rhizome_open_read(read_state, &m->filehash);
if (ret == 0)
ret = read_derive_key(m, bsk, read_state);
ret = read_derive_key(m, read_state);
return ret;
}
@ -1099,11 +1113,11 @@ int rhizome_open_decrypt_read(rhizome_manifest *m, rhizome_bk_t *bsk, struct rhi
*
* Returns -1 on error, 0 if extracted successfully, 1 if not found.
*/
int rhizome_extract_file(rhizome_manifest *m, const char *filepath, rhizome_bk_t *bsk)
int rhizome_extract_file(rhizome_manifest *m, const char *filepath)
{
struct rhizome_read read_state;
bzero(&read_state, sizeof read_state);
int ret = rhizome_open_decrypt_read(m, bsk, &read_state);
int ret = rhizome_open_decrypt_read(m, &read_state);
if (ret == 0)
ret = write_file(&read_state, filepath);
rhizome_read_close(&read_state);
@ -1168,28 +1182,24 @@ int rhizome_journal_pipe(struct rhizome_write *write, const rhizome_filehash_t *
}
// open an existing journal bundle, advance the head pointer, duplicate the existing content and get ready to add more.
int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, uint64_t new_size)
int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m, uint64_t advance_by, uint64_t new_size)
{
int ret = 0;
if (advance_by > m->fileLength)
assert(m->filesize != RHIZOME_SIZE_UNSET);
assert(m->is_journal);
if (advance_by > m->filesize)
return WHY("Cannot advance past the existing content");
uint64_t old_length = m->fileLength;
uint64_t copy_length = old_length - advance_by;
m->fileLength = m->fileLength + new_size - advance_by;
rhizome_manifest_set_ll(m, "filesize", m->fileLength);
uint64_t copy_length = m->filesize - advance_by;
rhizome_manifest_set_filesize(m, m->filesize + new_size - advance_by);
if (advance_by>0){
m->journalTail += advance_by;
rhizome_manifest_set_ll(m,"tail",m->journalTail);
}
if (advance_by > 0)
rhizome_manifest_set_tail(m, m->tail + advance_by);
m->version = m->fileLength;
rhizome_manifest_set_ll(m,"version",m->version);
rhizome_manifest_set_version(m, m->filesize);
ret = rhizome_open_write(write, NULL, m->fileLength, RHIZOME_PRIORITY_DEFAULT);
ret = rhizome_open_write(write, NULL, m->filesize, RHIZOME_PRIORITY_DEFAULT);
if (ret)
goto failure;
@ -1200,7 +1210,7 @@ int rhizome_write_open_journal(struct rhizome_write *write, rhizome_manifest *m,
goto failure;
}
ret = rhizome_write_derive_key(m, bsk, write);
ret = rhizome_write_derive_key(m, write);
if (ret)
goto failure;
@ -1212,12 +1222,12 @@ failure:
return ret;
}
int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, unsigned char *buffer, size_t len)
int rhizome_append_journal_buffer(rhizome_manifest *m, uint64_t advance_by, unsigned char *buffer, size_t len)
{
struct rhizome_write write;
bzero(&write, sizeof write);
int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, (uint64_t) len);
int ret = rhizome_write_open_journal(&write, m, advance_by, (uint64_t) len);
if (ret)
return -1;
@ -1231,8 +1241,7 @@ int rhizome_append_journal_buffer(rhizome_manifest *m, rhizome_bk_t *bsk, uint64
if (ret)
goto failure;
m->filehash = write.id;
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
rhizome_manifest_set_filehash(m, &write.id);
return 0;
failure:
@ -1241,15 +1250,15 @@ failure:
return ret;
}
int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t advance_by, const char *filename)
int rhizome_append_journal_file(rhizome_manifest *m, uint64_t advance_by, const char *filename)
{
struct stat stat;
if (lstat(filename,&stat))
return WHYF("Could not stat() payload file '%s'",filename);
return WHYF_perror("stat(%s)", alloca_str_toprint(filename));
struct rhizome_write write;
bzero(&write, sizeof write);
int ret = rhizome_write_open_journal(&write, m, bsk, advance_by, stat.st_size);
int ret = rhizome_write_open_journal(&write, m, advance_by, stat.st_size);
if (ret)
return -1;
@ -1263,8 +1272,7 @@ int rhizome_append_journal_file(rhizome_manifest *m, rhizome_bk_t *bsk, uint64_t
if (ret)
goto failure;
m->filehash = write.id;
rhizome_manifest_set(m, "filehash", alloca_tohex_rhizome_filehash_t(m->filehash));
rhizome_manifest_set_filehash(m, &write.id);
return 0;

41
rhizome_sync.c
View File

@ -93,7 +93,7 @@ static void rhizome_sync_request(struct subscriber *subscriber, uint64_t token,
mdp.out.payload_length = ob_position(b);
if (config.debug.rhizome)
DEBUGF("Sending request to %s for BARs from %"PRIu64" %s", alloca_tohex_sid_t(subscriber->sid), token, forwards?"forwards":"backwards");
overlay_mdp_dispatch(&mdp,0,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
ob_free(b);
}
@ -147,7 +147,7 @@ static void rhizome_sync_send_requests(struct subscriber *subscriber, struct rhi
break;
}
if (mdp.out.payload_length!=0)
overlay_mdp_dispatch(&mdp,0,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
// send request for more bars if we have room to cache them
if (state->bar_count >= CACHE_BARS)
@ -321,21 +321,16 @@ static void sync_process_bar_list(struct subscriber *subscriber, struct rhizome_
}
static int append_response(struct overlay_buffer *b, uint64_t token, const unsigned char *bar)
static void append_response(struct overlay_buffer *b, uint64_t token, const unsigned char *bar)
{
if (ob_append_packed_ui64(b, token))
return -1;
if (bar){
if (ob_append_bytes(b, bar, RHIZOME_BAR_BYTES))
return -1;
}else{
ob_append_packed_ui64(b, token);
if (bar)
ob_append_bytes(b, bar, RHIZOME_BAR_BYTES);
else{
unsigned char *ptr = ob_append_space(b, RHIZOME_BAR_BYTES);
if (!ptr)
return -1;
bzero(ptr, RHIZOME_BAR_BYTES);
if (ptr)
bzero(ptr, RHIZOME_BAR_BYTES);
}
ob_checkpoint(b);
return 0;
}
static uint64_t max_token=0;
@ -400,24 +395,26 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
// make sure we include the exact rowid that was requested, even if we just deleted / replaced the manifest
if (count==0 && rowid!=token){
if (token!=HEAD_FLAG){
if (append_response(b, token, NULL))
ob_checkpoint(b);
append_response(b, token, NULL);
if (ob_overrun(b))
ob_rewind(b);
else{
else {
count++;
last = token;
}
}else
token = rowid;
}
if (append_response(b, rowid, bar))
ob_checkpoint(b);
append_response(b, rowid, bar);
if (ob_overrun(b))
ob_rewind(b);
else {
last = rowid;
count++;
}
}
if (count >= max_count && rowid <= max_token)
break;
}
@ -427,7 +424,9 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
// send a zero lower bound if we reached the end of our manifest list
if (count && count < max_count && !forwards){
if (append_response(b, 0, NULL))
ob_checkpoint(b);
append_response(b, 0, NULL);
if (ob_overrun(b))
ob_rewind(b);
else {
last = 0;
@ -441,7 +440,7 @@ static void sync_send_response(struct subscriber *dest, int forwards, uint64_t t
mdp.out.payload_length = ob_position(b);
if (config.debug.rhizome_ads)
DEBUGF("Sending %d BARs from %"PRIu64" to %"PRIu64, count, token, last);
overlay_mdp_dispatch(&mdp,0,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
}
ob_free(b);
OUT();

146
route_link.c
View File

@ -4,6 +4,7 @@
#include "overlay_packet.h"
#include "str.h"
#include "conf.h"
#include "keyring.h"
#include <assert.h>
/*
@ -254,6 +255,7 @@ static struct neighbour *get_neighbour(struct subscriber *subscriber, char creat
n->mdp_ack_sequence = -1;
// TODO measure min/max rtt
n->rtt = 120;
n->next_neighbour_update = gettime_ms() + 10;
neighbours = n;
if (config.debug.linkstate)
DEBUGF("LINK STATE; new neighbour %s", alloca_tohex_sid_t(n->subscriber->sid));
@ -466,48 +468,27 @@ static int append_link_state(struct overlay_buffer *payload, char flags,
flags|=FLAG_HAS_ACK;
if (drop_rate!=-1)
flags|=FLAG_HAS_DROP_RATE;
int length_pos = ob_position(payload);
if (ob_append_byte(payload, 0))
return -1;
if (ob_append_byte(payload, flags))
return -1;
if (overlay_address_append(NULL, payload, receiver))
return -1;
if (ob_append_byte(payload, version))
return -1;
ob_append_byte(payload, 0);
ob_append_byte(payload, flags);
overlay_address_append(NULL, payload, receiver);
ob_append_byte(payload, version);
if (transmitter)
if (overlay_address_append(NULL, payload, transmitter))
return -1;
if (interface!=-1)
if (ob_append_byte(payload, interface))
return -1;
if (ack_sequence!=-1){
if (ob_append_byte(payload, ack_sequence))
return -1;
if (ob_append_ui32(payload, ack_mask))
return -1;
overlay_address_append(NULL, payload, transmitter);
if (interface != -1)
ob_append_byte(payload, interface);
if (ack_sequence != -1){
ob_append_byte(payload, ack_sequence);
ob_append_ui32(payload, ack_mask);
}
if (drop_rate!=-1)
if (ob_append_byte(payload, drop_rate))
return -1;
if (drop_rate != -1)
ob_append_byte(payload, drop_rate);
// TODO insert future fields here
if (ob_overrun(payload))
return -1;
// patch the record length
int end_pos = ob_position(payload);
if (ob_set(payload, length_pos, end_pos - length_pos))
return -1;
ob_set(payload, length_pos, end_pos - length_pos);
ob_checkpoint(payload);
return 0;
}
@ -709,12 +690,15 @@ static int send_legacy_self_announce_ack(struct neighbour *neighbour, struct lin
frame->ttl = 6;
frame->destination = neighbour->subscriber;
frame->source = my_subscriber;
frame->payload = ob_new();
if ((frame->payload = ob_new()) == NULL) {
op_free(frame);
return -1;
}
ob_append_ui32(frame->payload, neighbour->last_update);
ob_append_ui32(frame->payload, now);
ob_append_byte(frame->payload, link->neighbour_interface);
frame->queue=OQ_MESH_MANAGEMENT;
if (overlay_payload_enqueue(frame)){
if (overlay_payload_enqueue(frame) == -1) {
op_free(frame);
return -1;
}
@ -786,12 +770,16 @@ static int send_neighbour_link(struct neighbour *n)
send_legacy_self_announce_ack(n, n->best_link, now);
n->last_update = now;
} else {
struct overlay_frame *frame=emalloc_zero(sizeof(struct overlay_frame));
struct overlay_frame *frame = emalloc_zero(sizeof(struct overlay_frame));
frame->type=OF_TYPE_DATA;
frame->source=my_subscriber;
frame->ttl=1;
frame->queue=OQ_MESH_MANAGEMENT;
frame->payload = ob_new();
if ((frame->payload = ob_new()) == NULL) {
op_free(frame);
RETURN(-1);
}
frame->send_hook = neighbour_link_sent;
frame->send_context = n->subscriber;
frame->resend=-1;
@ -824,7 +812,7 @@ static int send_neighbour_link(struct neighbour *n)
append_link_state(frame->payload, flags, n->subscriber, my_subscriber, n->best_link->neighbour_interface, 1,
n->best_link->ack_sequence, n->best_link->ack_mask, -1);
if (overlay_payload_enqueue(frame))
if (overlay_payload_enqueue(frame) == -1)
op_free(frame);
n->best_link->ack_counter = ACK_WINDOW;
@ -884,30 +872,31 @@ static void link_send(struct sched_ent *alarm)
frame->source=my_subscriber;
frame->ttl=1;
frame->queue=OQ_MESH_MANAGEMENT;
frame->payload = ob_new();
ob_limitsize(frame->payload, 400);
overlay_mdp_encode_ports(frame->payload, MDP_PORT_LINKSTATE, MDP_PORT_LINKSTATE);
ob_checkpoint(frame->payload);
int pos = ob_position(frame->payload);
enum_subscribers(NULL, append_link, frame->payload);
ob_rewind(frame->payload);
if (ob_position(frame->payload) == pos)
op_free(frame);
else if (overlay_payload_enqueue(frame))
op_free(frame);
if (neighbours){
alarm->deadline = alarm->alarm;
schedule(alarm);
}else
alarm->alarm=0;
if ((frame->payload = ob_new()) == NULL)
WHY("Cannot send link details");
else {
ob_limitsize(frame->payload, 400);
overlay_mdp_encode_ports(frame->payload, MDP_PORT_LINKSTATE, MDP_PORT_LINKSTATE);
ob_checkpoint(frame->payload);
int pos = ob_position(frame->payload);
enum_subscribers(NULL, append_link, frame->payload);
ob_rewind(frame->payload);
if (ob_position(frame->payload) == pos)
op_free(frame);
else if (overlay_payload_enqueue(frame))
op_free(frame);
if (neighbours){
alarm->deadline = alarm->alarm;
schedule(alarm);
}else
alarm->alarm=0;
}
}
static void update_alarm(time_ms_t limit){
static void update_alarm(struct __sourceloc __whence, time_ms_t limit)
{
if (limit == 0)
FATALF("limit == 0");
if (link_send_alarm.alarm>limit || link_send_alarm.alarm==0){
unschedule(&link_send_alarm);
link_send_alarm.alarm = limit;
@ -924,9 +913,11 @@ int link_stop_routing(struct subscriber *subscriber)
subscriber->identity=NULL;
if (subscriber==my_subscriber)
my_subscriber=NULL;
struct link_state *state = get_link_state(subscriber);
state->next_update = gettime_ms();
update_alarm(state->next_update);
if (subscriber->link_state){
struct link_state *state = get_link_state(subscriber);
state->next_update = gettime_ms();
update_alarm(__WHENCE__, state->next_update);
}
return 0;
}
@ -1044,7 +1035,8 @@ int link_state_should_forward_broadcast(struct subscriber *transmitter)
}
// when we receive a packet from a neighbour with ourselves as the next hop, make sure we send an ack soon(ish)
int link_state_ack_soon(struct subscriber *subscriber){
int link_state_ack_soon(struct subscriber *subscriber)
{
IN();
struct neighbour *neighbour = get_neighbour(subscriber, 0);
if (!neighbour)
@ -1059,8 +1051,8 @@ int link_state_ack_soon(struct subscriber *subscriber){
if (config.debug.ack)
DEBUGF("Asking for next ACK Real Soon Now");
}
update_alarm(__WHENCE__, neighbour->next_neighbour_update);
}
update_alarm(neighbour->next_neighbour_update);
OUT();
return 0;
}
@ -1101,7 +1093,8 @@ int link_unicast_ack(struct subscriber *subscriber, struct overlay_interface *in
return 0;
}
static struct link_out *create_out_link(struct neighbour *neighbour, overlay_interface *interface, struct sockaddr_in *addr, char unicast){
static struct link_out *create_out_link(struct neighbour *neighbour, overlay_interface *interface, struct sockaddr_in *addr, char unicast)
{
struct link_out *ret=emalloc_zero(sizeof(struct link_out));
if (ret){
ret->_next=neighbour->out_links;
@ -1110,15 +1103,14 @@ static struct link_out *create_out_link(struct neighbour *neighbour, overlay_int
ret->destination = create_unicast_destination(*addr, interface);
else
ret->destination = add_destination_ref(interface->destination);
if (config.debug.linkstate)
DEBUGF("LINK STATE; Create possible %s link_out for neighbour %s on interface %s",
unicast?"unicast":"broadcast",
alloca_tohex_sid_t(neighbour->subscriber->sid),
interface->name);
ret->timeout = gettime_ms()+ret->destination->tick_ms*3;
update_alarm(gettime_ms()+5);
time_ms_t now = gettime_ms();
ret->timeout = now + ret->destination->tick_ms * 3;
update_alarm(__WHENCE__, now + 5);
}
return ret;
}
@ -1206,7 +1198,7 @@ int link_received_packet(struct decode_context *context, int sender_seq, char un
send_neighbour_link(neighbour);
}
update_alarm(neighbour->next_neighbour_update);
update_alarm(__WHENCE__, neighbour->next_neighbour_update);
return 0;
}
@ -1385,7 +1377,7 @@ int link_receive(struct overlay_frame *frame, overlay_mdp_frame *mdp)
if (config.debug.ack)
DEBUGF("LINK STATE; neighbour %s missed ack %d, queue another", alloca_tohex_sid_t(sender->sid), neighbour->last_update_seq);
neighbour->next_neighbour_update=now+5;
update_alarm(neighbour->next_neighbour_update);
update_alarm(__WHENCE__, neighbour->next_neighbour_update);
}
}
}
@ -1426,8 +1418,8 @@ void link_explained(struct subscriber *subscriber)
{
time_ms_t now = gettime_ms();
struct link_state *state = get_link_state(subscriber);
state->next_update = now+5;
update_alarm(now+5);
state->next_update = now + 5;
update_alarm(__WHENCE__, now + 5);
}
void link_interface_down(struct overlay_interface *interface)

255
serval.h
View File

@ -33,42 +33,41 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/types.h>
#ifdef WIN32
#include "win32/win32.h"
# include "win32/win32.h"
#else
#include <unistd.h>
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_NET_ROUTE_H
#include <net/route.h>
#endif
#ifdef HAVE_LINUX_IF_H
#include <linux/if.h>
#else
#ifdef HAVE_NET_IF_H
#include <net/if.h>
#endif
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#ifdef HAVE_LINUX_NETLINK_H
#include <linux/netlink.h>
#endif
#ifdef HAVE_LINUX_RTNETLINK_H
#include <linux/rtnetlink.h>
#endif
#ifdef HAVE_IFADDRS_H
#include <ifaddrs.h>
#endif
#ifdef HAVE_SYS_SOCKIO_H
#include <sys/sockio.h>
#endif
#ifdef HAVE_SYS_UCRED_H
#include <sys/ucred.h>
#endif
#endif
# include <unistd.h>
# ifdef HAVE_SYS_SOCKET_H
# include <sys/socket.h>
# endif
# ifdef HAVE_NET_ROUTE_H
# include <net/route.h>
# endif
# ifdef HAVE_LINUX_IF_H
# include <linux/if.h>
# else
# ifdef HAVE_NET_IF_H
# include <net/if.h>
# endif
# endif
# ifdef HAVE_NETINET_IN_H
# include <netinet/in.h>
# endif
# ifdef HAVE_LINUX_NETLINK_H
# include <linux/netlink.h>
# endif
# ifdef HAVE_LINUX_RTNETLINK_H
# include <linux/rtnetlink.h>
# endif
# ifdef HAVE_IFADDRS_H
# include <ifaddrs.h>
# endif
# ifdef HAVE_SYS_SOCKIO_H
# include <sys/sockio.h>
# endif
# ifdef HAVE_SYS_UCRED_H
# include <sys/ucred.h>
# endif
#endif //!WIN32
#if !defined(FORASTERISK) && !defined(s_addr)
#ifdef HAVE_ARPA_INET_H
@ -103,7 +102,6 @@ struct in_addr {
#endif
#include <fcntl.h>
#include <ctype.h>
#include <sys/stat.h>
#include "fdqueue.h"
@ -159,17 +157,12 @@ typedef struct sid_binary {
#define is_sid_t_any(SID) is_all_matching((SID).binary, sizeof (*(sid_t*)0).binary, 0)
#define alloca_tohex_sid_t(sid) alloca_tohex((sid).binary, sizeof (*(sid_t*)0).binary)
#define alloca_tohex_sid_t_trunc(sid,strlen) tohex((char *)alloca((strlen)+2), (strlen), (sid).binary)
#define alloca_tohex_sid_t_trunc(sid,strlen) tohex((char *)alloca((strlen)+1), (strlen), (sid).binary)
int cmp_sid_t(const sid_t *a, const sid_t *b);
int str_to_sid_t(sid_t *sid, const char *hex);
int strn_to_sid_t(sid_t *sid, const char *hex, const char **endp);
int str_is_subscriber_id(const char *sid);
int strn_is_subscriber_id(const char *sid, size_t *lenp);
int str_is_did(const char *did);
int strn_is_did(const char *did, size_t *lenp);
#define alloca_tohex_sas(sas) alloca_tohex((sas), SAS_SIZE)
/*
@ -197,28 +190,6 @@ int formf_serval_instance_path(struct __sourceloc, char *buf, size_t bufsiz, con
int vformf_serval_instance_path(struct __sourceloc, char *buf, size_t bufsiz, const char *fmt, va_list);
void serval_setinstancepath(const char *instancepath);
/* Basic socket operations.
*/
int _make_local_sockaddr(struct __sourceloc, struct sockaddr_un *sockname, socklen_t *addrlen, const char *fmt, ...)
__attribute__((format(printf, 4, 5)));
int _esocket(struct __sourceloc, int domain, int type, int protocol);
int _socket_bind(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
int _socket_connect(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
int _socket_listen(struct __sourceloc, int sock, int backlog);
int _socket_set_reuseaddr(struct __sourceloc, int sock, int reuseP);
int _socket_set_rcvbufsize(struct __sourceloc, int sock, unsigned buffer_size);
#define make_local_sockaddr(sockname, addrlenp, fmt,...) _make_local_sockaddr(__WHENCE__, (sockname), (addrlenp), (fmt), ##__VA_ARGS__)
#define esocket(domain, type, protocol) _esocket(__WHENCE__, (domain), (type), (protocol))
#define socket_bind(sock, addr, addrlen) _socket_bind(__WHENCE__, (sock), (addr), (addrlen))
#define socket_connect(sock, addr, addrlen) _socket_connect(__WHENCE__, (sock), (addr), (addrlen))
#define socket_listen(sock, backlog) _socket_listen(__WHENCE__, (sock), (backlog))
#define socket_set_reuseaddr(sock, reuseP) _socket_set_reuseaddr(__WHENCE__, (sock), (reuseP))
#define socket_set_rcvbufsize(sock, buffer_size) _socket_set_rcvbufsize(__WHENCE__, (sock), (buffer_size))
int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, struct sockaddr_un *dst_addr, socklen_t *dst_addrlen);
int cmp_sockaddr(const struct sockaddr *, socklen_t, const struct sockaddr *, socklen_t);
#define SERVER_CONFIG_RELOAD_INTERVAL_MS 1000
struct cli_parsed;
@ -240,100 +211,9 @@ extern char *instrumentation_file;
extern char *batman_socket;
extern char *batman_peerfile;
struct subscriber;
struct decode_context;
typedef struct keypair {
int type;
unsigned char *private_key;
size_t private_key_len;
unsigned char *public_key;
size_t public_key_len;
} keypair;
/* Contains just the list of private:public key pairs and types,
the pin used to extract them, and the slot in the keyring file
(so that it can be replaced/rewritten as required). */
#define PKR_MAX_KEYPAIRS 64
#define PKR_SALT_BYTES 32
#define PKR_MAC_BYTES 64
typedef struct keyring_identity {
char *PKRPin;
struct subscriber *subscriber;
time_ms_t challenge_expires;
unsigned char challenge[24];
unsigned int slot;
unsigned int keypair_count;
keypair *keypairs[PKR_MAX_KEYPAIRS];
} keyring_identity;
/* 64K identities, can easily be increased should the need arise,
but keep it low-ish for now so that the 64K pointers don't eat too
much ram on a small device. Should probably think about having
small and large device settings for some of these things */
#define KEYRING_MAX_IDENTITIES 65536
typedef struct keyring_context {
char *KeyRingPin;
unsigned char *KeyRingSalt;
int KeyRingSaltLen;
unsigned int identity_count;
keyring_identity *identities[KEYRING_MAX_IDENTITIES];
} keyring_context;
#define KEYRING_PAGE_SIZE 4096LL
#define KEYRING_BAM_BYTES 2048LL
#define KEYRING_BAM_BITS (KEYRING_BAM_BYTES<<3)
#define KEYRING_SLAB_SIZE (KEYRING_PAGE_SIZE*KEYRING_BAM_BITS)
typedef struct keyring_bam {
off_t file_offset;
unsigned char bitmap[KEYRING_BAM_BYTES];
struct keyring_bam *next;
} keyring_bam;
#define KEYRING_MAX_CONTEXTS 256
typedef struct keyring_file {
int context_count;
keyring_bam *bam;
keyring_context *contexts[KEYRING_MAX_CONTEXTS];
FILE *file;
off_t file_size;
} keyring_file;
void keyring_free(keyring_file *k);
void keyring_release_identity(keyring_file *k, int cn, int id);
#define KEYTYPE_CRYPTOBOX 0x01 // must be lowest
#define KEYTYPE_CRYPTOSIGN 0x02
#define KEYTYPE_RHIZOME 0x03
/* DIDs aren't really keys, but the keyring is a real handy place to keep them,
and keep them private if people so desire */
#define KEYTYPE_DID 0x04
/* handle to keyring file for use in running instance */
extern keyring_file *keyring;
/* Public calls to keyring management */
keyring_file *keyring_open(const char *path, int writeable);
keyring_file *keyring_open_instance();
keyring_file *keyring_open_instance_cli(const struct cli_parsed *parsed);
int keyring_enter_pin(keyring_file *k, const char *pin);
int keyring_set_did(keyring_identity *id, const char *did, const char *name);
int keyring_sanitise_position(const keyring_file *k,int *cn,int *in,int *kp);
int keyring_next_keytype(const keyring_file *k, int *cn, int *in, int *kp, int keytype);
int keyring_next_identity(const keyring_file *k,int *cn,int *in,int *kp);
int keyring_identity_find_keytype(const keyring_file *k, int cn, int in, int keytype);
int keyring_find_did(const keyring_file *k,int *cn,int *in,int *kp,char *did);
int keyring_find_sid(const keyring_file *k,int *cn,int *in,int *kp, const sid_t *sidp);
unsigned char *keyring_find_sas_private(keyring_file *k, const sid_t *sidp, unsigned char **sas_public);
int keyring_send_sas_request(struct subscriber *subscriber);
int keyring_commit(keyring_file *k);
keyring_identity *keyring_create_identity(keyring_file *k,keyring_context *c, const char *pin);
int keyring_seed(keyring_file *k);
void keyring_identity_extract(const keyring_identity *id, const sid_t **sidp, const char **didp, const char **namep);
int keyring_load(keyring_file *k, const char *keyring_pin, unsigned entry_pinc, const char **entry_pinv, FILE *input);
int keyring_dump(keyring_file *k, XPRINTF xpf, int include_secret);
/* Make sure we have space to put bytes of the packet as we go along */
#define CHECK_PACKET_LEN(B) {if (((*packet_len)+(B))>=packet_maxlen) { return WHY("Packet composition ran out of space."); } }
@ -392,12 +272,6 @@ struct slip_decode_state{
uint32_t crc;
int src_offset;
int dst_offset;
int mavlink_payload_length;
int mavlink_seq;
int mavlink_payload_start;
int mavlink_payload_offset;
uint8_t mavlink_payload[1024];
};
struct overlay_interface;
@ -470,21 +344,7 @@ typedef struct overlay_interface {
int recv_count;
int tx_count;
// stream socket tx state;
struct overlay_buffer *tx_packet;
unsigned char txbuffer[OVERLAY_INTERFACE_RX_BUFFER_SIZE];
int tx_bytes_pending;
// Throttle TX rate if required (stream interfaces only for now)
uint32_t throttle_bytes_per_second;
uint32_t throttle_burst_write_size;
uint64_t next_tx_allowed;
int32_t remaining_space;
time_ms_t next_heartbeat;
int mavlink_seq;
int radio_rssi;
int remote_rssi;
struct slip_decode_state slip_decode_state;
struct radio_link_state *radio_link_state;
// copy of ifconfig flags
uint16_t drop_packets;
@ -544,7 +404,7 @@ void insertTransactionInCache(unsigned char *transaction_id);
int overlay_forward_payload(struct overlay_frame *f);
int packetOkOverlay(struct overlay_interface *interface,unsigned char *packet, size_t len,
int recvttl, struct sockaddr *recvaddr, socklen_t recvaddrlen);
int recvttl, struct socket_address *recvaddr);
int parseMdpPacketHeader(struct decode_context *context, struct overlay_frame *frame,
struct overlay_buffer *buffer, struct subscriber **nexthop);
int parseEnvelopeHeader(struct decode_context *context, struct overlay_interface *interface,
@ -601,18 +461,14 @@ int rhizome_opendb();
int parseCommandLine(struct cli_context *context, const char *argv0, int argc, const char *const *argv);
int overlay_mdp_get_fds(struct pollfd *fds,int *fdcount,int fdmax);
int overlay_mdp_reply_error(int sock,
struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
int error_number,char *message);
typedef uint32_t mdp_port_t;
#define PRImdp_port_t "08" PRIx32
#define PRImdp_port_t "#08" PRIx32
typedef struct sockaddr_mdp {
sid_t sid;
mdp_port_t port;
} sockaddr_mdp;
unsigned char *keyring_get_nm_bytes(const sid_t *known_sidp, const sid_t *unknown_sidp);
typedef struct overlay_mdp_data_frame {
sockaddr_mdp src;
@ -664,19 +520,20 @@ typedef struct overlay_mdp_frame {
};
} overlay_mdp_frame;
int keyring_mapping_request(keyring_file *k, struct overlay_frame *frame, overlay_mdp_frame *req);
int keyring_send_unlock(struct subscriber *subscriber);
void keyring_release_subscriber(keyring_file *k, const sid_t *sid);
/* Server-side MDP functions */
int overlay_mdp_swap_src_dst(overlay_mdp_frame *mdp);
int overlay_mdp_reply(int sock,struct sockaddr_un *recvaddr, socklen_t recvaddrlen,
overlay_mdp_frame *mdpreply);
int overlay_mdp_dispatch(overlay_mdp_frame *mdp,int userGeneratedFrameP,
struct sockaddr_un *recvaddr, socklen_t recvaddrlen);
int overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port);
struct socket_address;
int overlay_mdp_dispatch(overlay_mdp_frame *mdp, struct socket_address *client);
void overlay_mdp_encode_ports(struct overlay_buffer *plaintext, mdp_port_t dst_port, mdp_port_t src_port);
int overlay_mdp_dnalookup_reply(const sockaddr_mdp *dstaddr, const sid_t *resolved_sidp, const char *uri, const char *did, const char *name);
struct mdp_header;
int mdp_bind_internal(struct subscriber *subscriber, mdp_port_t port,
int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len));
int mdp_unbind_internal(struct subscriber *subscriber, mdp_port_t port,
int (*internal)(const struct mdp_header *header, const uint8_t *payload, size_t len));
struct vomp_call_state;
void set_codec_flag(int codec, unsigned char *flags);
@ -786,7 +643,7 @@ int overlay_packetradio_tx_packet(struct overlay_frame *frame);
void overlay_dummy_poll(struct sched_ent *alarm);
void server_config_reload(struct sched_ent *alarm);
void server_shutdown_check(struct sched_ent *alarm);
int overlay_mdp_try_interal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp);
int overlay_mdp_try_internal_services(struct overlay_frame *frame, overlay_mdp_frame *mdp);
int overlay_send_probe(struct subscriber *peer, struct network_destination *destination, int queue);
int overlay_send_stun_request(struct subscriber *server, struct subscriber *request);
void fd_periodicstats(struct sched_ent *alarm);
@ -811,13 +668,6 @@ int limit_init(struct limit_state *state, int rate_micro_seconds);
int olsr_init_socket(void);
int olsr_send(struct overlay_frame *frame);
void write_uint64(unsigned char *o,uint64_t v);
void write_uint16(unsigned char *o,uint16_t v);
void write_uint32(unsigned char *o,uint32_t v);
uint64_t read_uint64(unsigned char *o);
uint32_t read_uint32(unsigned char *o);
uint16_t read_uint16(unsigned char *o);
int pack_uint(unsigned char *buffer, uint64_t v);
int measure_packed_uint(uint64_t v);
int unpack_uint(unsigned char *buffer, int buff_size, uint64_t *v);
@ -828,8 +678,7 @@ int slip_decode(struct slip_decode_state *state);
int upper7_decode(struct slip_decode_state *state,unsigned char byte);
uint32_t Crc32_ComputeBuf( uint32_t inCrc32, const void *buf,
size_t bufLen );
int rhizome_active_fetch_count();
int rhizome_active_fetch_bytes_received(int q);
void rhizome_fetch_log_short_status();
extern int64_t bundles_available;
extern char crash_handler_clue[1024];
@ -851,8 +700,4 @@ int link_stop_routing(struct subscriber *subscriber);
int generate_nonce(unsigned char *nonce,int bytes);
int mavlink_decode(struct overlay_interface *interface, struct slip_decode_state *state,uint8_t c);
int mavlink_heartbeat(unsigned char *frame,int *outlen);
int mavlink_encode_packet(struct overlay_interface *interface);
#endif // __SERVALD_SERVALD_H

27
server.c
View File

@ -21,7 +21,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <unistd.h>
#include <time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/stat.h>
#include "serval.h"
@ -108,11 +107,9 @@ int server(const struct cli_parsed *parsed)
sigemptyset(&sig.sa_mask); // Block the same signals during handler
sigaddset(&sig.sa_mask, SIGHUP);
sigaddset(&sig.sa_mask, SIGINT);
sigaddset(&sig.sa_mask, SIGQUIT);
sig.sa_flags = 0;
sigaction(SIGHUP, &sig, NULL);
sigaction(SIGINT, &sig, NULL);
sigaction(SIGQUIT, &sig, NULL);
/* Record PID to advertise that the server is now running */
char filename[1024];
@ -233,19 +230,23 @@ int server_check_stopfile()
void serverCleanUp()
{
/* Try to remove shutdown and PID files and exit */
server_remove_stopfile();
char filename[1024];
if (FORM_SERVAL_INSTANCE_PATH(filename, PIDFILE_NAME))
unlink(filename);
if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.socket")) {
unlink(filename);
if (serverMode){
rhizome_close_db();
dna_helper_shutdown();
}
rhizome_close_db();
char filename[1024];
if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.socket"))
unlink(filename);
dna_helper_shutdown();
if (FORM_SERVAL_INSTANCE_PATH(filename, "mdp.2.socket"))
unlink(filename);
if (FORM_SERVAL_INSTANCE_PATH(filename, "monitor.socket"))
unlink(filename);
/* Try to remove shutdown and PID files and exit */
server_remove_stopfile();
}
static void signame(char *buf, size_t len, int signal)

1
slip.c
View File

@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "serval.h"
#include "conf.h"
#include "log.h"
#include "dataformats.h"
#define DEBUG_packet_visualise(M,P,N) logServalPacket(LOG_LEVEL_DEBUG, __WHENCE__, (M), (P), (N))

118
socket.c
View File

@ -25,6 +25,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include "conf.h"
#include "log.h"
#include "strbuf_helpers.h"
#include "socket.h"
/* Form the name of an AF_UNIX (local) socket in the instance directory as an absolute path.
* Under Linux, this will create a socket name in the abstract namespace. This permits us to use
@ -40,25 +41,28 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* @author Andrew Bettison <andrew@servalproject.com>
* @author Daniel O'Connor <daniel@servalproject.com>
*/
int _make_local_sockaddr(struct __sourceloc __whence, struct sockaddr_un *addr, socklen_t *addrlen, const char *fmt, ...)
int _make_local_sockaddr(struct __sourceloc __whence, struct socket_address *addr, const char *fmt, ...)
{
bzero(addr, sizeof(*addr));
addr->local.sun_family = AF_UNIX;
va_list ap;
va_start(ap, fmt);
int r = vformf_serval_instance_path(__WHENCE__, addr->sun_path, sizeof addr->sun_path, fmt, ap);
int r = vformf_serval_instance_path(__WHENCE__, addr->local.sun_path, sizeof addr->local.sun_path, fmt, ap);
va_end(ap);
if (!r)
return WHY("socket name overflow");
if (real_sockaddr(addr, sizeof addr->sun_family + strlen(addr->sun_path) + 1, addr, addrlen) == -1)
return -1;
addr->addrlen=sizeof addr->local.sun_family + strlen(addr->local.sun_path) + 1;
// TODO perform real path transformation in making the serval instance path
// if (real_sockaddr(addr, addr) == -1)
// return -1;
#ifdef USE_ABSTRACT_NAMESPACE
// For the abstract name we use the absolute path name with the initial '/' replaced by the
// leading nul. This ensures that different instances of the Serval daemon have different socket
// names.
addr->sun_path[0] = '\0'; // mark as Linux abstract socket
--*addrlen; // do not count trailing nul in abstract socket name
addr->local.sun_path[0] = '\0'; // mark as Linux abstract socket
--addr->addrlen; // do not count trailing nul in abstract socket name
#endif // USE_ABSTRACT_NAMESPACE
addr->sun_family = AF_UNIX;
return 0;
}
@ -74,31 +78,33 @@ int _make_local_sockaddr(struct __sourceloc __whence, struct sockaddr_un *addr,
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, struct sockaddr_un *dst_addr, socklen_t *dst_addrlen)
int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr)
{
int src_path_len = src_addrlen - sizeof src_addr->sun_family;
if ( src_addrlen >= sizeof src_addr->sun_family + 1
&& src_addr->sun_family == AF_UNIX
&& src_addr->sun_path[0] != '\0'
&& src_addr->sun_path[src_path_len - 1] == '\0'
int src_path_len = src_addr->addrlen - sizeof src_addr->local.sun_family;
if ( src_addr->addrlen >= sizeof src_addr->local.sun_family + 1
&& src_addr->local.sun_family == AF_UNIX
&& src_addr->local.sun_path[0] != '\0'
&& src_addr->local.sun_path[src_path_len - 1] == '\0'
) {
char real_path[PATH_MAX];
size_t real_path_len;
if (realpath(src_addr->sun_path, real_path) == NULL)
return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->sun_path));
else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->sun_path)
return WHYF("sockaddr overrun: realpath(%s) returned %s", alloca_str_toprint(src_addr->sun_path), alloca_str_toprint(real_path));
if (realpath(src_addr->local.sun_path, real_path) == NULL)
return WHYF_perror("realpath(%s)", alloca_str_toprint(src_addr->local.sun_path));
else if ((real_path_len = strlen(real_path) + 1) > sizeof dst_addr->local.sun_path)
return WHYF("sockaddr overrun: realpath(%s) returned %s",
alloca_str_toprint(src_addr->local.sun_path), alloca_str_toprint(real_path));
else if ( real_path_len != src_path_len
|| memcmp(real_path, src_addr->sun_path, src_path_len) != 0
|| memcmp(real_path, src_addr->local.sun_path, src_path_len) != 0
) {
memcpy(dst_addr->sun_path, real_path, real_path_len);
*dst_addrlen = real_path_len + sizeof dst_addr->sun_family;
memcpy(dst_addr->local.sun_path, real_path, real_path_len);
dst_addr->addrlen = real_path_len + sizeof dst_addr->local.sun_family;
return 1;
}
}
if (dst_addr != src_addr)
memcpy(dst_addr, src_addr, src_addrlen);
*dst_addrlen = src_addrlen;
if (dst_addr != src_addr){
memcpy(&dst_addr->addr, &src_addr->addr, src_addr->addrlen);
dst_addr->addrlen = src_addr->addrlen;
}
return 0;
}
@ -107,43 +113,43 @@ int real_sockaddr(const struct sockaddr_un *src_addr, socklen_t src_addrlen, str
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int cmp_sockaddr(const struct sockaddr *addrA, socklen_t addrlenA, const struct sockaddr *addrB, socklen_t addrlenB)
int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address *addrB)
{
// Two zero-length sockaddrs are equal.
if (addrlenA == 0 && addrlenB == 0)
if (addrA->addrlen == 0 && addrB->addrlen == 0)
return 0;
// If either sockaddr is truncated, then we compare the bytes we have.
if (addrlenA < sizeof addrA->sa_family || addrlenB < sizeof addrB->sa_family) {
int c = memcmp(addrA, addrB, addrlenA < addrlenB ? addrlenA : addrlenB);
if (addrA->addrlen < sizeof addrA->addr.sa_family || addrB->addrlen < sizeof addrB->addr.sa_family) {
int c = memcmp(addrA, addrB, addrA->addrlen < addrB->addrlen ? addrA->addrlen : addrB->addrlen);
if (c == 0)
c = addrlenA < addrlenB ? -1 : addrlenA > addrlenB ? 1 : 0;
c = addrA->addrlen < addrB->addrlen ? -1 : addrA->addrlen > addrB->addrlen ? 1 : 0;
return c;
}
// Order first by address family.
if (addrA->sa_family < addrB->sa_family)
if (addrA->addr.sa_family < addrB->addr.sa_family)
return -1;
if (addrA->sa_family > addrB->sa_family)
if (addrA->addr.sa_family > addrB->addr.sa_family)
return 1;
// Both addresses are in the same family...
switch (addrA->sa_family) {
switch (addrA->addr.sa_family) {
case AF_UNIX: {
unsigned pathlenA = addrlenA - sizeof ((const struct sockaddr_un *)addrA)->sun_family;
unsigned pathlenB = addrlenB - sizeof ((const struct sockaddr_un *)addrB)->sun_family;
unsigned pathlenA = addrA->addrlen - sizeof (addrA->local.sun_family);
unsigned pathlenB = addrB->addrlen - sizeof (addrB->local.sun_family);
int c;
if ( pathlenA > 1 && pathlenB > 1
&& ((const struct sockaddr_un *)addrA)->sun_path[0] == '\0'
&& ((const struct sockaddr_un *)addrB)->sun_path[0] == '\0'
&& addrA->local.sun_path[0] == '\0'
&& addrB->local.sun_path[0] == '\0'
) {
// Both abstract sockets - just compare names, nul bytes are not terminators.
c = memcmp(&((const struct sockaddr_un *)addrA)->sun_path[1],
&((const struct sockaddr_un *)addrB)->sun_path[1],
c = memcmp(&addrA->local.sun_path[1],
&addrB->local.sun_path[1],
(pathlenA < pathlenB ? pathlenA : pathlenB) - 1);
} else {
// Either or both are named local file sockets. If the file names are identical up to the
// first nul, then the addresses are equal. This collates abstract socket names, whose first
// character is a nul, ahead of all non-empty file socket names.
c = strncmp(((const struct sockaddr_un *)addrA)->sun_path,
((const struct sockaddr_un *)addrB)->sun_path,
c = strncmp(addrA->local.sun_path,
addrB->local.sun_path,
(pathlenA < pathlenB ? pathlenA : pathlenB));
}
if (c == 0)
@ -153,9 +159,10 @@ int cmp_sockaddr(const struct sockaddr *addrA, socklen_t addrlenA, const struct
break;
}
// Fall back to comparing raw data bytes.
int c = memcmp(addrA->sa_data, addrB->sa_data, (addrlenA < addrlenB ? addrlenA : addrlenB) - sizeof addrA->sa_family);
int c = memcmp(addrA->addr.sa_data, addrB->addr.sa_data,
(addrA->addrlen < addrB->addrlen ? addrA->addrlen : addrB->addrlen) - sizeof addrA->addr.sa_family);
if (c == 0)
c = addrlenA < addrlenB ? -1 : addrlenA > addrlenB ? 1 : 0;
c = addrA->addrlen < addrB->addrlen ? -1 : addrA->addrlen > addrB->addrlen ? 1 : 0;
return c;
}
@ -225,3 +232,32 @@ int _socket_set_rcvbufsize(struct __sourceloc __whence, int sock, unsigned buffe
DEBUGF("setsockopt(%d, SOL_SOCKET, SO_RCVBUF, &%u, %u)", sock, buffer_size, (unsigned)sizeof buffer_size);
return 0;
}
ssize_t _send_message(struct __sourceloc __whence, int fd, const struct socket_address *address, const struct fragmented_data *data)
{
struct msghdr hdr={
.msg_name=(void *)&address->addr,
.msg_namelen=address->addrlen,
.msg_iov=(struct iovec*)data->iov,
.msg_iovlen=data->fragment_count,
};
ssize_t ret = sendmsg(fd, &hdr, 0);
if (ret==-1)
WHYF_perror("sendmsg(%d,%s,%lu)", fd, alloca_socket_address(address), (unsigned long)address->addrlen);
return ret;
}
ssize_t _recv_message(struct __sourceloc __whence, int fd, struct socket_address *address, struct fragmented_data *data)
{
struct msghdr hdr={
.msg_name=(void *)&address->addr,
.msg_namelen=address->addrlen,
.msg_iov=data->iov,
.msg_iovlen=data->fragment_count,
};
ssize_t ret = recvmsg(fd, &hdr, 0);
if (ret==-1)
WHYF_perror("recvmsg(%d,%s,%lu)", fd, alloca_socket_address(address), (unsigned long)address->addrlen);
return ret;
}

57
socket.h Normal file
View File

@ -0,0 +1,57 @@
#ifndef __SERVALD_SOCKET_H
#define __SERVALD_SOCKET_H
#ifndef WIN32
#include <sys/un.h>
#endif
struct socket_address{
socklen_t addrlen;
union{
struct sockaddr addr;
struct sockaddr_un local; // name "unix" is a predefined macro
struct sockaddr_in inet;
struct sockaddr_storage store;
};
};
/* Basic socket operations.
*/
int _make_local_sockaddr(struct __sourceloc, struct socket_address *addr, const char *fmt, ...)
__attribute__((format(printf, 3, 4)));
int _esocket(struct __sourceloc, int domain, int type, int protocol);
int _socket_bind(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
int _socket_connect(struct __sourceloc, int sock, const struct sockaddr *addr, socklen_t addrlen);
int _socket_listen(struct __sourceloc, int sock, int backlog);
int _socket_set_reuseaddr(struct __sourceloc, int sock, int reuseP);
int _socket_set_rcvbufsize(struct __sourceloc, int sock, unsigned buffer_size);
#define make_local_sockaddr(sockname, fmt,...) _make_local_sockaddr(__WHENCE__, (sockname), (fmt), ##__VA_ARGS__)
#define esocket(domain, type, protocol) _esocket(__WHENCE__, (domain), (type), (protocol))
#define socket_bind(sock, addr, addrlen) _socket_bind(__WHENCE__, (sock), (addr), (addrlen))
#define socket_connect(sock, addr, addrlen) _socket_connect(__WHENCE__, (sock), (addr), (addrlen))
#define socket_listen(sock, backlog) _socket_listen(__WHENCE__, (sock), (backlog))
#define socket_set_reuseaddr(sock, reuseP) _socket_set_reuseaddr(__WHENCE__, (sock), (reuseP))
#define socket_set_rcvbufsize(sock, buffer_size) _socket_set_rcvbufsize(__WHENCE__, (sock), (buffer_size))
int real_sockaddr(const struct socket_address *src_addr, struct socket_address *dst_addr);
int cmp_sockaddr(const struct socket_address *addrA, const struct socket_address *addrB);
// helper functions for manipulating fragmented packet data
#define MAX_FRAGMENTS 8
struct fragmented_data{
int fragment_count;
struct iovec iov[MAX_FRAGMENTS];
};
int prepend_fragment(struct fragmented_data *data, const uint8_t *payload, size_t len);
int append_fragment(struct fragmented_data *data, const uint8_t *payload, size_t len);
size_t copy_fragment(struct fragmented_data *src, uint8_t *dest, size_t length);
ssize_t _send_message(struct __sourceloc, int fd, const struct socket_address *address, const struct fragmented_data *data);
ssize_t _recv_message(struct __sourceloc, int fd, struct socket_address *address, struct fragmented_data *data);
#define send_message(fd, address, data) _send_message(__WHENCE__, (fd), (address), (data))
#define recv_message(fd, address, data) _recv_message(__WHENCE__, (fd), (address), (data))
#endif

6
sourcefiles.mk
View File

@ -19,7 +19,7 @@ SERVAL_SOURCES = \
$(SERVAL_BASE)log.c \
$(SERVAL_BASE)lsif.c \
$(SERVAL_BASE)main.c \
$(SERVAL_BASE)mavlink.c \
$(SERVAL_BASE)radio_link.c \
$(SERVAL_BASE)meshms.c \
$(SERVAL_BASE)mdp_client.c \
$(SERVAL_BASE)os.c \
@ -69,10 +69,12 @@ SERVAL_SOURCES = \
$(SERVAL_BASE)strbuf.c \
$(SERVAL_BASE)strbuf_helpers.c \
$(SERVAL_BASE)strlcpy.c \
$(SERVAL_BASE)uuid.c \
$(SERVAL_BASE)vomp.c \
$(SERVAL_BASE)vomp_console.c \
$(SERVAL_BASE)xprintf.c \
$(SERVAL_BASE)fec-3.0.1/ccsds_tables.c \
$(SERVAL_BASE)fec-3.0.1/decode_rs_8.c \
$(SERVAL_BASE)fec-3.0.1/encode_rs_8.c \
$(SERVAL_BASE)fec-3.0.1/init_rs_char.c
$(SERVAL_BASE)fec-3.0.1/init_rs_char.c \
$(SERVAL_BASE)context1.c

471
str.c
View File

@ -17,37 +17,33 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#define __STR_INLINE
#define __SERVAL_DNA_STR_INLINE
#include "str.h"
#include "strbuf_helpers.h"
#include "constants.h"
#include <stdlib.h>
#include <stdio.h>
#include <sys/uio.h>
#include <string.h>
#include <ctype.h>
#include <assert.h>
#include <limits.h>
#include <errno.h>
const char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
const char hexdigit_upper[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
const char hexdigit_lower[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
char *tohex(char *dstHex, size_t dstStrLen, const unsigned char *srcBinary)
{
char *p;
size_t i;
for (p = dstHex, i = 0; i < dstStrLen; ++i)
*p++ = (i & 1) ? hexdigit[*srcBinary++ & 0xf] : hexdigit[*srcBinary >> 4];
*p++ = (i & 1) ? hexdigit_upper[*srcBinary++ & 0xf] : hexdigit_upper[*srcBinary >> 4];
*p = '\0';
return dstHex;
}
/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] to nbinary bytes of data. Can be used to
* perform the conversion in-place, eg, fromhex(buf, (char*)buf, n); Returns -1 if a non-hex-digit
* character is encountered, otherwise returns the number of binary bytes produced (= nbinary).
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary)
{
if (strn_fromhex(dstBinary, nbinary, srcHex, NULL) == nbinary)
@ -55,13 +51,6 @@ size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary)
return -1;
}
/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] followed by a nul '\0' character to nbinary
* bytes of data. Can be used to perform the conversion in-place, eg, fromhex(buf, (char*)buf, n);
* Returns -1 if a non-hex-digit character is encountered or the character immediately following the
* last hex digit is not a nul, otherwise returns zero.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary)
{
const char *p;
@ -70,21 +59,6 @@ int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary)
return -1;
}
/* Decode pairs of ASCII hex characters [0-9A-Fa-f] into binary data with an optional upper limit on
* the number of binary bytes produced (destination buffer size). Returns the number of binary
* bytes decoded. If 'afterHex' is not NULL, then sets *afterHex to point to the source character
* immediately following the last hex digit consumed.
*
* Can be used to perform a conversion in-place, eg:
*
* strn_fromhex((unsigned char *)buf, n, (const char *)buf, NULL);
*
* Can also be used to count hex digits without converting, eg:
*
* strn_fromhex(NULL, -1, buf, NULL);
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *srcHex, const char **afterHex)
{
unsigned char *dstorig = dstBinary;
@ -106,6 +80,441 @@ size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *srcHex
return dstBinary - dstorig;
}
const char base64_symbols[65] = {
'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P',
'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f',
'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v',
'w','x','y','z','0','1','2','3','4','5','6','7','8','9','+','/',
'='
};
const char base64url_symbols[65] = {
'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P',
'Q','R','S','T','U','V','W','X','Y','Z','a','b','c','d','e','f',
'g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v',
'w','x','y','z','0','1','2','3','4','5','6','7','8','9','-','_',
'='
};
static size_t _base64_encodev(const char symbols[], char *dstBase64, const struct iovec *const iov, int const iovcnt)
{
char *dst = dstBase64;
unsigned place = 0;
unsigned char buf = 0;
int iovc = 0;
for (iovc = 0; iovc != iovcnt; ++iovc) {
unsigned char *src = iov[iovc].iov_base;
size_t cnt = iov[iovc].iov_len;
for (; cnt; --cnt, ++src) {
switch (place) {
case 0:
*dst++ = symbols[*src >> 2];
buf = (*src << 4) & 0x3f;
place = 1;
break;
case 1:
*dst++ = symbols[(*src >> 4) | buf];
buf = (*src << 2) & 0x3f;
place = 2;
break;
case 2:
*dst++ = symbols[(*src >> 6) | buf];
*dst++ = symbols[*src & 0x3f];
place = 0;
break;
}
}
}
if (place)
*dst++ = symbols[buf];
switch (place) {
case 2:
*dst++ = symbols[64];
case 1:
*dst++ = symbols[64];
}
return dst - dstBase64;
}
size_t base64_encodev(char *dstBase64, const struct iovec *const iov, int const iovcnt)
{
return _base64_encodev(base64_symbols, dstBase64, iov, iovcnt);
}
size_t base64url_encodev(char *dstBase64, const struct iovec *const iov, int const iovcnt)
{
return _base64_encodev(base64url_symbols, dstBase64, iov, iovcnt);
}
size_t base64_encode(char *const dstBase64, const unsigned char *src, size_t srclen)
{
struct iovec iov;
iov.iov_base = (void *) src;
iov.iov_len = srclen;
return _base64_encodev(base64_symbols, dstBase64, &iov, 1);
}
size_t base64url_encode(char *const dstBase64, const unsigned char *src, size_t srclen)
{
struct iovec iov;
iov.iov_base = (void *) src;
iov.iov_len = srclen;
return _base64_encodev(base64url_symbols, dstBase64, &iov, 1);
}
char *to_base64_str(char *const dstBase64, const unsigned char *srcBinary, size_t srcBytes)
{
dstBase64[base64_encode(dstBase64, srcBinary, srcBytes)] = '\0';
return dstBase64;
}
char *to_base64url_str(char *const dstBase64, const unsigned char *srcBinary, size_t srcBytes)
{
dstBase64[base64url_encode(dstBase64, srcBinary, srcBytes)] = '\0';
return dstBase64;
}
static size_t _base64_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen,
const char **afterp, int flags, int (*skip_pred)(char),
int (*isdigit_pred)(char), int (*ispad_pred)(char), uint8_t (*todigit)(char)
)
{
uint8_t buf = 0;
size_t digits = 0;
unsigned pads = 0;
size_t bytes = 0;
const char *const srcend = srcBase64 + srclen;
const char *src = srcBase64;
const char *first_pad = NULL;
for (; srclen == 0 || (src < srcend); ++src) {
int isdigit = isdigit_pred(*src);
int ispad = ispad_pred(*src);
if (!isdigit && !ispad && skip_pred && skip_pred(*src))
continue;
assert(pads <= 2);
if (pads == 2)
break;
int place = digits & 3;
if (pads == 1) {
if (place == 3)
break;
assert(place == 2);
if (ispad) {
++pads;
continue; // consume trailing space before ending
}
// If only one pad character was present but there should be two, then don't consume the first
// one.
assert(first_pad != NULL);
src = first_pad;
break;
}
assert(pads == 0);
if (ispad && place >= 2) {
first_pad = src;
++pads;
continue;
}
if (!isdigit)
break;
++digits;
if (dstBinary && bytes < dstsiz) {
uint8_t d = todigit(*src);
switch (place) {
case 0:
buf = d << 2;
break;
case 1:
dstBinary[bytes++] = buf | (d >> 4);
buf = d << 4;
break;
case 2:
dstBinary[bytes++] = buf | (d >> 2);
buf = d << 6;
break;
case 3:
dstBinary[bytes++] = buf | d;
break;
}
} else if (flags & B64_CONSUME_ALL) {
switch (place) {
case 1: case 2: case 3: ++bytes;
}
} else
break;
}
if (afterp)
*afterp = src;
else if (*src)
return 0;
return bytes;
}
size_t base64_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen,
const char **afterp, int flags, int (*skip_pred)(char))
{
return _base64_decode(dstBinary, dstsiz, srcBase64, srclen, afterp, flags, skip_pred, is_base64_digit, is_base64_pad, base64_digit);
}
size_t base64url_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen,
const char **afterp, int flags, int (*skip_pred)(char))
{
return _base64_decode(dstBinary, dstsiz, srcBase64, srclen, afterp, flags, skip_pred, is_base64url_digit, is_base64url_pad, base64url_digit);
}
#define _B64 _SERVAL_CTYPE_0_BASE64
#define _B64U _SERVAL_CTYPE_0_BASE64URL
uint8_t _serval_ctype_0[UINT8_MAX] = {
['A'] = _B64 | _B64U | 0,
['B'] = _B64 | _B64U | 1,
['C'] = _B64 | _B64U | 2,
['D'] = _B64 | _B64U | 3,
['E'] = _B64 | _B64U | 4,
['F'] = _B64 | _B64U | 5,
['G'] = _B64 | _B64U | 6,
['H'] = _B64 | _B64U | 7,
['I'] = _B64 | _B64U | 8,
['J'] = _B64 | _B64U | 9,
['K'] = _B64 | _B64U | 10,
['L'] = _B64 | _B64U | 11,
['M'] = _B64 | _B64U | 12,
['N'] = _B64 | _B64U | 13,
['O'] = _B64 | _B64U | 14,
['P'] = _B64 | _B64U | 15,
['Q'] = _B64 | _B64U | 16,
['R'] = _B64 | _B64U | 17,
['S'] = _B64 | _B64U | 18,
['T'] = _B64 | _B64U | 19,
['U'] = _B64 | _B64U | 20,
['V'] = _B64 | _B64U | 21,
['W'] = _B64 | _B64U | 22,
['X'] = _B64 | _B64U | 23,
['Y'] = _B64 | _B64U | 24,
['Z'] = _B64 | _B64U | 25,
['a'] = _B64 | _B64U | 26,
['b'] = _B64 | _B64U | 27,
['c'] = _B64 | _B64U | 28,
['d'] = _B64 | _B64U | 29,
['e'] = _B64 | _B64U | 30,
['f'] = _B64 | _B64U | 31,
['g'] = _B64 | _B64U | 32,
['h'] = _B64 | _B64U | 33,
['i'] = _B64 | _B64U | 34,
['j'] = _B64 | _B64U | 35,
['k'] = _B64 | _B64U | 36,
['l'] = _B64 | _B64U | 37,
['m'] = _B64 | _B64U | 38,
['n'] = _B64 | _B64U | 39,
['o'] = _B64 | _B64U | 40,
['p'] = _B64 | _B64U | 41,
['q'] = _B64 | _B64U | 42,
['r'] = _B64 | _B64U | 43,
['s'] = _B64 | _B64U | 44,
['t'] = _B64 | _B64U | 45,
['u'] = _B64 | _B64U | 46,
['v'] = _B64 | _B64U | 47,
['w'] = _B64 | _B64U | 48,
['x'] = _B64 | _B64U | 49,
['y'] = _B64 | _B64U | 50,
['z'] = _B64 | _B64U | 51,
['0'] = _B64 | _B64U | 52,
['1'] = _B64 | _B64U | 53,
['2'] = _B64 | _B64U | 54,
['3'] = _B64 | _B64U | 55,
['4'] = _B64 | _B64U | 56,
['5'] = _B64 | _B64U | 57,
['6'] = _B64 | _B64U | 58,
['7'] = _B64 | _B64U | 59,
['8'] = _B64 | _B64U | 60,
['9'] = _B64 | _B64U | 61,
['+'] = _B64 | 62,
['/'] = _B64 | 63,
['-'] = _B64U | 62,
['_'] = _B64U | 63,
};
#define _SEP _SERVAL_CTYPE_1_HTTP_SEPARATOR
#define _URI_SCHEME _SERVAL_CTYPE_1_URI_SCHEME
#define _URI_UNRES _SERVAL_CTYPE_1_URI_UNRESERVED
#define _URI_RES _SERVAL_CTYPE_1_URI_RESERVED
uint8_t _serval_ctype_1[UINT8_MAX] = {
['A'] = _URI_SCHEME | _URI_UNRES | 0xA,
['B'] = _URI_SCHEME | _URI_UNRES | 0xB,
['C'] = _URI_SCHEME | _URI_UNRES | 0xC,
['D'] = _URI_SCHEME | _URI_UNRES | 0xD,
['E'] = _URI_SCHEME | _URI_UNRES | 0xE,
['F'] = _URI_SCHEME | _URI_UNRES | 0xF,
['G'] = _URI_SCHEME | _URI_UNRES,
['H'] = _URI_SCHEME | _URI_UNRES,
['I'] = _URI_SCHEME | _URI_UNRES,
['J'] = _URI_SCHEME | _URI_UNRES,
['K'] = _URI_SCHEME | _URI_UNRES,
['L'] = _URI_SCHEME | _URI_UNRES,
['M'] = _URI_SCHEME | _URI_UNRES,
['N'] = _URI_SCHEME | _URI_UNRES,
['O'] = _URI_SCHEME | _URI_UNRES,
['P'] = _URI_SCHEME | _URI_UNRES,
['Q'] = _URI_SCHEME | _URI_UNRES,
['R'] = _URI_SCHEME | _URI_UNRES,
['S'] = _URI_SCHEME | _URI_UNRES,
['T'] = _URI_SCHEME | _URI_UNRES,
['U'] = _URI_SCHEME | _URI_UNRES,
['V'] = _URI_SCHEME | _URI_UNRES,
['W'] = _URI_SCHEME | _URI_UNRES,
['X'] = _URI_SCHEME | _URI_UNRES,
['Y'] = _URI_SCHEME | _URI_UNRES,
['Z'] = _URI_SCHEME | _URI_UNRES,
['a'] = _URI_SCHEME | _URI_UNRES | 0xa,
['b'] = _URI_SCHEME | _URI_UNRES | 0xb,
['c'] = _URI_SCHEME | _URI_UNRES | 0xc,
['d'] = _URI_SCHEME | _URI_UNRES | 0xd,
['e'] = _URI_SCHEME | _URI_UNRES | 0xe,
['f'] = _URI_SCHEME | _URI_UNRES | 0xf,
['g'] = _URI_SCHEME | _URI_UNRES,
['h'] = _URI_SCHEME | _URI_UNRES,
['i'] = _URI_SCHEME | _URI_UNRES,
['j'] = _URI_SCHEME | _URI_UNRES,
['k'] = _URI_SCHEME | _URI_UNRES,
['l'] = _URI_SCHEME | _URI_UNRES,
['m'] = _URI_SCHEME | _URI_UNRES,
['n'] = _URI_SCHEME | _URI_UNRES,
['o'] = _URI_SCHEME | _URI_UNRES,
['p'] = _URI_SCHEME | _URI_UNRES,
['q'] = _URI_SCHEME | _URI_UNRES,
['r'] = _URI_SCHEME | _URI_UNRES,
['s'] = _URI_SCHEME | _URI_UNRES,
['t'] = _URI_SCHEME | _URI_UNRES,
['u'] = _URI_SCHEME | _URI_UNRES,
['v'] = _URI_SCHEME | _URI_UNRES,
['w'] = _URI_SCHEME | _URI_UNRES,
['x'] = _URI_SCHEME | _URI_UNRES,
['y'] = _URI_SCHEME | _URI_UNRES,
['z'] = _URI_SCHEME | _URI_UNRES,
['0'] = _URI_SCHEME | _URI_UNRES | 0,
['1'] = _URI_SCHEME | _URI_UNRES | 1,
['2'] = _URI_SCHEME | _URI_UNRES | 2,
['3'] = _URI_SCHEME | _URI_UNRES | 3,
['4'] = _URI_SCHEME | _URI_UNRES | 4,
['5'] = _URI_SCHEME | _URI_UNRES | 5,
['6'] = _URI_SCHEME | _URI_UNRES | 6,
['7'] = _URI_SCHEME | _URI_UNRES | 7,
['8'] = _URI_SCHEME | _URI_UNRES | 8,
['9'] = _URI_SCHEME | _URI_UNRES | 9,
['\t'] = _SEP,
[' '] = _SEP,
['_'] = _URI_UNRES,
['='] = _SEP | _URI_RES,
['<'] = _SEP,
['>'] = _SEP,
[';'] = _SEP | _URI_RES,
[':'] = _SEP | _URI_RES,
['\\'] = _SEP,
['\''] = _URI_RES,
['"'] = _SEP,
['/'] = _SEP | _URI_RES,
['['] = _SEP | _URI_RES,
[']'] = _SEP | _URI_RES,
['{'] = _SEP,
['}'] = _SEP,
['('] = _SEP | _URI_RES,
[')'] = _SEP | _URI_RES,
[','] = _SEP | _URI_RES,
['.'] = _URI_SCHEME | _URI_UNRES,
['?'] = _SEP | _URI_RES,
['!'] = _URI_RES,
['+'] = _URI_SCHEME | _URI_RES,
['-'] = _URI_SCHEME | _URI_UNRES,
['*'] = _URI_RES,
['$'] = _URI_RES,
['&'] = _URI_RES,
['#'] = _URI_RES,
['@'] = _SEP | _URI_RES,
['~'] = _URI_UNRES,
};
#define _BND _SERVAL_CTYPE_2_MULTIPART_BOUNDARY
uint8_t _serval_ctype_2[UINT8_MAX] = {
['A'] = _BND,
['B'] = _BND,
['C'] = _BND,
['D'] = _BND,
['E'] = _BND,
['F'] = _BND,
['G'] = _BND,
['H'] = _BND,
['I'] = _BND,
['J'] = _BND,
['K'] = _BND,
['L'] = _BND,
['M'] = _BND,
['N'] = _BND,
['O'] = _BND,
['P'] = _BND,
['Q'] = _BND,
['R'] = _BND,
['S'] = _BND,
['T'] = _BND,
['U'] = _BND,
['V'] = _BND,
['W'] = _BND,
['X'] = _BND,
['Y'] = _BND,
['Z'] = _BND,
['a'] = _BND,
['b'] = _BND,
['c'] = _BND,
['d'] = _BND,
['e'] = _BND,
['f'] = _BND,
['g'] = _BND,
['h'] = _BND,
['i'] = _BND,
['j'] = _BND,
['k'] = _BND,
['l'] = _BND,
['m'] = _BND,
['n'] = _BND,
['o'] = _BND,
['p'] = _BND,
['q'] = _BND,
['r'] = _BND,
['s'] = _BND,
['t'] = _BND,
['u'] = _BND,
['v'] = _BND,
['w'] = _BND,
['x'] = _BND,
['y'] = _BND,
['z'] = _BND,
['0'] = _BND,
['1'] = _BND,
['2'] = _BND,
['3'] = _BND,
['4'] = _BND,
['5'] = _BND,
['6'] = _BND,
['7'] = _BND,
['8'] = _BND,
['9'] = _BND,
['+'] = _BND,
['/'] = _BND,
['='] = _BND,
['-'] = _BND,
['.'] = _BND,
[':'] = _BND,
['_'] = _BND,
['('] = _BND,
[')'] = _BND,
[','] = _BND,
['?'] = _BND,
[' '] = _BND,
};
/* Does this whole buffer contain the same value? */
int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value)
{

289
str.h
View File

@ -17,8 +17,8 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef __STR_H__
#define __STR_H__
#ifndef __SERVAL_DNA_STR_H__
#define __SERVAL_DNA_STR_H__
#include <string.h>
#include <stdint.h>
@ -26,20 +26,34 @@
#include <ctype.h>
#include <alloca.h>
#ifndef __STR_INLINE
#ifndef __SERVAL_DNA_STR_INLINE
# if __GNUC__ && !__GNUC_STDC_INLINE__
# define __STR_INLINE extern inline
# define __SERVAL_DNA_STR_INLINE extern inline
# else
# define __STR_INLINE inline
# define __SERVAL_DNA_STR_INLINE inline
# endif
#endif
/* -------------------- Useful functions and macros -------------------- */
#define alloca_strdup(str) strcpy(alloca(strlen(str) + 1), (str))
int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value);
char *str_toupper_inplace(char *s);
char *str_tolower_inplace(char *s);
/* -------------------- Hexadecimal strings -------------------- */
extern const char hexdigit_upper[16];
extern const char hexdigit_lower[16];
/* Return true iff 'len' bytes starting at 'text' are hex digits, upper or lower case.
* Does not check the following byte.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
__STR_INLINE int is_xsubstring(const char *text, int len)
__SERVAL_DNA_STR_INLINE int is_xsubstring(const char *text, int len)
{
while (len--)
if (!isxdigit(*text++))
@ -52,7 +66,7 @@ __STR_INLINE int is_xsubstring(const char *text, int len)
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
__STR_INLINE int is_xstring(const char *text, int len)
__SERVAL_DNA_STR_INLINE int is_xstring(const char *text, int len)
{
while (len--)
if (!isxdigit(*text++))
@ -60,43 +74,205 @@ __STR_INLINE int is_xstring(const char *text, int len)
return *text == '\0';
}
extern const char hexdigit[16];
/* Converts a given binary blob to uppercase ASCII hexadecimal with a NUL terminator on the end.
* 'dstHex' must point to a buffer of at least 'dstStrLen' + 1 bytes.
*/
char *tohex(char *dstHex, size_t dstStrlen, const unsigned char *srcBinary);
size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, const char **afterp);
#define alloca_tohex(buf,bytes) tohex((char *)alloca((bytes)*2+1), (bytes) * 2, (buf))
#define alloca_strdup(str) strcpy(alloca(strlen(str) + 1), (str))
/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] to nbinary bytes of data. Can be used to
* perform the conversion in-place, eg, fromhex(buf, (char*)buf, n); Returns -1 if a non-hex-digit
* character is encountered, otherwise returns the number of binary bytes produced (= nbinary).
* Does not insist that the last hex digit is followed by a NUL or any particular character.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
size_t fromhex(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
__STR_INLINE int hexvalue(char c)
{
switch (c) {
case '0': return 0;
case '1': return 1;
case '2': return 2;
case '3': return 3;
case '4': return 4;
case '5': return 5;
case '6': return 6;
case '7': return 7;
case '8': return 8;
case '9': return 9;
case 'a': case 'A': return 10;
case 'b': case 'B': return 11;
case 'c': case 'C': return 12;
case 'd': case 'D': return 13;
case 'e': case 'E': return 14;
case 'f': case 'F': return 15;
}
return -1;
/* Convert nbinary*2 ASCII hex characters [0-9A-Fa-f] followed by a NUL '\0' character to nbinary
* bytes of data. Can be used to perform the conversion in-place, eg, fromhex(buf, (char*)buf, n);
* Returns -1 if a non-hex-digit character is encountered or the character immediately following the
* last hex digit is not a NUL, otherwise returns zero.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int fromhexstr(unsigned char *dstBinary, const char *srcHex, size_t nbinary);
/* Decode pairs of ASCII hex characters [0-9A-Fa-f] into binary data with an optional upper limit on
* the number of binary bytes produced (destination buffer size). Returns the number of binary
* bytes decoded. If 'afterHex' is not NULL, then sets *afterHex to point to the source character
* immediately following the last hex digit consumed.
*
* Can be used to perform a conversion in-place, eg:
*
* strn_fromhex((unsigned char *)buf, n, (const char *)buf, NULL);
*
* Can also be used to count hex digits without converting, eg:
*
* strn_fromhex(NULL, -1, buf, NULL);
*
* The fromhex() and fromhexstr() functions are both implemented using strn_fromhex().
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
size_t strn_fromhex(unsigned char *dstBinary, ssize_t dstlen, const char *src, const char **afterp);
/* -------------------- Base64 encoding and decoding -------------------- */
/* Return the number of bytes required to represent 'binaryBytes' bytes of binary data encoded
* into Base64 form.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
#define BASE64_ENCODED_LEN(binaryBytes) (((size_t)(binaryBytes) + 2) / 3 * 4)
/* Array of encoding symbols. Entry [64] is the pad character (usually '=').
*/
const char base64_symbols[65];
const char base64url_symbols[65];
/* Encode 'srcBytes' bytes of binary data at 'srcBinary' into Base64 representation at 'dstBase64'
* (or Base64-URL representation at 'dstBase64url'), which must point to at least
* 'BASE64_ENCODED_LEN(srcBytes)' bytes. The encoding is terminated by a "=" or "==" pad to bring
* the total number of encoded bytes up to a multiple of 4.
*
* Returns the total number of encoded bytes writtent at 'dstBase64'.
*
* The base64_encodev() is a multi-buffer gather variant, analagous to readv(2) and writev(2).
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
size_t base64_encode(char *dstBase64, const unsigned char *srcBinary, size_t srcBytes);
size_t base64url_encode(char *dstBase64url, const unsigned char *srcBinary, size_t srcBytes);
struct iovec;
size_t base64_encode(char *dstBase64, const unsigned char *srcBinary, size_t srcBytes);
size_t base64url_encodev(char *dstBase64url, const struct iovec *iov, int iovcnt);
/* The same as base64_encode() but appends a terminating NUL character to the encoded string,
* so 'dstBase64' must point to at least 'BASE64_ENCODED_LEN(srcBytes) + 1' bytes.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
char *to_base64_str(char *dstBase64, const unsigned char *srcBinary, size_t srcBytes);
char *to_base64url_str(char *dstBase64url, const unsigned char *srcBinary, size_t srcBytes);
#define alloca_base64(buf,len) to_base64_str(alloca(BASE64_ENCODED_LEN(len) + 1), (buf), (len))
#define alloca_base64url(buf,len) to_base64url_str(alloca(BASE64_ENCODED_LEN(len) + 1), (buf), (len))
/* Decode the string at 'srcBase64' as ASCII Base64 or Base64-URL (as per RFC-4648), writing up to
* 'dstsiz' decoded binary bytes at 'dstBinary'. Returns the number of decoded binary bytes
* produced. If 'dstsiz' is zero or 'dstBinary' is NULL, no binary bytes are produced and returns
* zero.
*
* If the 'afterp' pointer is not NULL, then sets *afterp to point to the first character in
* 'srcBase64' where decoding stopped for whatever reason.
*
* If 'srclen' is 0, then the string at 'stcBase64' is assumed to be NUL-terminated, and decoding
* runs until the first non-Base64-digit is encountered. If 'srclen' is nonzero, then decoding will
* cease at the first non-Base64-digit or when 'srclen' bytes at 'srcBase64' have been decoded,
* whichever comes first.
*
* If 'skip_pred' is not NULL, then all leading, internal and trailing characters C which are not a
* valid Base64 digit or pad '=' will be skipped if skip_pred(C) returns true. Otherwise, decoding
* ends at C.
*
* If the B64_CONSUME_ALL flag is set, then once the 'dstsiz' limit is reached (or if 'dstBinary' is
* NULL), the Base64 decoding process continues without actually writing decoded bytes, but instead
* counts them and advances through the 'srcBase64' buffer as usual. The return value is then the
* number of binary bytes that would be decoded were all available Base64 decoded from 'srcBase64',
* and *afterp points to the first character beyond the end of the decoded source characters.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
size_t base64_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64, size_t srclen,
const char **afterp, int flags, int (*skip_pred)(char));
size_t base64url_decode(unsigned char *dstBinary, size_t dstsiz, const char *const srcBase64url, size_t srclen,
const char **afterp, int flags, int (*skip_pred)(char));
#define B64_CONSUME_ALL (1 << 0)
/* -------------------- Character classes -------------------- */
#define _SERVAL_CTYPE_0_BASE64_MASK 0x3f
#define _SERVAL_CTYPE_0_BASE64 (1 << 6)
#define _SERVAL_CTYPE_0_BASE64URL (1 << 7)
#define _SERVAL_CTYPE_1_HEX_MASK 0xf
#define _SERVAL_CTYPE_1_HTTP_SEPARATOR (1 << 4)
#define _SERVAL_CTYPE_1_URI_SCHEME (1 << 5)
#define _SERVAL_CTYPE_1_URI_UNRESERVED (1 << 6)
#define _SERVAL_CTYPE_1_URI_RESERVED (1 << 7)
#define _SERVAL_CTYPE_2_MULTIPART_BOUNDARY (1 << 0)
extern uint8_t _serval_ctype_0[UINT8_MAX];
extern uint8_t _serval_ctype_1[UINT8_MAX];
extern uint8_t _serval_ctype_2[UINT8_MAX];
__SERVAL_DNA_STR_INLINE int is_http_char(char c) {
return isascii(c);
}
int is_all_matching(const unsigned char *ptr, size_t len, unsigned char value);
__SERVAL_DNA_STR_INLINE int is_http_ctl(char c) {
return iscntrl(c);
}
char *str_toupper_inplace(char *s);
char *str_tolower_inplace(char *s);
__SERVAL_DNA_STR_INLINE int is_base64_digit(char c) {
return (_serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64) != 0;
}
__SERVAL_DNA_STR_INLINE int is_base64url_digit(char c) {
return (_serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64URL) != 0;
}
__SERVAL_DNA_STR_INLINE int is_base64_pad(char c) {
return c == '=';
}
__SERVAL_DNA_STR_INLINE int is_base64url_pad(char c) {
return c == '=';
}
__SERVAL_DNA_STR_INLINE uint8_t base64_digit(char c) {
return _serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64_MASK;
}
__SERVAL_DNA_STR_INLINE uint8_t base64url_digit(char c) {
return _serval_ctype_0[(unsigned char) c] & _SERVAL_CTYPE_0_BASE64_MASK;
}
__SERVAL_DNA_STR_INLINE int is_multipart_boundary(char c) {
return (_serval_ctype_2[(unsigned char) c] & _SERVAL_CTYPE_2_MULTIPART_BOUNDARY) != 0;
}
__SERVAL_DNA_STR_INLINE int is_valid_multipart_boundary_string(const char *s)
{
if (s[0] == '\0')
return 0;
for (; *s; ++s)
if (!is_multipart_boundary(*s))
return 0;
return s[-1] != ' ';
}
__SERVAL_DNA_STR_INLINE int is_http_separator(char c) {
return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_HTTP_SEPARATOR) != 0;
}
__SERVAL_DNA_STR_INLINE int is_http_token(char c) {
return is_http_char(c) && !is_http_ctl(c) && !is_http_separator(c);
}
/* Convert the given ASCII hex digit character into its radix value, eg, '0' ->
* 0, 'b' -> 11. If the argument is not an ASCII hex digit, returns -1.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
__SERVAL_DNA_STR_INLINE int hexvalue(char c) {
return isxdigit(c) ? _serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_HEX_MASK : -1;
}
/* -------------------- Printable string representation -------------------- */
char *toprint(char *dstStr, ssize_t dstBufSiz, const char *srcBuf, size_t srcBytes, const char quotes[2]);
char *toprint_str(char *dstStr, ssize_t dstBufSiz, const char *srcStr, const char quotes[2]);
@ -104,10 +280,14 @@ size_t toprint_len(const char *srcBuf, size_t srcBytes, const char quotes[2]);
size_t toprint_str_len(const char *srcStr, const char quotes[2]);
size_t strn_fromprint(unsigned char *dst, size_t dstsiz, const char *src, size_t srclen, char endquote, const char **afterp);
#define alloca_toprint(dstlen,buf,len) toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), "``") + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), "``")
#define alloca_toprint_quoted(dstlen,buf,len,quotes) toprint((char *)alloca((dstlen) == -1 ? toprint_len((const char *)(buf),(len), (quotes)) + 1 : (dstlen)), (dstlen), (const char *)(buf), (len), (quotes))
#define alloca_toprint(dstlen,buf,len) alloca_toprint_quoted(dstlen,buf,len,"``")
#define alloca_str_toprint_quoted(str, quotes) toprint_str((char *)alloca(toprint_str_len((str), (quotes)) + 1), -1, (str), (quotes))
#define alloca_str_toprint(str) alloca_str_toprint_quoted(str, "``")
/* -------------------- Useful string primitives -------------------- */
/* Like strchr(3), but only looks for 'c' in the first 'n' characters of 's', stopping at the first
* nul char in 's'.
*
@ -122,24 +302,24 @@ const char *strnchr(const char *s, size_t n, char c);
* @author Andrew Bettison <andrew@servalproject.com>
*/
__STR_INLINE ssize_t str_index_dfl(const char *s, char c, ssize_t dfl)
__SERVAL_DNA_STR_INLINE ssize_t str_index_dfl(const char *s, char c, ssize_t dfl)
{
const char *r = strchr(s, c);
return r ? r - s : dfl;
}
__STR_INLINE ssize_t str_rindex_dfl(const char *s, char c, ssize_t dfl)
__SERVAL_DNA_STR_INLINE ssize_t str_rindex_dfl(const char *s, char c, ssize_t dfl)
{
const char *r = strrchr(s, c);
return r ? r - s : dfl;
}
__STR_INLINE ssize_t str_index(const char *s, char c)
__SERVAL_DNA_STR_INLINE ssize_t str_index(const char *s, char c)
{
return str_index_dfl(s, c, -1);
}
__STR_INLINE ssize_t str_rindex(const char *s, char c)
__SERVAL_DNA_STR_INLINE ssize_t str_rindex(const char *s, char c)
{
return str_rindex_dfl(s, c, -1);
}
@ -267,25 +447,16 @@ int str_to_uint64_interval_ms(const char *str, int64_t *result, const char **aft
*/
int str_is_uri(const char *uri);
__STR_INLINE int is_uri_char_scheme(char c)
{
return isalpha(c) || isdigit(c) || c == '+' || c == '-' || c == '.';
__SERVAL_DNA_STR_INLINE int is_uri_char_scheme(char c) {
return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_URI_SCHEME) != 0;
}
__STR_INLINE int is_uri_char_unreserved(char c)
{
return isalpha(c) || isdigit(c) || c == '-' || c == '.' || c == '_' || c == '~';
__SERVAL_DNA_STR_INLINE int is_uri_char_unreserved(char c) {
return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_URI_UNRESERVED) != 0;
}
__STR_INLINE int is_uri_char_reserved(char c)
{
switch (c) {
case ':': case '/': case '?': case '#': case '[': case ']': case '@':
case '!': case '$': case '&': case '\'': case '(': case ')':
case '*': case '+': case ',': case ';': case '=':
return 1;
}
return 0;
__SERVAL_DNA_STR_INLINE int is_uri_char_reserved(char c) {
return (_serval_ctype_1[(unsigned char) c] & _SERVAL_CTYPE_1_URI_RESERVED) != 0;
}
/* Return true if the string resembles a URI scheme without the terminating colon.
@ -293,7 +464,7 @@ __STR_INLINE int is_uri_char_reserved(char c)
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
__STR_INLINE int str_is_uri_scheme(const char *scheme)
__SERVAL_DNA_STR_INLINE int str_is_uri_scheme(const char *scheme)
{
if (!isalpha(*scheme++))
return 0;
@ -359,4 +530,4 @@ int str_uri_authority_port(const char *auth, uint16_t *portp);
int parse_argv(char *cmdline, char delim, char **argv, int max_argv);
#endif
#endif // __SERVAL_DNA_STR_H__

4
strbuf.c
View File

@ -19,6 +19,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#define __STRBUF_INLINE
#include "strbuf.h"
#include "str.h"
static inline size_t min(size_t a, size_t b) {
return a < b ? a : b;
@ -76,7 +77,6 @@ strbuf strbuf_puts(strbuf sb, const char *text)
strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data)
{
static char hexdigit[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
char *p = sb->current;
sb->current += strlen;
if (sb->start) {
@ -84,7 +84,7 @@ strbuf strbuf_tohex(strbuf sb, size_t strlen, const unsigned char *data)
// The following loop could overwrite the '\0' at *sp->end.
size_t i;
for (i = 0; i < strlen && p < e; ++i)
*p++ = (i & 1) ? hexdigit[*data++ & 0xf] : hexdigit[*data >> 4];
*p++ = (i & 1) ? hexdigit_upper[*data++ & 0xf] : hexdigit_upper[*data >> 4];
// This will restore the '\0' at *sp->end if it was overwritten.
*e = '\0';
}

161
strbuf_helpers.c
View File

@ -36,6 +36,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#include <sys/uio.h>
#include "http_server.h"
#include "strbuf_helpers.h"
#include "str.h"
#include "socket.h"
static inline strbuf _toprint(strbuf sb, char c)
{
@ -315,7 +317,7 @@ strbuf strbuf_append_socket_type(strbuf sb, int type)
strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr)
{
strbuf_sprintf(sb, " %u.%u.%u.%u",
strbuf_sprintf(sb, "%u.%u.%u.%u",
((unsigned char *) &addr->s_addr)[0],
((unsigned char *) &addr->s_addr)[1],
((unsigned char *) &addr->s_addr)[2],
@ -323,13 +325,21 @@ strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr)
return sb;
}
strbuf strbuf_append_sockaddr_in(strbuf sb, const struct sockaddr_in *addr)
{
assert(addr->sin_family == AF_INET);
strbuf_puts(sb, "AF_INET:");
strbuf_append_in_addr(sb, &addr->sin_addr);
strbuf_sprintf(sb, ":%u", ntohs(addr->sin_port));
return sb;
}
strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen)
{
strbuf_append_socket_domain(sb, addr->sa_family);
switch (addr->sa_family) {
case AF_UNIX: {
strbuf_puts(sb, "AF_UNIX:");
size_t len = addrlen > sizeof addr->sa_family ? addrlen - sizeof addr->sa_family : 0;
strbuf_putc(sb, ' ');
if (addr->sa_data[0]) {
strbuf_toprint_quoted_len(sb, "\"\"", addr->sa_data, len);
if (len < 2)
@ -346,24 +356,30 @@ strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t
break;
case AF_INET: {
const struct sockaddr_in *addr_in = (const struct sockaddr_in *) addr;
strbuf_putc(sb, ' ');
strbuf_append_in_addr(sb, &addr_in->sin_addr);
strbuf_sprintf(sb, ":%u", ntohs(addr_in->sin_port));
strbuf_append_sockaddr_in(sb, addr_in);
if (addrlen != sizeof(struct sockaddr_in))
strbuf_sprintf(sb, " (addrlen=%d should be %zd)", (int)addrlen, sizeof(struct sockaddr_in));
}
break;
default: {
strbuf_append_socket_domain(sb, addr->sa_family);
size_t len = addrlen > sizeof addr->sa_family ? addrlen - sizeof addr->sa_family : 0;
int i;
for (i = 0; i < len; ++i)
strbuf_sprintf(sb, " %02x", addr->sa_data[i]);
for (i = 0; i < len; ++i) {
strbuf_putc(sb, i ? ',' : ':');
strbuf_sprintf(sb, "%02x", addr->sa_data[i]);
}
}
break;
}
return sb;
}
strbuf strbuf_append_socket_address(strbuf sb, const struct socket_address *addr)
{
return strbuf_append_sockaddr(sb, &addr->addr, addr->addrlen);
}
strbuf strbuf_append_strftime(strbuf sb, const char *format, const struct tm *tm)
{
// First, try calling strftime(3) directly on the buffer in the strbuf, if there is one and it
@ -402,6 +418,69 @@ strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt)
return sb;
}
strbuf strbuf_append_quoted_string(strbuf sb, const char *str)
{
strbuf_putc(sb, '"');
for (; *str; ++str) {
if (*str == '"' || *str == '\\')
strbuf_putc(sb, '\\');
strbuf_putc(sb, *str);
}
strbuf_putc(sb, '"');
return sb;
}
strbuf strbuf_json_null(strbuf sb)
{
strbuf_puts(sb, "null");
return sb;
}
strbuf strbuf_json_string(strbuf sb, const char *str)
{
if (str) {
strbuf_putc(sb, '"');
for (; *str; ++str) {
if (*str == '"' || *str == '\\') {
strbuf_putc(sb, '\\');
strbuf_putc(sb, *str);
}
else if (*str == '\b')
strbuf_puts(sb, "\\b");
else if (*str == '\f')
strbuf_puts(sb, "\\f");
else if (*str == '\n')
strbuf_puts(sb, "\\n");
else if (*str == '\r')
strbuf_puts(sb, "\\r");
else if (*str == '\t')
strbuf_puts(sb, "\\t");
else if (iscntrl(*str))
strbuf_sprintf(sb, "\\u%04X", (unsigned char) *str);
else
strbuf_putc(sb, *str);
}
strbuf_putc(sb, '"');
} else
strbuf_json_null(sb);
return sb;
}
strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len)
{
if (buf) {
strbuf_putc(sb, '"');
size_t i;
for (i = 0; i != len; ++i) {
strbuf_putc(sb, hexdigit_upper[*buf >> 4]);
strbuf_putc(sb, hexdigit_upper[*buf++ & 0xf]);
}
strbuf_putc(sb, '"');
} else
strbuf_json_null(sb);
return sb;
}
strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels)
{
unsigned i;
@ -427,31 +506,47 @@ strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, uns
return sb;
}
strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd)
strbuf strbuf_append_mime_content_type(strbuf sb, const struct mime_content_type *ct)
{
strbuf_puts(sb, "type=");
strbuf_toprint_quoted(sb, "``", cd->type);
strbuf_puts(sb, " name=");
strbuf_toprint_quoted(sb, "``", cd->name);
strbuf_puts(sb, " filename=");
strbuf_toprint_quoted(sb, "``", cd->filename);
strbuf_puts(sb, " size=");
strbuf_sprintf(sb, "%"PRIhttp_size_t, cd->size);
struct tm tm;
strbuf_puts(sb, " creation_date=");
if (cd->creation_date)
strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->creation_date, &tm));
else
strbuf_puts(sb, "0");
strbuf_puts(sb, " modification_date=");
if (cd->modification_date)
strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->modification_date, &tm));
else
strbuf_puts(sb, "0");
strbuf_puts(sb, " read_date=");
if (cd->read_date)
strbuf_append_strftime(sb, "%a, %d %b %Y %T %z", gmtime_r(&cd->read_date, &tm));
else
strbuf_puts(sb, "0");
strbuf_puts(sb, ct->type);
strbuf_putc(sb, '/');
strbuf_puts(sb, ct->subtype);
if (ct->charset) {
strbuf_puts(sb, "; charset=");
strbuf_append_quoted_string(sb, ct->charset);
}
if (ct->multipart_boundary) {
strbuf_puts(sb, "; boundary=");
strbuf_append_quoted_string(sb, ct->multipart_boundary);
}
return sb;
}
strbuf strbuf_append_mime_content_disposition(strbuf sb, const struct mime_content_disposition *cd)
{
strbuf_puts(sb, cd->type);
if (cd->name) {
strbuf_puts(sb, "; name=");
strbuf_append_quoted_string(sb, cd->name);
}
if (cd->filename) {
strbuf_puts(sb, "; filename=");
strbuf_append_quoted_string(sb, cd->filename);
}
if (cd->size)
strbuf_sprintf(sb, "; size=%"PRIhttp_size_t, cd->size);
struct tm tm;
if (cd->creation_date) {
strbuf_puts(sb, " creation_date=");
strbuf_append_strftime(sb, "\"%a, %d %b %Y %T %z\"", gmtime_r(&cd->creation_date, &tm));
}
if (cd->modification_date) {
strbuf_puts(sb, " modification_date=");
strbuf_append_strftime(sb, "\"%a, %d %b %Y %T %z\"", gmtime_r(&cd->modification_date, &tm));
}
if (cd->read_date) {
strbuf_puts(sb, " read_date=");
strbuf_append_strftime(sb, "\"%a, %d %b %Y %T %z\"", gmtime_r(&cd->read_date, &tm));
}
return sb;
}

33
strbuf_helpers.h
View File

@ -128,10 +128,21 @@ strbuf strbuf_append_in_addr(strbuf sb, const struct in_addr *addr);
/* Append a textual description of a struct sockaddr_in.
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct sockaddr_in;
strbuf strbuf_append_sockaddr_in(strbuf sb, const struct sockaddr_in *addr);
#define alloca_sockaddr_in(addr) strbuf_str(strbuf_append_sockaddr_in(strbuf_alloca(45), (const struct sockaddr_in *)(addr)))
/* Append a textual description of a struct sockaddr.
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct sockaddr;
strbuf strbuf_append_sockaddr(strbuf sb, const struct sockaddr *addr, socklen_t addrlen);
#define alloca_sockaddr(addr, addrlen) strbuf_str(strbuf_append_sockaddr(strbuf_alloca(200), (const struct sockaddr *)(addr), (addrlen)))
struct socket_address;
strbuf strbuf_append_socket_address(strbuf sb, const struct socket_address *addr);
#define alloca_socket_address(addr) strbuf_str(strbuf_append_socket_address(strbuf_alloca(200), (addr)))
/* Append a strftime(3) string.
* @author Andrew Bettison <andrew@servalproject.com>
*/
@ -145,6 +156,19 @@ struct iovec;
strbuf strbuf_append_iovec(strbuf sb, const struct iovec *iov, int iovcnt);
#define alloca_iovec(iov,cnt) strbuf_str(strbuf_append_iovec(strbuf_alloca(200), (iov), (cnt)))
/* Append a string using HTTP quoted-string format: delimited by double quotes (") and
* internal double quotes and backslash escaped by leading backslash.
* @author Andrew Bettison <andrew@servalproject.com>
*/
strbuf strbuf_append_quoted_string(strbuf sb, const char *str);
/* Append various JSON elements.
* @author Andrew Bettison <andrew@servalproject.com>
*/
strbuf strbuf_json_null(strbuf sb);
strbuf strbuf_json_string(strbuf sb, const char *str);
strbuf strbuf_json_hex(strbuf sb, const unsigned char *buf, size_t len);
/* Append a representation of a struct http_range[] array.
* @author Andrew Bettison <andrew@servalproject.com>
*/
@ -152,7 +176,14 @@ struct http_range;
strbuf strbuf_append_http_ranges(strbuf sb, const struct http_range *ranges, unsigned nels);
#define alloca_http_ranges(ra) strbuf_str(strbuf_append_http_ranges(strbuf_alloca(25*NELS(ra)), (ra), NELS(ra)))
/* Append a representation of a struct mime_content_disposition struct.
/* Append a representation of a struct mime_content_type in HTTP header format.
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct mime_content_type;
strbuf strbuf_append_mime_content_type(strbuf, const struct mime_content_type *);
#define alloca_mime_content_type(ct) strbuf_str(strbuf_append_mime_content_type(strbuf_alloca(500), (ct)))
/* Append a representation of a struct mime_content_disposition, in HTTP header format.
* @author Andrew Bettison <andrew@servalproject.com>
*/
struct mime_content_disposition;

7
testdefs.sh
View File

@ -57,14 +57,17 @@ extract_stdout_keyvalue_optional() {
esac
local _label_re=$(escape_grep_basic "$_label")
local _delim_re=$(escape_grep_basic "$_delim")
local _line=$(replayStdout | $GREP "^$_label_re$_delim_re")
local _line=$($GREP "^$_label_re$_delim_re" "$TFWSTDOUT")
local _value=
local _return=1
if [ -n "$_line" ]; then
_value="${_line#*$_delim}"
_return=0
fi
[ -n "$_var" ] && eval $_var="\$_value"
if [ -n "$_var" ]; then
eval $_var="\$_value"
eval tfw_log "$_var=\$(shellarg "\${$_var}")"
fi
return $_return
}

81
testdefs_rhizome.sh
View File

@ -78,6 +78,63 @@ assert_rhizome_list() {
rhizome_list_file_count=$(( $(replayStdout | wc -l) - 2 ))
}
# Parse the standard output produced by the immediately preceding "rhizome list"
# command into the following shell variables:
# NCOLS the number of columns
# NROWS the number of data rows (not counting headers)
# HEADER[c] the C-th header label, 0 <= C <= NCOLS-1
# <label>[R] where <label> is a header label with all non-alphanumerics
# replaced by underscore '_' and all alphas converted to upper case, eg,
# .author -> _AUTHOR, is the value of that column in the R-th row, 0 <=
# R < NROWS
#
# Warning: overwrites existing shell variables. Names of overwritten shell
# variables are derived directly from the output of rhizome list, so cannot be
# controlled. If a prefix is supplied, all variables are prefixed with that.
rhizome_list_unpack() {
local prefix="$1"
{
local n
read n
eval ${prefix}NCOLS=\"\$n\"
declare -a ${prefix}HEADER
local -a header
local oIFS="$IFS"
IFS=:
read -r -a header
IFS="$oIFS"
eval ${prefix}HEADER="(\"\${header[@]}\")"
local hdr
local -a colvars=()
for hdr in "${header[@]}"; do
case "$hdr" in
id)
hdr=BID;;
*)
hdr="${hdr//[^A-Za-z0-9_]/_}"
# hdr="${hdr^^*}" would do in Bash-4.0 and later
hdr="$(echo "$hdr" | sed -e 's/.*/\U&/')"
;;
esac
colvars+=("$hdr")
done
local -a row
IFS=:
local i=0
while eval read -r -a row; do
local j=0
local val
for val in "${row[@]}"; do
eval ${prefix}${colvars[$j]}[$i]=\"\$val\"
let ++j
done
let ++i
done
IFS="$oIFS"
eval ${prefix}NROWS=$i
} < "$TFWSTDOUT"
}
rhizome_list_dump() {
local ncols
local -a headers
@ -245,10 +302,30 @@ extract_stdout_secret() {
extract_stdout_keyvalue "$1" .secret "$rexp_bundlesecret"
}
extract_stdout_rowid() {
extract_stdout_keyvalue "$1" .rowid "$rexp_rowid"
}
extract_stdout_inserttime() {
extract_stdout_keyvalue "$1" .inserttime "$rexp_date"
}
extract_stdout_BK() {
extract_stdout_keyvalue "$1" BK "$rexp_bundlekey"
}
extract_stdout_date() {
extract_stdout_keyvalue "$1" date "$rexp_date"
}
extract_stdout_filesize() {
extract_stdout_keyvalue "$1" filesize "$rexp_filesize"
}
extract_stdout_filehash() {
extract_stdout_keyvalue "$1" filehash "$rexp_filehash"
}
extract_manifest() {
local _var="$1"
local _manifestfile="$2"
@ -289,6 +366,10 @@ extract_manifest_version() {
extract_manifest "$1" "$2" version "$rexp_version"
}
extract_manifest_date() {
extract_manifest "$1" "$2" date "$rexp_date"
}
extract_manifest_crypt() {
extract_manifest "$1" "$2" crypt "$rexp_crypt"
}

1115
testframework.sh
View File

File diff suppressed because it is too large Load Diff

4
tests/dnaprotocol
View File

@ -47,8 +47,10 @@ set_server_vars() {
set log.console.show_time on \
set mdp.iftype.wifi.tick_ms 100 \
set rhizome.enable No \
set debug.overlayinterfaces Yes \
set debug.overlayinterfaces No \
set debug.overlaybuffer No \
set debug.packetformats No \
set debug.overlayframes No \
set debug.overlayrouting No \
set debug.packettx No \
set debug.packetrx No \

128
tests/keyring
View File

@ -29,6 +29,12 @@ setup() {
set_instance +A
}
teardown_servald() {
kill_all_servald_processes
assert_no_servald_processes
report_servald_server
}
setup_instances() {
for arg; do
set_instance $arg
@ -56,6 +62,46 @@ test_KeyringCreate() {
assert_keyring_list 0
}
doc_DidName="Create an identity & set the name and number"
test_DidName() {
executeOk_servald keyring add ''
assertStdoutGrep --matches=1 "^sid:"
assertStdoutLineCount '==' 1
extract_stdout_keyvalue SID sid "$rexp_sid"
executeOk_servald keyring set did "$SID" '123456' 'Display Name'
assertStdoutGrep --matches=1 "^sid:$SID\$"
assertStdoutGrep --matches=1 "^did:123456\$"
assertStdoutGrep --matches=1 "^name:Display Name\$"
assertStdoutLineCount '==' 3
executeOk_servald keyring list
assertStdoutGrep --stderr --matches=1 "^$SID:123456:Display Name\$"
}
keyring_set_tag() {
executeOk_servald keyring set tag "$1" "$2" "$3"
assertStdoutGrep --matches=1 "^sid:$1\$"
assertStdoutGrep --matches=1 "^$2:$3\$"
}
doc_SetTag="Set a named tag against an identity"
test_SetTag() {
executeOk_servald keyring add ''
assertStdoutGrep --matches=1 "^sid:"
assertStdoutLineCount '==' 1
extract_stdout_keyvalue SID sid "$rexp_sid"
keyring_set_tag "$SID" 'tag1' 'First Value'
assertStdoutLineCount '==' 2
keyring_set_tag "$SID" 'tag2' 'Second Value'
executeOk_servald keyring set tag "$SID" 'tag2' 'Second Value'
assertStdoutGrep --matches=1 "^tag1:First Value\$"
assertStdoutLineCount '==' 3
keyring_set_tag "$SID" 'tag1' 'Third Value'
assertStdoutGrep --matches=1 "^tag2:Second Value\$"
assertStdoutLineCount '==' 3
executeOk_servald keyring dump --secret
tfw_cat --stdout
}
doc_Pinless="No keyring PIN with PIN-less identities"
test_Pinless() {
executeOk_servald keyring add ''
@ -141,9 +187,7 @@ finally_KeyringAutoCreate() {
stop_servald_server
}
teardown_KeyringAutoCreate() {
kill_all_servald_processes
assert_no_servald_processes
report_servald_server
teardown_servald
}
doc_KeyringPinServer="Start daemon with a keyring PIN"
@ -161,9 +205,7 @@ finally_KeyringPinServer() {
stop_servald_server
}
teardown_KeyringPinServer() {
kill_all_servald_processes
assert_no_servald_processes
report_servald_server
teardown_servald
}
doc_EntryPinServer="Start daemon with an entry PIN"
@ -181,9 +223,7 @@ finally_KeyringPinServer() {
stop_servald_server
}
teardown_KeyringPinServer() {
kill_all_servald_processes
assert_no_servald_processes
report_servald_server
teardown_servald
}
doc_KeyringEntryPinServer="Start daemon with a keyring and an entry PIN"
@ -201,9 +241,7 @@ finally_KeyringKeyringPinServer() {
stop_servald_server
}
teardown_KeyringKeyringPinServer() {
kill_all_servald_processes
assert_no_servald_processes
report_servald_server
teardown_servald
}
doc_KeyringEntryPinServer="Start daemon, unlock and lock identities"
@ -224,34 +262,64 @@ test_KeyringEntryPinServer() {
assertStdoutLineCount == 1
assertStdoutGrep --fixed-strings "$SIDA"
executeOk_servald id enter pin 'one'
executeOk_servald id self
executeOk_servald id list
assertStdoutLineCount == 2
assertStdoutGrep --fixed-strings "$SIDA"
assertStdoutGrep --fixed-strings "$ONE"
assertStdoutGrep --fixed-strings "sid:$SIDA"
assertStdoutGrep --fixed-strings "sid:$ONE"
executeOk_servald id enter pin 'two'
executeOk_servald id self
executeOk_servald id list
assertStdoutLineCount == 4
assertStdoutGrep --fixed-strings "$SIDA"
assertStdoutGrep --fixed-strings "$ONE"
assertStdoutGrep --fixed-strings "$TWOA"
assertStdoutGrep --fixed-strings "$TWOB"
assertStdoutGrep --fixed-strings "sid:$SIDA"
assertStdoutGrep --fixed-strings "sid:$ONE"
assertStdoutGrep --fixed-strings "sid:$TWOA"
assertStdoutGrep --fixed-strings "sid:$TWOB"
executeOk_servald id relinquish pin 'one'
executeOk_servald id self
executeOk_servald id list
assertStdoutLineCount == 3
assertStdoutGrep --fixed-strings "$SIDA"
assertStdoutGrep --fixed-strings "$TWOA"
assertStdoutGrep --fixed-strings "$TWOB"
assertStdoutGrep --fixed-strings "sid:$SIDA"
assertStdoutGrep --fixed-strings "sid:$TWOA"
assertStdoutGrep --fixed-strings "sid:$TWOB"
executeOk_servald id relinquish sid "$TWOB"
tfw_cat --stderr
executeOk_servald id self
executeOk_servald id list
assertStdoutLineCount == 2
assertStdoutGrep --fixed-strings "$SIDA"
assertStdoutGrep --fixed-strings "$TWOA"
assertStdoutGrep --fixed-strings "sid:$SIDA"
assertStdoutGrep --fixed-strings "sid:$TWOA"
}
teardown_KeyringEntryPinServer() {
kill_all_servald_processes
assert_no_servald_processes
report_servald_server
teardown_servald
}
doc_ListTags="Search for unlocked identities by their tags & values"
setup_ListTags() {
setup
executeOk_servald keyring add
extract_stdout_keyvalue ONE sid "$rexp_sid"
keyring_set_tag "$ONE" 'tag1' 'First Value'
executeOk_servald keyring add
extract_stdout_keyvalue TWO sid "$rexp_sid"
keyring_set_tag "$TWO" 'tag1' 'Second Value'
executeOk_servald keyring add
extract_stdout_keyvalue THREE sid "$rexp_sid"
keyring_set_tag "$THREE" 'tag2' 'Third Value'
start_servald_server
}
test_ListTags() {
executeOk_servald id list
assertStdoutLineCount == 3
assertStdoutGrep --fixed-strings "sid:$ONE"
assertStdoutGrep --fixed-strings "sid:$TWO"
assertStdoutGrep --fixed-strings "sid:$THREE"
executeOk_servald id list 'tag1'
assertStdoutLineCount == 2
assertStdoutGrep --fixed-strings "sid:$ONE"
assertStdoutGrep --fixed-strings "sid:$TWO"
executeOk_servald id list 'tag1' 'First Value'
assertStdoutLineCount == 1
assertStdoutGrep --fixed-strings "sid:$ONE"
}
teardown_ListTags() {
teardown_servald
}
doc_Load="Load keyring entries from a keyring dump"

68
tests/meshms
View File

@ -28,15 +28,22 @@ teardown() {
assert_no_servald_processes
}
setup_logging() {
executeOk_servald config \
set debug.meshms on \
set debug.rhizome on \
set debug.rhizome_manifest on \
set debug.rejecteddata on \
set log.console.level debug \
set log.console.show_time on
}
doc_MessageDelivery="Send messages, ack and read them in a 2 party conversation"
setup_MessageDelivery() {
setup_servald
set_instance +A
create_identities 2
executeOk_servald config \
set debug.meshms on \
set debug.rhizome on \
set log.console.level debug
setup_logging
}
test_MessageDelivery() {
# 1. empty list
@ -60,8 +67,12 @@ test_MessageDelivery() {
assertStdoutGrep --stdout --matches=1 "^0:19:<:How are you\$"
assertStdoutGrep --stdout --matches=1 "^1:5:<:Hi\$"
assertStdoutLineCount '==' 4
CONV_BID=$(replayStderr | sed -n -e '/MESHMS CONVERSATION BUNDLE/s/.*bid=\([0-9A-F]*\).*/\1/p')
CONV_SECRET=$(replayStderr | sed -n -e '/MESHMS CONVERSATION BUNDLE/s/.*secret=\([0-9A-F]*\).*/\1/p')
tfw_log "CONV_BID=$CONV_BID CONV_SECRET=$CONV_SECRET"
# 5. mark the first message as read
executeOk_servald meshms read messages $SIDA2 $SIDA1 5
check_meshms_bundles
executeOk_servald meshms list messages $SIDA2 $SIDA1
assertStdoutGrep --stdout --matches=1 "^0:19:<:How are you\$"
assertStdoutGrep --stdout --matches=1 "^1:5:MARK:read\$"
@ -69,6 +80,7 @@ test_MessageDelivery() {
assertStdoutLineCount '==' 5
# 6. mark all messages as read
executeOk_servald meshms read messages $SIDA2
check_meshms_bundles
executeOk_servald meshms list messages $SIDA2 $SIDA1
assertStdoutGrep --stdout --matches=1 "^0:19:MARK:read\$"
assertStdoutGrep --stdout --matches=1 "^1:19:<:How are you\$"
@ -82,17 +94,24 @@ test_MessageDelivery() {
assertStdoutLineCount '==' 5
}
doc_MessageThreading="Messages sent at the same time, thread differently"
setup_MessageThreading() {
setup_servald
foreach_instance +A +B create_single_identity
set_instance +A
executeOk_servald meshms send message $SIDA $SIDB "Hello can you hear me"
executeOk_servald meshms send message $SIDA $SIDB "Still waiting"
set_instance +B
executeOk_servald meshms send message $SIDB $SIDA "Help Im trapped in a test case factory"
executeOk_servald meshms send message $SIDB $SIDA "Never mind"
start_servald_instances +A +B
check_meshms_bundles() {
# Dump the MeshMS bundles to the log and check consistency
# The only "file" bundle should be the conversation list
executeOk_servald rhizome list file
rhizome_list_unpack X
assert [ $XNROWS -eq 1 ]
assert [ ${XBID[0]} = $CONV_BID ]
executeOk_servald rhizome extract bundle $CONV_BID manifest.conv payload.conv $CONV_SECRET
tfw_cat -v manifest.conv --hexdump payload.conv
# The only "MeshMS2" bundles should be the two ply bundles
executeOk_servald rhizome list MeshMS2
rhizome_list_unpack X
assert [ $XNROWS -eq 2 ]
local bid
for bid in ${XBID[*]}; do
executeOk_servald rhizome extract bundle $bid manifest.$bid payload.$bid
tfw_cat -v manifest.$bid --hexdump payload.$bid
done
}
has_unread_messages() {
@ -109,6 +128,19 @@ messages_delivered() {
fi
}
doc_MessageThreading="Messages sent at the same time, thread differently"
setup_MessageThreading() {
setup_servald
foreach_instance +A +B create_single_identity
foreach_instance +A +B setup_logging
set_instance +A
executeOk_servald meshms send message $SIDA $SIDB "Hello can you hear me"
executeOk_servald meshms send message $SIDA $SIDB "Still waiting"
set_instance +B
executeOk_servald meshms send message $SIDB $SIDA "Help Im trapped in a test case factory"
executeOk_servald meshms send message $SIDB $SIDA "Never mind"
start_servald_instances +A +B
}
test_MessageThreading() {
set_instance +B
wait_until has_unread_messages $SIDB
@ -118,7 +150,6 @@ test_MessageThreading() {
assertStdoutGrep --stdout --matches=1 "^2:54:>:Never mind\$"
assertStdoutGrep --stdout --matches=1 "^3:41:>:Help Im trapped in a test case factory\$"
assertStdoutLineCount '==' 6
set_instance +A
wait_until has_unread_messages $SIDA
wait_until messages_delivered $SIDA $SIDB
@ -136,10 +167,7 @@ setup_listConversations() {
setup_servald
set_instance +A
create_identities 5
executeOk_servald config \
set debug.rhizome on \
set debug.meshms on \
set log.console.level debug
setup_logging
# create 3 threads, with all permutations of incoming and outgoing messages
executeOk_servald meshms send message $SIDA1 $SIDA2 "Message1"
executeOk_servald meshms send message $SIDA3 $SIDA1 "Message2"

222
tests/rhizomehttp
View File

@ -25,6 +25,7 @@ source "${0%/*}/../testdefs_rhizome.sh"
shopt -s extglob
setup() {
CR=' '
setup_curl 7
setup_jq 1.2
setup_servald
@ -41,8 +42,20 @@ setup() {
get_rhizome_server_port PORTA +A
}
finally() {
stop_all_servald_servers
}
teardown() {
kill_all_servald_processes
assert_no_servald_processes
report_all_servald_servers
}
set_rhizome_config() {
executeOk_servald config \
set debug.httpd on \
set debug.rhizome_httpd on \
set debug.rhizome on \
set debug.verbose on \
set log.console.level debug
@ -50,51 +63,234 @@ set_rhizome_config() {
doc_AuthBasicMissing="Basic Authentication credentials are required"
test_AuthBasicMissing() {
execute --exit-status=67 curl \
--silent --fail --show-error \
executeOk curl \
--silent --show-error --write-out '%{http_code}' \
--output http.output \
--dump-header http.headers \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
assertStdoutIs '401'
assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR$"
}
teardown_AuthBasicMissing() {
tfw_cat http.headers http.output
teardown
}
doc_AuthBasicWrong="Basic Authentication credentials must be correct"
test_AuthBasicWrong() {
execute --exit-status=67 curl \
--silent --fail --show-error \
executeOk curl \
--silent --show-error --write-out '%{http_code}' \
--output http.output \
--dump-header http.headers \
--basic --user fred:nurks \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
assertStdoutIs '401'
assertGrep http.headers "^WWW-Authenticate: Basic realm=\"Serval Rhizome\"$CR$"
executeOk curl \
--silent --fail --show-error \
--silent --fail --show-error --write-out '%{http_code}' \
--output http.output \
--dump-header http.headers \
--basic --user ron:weasley \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
assertStdoutIs '200'
}
teardown_AuthBasicWrong() {
tfw_cat http.headers http.output
teardown
}
doc_RhizomeList="Fetch full Rhizome bundle list in JSON format"
setup_RhizomeList() {
for n in 1 2 3 4; do
create_file file$n ${n}k
add_bundles() {
local n
for ((n = $1; n <= $2; ++n)); do
create_file file$n $((1000 + $n))
executeOk_servald rhizome add file $SIDA file$n file$n.manifest
extract_stdout_manifestid BID[$n]
extract_stdout_version VERSION[$n]
extract_stdout_filesize SIZE[$n]
extract_stdout_filehash HASH[$n]
extract_stdout_date DATE[$n]
extract_stdout_rowid ROWID[$n]
[ "${ROWID[$n]}" -gt "${ROWID_MAX:-0}" ] && ROWID_MAX=${ROWID[$n]}
done
}
transform_bundlelist_json() {
# The following jq(1) incantation transforms the JSON array in
# bundlelist.json from the following form (which is optimised for
# transmission size):
# {
# "header":[
# ".token", "_id", "service", "id", "version","date",".inserttime",
# ".author",".fromhere","filesize","filehash","sender","recipient",
# "name"
# ],
# "rows":[
# [ "xx", rowid1, "service1", bundleid1, version1, .... ],
# [ null, rowid2, "service2", bundleid2, version2, .... ],
# ...
# [ null, rowidN, "serviceN", bundleidN, versionN, .... ]
# ]
# }
#
# into an array of JSON objects:
# [
# {
# "__index": 0,
# ".token": "xx",
# "_id": rowid1,
# "service": service1,
# "id": bundleid1,
# "version": version1,
# ...
# },
# {
# "__index": 1,
# ".token": null,
# "_id": rowid2,
# "service": service2,
# "id": bundleid2,
# "version": version2,
# ...
# },
# ...
# {
# "__index": 2,
# ".token": null,
# "_id": rowidN,
# "service": serviceN,
# "id": bundleidN,
# "version": versionN,
# ...
# }
# ]
# which is much easier to test with jq(1) expressions.
jq '
[
.header as $header |
.rows as $rows |
$rows | keys | .[] as $index |
[ $rows[$index] as $d | $d | keys | .[] as $i | {key:$header[$i], value:$d[$i]} ] |
from_entries |
.["__index"] = $index
]
' "$1" >"$2"
}
assertJq() {
local json="$1"
local jqscript="$2"
assert --message="$jqscript" [ "$(jq "$jqscript" "$json")" = true ]
}
doc_RhizomeList="Fetch small Rhizome bundle list in JSON format"
setup_RhizomeList() {
setup
NBUNDLES=100
add_bundles 0 $((NBUNDLES-1))
assert [ "$ROWID_MAX" -ge "$NBUNDLES" ]
}
test_RhizomeList() {
executeOk curl \
--silent --fail --show-error \
--output http.output \
--output bundlelist.json \
--dump-header http.headers \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
tfw_cat http.headers bundlelist.json
tfw_preserve bundlelist.json
assert [ "$(jq '.rows | length' bundlelist.json)" = $NBUNDLES ]
transform_bundlelist_json bundlelist.json array_of_objects.json
tfw_preserve array_of_objects.json
for ((n = 0; n != NBUNDLES; ++n)); do
if [ "${ROWID[$n]}" -eq "$ROWID_MAX" ]; then
# The first row must contain a non-null token string.
token=',".token":"","__index":0,'
else
token=',".token":null,'
fi
assertJq array_of_objects.json \
"contains([
{ name:\"file$n\",
service:\"file\",
id:\"${BID[$n]}\",
version:${VERSION[$n]},
filesize:${SIZE[$n]},
filehash:\"${HASH[$n]}\",
date:${DATE[$n]},
_id:${ROWID[$n]},
\".fromhere\":1,
\".author\":\"$SIDA\"
$token
}
])"
done
}
doc_RhizomeListSince="Fetch Rhizome bundle list since token in JSON format"
test_RhizomeListSinceJSON() {
:
curl_newsince() {
curl \
--silent --fail --show-error \
--no-buffer \
--output "$1" \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/newsince/$token/bundlelist.json"
}
doc_RhizomeNewSince="Fetch Rhizome bundle list since token in JSON format"
setup_RhizomeNewSince() {
setup
executeOk_servald config set rhizome.api.restful.newsince_timeout 60s
executeOk_servald config set rhizome.api.restful.newsince_poll_ms 500
add_bundles 0 5
executeOk curl \
--silent --fail --show-error \
--output bundlelist.json \
--dump-header http.headers \
--basic --user harry:potter \
"http://$addr_localhost:$PORTA/restful/rhizome/bundlelist.json"
assert [ "$(jq '.rows | length' bundlelist.json)" = 6 ]
transform_bundlelist_json bundlelist.json array_of_objects.json
token=$(jq --raw-output '.[0][".token"]' array_of_objects.json)
assert [ -n "$token" ]
}
test_RhizomeNewSince() {
fork %curl1 curl_newsince newsince1.json
fork %curl2 curl_newsince newsince2.json
fork %curl3 curl_newsince newsince3.json
wait_until [ -e newsince1.json -a -e newsince2.json -a -e newsince3.json ]
add_bundles 6 10
wait_until --timeout=10 grep "${BID[10]}" newsince1.json
wait_until --timeout=10 grep "${BID[10]}" newsince2.json
wait_until --timeout=10 grep "${BID[10]}" newsince3.json
fork_terminate_all
fork_wait_all
for i in 1 2 3; do
if [ $(jq . newsince$i | wc -c) -eq 0 ]; then
echo ']}' >>newsince$i.json
assert [ $(jq . newsince$i.json | wc -c) -ne 0 ]
fi
transform_bundlelist_json newsince$i.json objects$i.json
tfw_preserve newsince$i.json objects$i.json
for ((n = 0; n <= 5; ++n)); do
assertJq objects$i.json "contains([{id:\"${BID[$n]}\"}]) | not"
done
for ((n = 6; n <= 10; ++n)); do
assertJq objects$i.json \
"contains([
{ name:\"file$n\",
service:\"file\",
id:\"${BID[$n]}\",
version:${VERSION[$n]},
filesize:${SIZE[$n]},
filehash:\"${HASH[$n]}\",
date:${DATE[$n]},
_id:${ROWID[$n]},
\".fromhere\":1,
\".author\":\"$SIDA\",
\".token\":\"\"
}
])"
done
done
}
doc_RhizomeManifest="Fetch Rhizome bundle manifest"

131
tests/rhizomeops
View File

@ -42,6 +42,8 @@ setup_rhizome() {
set_rhizome_config() {
executeOk_servald config \
set debug.rhizome on \
set debug.rhizome_manifest on \
set debug.rejecteddata on \
set debug.verbose on \
set log.console.level debug
}
@ -206,24 +208,29 @@ setup_ExtractManifestAfterAdd() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAdd() {
executeOk_servald rhizome export manifest $manifestid file1x.manifest
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 8
assertStdoutLineCount '==' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file1x.manifest ]
assert diff file1.manifest file1x.manifest
@ -235,24 +242,29 @@ setup_ExtractManifestFileAfterAdd() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestFileAfterAdd() {
executeOk_servald rhizome export bundle $manifestid file1x.manifest file1x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 8
assertStdoutLineCount '==' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file1x.manifest ]
assert diff file1.manifest file1x.manifest
@ -267,30 +279,37 @@ setup_ExtractManifestFileFromExtBlob() {
executeOk_servald config set rhizome.external_blobs 1
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid1
executeOk_servald config set rhizome.external_blobs 0
echo "Another test file" >file2
executeOk_servald rhizome add file $SIDB1 file2 file2.manifest
extract_stdout_rowid rowid2
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
extract_manifest_id manifestid1 file1.manifest
extract_manifest_version version1 file1.manifest
extract_manifest_filehash filehash1 file1.manifest
extract_manifest_date date1 file1.manifest
extract_manifest_id manifestid2 file2.manifest
extract_manifest_version version2 file2.manifest
extract_manifest_filehash filehash2 file2.manifest
extract_manifest_date date2 file2.manifest
}
test_ExtractManifestFileFromExtBlob() {
executeOk_servald rhizome export bundle $manifestid1 file1x.manifest file1x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 8
assertStdoutLineCount '==' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid1\$"
assertStdoutGrep --matches=1 "^version:$version1\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid1\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash1\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date1\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file1x.manifest ]
assert diff file1.manifest file1x.manifest
@ -298,15 +317,18 @@ test_ExtractManifestFileFromExtBlob() {
assert diff file1 file1x
executeOk_servald rhizome export bundle $manifestid2 file2x.manifest file2x
tfw_cat --stdout --stderr
assertStdoutLineCount '==' 8
assertStdoutLineCount '==' 11
local size=$(( $(cat file2 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid2\$"
assertStdoutGrep --matches=1 "^version:$version2\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid2\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash2\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date2\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
assert [ -e file2x.manifest ]
assert diff file2.manifest file2x.manifest
@ -336,25 +358,30 @@ setup_ExtractManifestToStdout() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
extract_stdout_rowid rowid
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestToStdout() {
executeOk_servald rhizome export manifest $manifestid -
assertStdoutLineCount '>=' 9
assertStdoutLineCount '>=' 11
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --line=..8 --matches=1 "^service:file$"
assertStdoutGrep --line=..8 --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --line=..8 --matches=1 "^version:$version\$"
assertStdoutGrep --line=..8 --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --line=..8 --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --line=..8 --matches=1 "^filesize:$size\$"
assertStdoutGrep --line=..8 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..8 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=9 --matches=1 "^manifest:"
replayStdout | $SED -n '9s/^manifest://p' >file1x.manifest
replayStdout | $SED -n '10,$p' >>file1x.manifest
assertStdoutGrep --line=..11 --matches=1 "^service:file$"
assertStdoutGrep --line=..11 --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --line=..11 --matches=1 "^version:$version\$"
assertStdoutGrep --line=..11 --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --line=..11 --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --line=..11 --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --line=..11 --matches=1 "^filesize:$size\$"
assertStdoutGrep --line=..11 --matches=1 "^date:$date\$"
assertStdoutGrep --line=..11 --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --line=..11 --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --line=..11 --matches=1 "^\.readonly:0\$"
assertStdoutGrep --line=12 --matches=1 "^manifest:"
replayStdout | $SED -n '12s/^manifest://p' >file1x.manifest
replayStdout | $SED -n '13,$p' >>file1x.manifest
cat file1.manifest >file1n.manifest
echo >>file1n.manifest
tfw_cat file1n.manifest file1x.manifest
@ -367,23 +394,27 @@ setup_ExtractManifestAfterAddNoAuthor() {
setup_rhizome
echo "A test file" >file1
executeOk_servald rhizome add file '' file1 file1.manifest
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractManifestAfterAddNoAuthor() {
executeOk_servald rhizome export manifest $manifestid file1x.manifest
assert diff file1.manifest file1x.manifest
assertStdoutLineCount '==' 7
assertStdoutLineCount '==' 9
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.readonly:1\$"
}
@ -423,25 +454,30 @@ setup_ExtractFileAfterAdd() {
echo "A test file" >file1
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
tfw_cat --stderr
extract_stdout_rowid rowid
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1
extract_manifest_id manifestid file1.manifest
extract_manifest_version version file1.manifest
extract_manifest_filehash filehash file1.manifest
extract_manifest_date date file1.manifest
}
test_ExtractFileAfterAdd() {
executeOk_servald rhizome extract file $manifestid file1x
tfw_cat --stderr
assert diff file1 file1x
local size=$(( $(cat file1 | wc -c) + 0 ))
assertStdoutLineCount '==' 8
assertStdoutLineCount '==' 11
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB1\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
}
@ -509,10 +545,9 @@ setup_AddDeDuplicate() {
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
}
test_AddDeDuplicate() {
# Add first file again - nothing should change in its manifests, and it
# should appear that the add command succeeded (with perhaps some grumbling
# on stderr).
execute --exit-status=2 $servald rhizome add file $SIDB1 file1 file1.manifestA
# Add first file again - should return a "duplicate" status code and nothing
# should change in its manifests.
execute --exit-status=2 --stderr $servald rhizome add file $SIDB1 file1 file1.manifestA
assert [ -s file1.manifestA ]
assert_stdout_add_file file1
extract_stdout_secret file1_dup_secret
@ -522,7 +557,7 @@ test_AddDeDuplicate() {
assert diff file1.manifest file1.manifestA
assert [ $file1_secret = $file1_dup_secret ]
# Repeat for second file.
execute --exit-status=2 $servald rhizome add file $SIDB1 file2 file2.manifestA
execute --exit-status=2 --stderr $servald rhizome add file $SIDB1 file2 file2.manifestA
assert [ -s file2.manifestA ]
assert_stdout_add_file file2
extract_stdout_secret file2_dup_secret
@ -643,7 +678,7 @@ test_AddUpdateNoAuthor() {
execute $servald rhizome add file $SIDB1 file1_2 file1_2.manifest
tfw_cat --stderr
assertExitStatus '!=' 0
# Rhizome store contents have old payload.
# Rhizome store contents have old payload, with the original author.
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1 file2
}
@ -656,9 +691,10 @@ test_AddUpdateNoAuthorWithSecret() {
tfw_cat -v file1_2.manifest
executeOk_servald rhizome add file $SIDB1 file1_2 file1_2.manifest "$file1_secret"
tfw_cat --stderr
# Rhizome store contents have new payload.
# Rhizome store contents have new payload, but it has lost its author (no BK
# field any more).
executeOk_servald rhizome list
assert_rhizome_list --fromhere=1 --author=$SIDB1 file1_2 file2
assert_rhizome_list --fromhere=0 file1_2 --fromhere=1 --author=$SIDB1 file2
}
doc_AddUpdateAutoVersion="Add new payload to existing manifest with automatic version"
@ -677,18 +713,31 @@ test_AddUpdateAutoVersion() {
assert_rhizome_list --fromhere=1 file1_2 file2
}
doc_AddUnsupportedService="Add with unsupported service fails"
setup_AddUnsupportedService() {
doc_AddServiceInvalid="Add with invalid service fails"
setup_AddServiceInvalid() {
setup_servald
setup_rhizome
echo "Message1" >file1
echo -e 'service=Fubar' >file1.manifest
echo 'service=Fubar!' >file1.manifest
}
test_AddUnsupportedService() {
test_AddServiceInvalid() {
execute $servald rhizome add file $SIDB1 file1 file1.manifest
tfw_cat --stdout --stderr
assertExitStatus '!=' 0
}
doc_AddServiceUnsupported="Add with unsupported service succeeds"
setup_AddServiceUnsupported() {
setup_servald
setup_rhizome
echo "Message1" >file1
echo 'service=Fubar' >file1.manifest
}
test_AddServiceUnsupported() {
executeOk_servald rhizome add file $SIDB1 file1 file1.manifest
tfw_cat --stdout --stderr
}
doc_EncryptedPayload="Add and extract an encrypted payload"
setup_EncryptedPayload() {
setup_servald
@ -732,6 +781,7 @@ test_RecipientIsEncrypted() {
assert diff file1 file1x
extract_manifest_filehash filehash file1.manifest
executeOk_servald rhizome export file $filehash file1y
tfw_cat file1 -v file1y
assert ! diff file1 file1y
}
@ -1002,9 +1052,11 @@ setup_ImportOwnBundle() {
echo "Hello from B" >fileB
executeOk_servald rhizome add file $SIDB2 fileB fileB.manifest
assert_stdout_add_file fileB
extract_stdout_rowid rowid
extract_manifest_id manifestid fileB.manifest
extract_manifest_version version fileB.manifest
extract_manifest_filehash filehash fileB.manifest
extract_manifest_date date fileB.manifest
rm -f $SERVALINSTANCE_PATH/rhizome.db
executeOk_servald rhizome list
assert_rhizome_list
@ -1020,15 +1072,18 @@ test_ImportOwnBundle() {
tfw_cat --stderr
assert cmp fileB.manifest fileBx.manifest
assert cmp fileB fileBx
assertStdoutLineCount '==' 8
assertStdoutLineCount '==' 11
local size=$(( $(cat fileB | wc -c) + 0 ))
assertStdoutGrep --matches=1 "^service:file$"
assertStdoutGrep --matches=1 "^manifestid:$manifestid\$"
assertStdoutGrep --matches=1 "^version:$version\$"
assertStdoutGrep --matches=1 "^inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^\.rowid:$rowid\$"
assertStdoutGrep --matches=1 "^\.inserttime:$rexp_date\$"
assertStdoutGrep --matches=1 "^filehash:$filehash\$"
assertStdoutGrep --matches=1 "^filesize:$size\$"
assertStdoutGrep --matches=1 "^date:$date\$"
assertStdoutGrep --matches=1 "^\.author:$SIDB2\$"
assertStdoutGrep --matches=1 "^\.secret:$rexp_bundlesecret\$"
assertStdoutGrep --matches=1 "^\.readonly:0\$"
# Now bundle author should be known, so appears to be from here
executeOk_servald rhizome list

144
tests/rhizomeprotocol
View File

@ -43,6 +43,8 @@ configure_servald_server() {
set debug.rhizome on \
set debug.httpd on \
set debug.rhizome_httpd on \
set debug.rhizome_manifest on \
set debug.rhizome_ads on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set server.respawn_on_crash off \
@ -57,7 +59,7 @@ setup_common() {
}
receive_and_update_bundle() {
wait_until bundle_received_by $BID:$VERSION +B
wait_until "$@" bundle_received_by $BID:$VERSION +B
set_instance +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file1
@ -65,7 +67,7 @@ receive_and_update_bundle() {
set_instance +A
rhizome_update_file file1 file2
set_instance +B
wait_until bundle_received_by $BID:$VERSION +B
wait_until "$@" bundle_received_by $BID:$VERSION +B
executeOk_servald rhizome list
assert_rhizome_list --fromhere=0 file2
assert_rhizome_received file2
@ -75,7 +77,7 @@ doc_FileTransfer="New bundle and update transfer to one node"
setup_FileTransfer() {
setup_common
set_instance +A
rhizome_add_file file1 2048
rhizome_add_file file1 250000
start_servald_instances +A +B
foreach_instance +A assert_peers_are_instances +B
foreach_instance +B assert_peers_are_instances +A
@ -209,59 +211,6 @@ test_UnicastTransfer() {
receive_and_update_bundle
}
doc_SimulatedRadio="MDP Transfer over simulated radio link"
interface_up() {
$GREP "Interface .* is up" $instance_servald_log || return 1
return 0
}
start_radio_instance() {
executeOk_servald config \
set debug.rhizome on \
set debug.rhizome_ads on \
set debug.rhizome_httpd on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set debug.throttling on \
set debug.mavlink on \
set rhizome.advertise.interval 5000 \
set rhizome.rhizome_mdp_block_size 350 \
set log.console.level debug \
set log.console.show_pid on \
set log.console.show_time on \
set interfaces.1.type CATEAR \
set interfaces.1.mdp.tick_ms 5000 \
set interfaces.1.socket_type STREAM \
set interfaces.1.encapsulation SINGLE \
set interfaces.1.point_to_point on
start_servald_server
wait_until interface_up
}
setup_SimulatedRadio() {
setup_common
$servald_build_root/fakeradio 6 10000000 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
FAKERADIO_PID=$!
sleep 1
local END1=`head "$SERVALD_VAR/radioout" -n 1`
local END2=`tail "$SERVALD_VAR/radioout" -n 1`
tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
set_instance +A
rhizome_add_file file1 10000
executeOk_servald config \
set interfaces.1.file "$END1"
set_instance +B
executeOk_servald config \
set interfaces.1.file "$END2"
foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio() {
receive_and_update_bundle
}
teardown_SimulatedRadio() {
teardown
tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
kill $FAKERADIO_PID
tfw_cat "$SERVALD_VAR/radioerr"
}
doc_journalMDP="Transfer and update a journal bundle via MDP"
setup_journalMDP() {
@ -744,4 +693,87 @@ test_DirectSync() {
assert_rhizome_received fileA3
}
interface_up() {
$GREP "Interface .* is up" $instance_servald_log || return 1
return 0
}
start_radio_instance() {
executeOk_servald config \
set debug.rhizome on \
set debug.rhizome_ads on \
set debug.rhizome_tx on \
set debug.rhizome_rx on \
set debug.throttling on \
set debug.radio_link on \
set rhizome.advertise.interval 5000 \
set rhizome.rhizome_mdp_block_size 375 \
set log.console.level debug \
set log.console.show_pid on \
set log.console.show_time on \
set interfaces.1.type CATEAR \
set interfaces.1.mdp.tick_ms 5000 \
set interfaces.1.socket_type STREAM \
set interfaces.1.encapsulation SINGLE \
set interfaces.1.point_to_point on
start_servald_server
wait_until interface_up
}
start_fakeradio() {
$servald_build_root/fakeradio $1 $2 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
FAKERADIO_PID=$!
wait_until $GREP "^right:" "$SERVALD_VAR/radioout"
local _line=`head "$SERVALD_VAR/radioout" -n 1`
END1="${_line#*:}"
_line=`tail "$SERVALD_VAR/radioout" -n 1`
END2="${_line#*:}"
tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
}
doc_SimulatedRadio="MDP Transfer over simulated radio link (~90% packet arrival)"
setup_SimulatedRadio() {
setup_common
start_fakeradio 4 0.9
set_instance +A
rhizome_add_file file1 5000
executeOk_servald config \
set interfaces.1.file "$END1"
set_instance +B
executeOk_servald config \
set interfaces.1.file "$END2"
foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio() {
receive_and_update_bundle
}
teardown_SimulatedRadio() {
teardown
tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
kill $FAKERADIO_PID
tfw_cat "$SERVALD_VAR/radioerr"
}
doc_SimulatedRadio2="MDP Transfer over simulated radio link (~50% packet arrival)"
setup_SimulatedRadio2() {
setup_common
start_fakeradio 4 0.5
set_instance +A
rhizome_add_file file1 5000
executeOk_servald config \
set interfaces.1.file "$END1"
set_instance +B
executeOk_servald config \
set interfaces.1.file "$END2"
foreach_instance +A +B start_radio_instance
}
test_SimulatedRadio2() {
receive_and_update_bundle --timeout=120
}
teardown_SimulatedRadio2() {
teardown
tfw_log "Killing fakeradio, pid=$FAKERADIO_PID"
kill $FAKERADIO_PID
tfw_cat "$SERVALD_VAR/radioerr"
}
runTests "$@"

10
tests/rhizomestress
View File

@ -173,14 +173,14 @@ setup_StressRhizomeDirect() {
}
test_StressRhizomeDirect() {
set_instance +B
fork executeOk_servald rhizome direct sync
fork executeOk_servald --timeout=600 rhizome direct sync
set_instance +C
fork executeOk_servald rhizome direct pull
fork executeOk_servald --timeout=600 rhizome direct pull
set_instance +D
fork executeOk_servald rhizome direct push
fork executeOk_servald --timeout=600 rhizome direct push
set_instance +E
fork executeOk_servald rhizome direct sync
forkWaitAll
fork executeOk_servald --timeout=600 rhizome direct sync
fork_wait_all
set_instance +A
executeOk_servald rhizome list ''
tfw_quietly assert_rhizome_list --fromhere=1 file-A-!(*.manifest) --fromhere=0 file-[BDE]-!(*.manifest)

25
tests/routing
View File

@ -61,11 +61,11 @@ has_link() {
path_exists() {
local dest
for dest; do tru; done;
eval dest=\$$#
local dest_sidvar=SID${dest#+}
local next_inst=$1
shift
local I N
local I
for I; do
local sidvar=SID${I#+}
[ -n "${!sidvar}" ] || break
@ -81,6 +81,7 @@ start_routing_instance() {
set debug.mdprequests yes \
set debug.linkstate yes \
set debug.verbose yes \
set debug.subscriber yes \
set debug.overlayrouting yes \
set log.console.level debug \
set log.console.show_pid on \
@ -259,18 +260,23 @@ setup_slip_encoding() {
test_slip_encoding() {
executeOk_servald test slip --seed=1 --iterations=2000
}
start_fakeradio() {
$servald_build_root/fakeradio 4 1 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
FAKERADIO_PID=$!
wait_until $GREP "^right:" "$SERVALD_VAR/radioout"
local _line=`head "$SERVALD_VAR/radioout" -n 1`
END1="${_line#*:}"
_line=`tail "$SERVALD_VAR/radioout" -n 1`
END2="${_line#*:}"
tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
}
doc_simulate_extender="Simulate a mesh extender radio link"
setup_simulate_extender() {
setup_servald
assert_no_servald_processes
foreach_instance +A +B create_single_identity
$servald_build_root/fakeradio 1 20000000 > "$SERVALD_VAR/radioout" 2> "$SERVALD_VAR/radioerr" &
FAKERADIO_PID=$!
sleep 1
local END1=`head "$SERVALD_VAR/radioout" -n 1`
local END2=`tail "$SERVALD_VAR/radioout" -n 1`
tfw_log "Started fakeradio pid=$FAKERADIO_PID, end1=$END1, end2=$END2"
start_fakeradio
set_instance +A
executeOk_servald config \
set interfaces.1.file "$END1"
@ -281,10 +287,9 @@ setup_simulate_extender() {
executeOk_servald config \
set debug.throttling on \
set debug.packetradio on \
set debug.mavlink on \
set debug.radio_link on \
set interfaces.1.type CATEAR \
set interfaces.1.mdp.tick_ms 5000 \
set interfaces.1.mdp.packet_interval 5000 \
set interfaces.1.socket_type STREAM \
set interfaces.1.encapsulation SINGLE \
set interfaces.1.point_to_point on

41
timeit.c Normal file
View File

@ -0,0 +1,41 @@
/*******************************************************************************
*
* The BYTE UNIX Benchmarks - Release 3
* Module: timeit.c SID: 3.3 5/15/91 19:30:21
*******************************************************************************
* Bug reports, patches, comments, suggestions should be sent to:
*
* Ben Smith, Rick Grehan or Tom Yager
* ben@bytepb.byte.com rick_g@bytepb.byte.com tyager@bytepb.byte.com
*
*******************************************************************************
* Modification Log:
* May 12, 1989 - modified empty loops to avoid nullifying by optimizing
* compilers
* August 28, 1990 - changed timing relationship--now returns total number
* of iterations (ty)
* October 22, 1997 - code cleanup to remove ANSI C compiler warnings
* Andy Kahn <kahn@zk3.dec.com>
*
******************************************************************************/
/* this module is #included in other modules--no separate SCCS ID */
/*
* Timing routine
*
*/
#include <signal.h>
#include <unistd.h>
void wake_me(seconds, func)
int seconds;
void (*func)();
{
/* set up the signal handler */
signal(SIGALRM, func);
/* get the clock running */
alarm(seconds);
}

115
uuid.c Normal file
View File

@ -0,0 +1,115 @@
/*
Serval DNA Universally Unique Identifier support
Copyright (C) 2013 Serval Project Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#define __SERVALDNA_UUID_H_INLINE
#include "uuid.h"
#include "os.h"
#include "str.h"
#include <assert.h>
#ifdef HAVE_ARPA_INET_H
# include <arpa/inet.h>
#endif
enum uuid_version uuid_get_version(const uuid_t *uuid)
{
assert(uuid_is_valid(uuid));
switch (ntohs(uuid->u.record.time_hi_and_version) & 0xf000) {
case 0x1000: return UUID_VERSION_TIME_BASED;
case 0x2000: return UUID_VERSION_DCE_SECURITY;
case 0x3000: return UUID_VERSION_NAME_MD5;
case 0x4000: return UUID_VERSION_RANDOM;
case 0x5000: return UUID_VERSION_NAME_SHA1;
}
return UUID_VERSION_UNSUPPORTED;
}
void uuid_set_version(uuid_t *uuid, enum uuid_version version)
{
uint16_t version_bits;
switch (version) {
case UUID_VERSION_TIME_BASED: version_bits = 0x1000; break;
case UUID_VERSION_DCE_SECURITY: version_bits = 0x2000; break;
case UUID_VERSION_NAME_MD5: version_bits = 0x3000; break;
case UUID_VERSION_RANDOM: version_bits = 0x4000; break;
case UUID_VERSION_NAME_SHA1: version_bits = 0x5000; break;
default: abort();
}
assert(uuid_is_valid(uuid));
uuid->u.record.time_hi_and_version = htons((ntohs(uuid->u.record.time_hi_and_version) & 0xfff) | version_bits);
}
int uuid_generate_random(uuid_t *uuid)
{
if (urandombytes(uuid->u.binary, sizeof uuid->u.binary) == -1)
return -1;
// The following discards 6 random bits.
uuid->u.record.clock_seq_hi_and_reserved &= 0x3f;
uuid->u.record.clock_seq_hi_and_reserved |= 0x80;
uuid_set_version(uuid, UUID_VERSION_RANDOM);
return 0;
}
strbuf strbuf_uuid(strbuf sb, const uuid_t *uuid)
{
assert(uuid_is_valid(uuid));
unsigned i;
for (i = 0; i != sizeof uuid->u.binary; ++i) {
switch (i) {
case 4: case 6: case 8: case 10:
strbuf_putc(sb, '-');
default:
strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] >> 4]);
strbuf_putc(sb, hexdigit_lower[uuid->u.binary[i] & 0xf]);
}
}
return sb;
}
char *uuid_to_str(const uuid_t *uuid, char *const dst)
{
strbuf b = strbuf_local(dst, UUID_STRLEN + 1);
strbuf_uuid(b, uuid);
assert(!strbuf_overrun(b));
return dst;
}
int str_to_uuid(const char *const str, uuid_t *uuid, const char **afterp)
{
const char *end = str;
int ret = 0;
if ( strn_fromhex(uuid->u.binary, 4, end, &end) == 4
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 4, 2, end + 1, &end) == 2
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 6, 2, end + 1, &end) == 2
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 8, 2, end + 1, &end) == 2
&& *end == '-'
&& strn_fromhex(uuid->u.binary + 10, 6, end + 1, &end) == 6
) {
assert(end == str + UUID_STRLEN);
ret = uuid_is_valid(uuid);
}
if (afterp)
*afterp = end;
if (ret == 0 || (!afterp && *end))
return 0;
return 1;
}

128
uuid.h Normal file
View File

@ -0,0 +1,128 @@
/*
Serval DNA Universally Unique Identifier support
Copyright (C) 2013 Serval Project Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef __SERVALDNA_UUID_H
#define __SERVALDNA_UUID_H
#include <stdint.h>
#include <alloca.h>
#include <string.h>
#include "strbuf.h"
#ifndef __SERVALDNA_UUID_H_INLINE
# if __GNUC__ && !__GNUC_STDC_INLINE__
# define __SERVALDNA_UUID_H_INLINE extern inline
# else
# define __SERVALDNA_UUID_H_INLINE inline
# endif
#endif
/* A UUID is defined by RFC-4122 as a 128-bit identifier with the two most
* significant bits of the ninth byte being 1 and 0, which indicates the
* "variant" that is described by the RFC. Other variants exist, but are not
* supported here, and are treated as INVALID by the functions defined below.
* Any attempt to pass an invalid UUID to a function that requires a valid UUID
* as input will probably result in the calling process being aborted (see
* SIGABRT, abort(3)).
*
* In a valid UUID, the four lowest significant bits of the seventh byte define
* the "version" of the UUID, which essentially indicates how it was generated.
* The RFC defines five SUPPORTED versions. Any other version is UNSUPPORTED.
*
* The fields in the UUID 'record' structure are stored in network byte order,
* so code wishing to make use of the record structure must use ntohl(3) and
* ntohs(3) to read values, and htonl(3) and htons(3) to assign values.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
typedef struct uuid {
union {
struct {
uint32_t time_low;
uint16_t time_mid;
uint16_t time_hi_and_version;
uint8_t clock_seq_hi_and_reserved;
uint8_t clock_seq_low;
unsigned char node[6]; // uint48_t
} record;
unsigned char binary[16];
} u;
} uuid_t;
enum uuid_version {
UUID_VERSION_UNSUPPORTED = 0,
UUID_VERSION_TIME_BASED = 1,
UUID_VERSION_DCE_SECURITY = 2,
UUID_VERSION_NAME_MD5 = 3,
UUID_VERSION_RANDOM = 4,
UUID_VERSION_NAME_SHA1 = 5
};
__SERVALDNA_UUID_H_INLINE int cmp_uuid_t(const uuid_t *a, const uuid_t *b) {
return memcmp(a->u.binary, b->u.binary, sizeof a->u.binary);
}
__SERVALDNA_UUID_H_INLINE int uuid_is_valid(const uuid_t *any_uuid) {
return (any_uuid->u.record.clock_seq_hi_and_reserved & 0xc0) == 0x80;
}
enum uuid_version uuid_get_version(const uuid_t *valid_uuid);
void uuid_set_version(uuid_t *valid_uuid, enum uuid_version);
/* Returns -1 if error (eg, cannot open /dev/urandom), 0 if successful.
*/
int uuid_generate_random(uuid_t *dest_uuid);
/* Formats the given valid UUID in its canonical string representation:
* XXXXXXXX-VXXX-MXXX-XXXXXXXXXXXX
* where X is any hex digit
* V is 1, 2, 3, 4 or 5 (supported versions) or any other hex digit
* (unsupported versions)
* M is 8, 9, A or B (high two bits are variant 01)
*
* The 'dst' argument must point to a buffer of 37 bytes. The first 36 bytes
* are filled with the representation shown above, and the 37th byte dst[36] is
* set to NUL '\0'.
*
* Returns the 'dst' argument.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
char *uuid_to_str(const uuid_t *valid_uuid, char *dst);
#define UUID_STRLEN 36
#define alloca_uuid_str(uuid) uuid_to_str(&(uuid), alloca(UUID_STRLEN + 1))
/* Append a UUID to the given strbuf, formatted as per the uuid_to_str() function.
*/
strbuf strbuf_uuid(strbuf, const uuid_t *valid_uuid);
/* Parse a canonical UUID string (as generated by uuid_to_str()) into a valid
* UUID, which may or not be supported.
*
* Returns 1 if a valid UUID is parsed, storing the value in *result (unless result is NULL) and
* storing a pointer to the immediately succeeding character in *afterp. If afterp is NULL then
* returns 0 unless the immediately succeeding character is a NUL '\0'. If no UUID is parsed or
* if the UUID is not valid, then returns 0, leaving *result unchanged and
* setting setting *afterp to point to the character where parsing failed.
*
* @author Andrew Bettison <andrew@servalproject.com>
*/
int str_to_uuid(const char *str, uuid_t *result, const char **afterp);
#endif //__SERVALDNA_OS_H

16
version_string.sh
View File

@ -17,11 +17,6 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
if [ ! -e .git ]; then
echo "UNKNOWN-VERSION"
exit
fi
usage() {
echo "Usage: ${0##*/} [options]"'
@ -86,6 +81,11 @@ done
cd "$repo_path" >/dev/null
if [ ! -d .git ]; then
echo "UNKNOWN-VERSION"
exit 0
fi
get_author_label() {
# See git-commit-tree(1) for the semantics of working out the author's email
# address when committing.
@ -115,7 +115,11 @@ if [ -n "$dirty" ] && ! $allow_modified; then
fi
# Use the "git describe" command to form the version string and append $dirty.
if error="$( (desc="$(git describe --match="$version_tag_glob")" && echo "$desc$dirty") 2>&1 1>&5)" 5>&1; then
# This ugly construction is required for use on machines with bash version < 4.
error="$(git describe --match="$version_tag_glob" 2>&1 1>/dev/null)" || true
if [ -z "$error" ]; then
echo "$(git describe --match="$version_tag_glob")$dirty"
exit 0
fi

4
vomp.c
View File

@ -523,7 +523,7 @@ static int vomp_send_status_remote(struct vomp_call_state *call)
call->local.sequence++;
overlay_mdp_dispatch(&mdp,0,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
return 0;
}
@ -561,7 +561,7 @@ int vomp_received_audio(struct vomp_call_state *call, int audio_codec, int time,
mdp.out.queue=OQ_ISOCHRONOUS_VOICE;
overlay_mdp_dispatch(&mdp,0,NULL,0);
overlay_mdp_dispatch(&mdp, NULL);
return 0;
}