From 8d30acedc8cb30d4d01fc2e96266ad537162b943 Mon Sep 17 00:00:00 2001 From: Daniel O'Connor Date: Fri, 15 Jun 2012 15:01:14 +0930 Subject: [PATCH 01/37] Don't build libservald.so as it doesn't compile on 64 bit elf platforms and isn't used (it is only needed for JNI stuff on a phone) While I'm here split out CFLAGS generation to make compiling without optimisation easier (for debugging) --- Makefile.in | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile.in b/Makefile.in index 4ff63b64..b60e5c11 100644 --- a/Makefile.in +++ b/Makefile.in @@ -68,11 +68,15 @@ HDRS= fifo.h \ sqlite-amalgamation-3070900/sqlite3.h LDFLAGS=@LDFLAGS@ @PORTAUDIO_LIBS@ @SRC_LIBS@ @SPANDSP_LIBS@ @CODEC2_LIBS@ @PTHREAD_LIBS@ -CFLAGS= @CPPFLAGS@ @CFLAGS@ @PORTAUDIO_CFLAGS@ @SRC_CFLAGS@ @SPANDSP_CFLAGS@ @PTHREAD_CFLAGS@ $(VOIPTEST_CFLAGS) -Wall -Wno-unused-value + +CFLAGS= @CPPFLAGS@ @CFLAGS@ @PORTAUDIO_CFLAGS@ @SRC_CFLAGS@ @SPANDSP_CFLAGS@ @PTHREAD_CFLAGS@ $(VOIPTEST_CFLAGS) +CFLAGS+=-Wall -Wno-unused-value +#CFLAGS+=-Wunreachable-code +#CFLAGS+=-O0 + DEFS= @DEFS@ - -all: dna libservald.so +all: dna sqlite3.o: sqlite3.c @echo CC $< @@ -86,6 +90,8 @@ dna: $(OBJS) @echo LINK $@ @$(CC) $(CFLAGS) -Wall -o $@ $(OBJS) $(LDFLAGS) +# This does not build on 64 bit elf platforms as NaCL isn't built with -fPIC +# DOC 20120615 libservald.so: $(OBJS) @echo LINK $@ @$(CC) $(CFLAGS) -Wall -shared -o $@ $(OBJS) $(LDFLAGS) From 313a6fbadc4574f77d988f3d33ab5ae81b4b255c Mon Sep 17 00:00:00 2001 From: Daniel O'Connor Date: Fri, 15 Jun 2012 15:03:08 +0930 Subject: [PATCH 02/37] Don't use hard coded port number. --- peers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peers.c b/peers.c index ef20d717..2fbadcfe 100644 --- a/peers.c +++ b/peers.c @@ -192,7 +192,7 @@ int sendToPeers(unsigned char *packet,int packet_len,int method,int peerId,struc bzero(&peer_addr, sizeof(peer_addr)); peer_addr.sin_family=AF_INET; - peer_addr.sin_port = htons(4110); + peer_addr.sin_port = htons(PORT_DNA); if (method==REQ_PARALLEL) i=0; else { i=peerId; maxPeer=i; } for(;i<=maxPeer;i++) From d0e0864b0c11ee77ae4b2de58047aecfdae3051d Mon Sep 17 00:00:00 2001 From: Daniel O'Connor Date: Fri, 15 Jun 2012 15:03:52 +0930 Subject: [PATCH 03/37] Fix comment to be correct. --- overlay.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/overlay.c b/overlay.c index 11c4ead2..b7fe47c4 100644 --- a/overlay.c +++ b/overlay.c @@ -136,8 +136,7 @@ int overlayServerMode() for(i=0;i Date: Fri, 15 Jun 2012 15:04:36 +0930 Subject: [PATCH 04/37] Improve debug output (translate IPs & ports into human readable things). Revert change to use src_addr as this will break other platforms and needs more thought. Reformat overlay_interface_discover. --- overlay_interface.c | 111 +++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 54 deletions(-) diff --git a/overlay_interface.c b/overlay_interface.c index 043b96b2..764c7d78 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -17,6 +17,7 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ +#include #include #include "serval.h" @@ -180,17 +181,20 @@ int overlay_interface_args(const char *arg) int overlay_interface_init_socket(int interface, struct sockaddr_in src_addr, struct sockaddr_in broadcast) { + char srctxt[INET_ADDRSTRLEN]; + #define I(X) overlay_interfaces[interface].X I(broadcast_address) = broadcast; + I(src_address) = src_addr; I(fileP) = 0; I(fd) = socket(PF_INET,SOCK_DGRAM,0); - if (I(fd)<0) { + if (I(fd) < 0) { WHY_perror("socket()"); WHYF("Could not create UDP socket for interface: %s",strerror(errno)); goto error; } else - WHYF("interface #%d fd=%d",interface,I(fd)); + INFOF("interface #%d fd=%d",interface, I(fd)); int reuseP = 1; if (setsockopt(I(fd), SOL_SOCKET, SO_REUSEADDR, &reuseP, sizeof(reuseP)) < 0) { @@ -216,16 +220,13 @@ overlay_interface_init_socket(int interface, struct sockaddr_in src_addr, struct broadcast.sin_family = AF_INET; broadcast.sin_port = htons(I(port)); - /* XXX Is this right? Are we really setting the local side address? - I was in a plane when at the time, so couldn't Google it. - */ - if (debug&DEBUG_PACKETRX) fprintf(stderr,"src_addr=%08x\n",(unsigned int)src_addr.sin_addr.s_addr); - if (bind(I(fd),(struct sockaddr *)&src_addr,sizeof(src_addr))) { + if (bind(I(fd), (struct sockaddr *)&broadcast, sizeof(broadcast))) { WHY_perror("bind"); WHY("MP HLR server could not bind to requested UDP port (bind() failed)"); goto error; } - if (debug&(DEBUG_PACKETRX|DEBUG_IO)) fprintf(stderr,"Bound to port 0x%04x\n",src_addr.sin_port); + assert(inet_ntop(AF_INET, (const void *)&broadcast.sin_addr, srctxt, INET_ADDRSTRLEN) != NULL); + if (debug & (DEBUG_PACKETRX | DEBUG_IO)) INFOF("Bound to %s:%d", srctxt, ntohs(broadcast.sin_port)); return 0; @@ -360,7 +361,6 @@ int overlay_rx_messages() } else { /* Read from UDP socket */ int recvttl=1; - errno=0; plen=recvwithttl(overlay_interfaces[i].fd,packet,sizeof(packet), &recvttl,&src_addr,&addrlen); if (plen<0) { @@ -564,69 +564,72 @@ overlay_interface_register(char *name, return 0; } -time_t overlay_last_interface_discover_time=0; -int overlay_interface_discover() -{ - int have_route; - - /* Don't waste too much time and effort on interface discovery, - especially if we can't attach to a given interface for some reason. */ - if (overlay_last_interface_discover_time>time(0)) - overlay_last_interface_discover_time=time(0); - if ((time(0)-overlay_last_interface_discover_time)<2) return 0; - overlay_last_interface_discover_time=time(0); +static time_t overlay_last_interface_discover_time = 0; +int +overlay_interface_discover(void) { + int no_route, i; + time_t now; + struct interface_rules *r; + struct sockaddr_in dummyaddr; + + /* Don't waste too much time and effort on interface discovery, + especially if we can't attach to a given interface for some reason. */ + now = time(NULL); + if (overlay_last_interface_discover_time > now) + overlay_last_interface_discover_time = now; - /* The Android ndk doesn't have ifaddrs.h, so we have to use the netlink interface. - However, netlink is only available on Linux, so for BSD systems, e.g., Mac, we - need to use the ifaddrs method. + if ((now - overlay_last_interface_discover_time) < 2) + return 0; - Also, ifaddrs will work on non-linux systems which is considered critical. - */ + overlay_last_interface_discover_time = now; /* Mark all interfaces as not observed, so that we know if we need to cull any */ - int i; - for(i=0;inamespec[0]=='>') { - for(i=0;inamespec)) break; - if (inamespec,dummyaddr,dummyaddr, - 1000000,PORT_DNA,OVERLAY_INTERFACE_WIFI)) - { if (debug&DEBUG_OVERLAYINTERFACES) - WHY("Could not initialise newly seen interface"); } - else - if (debug&DEBUG_OVERLAYINTERFACES) fprintf(stderr,"Registered interface %s\n",r->namespec); - } + for (r = interface_filter; r != NULL; r = r->next) { + if (r->namespec[0] != '>') + continue; + + for(i = 0; i < overlay_interface_count; i++) + if (!strcasecmp(overlay_interfaces[i].name,r->namespec)) + break; + + if (i < overlay_interface_count) + /* We already know about this interface, so just update it */ + overlay_interfaces[i].observed = 1; + else { + /* New interface, so register it */ + if (overlay_interface_init(r->namespec,dummyaddr,dummyaddr, + 1000000,PORT_DNA,OVERLAY_INTERFACE_WIFI)) { + if (debug & DEBUG_OVERLAYINTERFACES) WHYF("Could not initialise newly seen interface %s", r->namespec); + } + else + if (debug & DEBUG_OVERLAYINTERFACES) INFOF("Registered interface %s",r->namespec); } - r=r->next; } - have_route = 1; + + /* Look for real interfaces */ + no_route = 1; #ifdef HAVE_IFADDRS_H - if (have_route != 0) - have_route = doifaddrs(); + if (no_route != 0) + no_route = doifaddrs(); #endif #ifdef SIOCGIFCONF - if (have_route != 0) - have_route = lsif(); + if (no_route != 0) + no_route = lsif(); #endif #ifdef linux - if (have_route != 0) - have_route = scrapeProcNetRoute(); + if (no_route != 0) + no_route = scrapeProcNetRoute(); #endif - if (have_route != 0) { - FATAL("Unable to get any routing information"); + if (no_route != 0) { + FATAL("Unable to get any interface information"); } return 0; From e8c648c724c9d2b8ee8234090c198d5366f57ce0 Mon Sep 17 00:00:00 2001 From: Daniel O'Connor Date: Fri, 15 Jun 2012 15:08:23 +0930 Subject: [PATCH 05/37] Remove reference to non existent variable. --- overlay_interface.c | 1 - 1 file changed, 1 deletion(-) diff --git a/overlay_interface.c b/overlay_interface.c index 764c7d78..f34233a7 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -185,7 +185,6 @@ overlay_interface_init_socket(int interface, struct sockaddr_in src_addr, struct #define I(X) overlay_interfaces[interface].X I(broadcast_address) = broadcast; - I(src_address) = src_addr; I(fileP) = 0; I(fd) = socket(PF_INET,SOCK_DGRAM,0); From 0dd5845cdd54a8e047080ca9945f042f9e69ba43 Mon Sep 17 00:00:00 2001 From: gardners Date: Fri, 15 Jun 2012 14:48:45 +0930 Subject: [PATCH 06/37] added instrumentation to detect when servald spends more than 5ms doing anything. Reports which bit of the overlay mesh main loop the time was spent in, to help track it down. --- log.c | 4 ++- overlay.c | 69 ++++++++++++++++++++++++++++++++++++++++++++- overlay_interface.c | 5 ++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/log.c b/log.c index 1654421f..43362021 100644 --- a/log.c +++ b/log.c @@ -37,9 +37,11 @@ void logMessage(int level, const char *file, unsigned int line, const char *func void vlogMessage(int level, const char *file, unsigned int line, const char *function, const char *fmt, va_list ap) { + va_list ap2; strbuf b = strbuf_alloca(8192); strbuf_sprintf(b, "%s:%u:%s() ", file ? trimbuildpath(file) : "NULL", line, function ? function : "NULL"); - strbuf_vsprintf(b, fmt, ap); + va_copy(ap2, ap); + strbuf_vsprintf(b, fmt, ap2); #ifdef ANDROID int alevel = ANDROID_LOG_UNKNOWN; switch (level) { diff --git a/overlay.c b/overlay.c index b7fe47c4..65473477 100644 --- a/overlay.c +++ b/overlay.c @@ -70,6 +70,42 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "serval.h" +/* @PGS/20120615 */ +int last_valid=0; +int last_line; +const char *last_file; +const char *last_func; +long long last_time; + +/* @PGS/20120615 */ +void TIMING_PAUSE() +{ + last_valid=0; +} + +/* @PGS/20120615 */ +void TIMING_CHECK(const char *file,const char *func,int line) +{ + long long now=overlay_gettime_ms(); + if (last_valid) { + if (now-last_time>5) { + // More than 5ms spent in a given task, complain + char msg[1024]; + snprintf(msg,1024,"Spent %lldms between %s:%d in %s() and here", + now-last_time,last_file,last_line,last_func); + logMessage(LOG_LEVEL_WARN,file,line,func,"%s",msg); + } + } + + last_valid=1; + last_file=file; + last_func=func; + last_line=line; + last_time=now; +} +#define TIMING_CHECK() TIMING_CHECK(__FILE__,__FUNCTION__,__LINE__) + + int overlayMode=0; overlay_txqueue overlay_tx[OQ_MAX]; @@ -121,18 +157,27 @@ int overlayServerMode() while(1) { + TIMING_CHECK(); + server_shutdown_check(); + TIMING_CHECK(); + /* Work out how long we can wait before we need to tick */ long long ms=overlay_time_until_next_tick(); memabuseCheck(); + TIMING_CHECK(); //int filesPresent=0; fds[0].fd=sock; fds[0].events=POLLIN; fdcount=1; rhizome_server_get_fds(fds,&fdcount,128); + TIMING_CHECK(); rhizome_fetching_get_fds(fds,&fdcount,128); + TIMING_CHECK(); overlay_mdp_get_fds(fds,&fdcount,128); + TIMING_CHECK(); monitor_get_fds(fds,&fdcount,128); + TIMING_CHECK(); for(i=0;i5) ms=5; } } - + TIMING_CHECK(); + /* Progressively update link scores to neighbours etc, and find out how long before we should next tick the route table. Basically the faster the CPU and the sparser the route table, the less often we @@ -168,9 +214,12 @@ int overlayServerMode() int vomp_tick_time=vomp_tick_interval(); if (ms>vomp_tick_time) ms=vomp_tick_time; + TIMING_CHECK(); if (debug&DEBUG_VERBOSE_IO) DEBUGF("Waiting via poll() for up to %lldms", ms); + TIMING_PAUSE(); int r = poll(fds, fdcount, ms); + TIMING_CHECK(); if (r == -1) WHY_perror("poll"); else if (debug&DEBUG_VERBOSE_IO) { @@ -181,7 +230,9 @@ int overlayServerMode() DEBUGF("fd #%d is ready (0x%x)\n", fds[i].fd, fds[i].revents); } /* Do high-priority audio handling first */ + TIMING_CHECK(); vomp_tick(); + TIMING_CHECK(); if (r > 0) { /* We have data, so try to receive it */ @@ -203,27 +254,43 @@ int overlayServerMode() } } + TIMING_CHECK(); overlay_rx_messages(); + TIMING_CHECK(); if (rhizome_enabled()) { + TIMING_CHECK(); rhizome_server_poll(); + TIMING_CHECK(); rhizome_fetch_poll(); + TIMING_CHECK(); overlay_mdp_poll(); + TIMING_CHECK(); monitor_poll(); + TIMING_CHECK(); } } else { /* No data before tick occurred, so do nothing. Well, for now let's just check anyway. */ if (debug&DEBUG_IO) fprintf(stderr,"poll() timeout.\n"); + TIMING_CHECK(); overlay_rx_messages(); + TIMING_CHECK(); if (rhizome_enabled()) { + TIMING_CHECK(); rhizome_server_poll(); + TIMING_CHECK(); rhizome_fetch_poll(); + TIMING_CHECK(); overlay_mdp_poll(); + TIMING_CHECK(); monitor_poll(); + TIMING_CHECK(); } } + TIMING_CHECK(); /* Check if we need to trigger any ticks on any interfaces */ overlay_check_ticks(); + TIMING_CHECK(); } return 0; diff --git a/overlay_interface.c b/overlay_interface.c index f34233a7..cfae3bcc 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -217,6 +217,11 @@ overlay_interface_init_socket(int interface, struct sockaddr_in src_addr, struct a bad signal. */ fcntl(I(fd), F_SETFL, fcntl(I(fd), F_GETFL, NULL) | O_CLOEXEC); + /* @PGS/20120615 + Use the broadcast address, so that we can reliably receive broadcast + traffic on all platforms. BUT on OSX we really need a non-broadcast socket + to send from, because you cannot send from a broadcast socket on OSX it seems. + */ broadcast.sin_family = AF_INET; broadcast.sin_port = htons(I(port)); if (bind(I(fd), (struct sockaddr *)&broadcast, sizeof(broadcast))) { From c2b041d9c3d68f3aee242bf9843d54ce56b0a8a8 Mon Sep 17 00:00:00 2001 From: gardners Date: Fri, 15 Jun 2012 15:07:16 +0930 Subject: [PATCH 07/37] added readme file indicating how to use dummy network interfaces. --- README.DUMMYNETS | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 README.DUMMYNETS diff --git a/README.DUMMYNETS b/README.DUMMYNETS new file mode 100644 index 00000000..62cccd3d --- /dev/null +++ b/README.DUMMYNETS @@ -0,0 +1,10 @@ +Sometimes it is helpful to run more than one servald instance on a given machine +for debugging purposes. To make this easier, there is a dummy interface driver +that servald knows about. To use it: + +1. create an empty file, e.g., dummynet0, somewhere convenient + +2. For each servald instance you wish to use it, set the interface specification to include the dummynet file. Use a specification like "+>pathtodummynetfile", where pathtodummynetfile is the relative path from the instance path of that servald instance to the dummynet file. For example, you might run: + % servald config set interfaces "+eth0,+>../dummynet0" + +3. Run each servald instance. They should now use the dummy network. From 0b85a06a3a718566734eb6ba6cf79f4599f689b0 Mon Sep 17 00:00:00 2001 From: gardners Date: Fri, 15 Jun 2012 15:09:12 +0930 Subject: [PATCH 08/37] Added explanation of changed semantics when using dummynets. --- README.DUMMYNETS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.DUMMYNETS b/README.DUMMYNETS index 62cccd3d..9a6d0420 100644 --- a/README.DUMMYNETS +++ b/README.DUMMYNETS @@ -1,3 +1,5 @@ +@PGS/20120615 + Sometimes it is helpful to run more than one servald instance on a given machine for debugging purposes. To make this easier, there is a dummy interface driver that servald knows about. To use it: @@ -8,3 +10,7 @@ that servald knows about. To use it: % servald config set interfaces "+eth0,+>../dummynet0" 3. Run each servald instance. They should now use the dummy network. + +NOTE: Because dummynets are files, not sockets, poll/select does not work on them. As a result the main overlay loop has slightly different behaviour and timing characteristics when a dummynet is in use. + +TODO: Convert dummynet interface to use a unixdomain socket, and a simple dummynet server that reflects packets among the clients connected, so that all socket semantics (including use of poll/select) are preserved. From f6cce7103d89cc03b1edad5b2b72c4497fe06d80 Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Mon, 18 Jun 2012 09:37:24 +0930 Subject: [PATCH 09/37] Fix error message --- monitor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitor.c b/monitor.c index ea18f123..8ba7c033 100644 --- a/monitor.c +++ b/monitor.c @@ -520,8 +520,8 @@ int monitor_process_data(int index) if (vomp_sample_size(c->sample_codec)!=c->data_offset) return - WHYF("Ignoring sample block of incorrect size (expected %d, got %d bytes)", - vomp_sample_size(c->sample_codec)!=c->data_offset); + WHYF("Ignoring sample block of incorrect size (expected %d, got %d bytes for codec %d)", + vomp_sample_size(c->sample_codec), c->data_offset, c->sample_codec); fcntl(c->socket,F_SETFL, fcntl(c->socket, F_GETFL, NULL)|O_NONBLOCK); From 6023cd4da24262882be2566c84aba5681b404fa0 Mon Sep 17 00:00:00 2001 From: gardners Date: Wed, 20 Jun 2012 12:57:12 +0930 Subject: [PATCH 10/37] moved timing check definitions to serval.h --- overlay.c | 3 +-- serval.h | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/overlay.c b/overlay.c index 65473477..139d7d19 100644 --- a/overlay.c +++ b/overlay.c @@ -84,7 +84,7 @@ void TIMING_PAUSE() } /* @PGS/20120615 */ -void TIMING_CHECK(const char *file,const char *func,int line) +void _TIMING_CHECK(const char *file,const char *func,int line) { long long now=overlay_gettime_ms(); if (last_valid) { @@ -103,7 +103,6 @@ void TIMING_CHECK(const char *file,const char *func,int line) last_line=line; last_time=now; } -#define TIMING_CHECK() TIMING_CHECK(__FILE__,__FUNCTION__,__LINE__) int overlayMode=0; diff --git a/serval.h b/serval.h index 2ec61d34..58cf4fc2 100755 --- a/serval.h +++ b/serval.h @@ -114,6 +114,11 @@ struct in_addr { /* bzero(3) is deprecated in favour of memset(3). */ #define bzero(addr,len) memset((addr), 0, (len)) +/* @PGS/20120615 */ +#define TIMING_CHECK() _TIMING_CHECK(__FILE__,__FUNCTION__,__LINE__) +void _TIMING_CHECK(const char *file,const char *func,int line); +void TIMING_PAUSE(); + /* UDP Port numbers for various Serval services. The overlay mesh works over DNA */ #define PORT_DNA 4110 From b48fdb6567fe9ea22de36156b071d87cff49dfb3 Mon Sep 17 00:00:00 2001 From: gardners Date: Wed, 20 Jun 2012 14:01:12 +0930 Subject: [PATCH 11/37] added deeper timing checks to reveal that rhizome advertisements are the slowest part -- need to add caching of manifests so that we don't hit the database every interface tick. --- commandline.c | 30 ++++++++++++++++++++++++++++++ overlay_interface.c | 29 +++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/commandline.c b/commandline.c index c9d65ec6..ca59f26e 100644 --- a/commandline.c +++ b/commandline.c @@ -1551,6 +1551,34 @@ int app_test_rfs(int argc, const char *const *argv, struct command_line_option * return 0; } +int app_crypt_test(int argc, const char *const *argv, struct command_line_option *o) +{ + unsigned char nonce[crypto_box_curve25519xsalsa20poly1305_NONCEBYTES]; + unsigned char k[crypto_box_curve25519xsalsa20poly1305_BEFORENMBYTES]; + + unsigned char plain_block[65536]; + + urandombytes(nonce,sizeof(nonce)); + urandombytes(k,sizeof(k)); + + int len,i; + + overlay_gettime_ms(); + + for(len=16;len<=65536;len*=2) { + unsigned long long start=overlay_gettime_ms(); + for (i=0;i<1000;i++) { + bzero(&plain_block[0],crypto_box_curve25519xsalsa20poly1305_ZEROBYTES); + crypto_box_curve25519xsalsa20poly1305_afternm + (plain_block,plain_block,len,nonce,k); + } + unsigned long long end=overlay_gettime_ms(); + printf("%d bytes - 100 tests took %lldms - mean time = %.2fms\n", + len,end-start,(end-start)*1.0/i); + } + return 0; +} + int app_node_info(int argc, const char *const *argv, struct command_line_option *o) { const char *sid; @@ -1764,6 +1792,8 @@ command_line_option command_line_options[]={ "Test RFS field calculation"}, {app_monitor_cli,{"monitor","[]",NULL},0, "Interactive servald monitor interface. Specify SID to auto-dial that peer and insert dummy audio data"}, + {app_crypt_test,{"crypt","test",NULL},0, + "Run cryptography speed test"}, #ifdef HAVE_VOIPTEST {app_pa_phone,{"phone",NULL},0, "Run phone test application"}, diff --git a/overlay_interface.c b/overlay_interface.c index cfae3bcc..c988eabe 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -791,6 +791,8 @@ int overlay_tick_interface(int i, long long now) #define MAX_FRAME_PAX 1024 overlay_frame *pax[MAX_FRAME_PAX]; + TIMING_CHECK(); + if (overlay_interfaces[i].bits_per_second<1) { /* An interface with no speed budget is for listening only, so doesn't get ticked */ return 0; @@ -827,6 +829,7 @@ int overlay_tick_interface(int i, long long now) Give priority to newly observed nodes so that good news travels quickly to help roaming. XXX - Don't forget about PONGing reachability reports to allow use of monodirectional links. */ +TIMING_CHECK(); overlay_stuff_packet_from_queue(i,e,OQ_MESH_MANAGEMENT,now,pax,&frame_pax,MAX_FRAME_PAX); /* We previously limited manifest space to 3/4 of MTU, but that causes problems for @@ -835,23 +838,30 @@ int overlay_tick_interface(int i, long long now) #warning reduce to <= mtu*3/4 once we have compacty binary canonical manifest format ob_limitsize(e,overlay_interfaces[i].mtu*4/4); +TIMING_CHECK(); + /* Add advertisements for ROUTES not Rhizome bundles. Rhizome bundle advertisements are lower priority */ overlay_route_add_advertisements(i,e); ob_limitsize(e,overlay_interfaces[i].mtu); + TIMING_CHECK(); + /* 4. XXX Add lower-priority queued data */ overlay_stuff_packet_from_queue(i,e,OQ_ISOCHRONOUS_VIDEO,now,pax,&frame_pax,MAX_FRAME_PAX); overlay_stuff_packet_from_queue(i,e,OQ_ORDINARY,now,pax,&frame_pax,MAX_FRAME_PAX); overlay_stuff_packet_from_queue(i,e,OQ_OPPORTUNISTIC,now,pax,&frame_pax,MAX_FRAME_PAX); /* 5. XXX Fill the packet up to a suitable size with anything that seems a good idea */ + TIMING_CHECK(); if (rhizome_enabled()) overlay_rhizome_add_advertisements(i,e); if (debug&DEBUG_PACKETCONSTRUCTION) dump("assembled packet",&e->bytes[0],e->length); + TIMING_CHECK(); + /* Now send the frame. This takes the form of a special DNA packet with a different service code, which we setup earlier. */ if (debug&DEBUG_OVERLAYINTERFACES) @@ -933,7 +943,7 @@ int overlay_tick_interface(int i, long long now) return 0; } else return WHY("overlay_broadcast_ensemble() failed"); - + TIMING_CHECK(); } @@ -943,15 +953,18 @@ overlay_check_ticks(void) { /* Check if any interface(s) are due for a tick */ int i; + TIMING_CHECK(); /* Check for changes to interfaces */ overlay_interface_discover(); - + TIMING_CHECK(); + long long now = overlay_gettime_ms(); /* Now check if the next tick time for the interfaces is no later than that time. If so, trigger a tick on the interface. */ if (debug & DEBUG_OVERLAYINTERFACES) INFOF("Examining %d interfaces.",overlay_interface_count); for(i = 0; i < overlay_interface_count; i++) { + TIMING_CHECK(); /* Only tick live interfaces */ if (overlay_interfaces[i].observed > 0) { if (debug & DEBUG_VERBOSE_IO) INFOF("Interface %s ticks every %dms, last at %lld.", @@ -959,12 +972,16 @@ overlay_check_ticks(void) { overlay_interfaces[i].tick_ms, overlay_interfaces[i].last_tick_ms); if (now >= overlay_interfaces[i].last_tick_ms + overlay_interfaces[i].tick_ms) { - /* This interface is due for a tick */ - overlay_tick_interface(i, now); - overlay_interfaces[i].last_tick_ms = now; + TIMING_CHECK(); + + /* This interface is due for a tick */ + overlay_tick_interface(i, now); + TIMING_CHECK(); + overlay_interfaces[i].last_tick_ms = now; } } else - if (debug & DEBUG_VERBOSE_IO) INFOF("Interface %s is awol.", overlay_interfaces[i].name); + if (debug & DEBUG_VERBOSE_IO) INFOF("Interface %s is awol.", overlay_interfaces[i].name); + TIMING_CHECK(); } return 0; From 926d4296c45f7c46b4fb462f753ef79d06f8bd4d Mon Sep 17 00:00:00 2001 From: Jeremy Lakeman Date: Wed, 20 Jun 2012 15:29:46 +0930 Subject: [PATCH 12/37] Added overall timer for main loop, minor tweaks to order and freq. of processes --- lsif.c | 23 +++--- overlay.c | 90 +++++++++++++----------- overlay_interface.c | 166 ++++++++++++++++++++------------------------ 3 files changed, 138 insertions(+), 141 deletions(-) diff --git a/lsif.c b/lsif.c index 1eabf120..501c5c2e 100644 --- a/lsif.c +++ b/lsif.c @@ -112,7 +112,7 @@ int scrapeProcNetRoute() int lsif(void) { - char buf[8192], addrtxt[INET_ADDRSTRLEN], bcasttxt[INET_ADDRSTRLEN]; + char buf[8192]; struct ifconf ifc; int sck, nInterfaces, ofs; struct ifreq *ifr; @@ -165,12 +165,14 @@ lsif(void) { bcopy(&ifr->ifr_ifru.ifru_addr, &local, sizeof(local)); bcopy(&ifr->ifr_ifru.ifru_broadaddr, &broadcast ,sizeof(broadcast)); - assert(inet_ntop(AF_INET, (const void *)&local.sin_addr, addrtxt, INET_ADDRSTRLEN) != NULL); - assert(inet_ntop(AF_INET, (const void *)&broadcast.sin_addr, bcasttxt, INET_ADDRSTRLEN) != NULL); - - if (debug & DEBUG_OVERLAYINTERFACES) INFOF("name=%s addr=%s, broad=%s\n", + if (debug & DEBUG_OVERLAYINTERFACES) { + char addrtxt[INET_ADDRSTRLEN], bcasttxt[INET_ADDRSTRLEN]; + assert(inet_ntop(AF_INET, (const void *)&local.sin_addr, addrtxt, INET_ADDRSTRLEN) != NULL); + assert(inet_ntop(AF_INET, (const void *)&broadcast.sin_addr, bcasttxt, INET_ADDRSTRLEN) != NULL); + INFOF("name=%s addr=%s, broad=%s\n", ifr->ifr_name, addrtxt, bcasttxt); + } overlay_interface_register(ifr->ifr_name, local, broadcast); nInterfaces++; } @@ -187,7 +189,6 @@ lsif(void) { int doifaddrs(void) { struct ifaddrs *ifaddr, *ifa; - char addrtxt[INET_ADDRSTRLEN], bcasttxt[INET_ADDRSTRLEN]; char *name; struct sockaddr_in local, netmask, broadcast; @@ -217,10 +218,12 @@ doifaddrs(void) { /* Compute broadcast address */ broadcast.sin_addr.s_addr |= (~netmask.sin_addr.s_addr); - assert(inet_ntop(AF_INET, (const void *)&local.sin_addr, addrtxt, INET_ADDRSTRLEN) != NULL); - assert(inet_ntop(AF_INET, (const void *)&broadcast.sin_addr, bcasttxt, INET_ADDRSTRLEN) != NULL); - - if (debug & DEBUG_OVERLAYINTERFACES) INFOF("name=%s addr=%s broad=%s", name, addrtxt, bcasttxt); + if (debug & DEBUG_OVERLAYINTERFACES){ + char addrtxt[INET_ADDRSTRLEN], bcasttxt[INET_ADDRSTRLEN]; + assert(inet_ntop(AF_INET, (const void *)&local.sin_addr, addrtxt, INET_ADDRSTRLEN) != NULL); + assert(inet_ntop(AF_INET, (const void *)&broadcast.sin_addr, bcasttxt, INET_ADDRSTRLEN) != NULL); + INFOF("name=%s addr=%s broad=%s", name, addrtxt, bcasttxt); + } overlay_interface_register(name,local,broadcast); } diff --git a/overlay.c b/overlay.c index 139d7d19..30ca8adf 100644 --- a/overlay.c +++ b/overlay.c @@ -88,7 +88,7 @@ void _TIMING_CHECK(const char *file,const char *func,int line) { long long now=overlay_gettime_ms(); if (last_valid) { - if (now-last_time>5) { + if (now-last_time>10) { // More than 5ms spent in a given task, complain char msg[1024]; snprintf(msg,1024,"Spent %lldms between %s:%d in %s() and here", @@ -104,6 +104,19 @@ void _TIMING_CHECK(const char *file,const char *func,int line) last_time=now; } +long long last_loop_time=0; +void LOOP_END() +{ + long long now = overlay_gettime_ms(); + if (last_loop_time!=0 && now - last_loop_time>15){ + DEBUGF("Last loop took %lldms",now - last_loop_time); + } +} + +void LOOP_START() +{ + last_loop_time = overlay_gettime_ms(); +} int overlayMode=0; @@ -146,6 +159,7 @@ int overlayServerMode() struct pollfd fds[128]; int fdcount; + int r; /* Create structures to use 1MB of RAM for testing */ overlay_route_init(1); @@ -213,26 +227,31 @@ int overlayServerMode() int vomp_tick_time=vomp_tick_interval(); if (ms>vomp_tick_time) ms=vomp_tick_time; - TIMING_CHECK(); - if (debug&DEBUG_VERBOSE_IO) - DEBUGF("Waiting via poll() for up to %lldms", ms); - TIMING_PAUSE(); - int r = poll(fds, fdcount, ms); - TIMING_CHECK(); - if (r == -1) - WHY_perror("poll"); - else if (debug&DEBUG_VERBOSE_IO) { - DEBUGF("poll() says %d file descriptors are ready", r); - int i; - for(i=0;i0){ + TIMING_CHECK(); + if (debug&DEBUG_VERBOSE_IO) + DEBUGF("Waiting via poll() for up to %lldms", ms); + TIMING_PAUSE(); + + r = poll(fds, fdcount, ms); + + TIMING_CHECK(); + if (r == -1) + WHY_perror("poll"); + else if (debug&DEBUG_VERBOSE_IO) { + DEBUGF("poll() says %d file descriptors are ready", r); + int i; + for(i=0;i 0) { /* We have data, so try to receive it */ if (debug&DEBUG_IO) { @@ -257,39 +276,28 @@ int overlayServerMode() overlay_rx_messages(); TIMING_CHECK(); if (rhizome_enabled()) { - TIMING_CHECK(); rhizome_server_poll(); TIMING_CHECK(); rhizome_fetch_poll(); TIMING_CHECK(); - overlay_mdp_poll(); - TIMING_CHECK(); - monitor_poll(); - TIMING_CHECK(); } + overlay_mdp_poll(); + TIMING_CHECK(); + monitor_poll(); + TIMING_CHECK(); } else { /* No data before tick occurred, so do nothing. Well, for now let's just check anyway. */ if (debug&DEBUG_IO) fprintf(stderr,"poll() timeout.\n"); + + /* Do high-priority audio handling first */ TIMING_CHECK(); - overlay_rx_messages(); + vomp_tick(); + TIMING_CHECK(); + /* Check if we need to trigger any ticks on any interfaces */ + overlay_check_ticks(); TIMING_CHECK(); - if (rhizome_enabled()) { - TIMING_CHECK(); - rhizome_server_poll(); - TIMING_CHECK(); - rhizome_fetch_poll(); - TIMING_CHECK(); - overlay_mdp_poll(); - TIMING_CHECK(); - monitor_poll(); - TIMING_CHECK(); - } } - TIMING_CHECK(); - /* Check if we need to trigger any ticks on any interfaces */ - overlay_check_ticks(); - TIMING_CHECK(); } return 0; diff --git a/overlay_interface.c b/overlay_interface.c index c988eabe..f1d2af0c 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -291,105 +291,91 @@ int overlay_interface_init(char *name,struct sockaddr_in src_addr,struct sockadd int overlay_rx_messages() { int i; - + /* Grab packets, unpackage and dispatch frames to consumers */ /* XXX Okay, so how are we managing out-of-process consumers? - They need some way to register their interest in listening to a port. - */ + They need some way to register their interest in listening to a port. + */ unsigned char packet[16384]; int plen=0; - int c[OVERLAY_MAX_INTERFACES]; - int count=0; - /* Look at all interfaces */ - for(i=0;i0); count+=c[i]; } - - /* Grab packets from interfaces in round-robin fashion until all have been grabbed, - or until we have spent too long (maybe 10ms?) */ - int now = overlay_gettime_ms(); - while(count>0) - { - for(i=0;i=length) + { + if (debug&DEBUG_OVERLAYINTERFACES) + fprintf(stderr,"At end of input on dummy interface #%d\n",i); + } + else + { + lseek(overlay_interfaces[i].fd,overlay_interfaces[i].offset,SEEK_SET); + if (debug&DEBUG_OVERLAYINTERFACES) + fprintf(stderr,"Reading from interface #%d log at offset %d, end of file at %lld.\n",i, + overlay_interfaces[i].offset,length); + if (read(overlay_interfaces[i].fd,&packet[0],2048)==2048) { - struct sockaddr src_addr; - unsigned int addrlen=sizeof(src_addr); - unsigned char transaction_id[8]; - - overlay_last_interface_number=i; - - /* Set socket non-blocking before we try to read from it */ - fcntl(overlay_interfaces[i].fd, F_SETFL, - fcntl(overlay_interfaces[i].fd, F_GETFL, NULL)|O_NONBLOCK); - - if (overlay_interfaces[i].fileP) { - /* Read from dummy interface file */ - long long length=lseek(overlay_interfaces[i].fd,0,SEEK_END); - if (overlay_interfaces[i].offset>=length) - { - if (debug&DEBUG_OVERLAYINTERFACES) - fprintf(stderr,"At end of input on dummy interface #%d\n",i); - } - else - { - lseek(overlay_interfaces[i].fd,overlay_interfaces[i].offset,SEEK_SET); - if (debug&DEBUG_OVERLAYINTERFACES) - fprintf(stderr,"Reading from interface #%d log at offset %d, end of file at %lld.\n",i, - overlay_interfaces[i].offset,length); - if (read(overlay_interfaces[i].fd,&packet[0],2048)==2048) - { - overlay_interfaces[i].offset+=2048; - plen=2048-128; - plen=packet[110]+(packet[111]<<8); - if (plen>(2048-128)) plen=-1; - if (debug&DEBUG_PACKETRX) { - fflush(stdout); - serval_packetvisualise(stderr, - "Read from dummy interface", - &packet[128],plen); - fflush(stderr); - } - bzero(&transaction_id[0],8); - bzero(&src_addr,sizeof(src_addr)); - if ((plen>=0)&&(packet[0]==0x01)&&!(packet[1]|packet[2]|packet[3])) { - { if (packetOk(i,&packet[128],plen,transaction_id, - -1 /* fake TTL */, - &src_addr,addrlen,1)) - WHY("Malformed or unsupported packet from dummy interface (packetOK() failed)"); } } - else WHY("Invalid packet version in dummy interface"); - } - else { - if (debug&DEBUG_IO) fprintf(stderr,"Read NOTHING from dummy interface\n"); - c[i]=0; count--; - } - } - } else { - /* Read from UDP socket */ - int recvttl=1; - plen=recvwithttl(overlay_interfaces[i].fd,packet,sizeof(packet), - &recvttl,&src_addr,&addrlen); - if (plen<0) { - c[i]=0; count--; - } else { - /* We have a frame from this interface */ - if (debug&DEBUG_PACKETRX) { - fflush(stdout); - serval_packetvisualise(stderr,"Read from real interface", - packet,plen); - fflush(stderr); - } - if (debug&DEBUG_OVERLAYINTERFACES)fprintf(stderr,"Received %d bytes on interface #%d (%s)\n",plen,i,overlay_interfaces[i].name); - - if (packetOk(i,packet,plen,NULL,recvttl,&src_addr,addrlen,1)) { - WHY("Malformed packet"); - serval_packetvisualise(stderr,"Malformed packet", packet,plen); - } - } + overlay_interfaces[i].offset+=2048; + plen=2048-128; + plen=packet[110]+(packet[111]<<8); + if (plen>(2048-128)) plen=-1; + if (debug&DEBUG_PACKETRX) { + fflush(stdout); + serval_packetvisualise(stderr, + "Read from dummy interface", + &packet[128],plen); + fflush(stderr); } + bzero(&transaction_id[0],8); + bzero(&src_addr,sizeof(src_addr)); + if ((plen>=0)&&(packet[0]==0x01)&&!(packet[1]|packet[2]|packet[3])) { + { if (packetOk(i,&packet[128],plen,transaction_id, + -1 /* fake TTL */, + &src_addr,addrlen,1)) + WHY("Malformed or unsupported packet from dummy interface (packetOK() failed)"); } } + else WHY("Invalid packet version in dummy interface"); } - /* Don't sit here forever, or else we will never send any packets */ - if (overlay_gettime_ms()>(now+10)) break; + else { + if (debug&DEBUG_IO) fprintf(stderr,"Read NOTHING from dummy interface\n"); + } + } + } else { + /* Read from UDP socket */ + int recvttl=1; + plen=recvwithttl(overlay_interfaces[i].fd,packet,sizeof(packet), + &recvttl,&src_addr,&addrlen); + if (plen>=0) { + /* We have a frame from this interface */ + if (debug&DEBUG_PACKETRX) { + fflush(stdout); + serval_packetvisualise(stderr,"Read from real interface", + packet,plen); + fflush(stderr); + } + if (debug&DEBUG_OVERLAYINTERFACES) + fprintf(stderr,"Received %d bytes on interface #%d (%s)\n",plen,i,overlay_interfaces[i].name); + + if (packetOk(i,packet,plen,NULL,recvttl,&src_addr,addrlen,1)) { + WHY("Malformed packet"); + serval_packetvisualise(stderr,"Malformed packet", packet,plen); + } + } } - + } + return 0; } From 500b9f9f28f546409e811e2725794f6f9cc4b539 Mon Sep 17 00:00:00 2001 From: gardners Date: Wed, 20 Jun 2012 16:07:07 +0930 Subject: [PATCH 13/37] fixed a memory leak, more to find. --- overlay_interface.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/overlay_interface.c b/overlay_interface.c index f1d2af0c..46a796ef 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -774,6 +774,7 @@ int overlay_queue_dump(overlay_txqueue *q) int overlay_tick_interface(int i, long long now) { int frame_pax=0; + overlay_buffer *e=NULL; #define MAX_FRAME_PAX 1024 overlay_frame *pax[MAX_FRAME_PAX]; @@ -789,7 +790,7 @@ int overlay_tick_interface(int i, long long now) /* Get a buffer ready, and limit it's size appropriately. XXX size limit should be reduced from MTU. XXX we should also take account of the volume of data likely to be in the TX buffer. */ - overlay_buffer *e=ob_new(overlay_interfaces[i].mtu); + e=ob_new(overlay_interfaces[i].mtu); if (!e) return WHY("ob_new() failed"); ob_limitsize(e,overlay_interfaces[i].mtu/4); @@ -798,7 +799,7 @@ int overlay_tick_interface(int i, long long now) unsigned char bytes[]={/* Magic */ 'O',0x10, /* Version */ 0x00,0x01}; if (ob_append_bytes(e,bytes,4)) { - ob_free(e); + ob_free(e); return WHY("ob_append_bytes() refused to append magic bytes."); } @@ -926,9 +927,13 @@ TIMING_CHECK(); } } } + if (e) ob_free(e); e=NULL; return 0; } - else return WHY("overlay_broadcast_ensemble() failed"); + else { + if (e) ob_free(e); e=NULL; + return WHY("overlay_broadcast_ensemble() failed"); + } TIMING_CHECK(); } From 8ea37af64d9c2ed2d32083ad986fc4564e8effb9 Mon Sep 17 00:00:00 2001 From: gardners Date: Wed, 20 Jun 2012 12:56:32 +0930 Subject: [PATCH 14/37] servald now can return arbitrary URIs in response to DNA requests. --- commandline.c | 41 ++++++++++++++++++++++++++--------------- overlay_mdp.c | 36 ++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/commandline.c b/commandline.c index ca59f26e..5db4a4a1 100644 --- a/commandline.c +++ b/commandline.c @@ -426,8 +426,10 @@ int app_dna_lookup(int argc, const char *const *argv, struct command_line_option if (create_serval_instance_dir() == -1) return -1; - int sid_count=0; - unsigned char sids[128][SID_SIZE]; + int uri_count=0; +#define MAXREPLIES 256 +#define MAXURILEN 256 + char uris[MAXREPLIES][MAXURILEN]; const char *did; if (cli_arg(argc, argv, o, "did", &did, NULL, "*") == -1) @@ -484,19 +486,28 @@ int app_dna_lookup(int argc, const char *const *argv, struct command_line_option WHYF(" Error message: %s", mdp.error.message); } else if ((rx.packetTypeAndFlags&MDP_TYPE_MASK)==MDP_TX) { - /* Display match unless it is a duplicate. - XXX - For wildcard searches, each sid will only show up once. */ - int i; - for(i=0;iprivate_key_len>64) /* skip excessively long DID records */ continue; - /* copy SID out into source address of frame */ /* and null-terminated DID */ - bcopy(&keyring->contexts[cn]->identities[in]->keypairs[0] - ->public_key[0],&mdp->out.src.sid[0],SID_SIZE); - bcopy(keyring->contexts[cn]->identities[in]->keypairs[kp] - ->private_key,&mdp->out.payload[0], - keyring->contexts[cn]->identities[in]->keypairs[kp] - ->private_key_len); - bcopy(keyring->contexts[cn]->identities[in]->keypairs[kp] - ->public_key,&mdp->out.payload[keyring - ->contexts[cn] - ->identities[in] - ->keypairs[kp] - ->private_key_len], - keyring->contexts[cn]->identities[in]->keypairs[kp] - ->private_key_len); - /* set length */ - mdp->out.payload_length= + unsigned char *unpackedDid= keyring->contexts[cn]->identities[in]->keypairs[kp] - ->private_key_len - +keyring->contexts[cn]->identities[in]->keypairs[kp] - ->public_key_len; + ->private_key; + unsigned char *packedSid= + keyring->contexts[cn]->identities[in]->keypairs[0] + ->public_key; + char *name= + (char *)keyring->contexts[cn]->identities[in]->keypairs[kp] + ->public_key; + /* copy SID out into source address of frame */ + bcopy(packedSid,&mdp->out.src.sid[0],SID_SIZE); + /* and build reply as did\nname\nURI */ + snprintf((char *)&mdp->out.payload[0],512,"%s\n%s\nsid://%s/%s", + unpackedDid,name,overlay_render_sid(packedSid), + unpackedDid); + mdp->out.payload_length=strlen((char *)mdp->out.payload)+1; + /* mark as outgoing MDP message */ mdp->packetTypeAndFlags&=MDP_FLAG_MASK; mdp->packetTypeAndFlags|=MDP_TX; From 30e254560a24090f37b930d957d920cde8c757fa Mon Sep 17 00:00:00 2001 From: gardners Date: Wed, 20 Jun 2012 15:02:58 +0930 Subject: [PATCH 15/37] node info now understands new dna response packet format. --- commandline.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/commandline.c b/commandline.c index 5db4a4a1..46feefbc 100644 --- a/commandline.c +++ b/commandline.c @@ -1680,16 +1680,19 @@ int app_node_info(int argc, const char *const *argv, struct command_line_option { int bytes=m2.in.payload_length; - - if ((bytes+1)>=sizeof(mdp.nodeinfo.did)+sizeof(mdp.nodeinfo.name)){ - WHYF("Result is too large"); - continue; + char did[512]; + char name[512]; + char uri[512]; + bzero(did,32); bzero(name,64); + /* Check if reply is validly formatted */ + if (sscanf((char *)m2.in.payload,"%[^\n]\n%[^\n]\n%400s", + did,name,uri)==3) { + did[31]=0; + bcopy(&did[0],&mdp.nodeinfo.did[0],32); + bcopy(&name[0],&mdp.nodeinfo.name[0],64); + mdp.nodeinfo.resolve_did=1; + break; } - bcopy(&m2.in.payload[0],&mdp.nodeinfo.did[0],32); - bcopy(&m2.in.payload[32],&mdp.nodeinfo.name[0],64); - mdp.nodeinfo.did[bytes]=0; - mdp.nodeinfo.resolve_did=1; - break; } } } From 3281f2c6528a9ec2e17ab2d20d7ff959b08d1cd3 Mon Sep 17 00:00:00 2001 From: gardners Date: Wed, 20 Jun 2012 17:45:40 +0930 Subject: [PATCH 16/37] fixed introduced bug where "node info" failed to provide automatic resolution of DIDs and names for local identities. --- commandline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commandline.c b/commandline.c index 46feefbc..e72c9a2c 100644 --- a/commandline.c +++ b/commandline.c @@ -1601,7 +1601,7 @@ int app_node_info(int argc, const char *const *argv, struct command_line_option mdp.packetTypeAndFlags=MDP_NODEINFO; if (argc>3) resolveDid=1; - mdp.nodeinfo.resolve_did=0; // so we know that we don't have a result yet. + mdp.nodeinfo.resolve_did=1; // Request resolution of DID and Name by local server if it can. /* get SID or SID prefix XXX - Doesn't correctly handle odd-lengthed SID prefixes (ignores last digit). From e5394dd14352430edfcffb42d41e1f83de55b2a2 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 08:07:03 +0930 Subject: [PATCH 17/37] added stubs for DNA lookup helper app (not yet complete). cleaned up some unused variables. --- Android.mk | 1 + Makefile.in | 1 + codecs.c | 2 +- commandline.c | 3 +-- overlay_mdp.c | 16 +++++++++++++++- serval.h | 4 ++++ 6 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Android.mk b/Android.mk index 0c16ee3b..574600a5 100644 --- a/Android.mk +++ b/Android.mk @@ -42,6 +42,7 @@ SERVALD_SRC_FILES = \ serval-dna/keyring.c \ serval-dna/vomp.c \ serval-dna/lsif.c \ + serval-dna/dna_helper.c \ serval-dna/monitor.c \ serval-dna/monitor-cli.c \ serval-dna/codecs.c \ diff --git a/Makefile.in b/Makefile.in index b60e5c11..3544a9c9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -45,6 +45,7 @@ SRCS= main.c \ lsif.c \ monitor.c \ monitor-cli.c \ + dna_helper.c \ codecs.c \ audiodevices.c \ audio_msm_g1.c \ diff --git a/codecs.c b/codecs.c index 03e0c1d8..5cf04582 100644 --- a/codecs.c +++ b/codecs.c @@ -48,6 +48,6 @@ int bufferAudioForPlayback(int codec,long long start_time,long long end_time, decode codecs etc here. */ /* send audio to device */ - int bytesWritten=audev->write(&data[0],dataLen); + // int bytesWritten=audev->write(&data[0],dataLen); return 0; } diff --git a/commandline.c b/commandline.c index e72c9a2c..d7194605 100644 --- a/commandline.c +++ b/commandline.c @@ -1678,8 +1678,7 @@ int app_node_info(int argc, const char *const *argv, struct command_line_option continue; } - { - int bytes=m2.in.payload_length; + { char did[512]; char name[512]; char uri[512]; diff --git a/overlay_mdp.c b/overlay_mdp.c index 0c190023..032bd1ba 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -509,6 +509,7 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) bcopy(&mdp->out.src.sid[0],&srcsid[0],SID_SIZE); /* now switch addresses around for any replies */ overlay_mdp_swap_src_dst(mdp); + int results=0; while(keyring_find_did(keyring,&cn,&in,&kp,did)) { /* package DID and Name into reply (we include the DID because @@ -542,8 +543,21 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) overlay_mdp_dispatch(mdp,0 /* system generated */, NULL,0); kp++; - + results++; } + if (!results) { + /* No local results, so see if servald has been configured to use + a DNA-helper that can provide additional mappings. This provides + a generalised interface for resolving telephone numbers into URIs. + The first use will be for resolving DIDs to SIP addresses for + OpenBTS boxes run by the OTI/Commotion project. + + The helper is run asynchronously, and the replies will be delivered + when results become available, so this function will return + immediately, so as not to cause blockages and delays in servald. + */ + dna_helper_enqueue(did,mdp->out.dst.sid); + } /* and switch addresses back around in case the caller was planning on using MDP structure again (this happens if there is a loop-back reply and the frame needs sending on, as happens with broadcasts. MDP ping diff --git a/serval.h b/serval.h index 58cf4fc2..6ae84980 100755 --- a/serval.h +++ b/serval.h @@ -1519,3 +1519,7 @@ int stopAudio(); #define SERVER_NOTRUNNING 3 #define SERVER_RUNNING 4 int server_probe(int *pid); + +int dna_helper_enqueue(char *did, unsigned char *requestorSid); +int dna_return_resolution(overlay_mdp_frame *mdp, unsigned char *fromSid, + const char *did,const char *name,const char *uri); From dceeed8d3545c842b0c1bc4905e1e863a92156f9 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 10:30:08 +0930 Subject: [PATCH 18/37] more work on dna helper interface. Also added missing dna_helper.c file from previous changes. moved sigpipe/sigio detection code to own file. --- Android.mk | 1 + Makefile.in | 1 + dna_helper.c | 118 +++++++++++++++++++++++++++++++++++++++++++++++++ rhizome_http.c | 15 ------- serval.h | 5 +++ sighandlers.c | 33 ++++++++++++++ 6 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 dna_helper.c create mode 100644 sighandlers.c diff --git a/Android.mk b/Android.mk index 574600a5..eb6a0577 100644 --- a/Android.mk +++ b/Android.mk @@ -43,6 +43,7 @@ SERVALD_SRC_FILES = \ serval-dna/vomp.c \ serval-dna/lsif.c \ serval-dna/dna_helper.c \ + serval-dna/sighandlers.c \ serval-dna/monitor.c \ serval-dna/monitor-cli.c \ serval-dna/codecs.c \ diff --git a/Makefile.in b/Makefile.in index 3544a9c9..0796bd36 100644 --- a/Makefile.in +++ b/Makefile.in @@ -46,6 +46,7 @@ SRCS= main.c \ monitor.c \ monitor-cli.c \ dna_helper.c \ + sighandlers.c \ codecs.c \ audiodevices.c \ audio_msm_g1.c \ diff --git a/dna_helper.c b/dna_helper.c new file mode 100644 index 00000000..e692b2ee --- /dev/null +++ b/dna_helper.c @@ -0,0 +1,118 @@ +/* +Serval Distributed Numbering Architecture (DNA) +Copyright (C) 2012 Paul Gardner-Stephen + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "serval.h" + +/* + The challenge with making an interface for calling an external program to + resolve a DID into a URI is that it really should be asynchronous, so that + servald can't pause due to delays in looking up DIDs by helper applications. + + This can be partially mitigated by having a cache, so that at least for repeated + requests the helper doesn't need to be called each time. This is very important + because the DNA protocol relies on pre-emptive retries to ensure reception of + a request over a lossy network. + + The second part of the solution is to create an asynchronous queue for requests, + by passing them via file descriptor to a single persistent instance of the DNA + helper application, and polling the output of that application for results, and + then passing them out to their destinations. This ensures that the process is + asynchronous and non-blocking, regardless of how much time the helper application + requires. Then the helper will just be another file descriptor to poll in the + main loop. + */ + +int dna_helper_stdin=-1; +int dna_helper_stdout=-1; + +int dna_helper_start(const char *command) +{ + return -1; +} + +int dna_helper_enqueue(char *did, unsigned char *requestorSid) +{ + /* Check if we have a helper configured. If not, then set + dna_helper_stdin to magic value of -2 so that we don't waste time + in future looking up the dna helper configuration value. */ + if (dna_helper_stdin==-2) return -1; + if (dna_helper_stdin==-1) { + const char *dna_helper = confValueGet("dna.helper",NULL); + if (!dna_helper||!dna_helper[0]) { + dna_helper_stdin=-2; + return -1; + } + + /* Okay, so we have a helper configured. + Run it */ + dna_helper_start(dna_helper); + if (dna_helper_stdin<0) + return -1; + } + + /* Write request to dna helper. + Request takes form: DIDSID-of-Requestor\n + By passing the requestor's SID to the helper, we don't need to maintain + any state, as all we have to do is wait for responses from the helper, + which will include the requestor's SID. + */ + signal(SIGPIPE,sigPipeHandler); + sigPipeFlag=0; + write(dna_helper_stdin,did,strlen(did)); + write(dna_helper_stdin," ",1); + write(dna_helper_stdin,overlay_render_sid(requestorSid),SID_SIZE*2); + write(dna_helper_stdin,"\n",1); + if (sigPipeFlag) { + /* Assume broken pipe due to dead helper. + Next request will cause it to be restarted. + (Losing the current request is not a big problem, because + DNA preemptively retries, anyway. + XXX In fact, we should probably have a limit to the number of restarts + in quick succession so that we don't waste lots of time with a buggy or + suicidal helper. + */ + close(dna_helper_stdin); + close(dna_helper_stdout); + dna_helper_stdin=-1; + dna_helper_stdout=-1; + return -1; + } + + return 0; +} + +int dna_return_resolution(overlay_mdp_frame *mdp, unsigned char *fromSid, + const char *did,const char *name,const char *uri) +{ + /* copy SID out into source address of frame */ + bcopy(fromSid,&mdp->out.src.sid[0],SID_SIZE); + + /* and build reply as did\nname\nURI */ + snprintf((char *)&mdp->out.payload[0],512,"%s\n%s\n%s", + did,name,uri); + mdp->out.payload_length=strlen((char *)mdp->out.payload)+1; + + /* Dispatch response */ + mdp->packetTypeAndFlags&=MDP_FLAG_MASK; + mdp->packetTypeAndFlags|=MDP_TX; + overlay_mdp_dispatch(mdp,0 /* system generated */, + NULL,0); + + return 0; +} diff --git a/rhizome_http.c b/rhizome_http.c index 44582a39..55bb058c 100644 --- a/rhizome_http.c +++ b/rhizome_http.c @@ -31,8 +31,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ int rhizome_server_socket=-1; -int sigPipeFlag=0; -int sigIoFlag=0; rhizome_http_request *rhizome_live_http_requests[RHIZOME_SERVER_MAX_LIVE_REQUESTS]; int rhizome_server_live_request_count=0; @@ -62,19 +60,6 @@ unsigned char favicon_bytes[]={ ,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; int favicon_len=318; -void sigPipeHandler(int signal) -{ - sigPipeFlag++; - return; -} - -void sigIoHandler(int signal) -{ - WHY("sigio"); - sigIoFlag++; - return; -} - int rhizome_server_start() { if (rhizome_server_socket>-1) return 0; diff --git a/serval.h b/serval.h index 6ae84980..fa397eef 100755 --- a/serval.h +++ b/serval.h @@ -1523,3 +1523,8 @@ int server_probe(int *pid); int dna_helper_enqueue(char *did, unsigned char *requestorSid); int dna_return_resolution(overlay_mdp_frame *mdp, unsigned char *fromSid, const char *did,const char *name,const char *uri); + +extern int sigPipeFlag; +extern int sigIoFlag; +void sigPipeHandler(int signal); +void sigIoHandler(int signal); diff --git a/sighandlers.c b/sighandlers.c new file mode 100644 index 00000000..e580a689 --- /dev/null +++ b/sighandlers.c @@ -0,0 +1,33 @@ +/* +Serval Distributed Numbering Architecture (DNA) +Copyright (C) 2012 Paul Gardner-Stephen + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +int sigPipeFlag=0; +int sigIoFlag=0; + +void sigPipeHandler(int signal) +{ + sigPipeFlag++; + return; +} + +void sigIoHandler(int signal) +{ + sigIoFlag++; + return; +} From 89d036822b34942da38ad5d42bc77f4e21ce5af8 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 11:42:34 +0930 Subject: [PATCH 19/37] added code to start dna helper, and detect when it has failed and restart it. --- dna_helper.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dna_helper.c b/dna_helper.c index e692b2ee..716b69fd 100644 --- a/dna_helper.c +++ b/dna_helper.c @@ -43,6 +43,39 @@ int dna_helper_stdout=-1; int dna_helper_start(const char *command) { + int stdin_fds[2]; + int stdout_fds[2]; + + if (pipe(stdin_fds)) return -1; + if (pipe(stdout_fds)) { + close(stdin_fds[0]); close(stdin_fds[1]); + return -1; + } + + int pid=-1; + if ((pid=fork())!=0) { + /* Child, should exec() to become helper after installing file descriptors. */ + if (dup2(stdin_fds[1],0)) exit(-1); /* replace stdin */ + if (dup2(stdout_fds[0],1)) exit(-1); /* replace stdout */ + if (dup2(stdout_fds[0],2)) exit(-1); /* replace stderr */ + execl(command,command,NULL); + /* execl() should never return, since it replaces this process with a new + one. Thus something bad must have happened. */ + exit(-1); + } else { + if (pid==-1) { + /* fork failed */ + close(stdin_fds[0]); close(stdin_fds[1]); + close(stdout_fds[0]); close(stdout_fds[1]); + return -1; + } else { + /* Parent, should put file descriptors into place for use */ + dna_helper_stdin=stdin_fds[0]; + dna_helper_stdout=stdout_fds[1]; + return 0; + } + } + return -1; } From 50eaeca51a8c3d80eb97b30bedd12d116e8b4c69 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 11:43:01 +0930 Subject: [PATCH 20/37] added sanity checking of poll timeout to prevent infinite wait if any ticks are overdue (this whole thing will be revamped soon to be simpler and more efficient, so this is a temporary solution). --- overlay.c | 1 + 1 file changed, 1 insertion(+) diff --git a/overlay.c b/overlay.c index 30ca8adf..7e32d987 100644 --- a/overlay.c +++ b/overlay.c @@ -235,6 +235,7 @@ int overlayServerMode() DEBUGF("Waiting via poll() for up to %lldms", ms); TIMING_PAUSE(); + if (ms>15000) ms=15000; r = poll(fds, fdcount, ms); TIMING_CHECK(); From 2cb84bb8dbdb2ca3ac1beeeffecc1bdcbb215a40 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 12:20:10 +0930 Subject: [PATCH 21/37] various fixes to new DNA URI response relating to parsing replies that lacked DID and/or name field. --- commandline.c | 51 +++++++++++++++++++++++++-------------------------- dna_helper.c | 26 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/commandline.c b/commandline.c index d7194605..3a683ea6 100644 --- a/commandline.c +++ b/commandline.c @@ -491,24 +491,24 @@ int app_dna_lookup(int argc, const char *const *argv, struct command_line_option char did[512]; char name[512]; char uri[512]; - /* Check if reply is validly formatted */ - if (sscanf((char *)rx.in.payload,"%[^\n]\n%[^\n]\n%400s", - did,name,uri)==3) { - /* Have we seen this response before? */ - int i; - for(i=0;i=count||i>=511) return WHY("DNA response does not contain name field"); + l=0; i++; + for(;i<511&&i=count||i>=511) return WHY("DNA response does not contain URI field"); + l=0; i++; + for(;i<511&&i Date: Thu, 21 Jun 2012 12:21:25 +0930 Subject: [PATCH 22/37] various simplifications of DNA reply generation and related code. --- overlay_mdp.c | 65 ++++++++++++++++++++++++++++++--------------------- serval.h | 2 ++ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/overlay_mdp.c b/overlay_mdp.c index 032bd1ba..bd4adf54 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -476,10 +476,22 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) } if (match>-1) { struct sockaddr_un addr; + printf("unix domain socket '%s'\n",mdp_bindings_sockets[match]); bcopy(mdp_bindings_sockets[match],&addr.sun_path[0],mdp_bindings_socket_name_lengths[match]); addr.sun_family=AF_UNIX; - int r=sendto(mdp_named_socket,mdp,overlay_mdp_relevant_bytes(mdp),0,(struct sockaddr*)&addr,sizeof(addr)); - if (r==overlay_mdp_relevant_bytes(mdp)) return 0; + errno=0; + int len=overlay_mdp_relevant_bytes(mdp); + int r=sendto(mdp_named_socket,mdp,len,0,(struct sockaddr*)&addr,sizeof(addr)); + if (r==overlay_mdp_relevant_bytes(mdp)) { + dump("Sent to MDP client",(unsigned char *)mdp,len); + perror("errno after sendto()"); + return 0; + } + if (errno==ENOENT) { + /* far-end of socket has died, so drop binding */ + printf("Closing dead MDP client '%s'\n",mdp_bindings_sockets[match]); + overlay_mdp_releasebindings(&addr,mdp_bindings_socket_name_lengths[match]); + } WHY_perror("sendto(e)"); return WHY("Failed to pass received MDP frame to client"); } else { @@ -505,13 +517,23 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) /* make sure it is null terminated */ did[pll]=0; /* remember source sid for putting back later */ - unsigned char srcsid[32]; - bcopy(&mdp->out.src.sid[0],&srcsid[0],SID_SIZE); - /* now switch addresses around for any replies */ - overlay_mdp_swap_src_dst(mdp); + overlay_mdp_frame mdpreply; + int results=0; while(keyring_find_did(keyring,&cn,&in,&kp,did)) { + bzero(&mdpreply,sizeof(mdpreply)); + + /* mark as outgoing MDP message */ + mdp->packetTypeAndFlags&=MDP_FLAG_MASK; + mdp->packetTypeAndFlags|=MDP_TX; + + /* Set source and destination addresses */ + bcopy(&mdp->out.dst.sid,mdpreply.out.src.sid,SID_SIZE); + bcopy(&mdp->out.src.sid,mdpreply.out.dst.sid,SID_SIZE); + mdpreply.out.src.port=mdp->out.dst.port; + mdpreply.out.dst.port=mdp->out.src.port; + /* package DID and Name into reply (we include the DID because it could be a wild-card DID search, but the SID is implied in the source address of our reply). */ @@ -530,18 +552,15 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) (char *)keyring->contexts[cn]->identities[in]->keypairs[kp] ->public_key; /* copy SID out into source address of frame */ - bcopy(packedSid,&mdp->out.src.sid[0],SID_SIZE); + bcopy(packedSid,&mdpreply.out.src.sid[0],SID_SIZE); /* and build reply as did\nname\nURI */ - snprintf((char *)&mdp->out.payload[0],512,"%s\n%s\nsid://%s/%s", + snprintf((char *)&mdpreply.out.payload[0],512,"%s\n%s\nsid://%s/%s", unpackedDid,name,overlay_render_sid(packedSid), unpackedDid); - mdp->out.payload_length=strlen((char *)mdp->out.payload)+1; + mdpreply.out.payload_length=strlen((char *)mdpreply.out.payload)+1; - /* mark as outgoing MDP message */ - mdp->packetTypeAndFlags&=MDP_FLAG_MASK; - mdp->packetTypeAndFlags|=MDP_TX; - overlay_mdp_dispatch(mdp,0 /* system generated */, - NULL,0); + /* deliver reply */ + overlay_mdp_dispatch(&mdpreply,0 /* system generated */,NULL,0); kp++; results++; } @@ -556,18 +575,8 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) when results become available, so this function will return immediately, so as not to cause blockages and delays in servald. */ - dna_helper_enqueue(did,mdp->out.dst.sid); + dna_helper_enqueue(did,mdp->out.src.sid); } - /* and switch addresses back around in case the caller was planning on - using MDP structure again (this happens if there is a loop-back reply - and the frame needs sending on, as happens with broadcasts. MDP ping - is a simple application where this occurs). - Similarly restore destination address & MDP payload content and - length */ - overlay_mdp_swap_src_dst(mdp); - bcopy(&srcsid[0],&mdp->out.src.sid[0],SID_SIZE); - bcopy(&did[0],&mdp->out.payload[0],pll); - mdp->out.payload_length=pll; return 0; } break; @@ -610,7 +619,8 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) default: /* Unbound socket. We won't be sending ICMP style connection refused messages, partly because they are a waste of bandwidth. */ - return WHY("Received packet for which no listening process exists"); + return WHYF("Received packet for which no listening process exists (MDP ports: src=%d, dst=%d", + mdp->out.src.port,mdp->out.dst.port); } } break; @@ -682,6 +692,8 @@ int overlay_mdp_sanitytest_sourceaddr(sockaddr_mdp *src,int userGeneratedFrameP, printf("addr=%s port=%u (0x%x)\n", overlay_render_sid(src->sid),src->port,src->port); + if (recvaddr) printf("recvaddr='%s'\n", + recvaddr->sun_path); return WHY("No such socket binding:unix domain socket tuple exists -- someone might be trying to spoof someone else's connection"); } @@ -1151,6 +1163,7 @@ int overlay_mdp_send(overlay_mdp_frame *mdp,int flags,int timeout_ms) return WHY("Timeout waiting for server response"); } + int ttl=-1; if (!overlay_mdp_recv(mdp,&ttl)) { /* If all is well, examine result and return error code provided */ diff --git a/serval.h b/serval.h index fa397eef..be3e23aa 100755 --- a/serval.h +++ b/serval.h @@ -1523,6 +1523,8 @@ int server_probe(int *pid); int dna_helper_enqueue(char *did, unsigned char *requestorSid); int dna_return_resolution(overlay_mdp_frame *mdp, unsigned char *fromSid, const char *did,const char *name,const char *uri); +int parseDnaReply(unsigned char *bytes,int count, + char *did,char *name,char *uri); extern int sigPipeFlag; extern int sigIoFlag; From 2aa06ea84b940eaf99dfbc3c4458aad9bc48eecc Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 13:28:21 +0930 Subject: [PATCH 23/37] added new test for dnaprotocol --- tests/dnaprotocol | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/dnaprotocol diff --git a/tests/dnaprotocol b/tests/dnaprotocol new file mode 100644 index 00000000..aed2d87c --- /dev/null +++ b/tests/dnaprotocol @@ -0,0 +1,60 @@ +#!/bin/bash + +# Tests for Serval DNA server operations. +# +# Copyright 2012 Paul Gardner-Stephen +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +source "${0%/*}/../testframework.sh" +source "${0%/*}/../testdefs.sh" + +setup() { + setup_servald + assert_no_servald_processes + set_instance +} + +teardown() { + stop_all_servald_servers + kill_all_servald_processes + assert_no_servald_processes +} + +setup_servald_instance() { + setup_servald + set_instance + touch "$SERVALINSTANCE_PATH/dummynet" + executeOk_servald config set interfaces "+>dummynet" + executeOk_servald keyring add + assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] + executeOk_servald keyring list + tfw_cat --stdout + sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") + assert --message='main identity known' [ -n "$sid" ] + executeOk_servald set did $sid 1234567 "Agent Smith" + start_servald_server +} + +doc_ServerStartCreateInstanceDir="DNA lookup of wildcard finds self" +setup_ServerStartCreateInstanceDir() { + setup +} +test_ServerStartCreateInstanceDir() { + setup_servald_instance + executeOk_servald dna lookup "\*" +} + +runTests "$@" From 2a6cdc0389398e8f7f6417796cf87804495f02f1 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 13:36:27 +0930 Subject: [PATCH 24/37] got wildcard search for self test functioning. --- tests/dnaprotocol | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 tests/dnaprotocol diff --git a/tests/dnaprotocol b/tests/dnaprotocol old mode 100644 new mode 100755 index aed2d87c..7014951b --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -48,13 +48,14 @@ setup_servald_instance() { start_servald_server } -doc_ServerStartCreateInstanceDir="DNA lookup of wildcard finds self" -setup_ServerStartCreateInstanceDir() { +doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self" +setup_DNAWildcardSearchFindsSelf() { setup } -test_ServerStartCreateInstanceDir() { +test_DNAWildcardSearchFindsSelf() { setup_servald_instance executeOk_servald dna lookup "\*" + assertStdoutLineCount '==' 1 } runTests "$@" From c3649c415f90f1c53db9228621e8060dcb05e6d9 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 13:37:34 +0930 Subject: [PATCH 25/37] fixed a bug with DNA response generation. --- overlay_mdp.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/overlay_mdp.c b/overlay_mdp.c index bd4adf54..c45b02e7 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -525,8 +525,7 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) bzero(&mdpreply,sizeof(mdpreply)); /* mark as outgoing MDP message */ - mdp->packetTypeAndFlags&=MDP_FLAG_MASK; - mdp->packetTypeAndFlags|=MDP_TX; + mdpreply.packetTypeAndFlags=MDP_TX; /* Set source and destination addresses */ bcopy(&mdp->out.dst.sid,mdpreply.out.src.sid,SID_SIZE); From fefe7d2e912b365b9b5fb07fa8a625fb2298e400 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 13:43:28 +0930 Subject: [PATCH 26/37] fixed dna query bugs and split wildcard query test into star and empty string versions. --- overlay_mdp.c | 4 +--- tests/dnaprotocol | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/overlay_mdp.c b/overlay_mdp.c index c45b02e7..a707a846 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -483,8 +483,6 @@ int overlay_saw_mdp_frame(int interface, overlay_mdp_frame *mdp,long long now) int len=overlay_mdp_relevant_bytes(mdp); int r=sendto(mdp_named_socket,mdp,len,0,(struct sockaddr*)&addr,sizeof(addr)); if (r==overlay_mdp_relevant_bytes(mdp)) { - dump("Sent to MDP client",(unsigned char *)mdp,len); - perror("errno after sendto()"); return 0; } if (errno==ENOENT) { @@ -1097,7 +1095,7 @@ int overlay_mdp_relevant_bytes(overlay_mdp_frame *mdp) end of the string, to avoid information leaks */ len=&mdp->error.message[0]-(char *)mdp; len+=strlen(mdp->error.message)+1; - INFOF("mdp return/error code: %d:%s",mdp->error.error,mdp->error.message); + if (mdp->error.error) INFOF("mdp return/error code: %d:%s",mdp->error.error,mdp->error.message); break; case MDP_VOMPEVENT: /* XXX too hard to work out precisely for now. */ diff --git a/tests/dnaprotocol b/tests/dnaprotocol index 7014951b..5c8d1d39 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -48,14 +48,25 @@ setup_servald_instance() { start_servald_server } -doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self" +doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self (star method)" setup_DNAWildcardSearchFindsSelf() { setup } test_DNAWildcardSearchFindsSelf() { setup_servald_instance - executeOk_servald dna lookup "\*" + executeOk_servald dna lookup "*" assertStdoutLineCount '==' 1 } +doc_DNAWildcardSearchFindsSelfEmpty="DNA lookup of wildcard finds self (empty query method)" +setup_DNAWildcardSearchFindsSelfEmpty() { + setup +} +test_DNAWildcardSearchFindsSelfEmpty() { + setup_servald_instance + executeOk_servald dna lookup "" + assertStdoutLineCount '==' 1 +} + + runTests "$@" From 5d3e9a21f7884470ed5a321f390dce03638c8b44 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 13:58:10 +0930 Subject: [PATCH 27/37] fixed bug with interface specifications to allow - in dummynet paths. --- overlay_interface.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/overlay_interface.c b/overlay_interface.c index 46a796ef..d9efdf56 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -114,7 +114,7 @@ int overlay_interface_arg(char *arg) if (!r) return WHY("calloc(struct interface rules),1) failed"); - if (sscanf(arg,"%[+-]%n%[^=+-]%n=%[^:]%n:%d%n:%[^:]%n", + if (sscanf(arg,"%[+-]%n%[^=:,]%n=%[^:]%n:%d%n:%[^:]%n", sign,&n,interface_name,&n,typestring,&n,&port,&n,speed,&n)>=1) { if (n') { I(fileP)=1; char dummyfile[1024]; - if (!FORM_SERVAL_INSTANCE_PATH(dummyfile, &name[1]) || (I(fd) = open(dummyfile,O_APPEND|O_RDWR)) < 1) { - return WHY("could not open dummy interface file for append"); - } + if (name[1]=='/') { + /* Absolute path */ + snprintf(dummyfile,1024,"%s",&name[1]); + } else + /* Relative to instance path */ + if (!FORM_SERVAL_INSTANCE_PATH(dummyfile, &name[1]) || (I(fd) = open(dummyfile,O_APPEND|O_RDWR)) < 1) { + return WHY("could not open dummy interface file for append"); + } /* Seek to end of file as initial reading point */ I(offset)=lseek(I(fd),0,SEEK_END); /* socket gets reused to hold file offset */ /* XXX later add pretend location information so that we can decide which "packets" to receive From 22ebdbc1ae9a8f89399a6b4037f6b14fe9fdb3a5 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 13:58:31 +0930 Subject: [PATCH 28/37] work on getting multiple dna instances running in test framework. --- tests/dnaprotocol | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/dnaprotocol b/tests/dnaprotocol index 5c8d1d39..eef8f362 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -35,7 +35,7 @@ teardown() { setup_servald_instance() { setup_servald - set_instance + set_instance A touch "$SERVALINSTANCE_PATH/dummynet" executeOk_servald config set interfaces "+>dummynet" executeOk_servald keyring add @@ -44,8 +44,20 @@ setup_servald_instance() { tfw_cat --stdout sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") assert --message='main identity known' [ -n "$sid" ] - executeOk_servald set did $sid 1234567 "Agent Smith" - start_servald_server + executeOk_servald set did $sid 5550001 "Agent Smith" + start_servald_server A + INSTANCEPATH_A=$SERVALINSTANCE_PATH + + set_instance B + executeOk_servald config set interfaces "+>$INSTANCEPATH_A/dummynet" + executeOk_servald keyring add + assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] + executeOk_servald keyring list + tfw_cat --stdout + sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") + assert --message='main identity known' [ -n "$sid" ] + executeOk_servald set did $sid 5552662 "Neo" + start_servald_server B } doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self (star method)" From 01712ab67e3233640e114a335b27ca5de13398a8 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 14:35:05 +0930 Subject: [PATCH 29/37] monitor.socket config option added to allow multiple servald instances on the same machine without conflicting abstract namespace unix domain sockets for monitor interface. --- monitor-cli.c | 3 ++- monitor.c | 14 ++++++++++---- serval.h | 2 ++ tests/dnaprotocol | 7 +++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/monitor-cli.c b/monitor-cli.c index 23cc9c77..739c5d0f 100644 --- a/monitor-cli.c +++ b/monitor-cli.c @@ -77,7 +77,8 @@ int app_monitor_cli(int argc, const char *const *argv, struct command_line_optio memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; addr.sun_path[0]=0; - snprintf(&addr.sun_path[1],100,"org.servalproject.servald.monitor.socket"); + snprintf(&addr.sun_path[1],100, + confValueGet("monitor.socket",DEFAULT_MONITOR_SOCKET_NAME)); int len = 1+strlen(&addr.sun_path[1]) + sizeof(addr.sun_family); char *p=(char *)&addr; printf("last char='%c' %02x\n",p[len-1],p[len-1]); diff --git a/monitor.c b/monitor.c index 8ba7c033..a6db2bf1 100644 --- a/monitor.c +++ b/monitor.c @@ -84,14 +84,19 @@ int monitor_setup_sockets() } #ifdef linux - /* Use abstract namespace as Android has no writable FS which supports sockets */ + /* Use abstract namespace as Android has no writable FS which supports sockets. + Abstract namespace is just plain better, anyway, as no dead files end up + hanging around. */ name.sun_path[0]=0; /* XXX: 104 comes from OSX sys/un.h - no #define (note Linux has UNIX_PATH_MAX and it's 108(!)) */ - snprintf(&name.sun_path[1],104-2,"org.servalproject.servald.monitor.socket"); + snprintf(&name.sun_path[1],104-2, + confValueGet("monitor.socket",DEFAULT_MONITOR_SOCKET_NAME)); /* Doesn't include trailing nul */ len = 1+strlen(&name.sun_path[1]) + sizeof(name.sun_family); #else - snprintf(name.sun_path,104-1,"%s/org.servalproject.servald.monitor.socket",serval_instancepath()); + snprintf(name.sun_path,104-1,"%s/", + serval_instancepath(), + confValueGet("monitor.socket",DEFAULT_MONITOR_SOCKET_NAME)); unlink(name.sun_path); /* Includes trailing nul */ len = 1+strlen(name.sun_path) + sizeof(name.sun_family); @@ -723,7 +728,8 @@ int server_probe(int *pid) memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; addr.sun_path[0]=0; - snprintf(&addr.sun_path[1],100,"org.servalproject.servald.monitor.socket"); + snprintf(&addr.sun_path[1],100, + confValueGet("monitor.socket",DEFAULT_MONITOR_SOCKET_NAME)); int len = 1+strlen(&addr.sun_path[1]) + sizeof(addr.sun_family); char *p=(char *)&addr; if (0) DEBUGF("last char='%c' %02x\n",p[len-1],p[len-1]); diff --git a/serval.h b/serval.h index be3e23aa..51b34c5d 100755 --- a/serval.h +++ b/serval.h @@ -1530,3 +1530,5 @@ extern int sigPipeFlag; extern int sigIoFlag; void sigPipeHandler(int signal); void sigIoHandler(int signal); + +#define DEFAULT_MONITOR_SOCKET_NAME "org.servalproject.servald.monitor.socket" diff --git a/tests/dnaprotocol b/tests/dnaprotocol index eef8f362..7cf4bf29 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -38,6 +38,7 @@ setup_servald_instance() { set_instance A touch "$SERVALINSTANCE_PATH/dummynet" executeOk_servald config set interfaces "+>dummynet" + executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.A" executeOk_servald keyring add assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] executeOk_servald keyring list @@ -46,10 +47,13 @@ setup_servald_instance() { assert --message='main identity known' [ -n "$sid" ] executeOk_servald set did $sid 5550001 "Agent Smith" start_servald_server A + executeOk_servald status + tfw_cat --stdout INSTANCEPATH_A=$SERVALINSTANCE_PATH set_instance B executeOk_servald config set interfaces "+>$INSTANCEPATH_A/dummynet" + executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.B" executeOk_servald keyring add assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] executeOk_servald keyring list @@ -57,7 +61,10 @@ setup_servald_instance() { sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") assert --message='main identity known' [ -n "$sid" ] executeOk_servald set did $sid 5552662 "Neo" + echo "# A/B Instance Paths: $SERVALINSTANCE_PATH $INSTANCEPATH_A" + tfw_cat --stdout start_servald_server B + executeOk_servald status } doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self (star method)" From 48a1d7e20ff65db13f55ab391601e7a0aaa54b0f Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 14:45:09 +0930 Subject: [PATCH 30/37] MDP socket name can now be changed by config option mdp.socket. DNA tests now passing again. --- overlay_mdp.c | 3 ++- serval.h | 1 + tests/dnaprotocol | 5 ++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/overlay_mdp.c b/overlay_mdp.c index a707a846..89e6a2fd 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -42,7 +42,8 @@ int overlay_mdp_setup_sockets() /* XXX The 100 should be replaced with the actual maximum allowed. Apparently POSIX requires it to be at least 100, but I would still feel more comfortable with using the appropriate constant. */ - snprintf(&name.sun_path[1],100,"org.servalproject.mesh.overlay.mdp"); + snprintf(&name.sun_path[1],100, + confValueGet("mdp.socket",DEFAULT_MDP_SOCKET_NAME); len = 1+strlen(&name.sun_path[1]) + sizeof(name.sun_family); mdp_abstract_socket = socket(AF_UNIX, SOCK_DGRAM, 0); diff --git a/serval.h b/serval.h index 51b34c5d..c6255373 100755 --- a/serval.h +++ b/serval.h @@ -1532,3 +1532,4 @@ void sigPipeHandler(int signal); void sigIoHandler(int signal); #define DEFAULT_MONITOR_SOCKET_NAME "org.servalproject.servald.monitor.socket" +#define DEFAULT_MDP_SOCKET_NAME "org.servalproject.servald.mdp.socket" diff --git a/tests/dnaprotocol b/tests/dnaprotocol index 7cf4bf29..fefcb87a 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -28,7 +28,8 @@ setup() { } teardown() { - stop_all_servald_servers +# TODO Disabled until stopping multiple-instances are supported +# TODO stop_all_servald_servers kill_all_servald_processes assert_no_servald_processes } @@ -39,6 +40,7 @@ setup_servald_instance() { touch "$SERVALINSTANCE_PATH/dummynet" executeOk_servald config set interfaces "+>dummynet" executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.A" + executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.A" executeOk_servald keyring add assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] executeOk_servald keyring list @@ -54,6 +56,7 @@ setup_servald_instance() { set_instance B executeOk_servald config set interfaces "+>$INSTANCEPATH_A/dummynet" executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.B" + executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.B" executeOk_servald keyring add assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] executeOk_servald keyring list From 5e338f6285a6b4852daf1a30582d77b4e41f2ea3 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 15:04:34 +0930 Subject: [PATCH 31/37] make tests faster to run. --- testdefs.sh | 15 +++++++------ tests/dnaprotocol | 54 ++++++++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/testdefs.sh b/testdefs.sh index 076d464c..e2b4e608 100644 --- a/testdefs.sh +++ b/testdefs.sh @@ -190,13 +190,16 @@ signal_all_servald_processes() { # first SIGTERM then SIGHUP and finally SIGKILL # - assert that no more servald processes are running kill_all_servald_processes() { - if signal_all_servald_processes TERM; then - sleep 2 - if signal_all_servald_processes HUP; then - sleep 2 +# PGS 20120621 - Made fast so that tests can be run quickly +# TODO: Make this better by checking that things really have died, and take +# exactly the time required, rather than using fixed delays +# if signal_all_servald_processes TERM; then +# sleep 2 +# if signal_all_servald_processes HUP; then +# sleep 2 signal_all_servald_processes KILL - fi - fi +# fi +# fi } # Utility function: diff --git a/tests/dnaprotocol b/tests/dnaprotocol index fefcb87a..dabb7ea6 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -35,12 +35,11 @@ teardown() { } setup_servald_instance() { - setup_servald - set_instance A + set_instance $1 touch "$SERVALINSTANCE_PATH/dummynet" - executeOk_servald config set interfaces "+>dummynet" - executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.A" - executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.A" + executeOk_servald config set interfaces "+>$2" + executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.$1" + executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.$1" executeOk_servald keyring add assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] executeOk_servald keyring list @@ -48,26 +47,33 @@ setup_servald_instance() { sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") assert --message='main identity known' [ -n "$sid" ] executeOk_servald set did $sid 5550001 "Agent Smith" - start_servald_server A + start_servald_server $1 executeOk_servald status tfw_cat --stdout - INSTANCEPATH_A=$SERVALINSTANCE_PATH +} - set_instance B - executeOk_servald config set interfaces "+>$INSTANCEPATH_A/dummynet" - executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.B" - executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.B" - executeOk_servald keyring add - assert [ -e "$SERVALINSTANCE_PATH/serval.keyring" ] - executeOk_servald keyring list - tfw_cat --stdout - sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") - assert --message='main identity known' [ -n "$sid" ] - executeOk_servald set did $sid 5552662 "Neo" - echo "# A/B Instance Paths: $SERVALINSTANCE_PATH $INSTANCEPATH_A" - tfw_cat --stdout - start_servald_server B - executeOk_servald status +setup_servald_instances() { + setup_servald + setup_servald_instance A dummynet + DUMMYNET=$SERVALINSTANCE_PATH/dummynet + setup_servald_instance B $DUMMYNET + + # Now make sure that they can see each other + sleep 2.5 # Should be plenty of time + set_instance A + echo "Dummynet file $DUMMYNET after 5 seconds: "`ls -l $DUMMYNET` + executeOk_servald id peers + assertStdoutLineCount '==' 1 +} + +doc_MultipleServalDTest="Can start multiple servald instances communicating via a dummy interface" + +setup_MultipleServalDTest() { + setup +} + +test_MultipleServalDTest() { + setup_servald_instances } doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self (star method)" @@ -75,7 +81,7 @@ setup_DNAWildcardSearchFindsSelf() { setup } test_DNAWildcardSearchFindsSelf() { - setup_servald_instance + setup_servald_instances executeOk_servald dna lookup "*" assertStdoutLineCount '==' 1 } @@ -85,7 +91,7 @@ setup_DNAWildcardSearchFindsSelfEmpty() { setup } test_DNAWildcardSearchFindsSelfEmpty() { - setup_servald_instance + setup_servald_instances executeOk_servald dna lookup "" assertStdoutLineCount '==' 1 } From 136a7bcf56bca7314b4c562a9f96689322691ad0 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 15:52:44 +0930 Subject: [PATCH 32/37] fixed bug with absolute-path dummynet files. added pid of producer to dummynet packets. --- overlay_interface.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/overlay_interface.c b/overlay_interface.c index d9efdf56..b1a24de4 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -276,9 +276,13 @@ int overlay_interface_init(char *name,struct sockaddr_in src_addr,struct sockadd snprintf(dummyfile,1024,"%s",&name[1]); } else /* Relative to instance path */ - if (!FORM_SERVAL_INSTANCE_PATH(dummyfile, &name[1]) || (I(fd) = open(dummyfile,O_APPEND|O_RDWR)) < 1) { - return WHY("could not open dummy interface file for append"); - } + if (!FORM_SERVAL_INSTANCE_PATH(dummyfile, &name[1])) + return WHY("could not form dummy interfance name"); + + if ((I(fd) = open(dummyfile,O_APPEND|O_RDWR)) < 1) { + return WHY("could not open dummy interface file for append"); + } + /* Seek to end of file as initial reading point */ I(offset)=lseek(I(fd),0,SEEK_END); /* socket gets reused to hold file offset */ /* XXX later add pretend location information so that we can decide which "packets" to receive @@ -428,9 +432,14 @@ int overlay_broadcast_ensemble(int interface_number, if (overlay_interfaces[interface_number].fileP) { char buf[2048]; + bzero(&buf[0],128); /* Version information */ buf[0]=1; buf[1]=0; buf[2]=0; buf[3]=0; + /* PID of creator */ + buf[4]=getpid()&0xff; buf[5]=getpid()>>8; + + /* TODO make a structure for all this stuff */ /* bytes 4-5 = half-power beam height (uint16) */ /* bytes 6-7 = half-power beam width (uint16) */ /* bytes 8-11 = range in metres, centre beam (uint32) */ From fa1cef7f3bd364252356ccccdb5af6454292b370 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 15:58:39 +0930 Subject: [PATCH 33/37] fixed bugs in dnaprotocol test script to work. --- tests/dnaprotocol | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/dnaprotocol b/tests/dnaprotocol index dabb7ea6..8c7a4254 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -36,7 +36,6 @@ teardown() { setup_servald_instance() { set_instance $1 - touch "$SERVALINSTANCE_PATH/dummynet" executeOk_servald config set interfaces "+>$2" executeOk_servald config set monitor.socket "org.servalproject.servald.monitor.socket.$1" executeOk_servald config set mdp.socket "org.servalproject.servald.mdp.socket.$1" @@ -46,7 +45,7 @@ setup_servald_instance() { tfw_cat --stdout sid=$(replayStdout | sed -ne "1s/^\($rexp_sid\):.*\$/\1/p") assert --message='main identity known' [ -n "$sid" ] - executeOk_servald set did $sid 5550001 "Agent Smith" + executeOk_servald set did $sid 5550001 "Agent $1 Smith" start_servald_server $1 executeOk_servald status tfw_cat --stdout @@ -54,16 +53,18 @@ setup_servald_instance() { setup_servald_instances() { setup_servald - setup_servald_instance A dummynet - DUMMYNET=$SERVALINSTANCE_PATH/dummynet + DUMMYNET=/tmp/dummy + touch $DUMMYNET + assert [ -e $DUMMYNET ] + setup_servald_instance A $DUMMYNET setup_servald_instance B $DUMMYNET # Now make sure that they can see each other - sleep 2.5 # Should be plenty of time + sleep 5 # Should be plenty of time set_instance A echo "Dummynet file $DUMMYNET after 5 seconds: "`ls -l $DUMMYNET` executeOk_servald id peers - assertStdoutLineCount '==' 1 + assertStdoutLineCount '==' 1 } doc_MultipleServalDTest="Can start multiple servald instances communicating via a dummy interface" @@ -76,24 +77,30 @@ test_MultipleServalDTest() { setup_servald_instances } -doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds self (star method)" +doc_DNAWildcardSearchFindsSelf="DNA lookup of wildcard finds (star method)" setup_DNAWildcardSearchFindsSelf() { setup } test_DNAWildcardSearchFindsSelf() { setup_servald_instances + set_instance A executeOk_servald dna lookup "*" - assertStdoutLineCount '==' 1 + assertStdoutLineCount '==' 2 + assertStdoutGrep --matches=1 "^sid://.*/5550001:5550001:Agent A Smith$" + assertStdoutGrep --matches=1 "^sid://.*/5550001:5550001:Agent B Smith$" } -doc_DNAWildcardSearchFindsSelfEmpty="DNA lookup of wildcard finds self (empty query method)" +doc_DNAWildcardSearchFindsSelfEmpty="DNA lookup of wildcard (empty query method)" setup_DNAWildcardSearchFindsSelfEmpty() { setup } test_DNAWildcardSearchFindsSelfEmpty() { setup_servald_instances + set_instance A executeOk_servald dna lookup "" - assertStdoutLineCount '==' 1 + assertStdoutLineCount '==' 2 + assertStdoutGrep --matches=1 "^sid://.*/5550001:5550001:Agent A Smith$" + assertStdoutGrep --matches=1 "^sid://.*/5550001:5550001:Agent B Smith$" } From 0254ca0ab8ed23b517265904292f8b42afa066f9 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 16:04:14 +0930 Subject: [PATCH 34/37] made dnaprotocol test more rigorous to ensure it really returns results from each SID/servald instance. Fixed bug in "keyring list" that was returning keyrings in lower case instead of normalised form using overlay_render_sid() --- commandline.c | 11 ++++++----- tests/dnaprotocol | 10 ++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/commandline.c b/commandline.c index 3a683ea6..a888c36c 100644 --- a/commandline.c +++ b/commandline.c @@ -1437,18 +1437,19 @@ int app_keyring_list(int argc, const char *const *argv, struct command_line_opti { int kpn; keypair *kp; - unsigned char *sid=NULL,*did=NULL; + unsigned char *sid=NULL,*did=NULL,*name=NULL; for(kpn=0;kpncontexts[cn]->identities[in]->keypair_count;kpn++) { kp=k->contexts[cn]->identities[in]->keypairs[kpn]; if (kp->type==KEYTYPE_CRYPTOBOX) sid=kp->public_key; - if (kp->type==KEYTYPE_DID) did=kp->private_key; + if (kp->type==KEYTYPE_DID) { did=kp->private_key; name=kp->public_key; } } - if (sid||did) { - int i; - if (sid) for(i=0;i Date: Thu, 21 Jun 2012 16:08:39 +0930 Subject: [PATCH 35/37] added test to make sure that DNA lookups by number work as advertised. --- tests/dnaprotocol | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/dnaprotocol b/tests/dnaprotocol index 6708fa13..624e6b2f 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -105,5 +105,25 @@ test_DNAWildcardSearchFindsSelfEmpty() { assertStdoutGrep --matches=1 "^sid://$SIDB/5550001:5550001:Agent B Smith$" } +doc_DNASpecificLookup="DNA Lookup by phone number" +setup_DNASpecificLookup() { + setup +} +test_DNASpecificLookup() { + setup_servald_instances + set_instance A + # Make sure we get no false positives + executeOk_servald dna lookup "5551234" + assertStdoutLineCount '==' 0 + executeOk_servald dna lookup "555000" + assertStdoutLineCount '==' 0 + executeOk_servald dna lookup "55500011" + assertStdoutLineCount '==' 0 + # Make sure we get the right results, and no duplicates + executeOk_servald dna lookup "5550001" + assertStdoutLineCount '==' 2 + assertStdoutGrep --matches=1 "^sid://$SIDA/5550001:5550001:Agent A Smith$" + assertStdoutGrep --matches=1 "^sid://$SIDB/5550001:5550001:Agent B Smith$" +} runTests "$@" From 8977142a30ab371f6310a2d40a1be7427c154948 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 16:21:16 +0930 Subject: [PATCH 36/37] fixed "timeout from MDP server" errors in node info output. added extra tests for node info. --- overlay_mdp.c | 2 +- tests/dnaprotocol | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/overlay_mdp.c b/overlay_mdp.c index 89e6a2fd..f20ebb85 100644 --- a/overlay_mdp.c +++ b/overlay_mdp.c @@ -1158,7 +1158,7 @@ int overlay_mdp_send(overlay_mdp_frame *mdp,int flags,int timeout_ms) mdp->packetTypeAndFlags=MDP_ERROR; mdp->error.error=1; snprintf(mdp->error.message,128,"Timeout waiting for reply to MDP packet (packet was successfully sent)."); - return WHY("Timeout waiting for server response"); + return -1; /* WHY("Timeout waiting for server response"); */ } diff --git a/tests/dnaprotocol b/tests/dnaprotocol index 624e6b2f..3729f56c 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -126,4 +126,44 @@ test_DNASpecificLookup() { assertStdoutGrep --matches=1 "^sid://$SIDB/5550001:5550001:Agent B Smith$" } +doc_DNANodeInfoLocalResolution="'node info' auto-resolves for local identities" +setup_DNANodeInfoLocalResolution() { + setup +} +test_DNANodeInfoLocalResolution() { + setup_servald_instances + set_instance A + + # node info for a local identity returns DID/Name since it is free, even + # if it isn't asked for. + executeOk_servald node info $SIDA + assertStdoutLineCount '==' 1 + assertStdoutGrep --matches=1 "Agent A Smith" + assertStdoutGrep --matches=0 "did-not-resolved" +} + +doc_DNANodeInfoRemoteResolution="'node info' for remote identities" +setup_DNANodeInfoRemoteResolution() { + setup +} +test_DNANodeInfoRemoteResolution() { + setup_servald_instances + set_instance A + + # if resolvedid is not specified for a remote identity, then don't resolve + # it. + executeOk_servald node info $SIDB + assertStdoutLineCount '==' 1 + assertStdoutGrep --matches=0 "Agent B Smith" + assertStdoutGrep --matches=1 "did-not-resolved" + + # But if it resolvedid is specified, then do resolve it using DNA + executeOk_servald node info $SIDB resolvedid + assertStdoutLineCount '==' 1 + assertStdoutGrep --matches=1 "Agent B Smith" + assertStdoutGrep --matches=0 "did-not-resolved" + +} + + runTests "$@" From 2d718a297fa6d9eddd41a6ae7fa3c7ece7deb955 Mon Sep 17 00:00:00 2001 From: gardners Date: Thu, 21 Jun 2012 17:02:36 +0930 Subject: [PATCH 37/37] reversed changes that broke DNA tests. Improved dnaprotocol test script slightly. --- overlay.c | 94 ++++++++++++------------- overlay_interface.c | 166 ++++++++++++++++++++++++-------------------- tests/dnaprotocol | 1 + 3 files changed, 135 insertions(+), 126 deletions(-) diff --git a/overlay.c b/overlay.c index 7e32d987..b1f6f9e4 100644 --- a/overlay.c +++ b/overlay.c @@ -88,7 +88,7 @@ void _TIMING_CHECK(const char *file,const char *func,int line) { long long now=overlay_gettime_ms(); if (last_valid) { - if (now-last_time>10) { + if (now-last_time>5) { // More than 5ms spent in a given task, complain char msg[1024]; snprintf(msg,1024,"Spent %lldms between %s:%d in %s() and here", @@ -104,19 +104,6 @@ void _TIMING_CHECK(const char *file,const char *func,int line) last_time=now; } -long long last_loop_time=0; -void LOOP_END() -{ - long long now = overlay_gettime_ms(); - if (last_loop_time!=0 && now - last_loop_time>15){ - DEBUGF("Last loop took %lldms",now - last_loop_time); - } -} - -void LOOP_START() -{ - last_loop_time = overlay_gettime_ms(); -} int overlayMode=0; @@ -159,7 +146,6 @@ int overlayServerMode() struct pollfd fds[128]; int fdcount; - int r; /* Create structures to use 1MB of RAM for testing */ overlay_route_init(1); @@ -227,32 +213,29 @@ int overlayServerMode() int vomp_tick_time=vomp_tick_interval(); if (ms>vomp_tick_time) ms=vomp_tick_time; - LOOP_END(); - - if (ms>0){ - TIMING_CHECK(); - if (debug&DEBUG_VERBOSE_IO) - DEBUGF("Waiting via poll() for up to %lldms", ms); - TIMING_PAUSE(); - - if (ms>15000) ms=15000; - r = poll(fds, fdcount, ms); - - TIMING_CHECK(); - if (r == -1) - WHY_perror("poll"); - else if (debug&DEBUG_VERBOSE_IO) { - DEBUGF("poll() says %d file descriptors are ready", r); - int i; - for(i=0;i15000) ms=15000; + int r = poll(fds, fdcount, ms); + TIMING_CHECK(); + if (r == -1) + WHY_perror("poll"); + else if (debug&DEBUG_VERBOSE_IO) { + DEBUGF("poll() says %d file descriptors are ready", r); + int i; + for(i=0;i 0) { /* We have data, so try to receive it */ if (debug&DEBUG_IO) { @@ -277,28 +260,39 @@ int overlayServerMode() overlay_rx_messages(); TIMING_CHECK(); if (rhizome_enabled()) { + TIMING_CHECK(); rhizome_server_poll(); TIMING_CHECK(); rhizome_fetch_poll(); TIMING_CHECK(); + overlay_mdp_poll(); + TIMING_CHECK(); + monitor_poll(); + TIMING_CHECK(); } - overlay_mdp_poll(); - TIMING_CHECK(); - monitor_poll(); - TIMING_CHECK(); } else { /* No data before tick occurred, so do nothing. Well, for now let's just check anyway. */ if (debug&DEBUG_IO) fprintf(stderr,"poll() timeout.\n"); - - /* Do high-priority audio handling first */ TIMING_CHECK(); - vomp_tick(); - TIMING_CHECK(); - /* Check if we need to trigger any ticks on any interfaces */ - overlay_check_ticks(); + overlay_rx_messages(); TIMING_CHECK(); + if (rhizome_enabled()) { + TIMING_CHECK(); + rhizome_server_poll(); + TIMING_CHECK(); + rhizome_fetch_poll(); + TIMING_CHECK(); + overlay_mdp_poll(); + TIMING_CHECK(); + monitor_poll(); + TIMING_CHECK(); + } } + TIMING_CHECK(); + /* Check if we need to trigger any ticks on any interfaces */ + overlay_check_ticks(); + TIMING_CHECK(); } return 0; diff --git a/overlay_interface.c b/overlay_interface.c index b1a24de4..cd29caa3 100644 --- a/overlay_interface.c +++ b/overlay_interface.c @@ -300,91 +300,105 @@ int overlay_interface_init(char *name,struct sockaddr_in src_addr,struct sockadd int overlay_rx_messages() { int i; - + /* Grab packets, unpackage and dispatch frames to consumers */ /* XXX Okay, so how are we managing out-of-process consumers? - They need some way to register their interest in listening to a port. - */ + They need some way to register their interest in listening to a port. + */ unsigned char packet[16384]; int plen=0; + int c[OVERLAY_MAX_INTERFACES]; + int count=0; - /* Grab one packet from each interface in round-robin fashion */ - for(i=0;i=length) - { - if (debug&DEBUG_OVERLAYINTERFACES) - fprintf(stderr,"At end of input on dummy interface #%d\n",i); - } - else - { - lseek(overlay_interfaces[i].fd,overlay_interfaces[i].offset,SEEK_SET); - if (debug&DEBUG_OVERLAYINTERFACES) - fprintf(stderr,"Reading from interface #%d log at offset %d, end of file at %lld.\n",i, - overlay_interfaces[i].offset,length); - if (read(overlay_interfaces[i].fd,&packet[0],2048)==2048) + /* Look at all interfaces */ + for(i=0;i0); count+=c[i]; } + + /* Grab packets from interfaces in round-robin fashion until all have been grabbed, + or until we have spent too long (maybe 10ms?) */ + int now = overlay_gettime_ms(); + while(count>0) + { + for(i=0;i(2048-128)) plen=-1; - if (debug&DEBUG_PACKETRX) { - fflush(stdout); - serval_packetvisualise(stderr, - "Read from dummy interface", - &packet[128],plen); - fflush(stderr); - } - bzero(&transaction_id[0],8); - bzero(&src_addr,sizeof(src_addr)); - if ((plen>=0)&&(packet[0]==0x01)&&!(packet[1]|packet[2]|packet[3])) { - { if (packetOk(i,&packet[128],plen,transaction_id, - -1 /* fake TTL */, - &src_addr,addrlen,1)) - WHY("Malformed or unsupported packet from dummy interface (packetOK() failed)"); } } - else WHY("Invalid packet version in dummy interface"); - } - else { - if (debug&DEBUG_IO) fprintf(stderr,"Read NOTHING from dummy interface\n"); - } - } - } else { - /* Read from UDP socket */ - int recvttl=1; - plen=recvwithttl(overlay_interfaces[i].fd,packet,sizeof(packet), - &recvttl,&src_addr,&addrlen); - if (plen>=0) { - /* We have a frame from this interface */ - if (debug&DEBUG_PACKETRX) { - fflush(stdout); - serval_packetvisualise(stderr,"Read from real interface", - packet,plen); - fflush(stderr); - } - if (debug&DEBUG_OVERLAYINTERFACES) - fprintf(stderr,"Received %d bytes on interface #%d (%s)\n",plen,i,overlay_interfaces[i].name); + struct sockaddr src_addr; + unsigned int addrlen=sizeof(src_addr); + unsigned char transaction_id[8]; - if (packetOk(i,packet,plen,NULL,recvttl,&src_addr,addrlen,1)) { - WHY("Malformed packet"); - serval_packetvisualise(stderr,"Malformed packet", packet,plen); + overlay_last_interface_number=i; + + /* Set socket non-blocking before we try to read from it */ + fcntl(overlay_interfaces[i].fd, F_SETFL, + fcntl(overlay_interfaces[i].fd, F_GETFL, NULL)|O_NONBLOCK); + + if (overlay_interfaces[i].fileP) { + /* Read from dummy interface file */ + long long length=lseek(overlay_interfaces[i].fd,0,SEEK_END); + if (overlay_interfaces[i].offset>=length) + { + if (debug&DEBUG_OVERLAYINTERFACES) + fprintf(stderr,"At end of input on dummy interface #%d\n",i); + } + else + { + lseek(overlay_interfaces[i].fd,overlay_interfaces[i].offset,SEEK_SET); + if (debug&DEBUG_OVERLAYINTERFACES) + fprintf(stderr,"Reading from interface #%d log at offset %d, end of file at %lld.\n",i, + overlay_interfaces[i].offset,length); + if (read(overlay_interfaces[i].fd,&packet[0],2048)==2048) + { + overlay_interfaces[i].offset+=2048; + plen=2048-128; + plen=packet[110]+(packet[111]<<8); + if (plen>(2048-128)) plen=-1; + if (debug&DEBUG_PACKETRX) { + fflush(stdout); + serval_packetvisualise(stderr, + "Read from dummy interface", + &packet[128],plen); + fflush(stderr); + } + bzero(&transaction_id[0],8); + bzero(&src_addr,sizeof(src_addr)); + if ((plen>=0)&&(packet[0]==0x01)&&!(packet[1]|packet[2]|packet[3])) { + { if (packetOk(i,&packet[128],plen,transaction_id, + -1 /* fake TTL */, + &src_addr,addrlen,1)) + WHY("Malformed or unsupported packet from dummy interface (packetOK() failed)"); } } + else WHY("Invalid packet version in dummy interface"); + } + else { + if (debug&DEBUG_IO) fprintf(stderr,"Read NOTHING from dummy interface\n"); + c[i]=0; count--; + } + } + } else { + /* Read from UDP socket */ + int recvttl=1; + plen=recvwithttl(overlay_interfaces[i].fd,packet,sizeof(packet), + &recvttl,&src_addr,&addrlen); + if (plen<0) { + c[i]=0; count--; + } else { + /* We have a frame from this interface */ + if (debug&DEBUG_PACKETRX) { + fflush(stdout); + serval_packetvisualise(stderr,"Read from real interface", + packet,plen); + fflush(stderr); + } + if (debug&DEBUG_OVERLAYINTERFACES)fprintf(stderr,"Received %d bytes on interface #%d (%s)\n",plen,i,overlay_interfaces[i].name); + + if (packetOk(i,packet,plen,NULL,recvttl,&src_addr,addrlen,1)) { + WHY("Malformed packet"); + serval_packetvisualise(stderr,"Malformed packet", packet,plen); + } + } + } } - } + /* Don't sit here forever, or else we will never send any packets */ + if (overlay_gettime_ms()>(now+10)) break; } - } - + return 0; } diff --git a/tests/dnaprotocol b/tests/dnaprotocol index 3729f56c..868b341e 100755 --- a/tests/dnaprotocol +++ b/tests/dnaprotocol @@ -54,6 +54,7 @@ setup_servald_instance() { setup_servald_instances() { setup_servald DUMMYNET=/tmp/dummy + rm $DUMMYNET touch $DUMMYNET assert [ -e $DUMMYNET ] setup_servald_instance A $DUMMYNET