0.4.2: cleanup release

Version 0.4.2 is largely a cleanup release. Changes are minor:

 * Programatically replace libcrypto's random number generator with our
   own (using /dev/urandom or Windows CAPI) since libcrypto's RNG likes
   to use uninitialized memory as one of its entropy sources. This causes
   massive floods of valgrind (debugger) errors during memory error
   profiling analysis.

 * Clean up some other code to eliminate valgrind errors.

Valgrind now runs on Linux with only one error. This error is in
EthernetTap and is a false positive.

AUTHORS.txt

@@ -16,12 +16,6 @@ ZeroTier One includes or links with the following third party software:
  * LZ4 compression algorithm by Yann Collet (BSD license)
    http://code.google.com/p/lz4/
 
- * JsonCPP by Baptiste Lepilleur (public domain)
-   http://jsoncpp.sourceforge.net
-
- * http-parser, a simple C http parser library (MIT license)
-   https://github.com/joyent/http-parser
-
  * OpenSSL libcrypto (BSD-style OpenSSL license)
    http://www.openssl.org/
 

Makefile.linux

@@ -6,12 +6,12 @@ ARCH=$(shell uname -m)
 DEFS=-DZT_ARCH="$(ARCH)" -DZT_OSNAME="linux" -DZT_TRACE
 
 # Uncomment for a release optimized build
-CFLAGS=-Wall -O6 -fno-unroll-loops -pthread $(INCLUDES) -DNDEBUG $(DEFS)
-STRIP=strip --strip-all
+#CFLAGS=-Wall -O3 -fno-unroll-loops -fstack-protector -pthread $(INCLUDES) -DNDEBUG $(DEFS)
+#STRIP=strip --strip-all
 
 # Uncomment for a debug build
-#CFLAGS=-Wall -g -pthread $(INCLUDES) -DZT_TRACE -DZT_LOG_STDOUT $(DEFS)
-#STRIP=echo
+CFLAGS=-Wall -g -pthread $(INCLUDES) -DZT_TRACE $(DEFS)
+STRIP=echo
 
 CXXFLAGS=$(CFLAGS) -fno-rtti
 
@@ -20,16 +20,20 @@ CXXFLAGS=$(CFLAGS) -fno-rtti
 # separate binaries for the RedHat and Debian universes to distribute via
 # auto-update. This way we get one Linux binary for all systems of a given
 # architecture.
-LIBS=ext/bin/libcrypto/linux-$(ARCH)/libcrypto.a
+LIBS=ext/bin/libcrypto/linux-$(ARCH)/libcrypto.a -lm -ldl
 
 include objects.mk
 
-all:	one launcher
+all:	one cli launcher
 
 one:	$(OBJS)
 	$(CXX) $(CXXFLAGS) -o zerotier-one main.cpp $(OBJS) $(LIBS)
 	$(STRIP) zerotier-one
 
+cli:	$(OBJS)
+	$(CXX) $(CXXFLAGS) -o zerotier-cli cli.cpp $(OBJS) $(LIBS)
+	$(STRIP) zerotier-cli
+
 selftest:	$(OBJS)
 	$(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.cpp $(OBJS) $(LIBS)
 	$(STRIP) zerotier-selftest
@@ -38,10 +42,6 @@ idtool:	$(OBJS)
 	$(CXX) $(CXXFLAGS) -o zerotier-idtool idtool.cpp $(OBJS) $(LIBS)
 	$(STRIP) zerotier-idtool
 
-packtool:	$(OBJS)
-	$(CXX) $(CXXFLAGS) -o zerotier-packtool packtool.cpp $(OBJS) $(LIBS)
-	$(STRIP) zerotier-packtool
-
 launcher:
 	$(CC) -Os -o zerotier-launcher launcher.c
 	$(STRIP) zerotier-launcher

Makefile.mac

@@ -5,7 +5,7 @@ INCLUDES=-Iext/bin/libcrypto/include -Iext/jsoncpp/include
 DEFS=-DZT_ARCH="x86_combined" -DZT_OSNAME="mac" -DZT_TRACE
 
 # Uncomment for a release optimized universal binary build
-CFLAGS=-arch i386 -arch x86_64 -Wall -O6 -ftree-vectorize -pthread -mmacosx-version-min=10.6 -DNDEBUG $(INCLUDES) $(DEFS)
+CFLAGS=-arch i386 -arch x86_64 -Wall -O3 -ftree-vectorize -fstack-protector -pthread -mmacosx-version-min=10.6 -DNDEBUG $(INCLUDES) $(DEFS)
 STRIP=strip
 
 # Uncomment for a debug build
@@ -13,19 +13,20 @@ STRIP=strip
 #STRIP=echo
 
 CXXFLAGS=$(CFLAGS) -fno-rtti
-
-# We statically link against libcrypto since Apple has apparently decided
-# to deprecate it and may remove it in future OS releases.
-LIBS=ext/bin/libcrypto/mac-x86_combined/libcrypto.a
+LIBS=-lcrypto -lm
 
 include objects.mk
 
-all: one launcher mac-tap
+all: one cli launcher mac-tap
 
 one:	$(OBJS)
 	$(CXX) $(CXXFLAGS) -o zerotier-one main.cpp $(OBJS) $(LIBS)
 	$(STRIP) zerotier-one
 
+cli:	$(OBJS)
+	$(CXX) $(CXXFLAGS) -o zerotier-cli cli.cpp $(OBJS) $(LIBS)
+	$(STRIP) zerotier-cli
+
 selftest: $(OBJS)
 	$(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.cpp $(OBJS) $(LIBS)
 	$(STRIP) zerotier-selftest
@@ -34,10 +35,6 @@ idtool: $(OBJS)
 	$(CXX) $(CXXFLAGS) -o zerotier-idtool idtool.cpp $(OBJS) $(LIBS)
 	$(STRIP) zerotier-idtool
 
-packtool: $(OBJS)
-	$(CXX) $(CXXFLAGS) -o zerotier-packtool packtool.cpp $(OBJS) $(LIBS)
-	$(STRIP) zerotier-packtool
-
 mac-tap: FORCE
 	cd mac-tap/tuntap ; make tap.kext
 
@@ -55,6 +52,9 @@ launcher-fakebin:
 	$(CC) $(CFLAGS) -DZEROTIER_FAKE_VERSION_MAJOR=1 -DZEROTIER_FAKE_VERSION_MINOR=2 -DZEROTIER_FAKE_VERSION_REV
 	$(CC) $(CFLAGS) -DZEROTIER_FAKE_VERSION_MAJOR=1 -DZEROTIER_FAKE_VERSION_MINOR=2 -DZEROTIER_FAKE_VERSION_REV
 
+makekeypair: $(OBJS)
+	$(CXX) $(CXXFLAGS) -o zerotier-makekeypair makekeypair.cpp $(OBJS) $(LIBS)
+
 clean:
 	rm -rf *.dSYM
 	rm -f $(OBJS) zerotier-*

README.md

@@ -47,8 +47,8 @@ Check out the [blog](http://blog.zerotier.com/) for announcements, in-depth arti
 **Q:** Can you see my traffic? What about other users? Can you sniff the LAN?  
 **A:** No. All unicast (direct computer to computer) traffic is encrypted end-to-end (even if it's being relayed), and the ZeroTier virtual LAN behaves like a LAN with a secure enterprise-grade switch that does not allow unicast conversations to be sniffed. Multicast and broadcast traffic will of course be seen by many recipients, but that's the idea.
 
-**Q:** You say "almost unlimited size." Isn't multicast and broadcast traffic eventually going to be too much? What happens then?  
-**A:** ZeroTier One uses an algorithm called *implicit social switching*. The overall maximum number of recipients for a multicast is limited, so if there are too many listeners to a given multicast address then obviously not everyone will receive every message. So who does? Social switching causes multicasts to propagate in the direction of peers to whom you have recently communicated. As a result, multicasts tend to propagate along lines of association. The people most likely to get your service announcements are those with whom you frequently connect.
+**Q:** What about privacy? Does this hide my location on the network?  
+**A:** ZeroTier is not a connection anonymizer. Other than encryption, it doesn't do anything special to hide your identity or network location. If you want strong privacy protection there are already very advanced tools like [Tor](https://www.torproject.org) for that, and this isn't trying to duplicate their functionality. At the same time, ZT does not do anything special to harm your privacy either. It's not spyware or snoop-ware.
 
 **Q:** Is this designed to replace IP, BGP, IPv6, routers, etc.?  
 **A:** No. Its purpose is to act as a collaboration tool, a VPN alternative, a network mobility tool, a testbed for the development of software that takes advantage of fully open networking, a virtual LAN party for gamers, and so on, but it's not intended (or able) to replace the core of the Internet.
@@ -89,4 +89,7 @@ Check out the [blog](http://blog.zerotier.com/) for announcements, in-depth arti
 
 ----
 
+<a href="http://flattr.com/thing/1611614/ZeroTier-Networks" target="_blank"><img src="http://api.flattr.com/button/flattr-badge-large.png" alt="Flattr this" title="Flattr this" border="0" /></a>  
+[![githalytics.com alpha](https://cruel-carlota.pagodabox.com/59b2cbb9c154bf84bddb4b714402e548 "githalytics.com")](http://githalytics.com/zerotier/ZeroTierOne)
+
 (c)2012-2013 [ZeroTier Networks LLC](https://www.zerotier.com/)

cli.cpp

@@ -0,0 +1,143 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef __WINDOWS__
+#include <unistd.h>
+#endif
+
+#include "node/Node.hpp"
+#include "node/Constants.hpp"
+#include "node/Utils.hpp"
+#include "node/Thread.hpp"
+#include "node/Condition.hpp"
+
+using namespace ZeroTier;
+
+static void printHelp(FILE *out,const char *exename)
+{
+	fprintf(out,"Usage: %s [-switches] <command>"ZT_EOL_S,exename);
+	fprintf(out,ZT_EOL_S);
+	fprintf(out,"Switches:"ZT_EOL_S);
+	fprintf(out,"  -t<token>        - Specify token on command line"ZT_EOL_S);
+	fprintf(out,"  -T<file>         - Read token from file"ZT_EOL_S);
+	fprintf(out,ZT_EOL_S);
+	fprintf(out,"Use the 'help' command to get help from ZeroTier One itself."ZT_EOL_S);
+}
+
+static volatile unsigned int numResults = 0;
+static Condition doneCondition;
+
+static void resultHandler(void *arg,unsigned long id,const char *line)
+{
+	++numResults;
+	if (strlen(line))
+		fprintf(stdout,"%s"ZT_EOL_S,line);
+	else doneCondition.signal();
+}
+
+int main(int argc,char **argv)
+{
+	if (argc <= 1) {
+		printHelp(stdout,argv[0]);
+		return -1;
+	}
+
+	std::string authToken;
+	std::string command;
+	bool pastSwitches = false;
+	for(int i=1;i<argc;++i) {
+		if ((argv[i][0] == '-')&&(!pastSwitches)) {
+			if (strlen(argv[i]) <= 1) {
+				printHelp(stdout,argv[0]);
+				return -1;
+			}
+			switch(argv[i][1]) {
+				case 't':
+					authToken.assign(argv[i] + 2);
+					break;
+				case 'T':
+					if (!Utils::readFile(argv[i] + 2,authToken)) {
+						fprintf(stdout,"FATAL ERROR: unable to read token from '%s'"ZT_EOL_S,argv[i] + 2);
+						return -2;
+					}
+					break;
+				default:
+					return -1;
+			}
+		} else {
+			pastSwitches = true;
+			if (command.length())
+				command.push_back(' ');
+			command.append(argv[i]);
+		}
+	}
+
+	if (!authToken.length()) {
+		const char *home = getenv("HOME");
+		if (home) {
+			std::string dotZeroTierAuthToken(home);
+			dotZeroTierAuthToken.push_back(ZT_PATH_SEPARATOR);
+			dotZeroTierAuthToken.append(".zerotierOneAuthToken");
+			if (!Utils::readFile(dotZeroTierAuthToken.c_str(),authToken)) {
+#ifndef __WINDOWS__
+#ifdef __APPLE__
+				const char *systemAuthTokenPath = "/Library/Application Support/ZeroTier/One/authtoken.secret";
+#else
+				const char *systemAuthTokenPath = "/var/lib/zerotier-one/authtoken.secret";
+#endif
+				if (!Utils::readFile(systemAuthTokenPath,authToken)) {
+					fprintf(stdout,"FATAL ERROR: no token specified on command line and could not read '%s' or '%s'"ZT_EOL_S,dotZeroTierAuthToken.c_str(),systemAuthTokenPath);
+					return -2;
+				}
+#else // __WINDOWS__
+				fprintf(stdout,"FATAL ERROR: no token specified on command line and could not read '%s'"ZT_EOL_S,dotZeroTierAuthToken.c_str());
+				return -2;
+#endif // __WINDOWS__
+			}
+		}
+	}
+	if (!authToken.length()) {
+		fprintf(stdout,"FATAL ERROR: could not find auth token"ZT_EOL_S);
+		return -2;
+	}
+
+	Node::LocalClient client(authToken.c_str(),&resultHandler,(void *)0);
+	client.send(command.c_str());
+
+	doneCondition.wait(1000);
+
+	if (!numResults) {
+		fprintf(stdout,"ERROR: no results received. Is ZeroTier One running?"ZT_EOL_S);
+		return -1;
+	}
+
+	return 0;
+}

ext/bin/libcrypto/linux-armv6l

@@ -0,0 +1 @@
+linux-armv4/

ext/bin/libcrypto/openssl-config-linux-armv4.sh

@@ -0,0 +1,4 @@
+make clean
+./Configure no-sock no-ssl2 no-ssl3 no-err no-krb5 no-engine no-hw no-tlsext no-jpake no-capieng no-idea no-camellia no-seed no-bf no-cast no-des no-rc2 no-rc4 no-rc5 no-md2 no-md4 no-ripemd no-mdc2 no-rsa no-dsa no-dh no-shared no-zlib no-dso no-cms no-ocsp no-txt_db no-pem no-mdc2 no-ui no-ts linux-armv4
+make depend
+make build_crypto

ext/http-parser/AUTHORS

@@ -1,41 +0,0 @@
-# Authors ordered by first contribution.
-Ryan Dahl <ry@tinyclouds.org>
-Jeremy Hinegardner <jeremy@hinegardner.org>
-Sergey Shepelev <temotor@gmail.com>
-Joe Damato <ice799@gmail.com>
-tomika <tomika_nospam@freemail.hu>
-Phoenix Sol <phoenix@burninglabs.com>
-Cliff Frey <cliff@meraki.com>
-Ewen Cheslack-Postava <ewencp@cs.stanford.edu>
-Santiago Gala <sgala@apache.org>
-Tim Becker <tim.becker@syngenio.de>
-Jeff Terrace <jterrace@gmail.com>
-Ben Noordhuis <info@bnoordhuis.nl>
-Nathan Rajlich <nathan@tootallnate.net>
-Mark Nottingham <mnot@mnot.net>
-Aman Gupta <aman@tmm1.net>
-Tim Becker <tim.becker@kuriositaet.de>
-Sean Cunningham <sean.cunningham@mandiant.com>
-Peter Griess <pg@std.in>
-Salman Haq <salman.haq@asti-usa.com>
-Cliff Frey <clifffrey@gmail.com>
-Jon Kolb <jon@b0g.us>
-Fouad Mardini <f.mardini@gmail.com>
-Paul Querna <pquerna@apache.org>
-Felix Geisendörfer <felix@debuggable.com>
-koichik <koichik@improvement.jp>
-Andre Caron <andre.l.caron@gmail.com>
-Ivo Raisr <ivosh@ivosh.net>
-James McLaughlin <jamie@lacewing-project.org>
-David Gwynne <loki@animata.net>
-Thomas LE ROUX <thomas@november-eleven.fr>
-Randy Rizun <rrizun@ortivawireless.com>
-Andre Louis Caron <andre.louis.caron@usherbrooke.ca>
-Simon Zimmermann <simonz05@gmail.com>
-Erik Dubbelboer <erik@dubbelboer.com>
-Martell Malone <martellmalone@gmail.com>
-Bertrand Paquet <bpaquet@octo.com>
-BogDan Vatra <bogdan@kde.org>
-Peter Faiman <peter@thepicard.org>
-Corey Richardson <corey@octayn.net>
-Tóth Tamás <tomika_nospam@freemail.hu>

ext/http-parser/CONTRIBUTIONS

@@ -1,4 +0,0 @@
-Contributors must agree to the Contributor License Agreement before patches
-can be accepted.
-
-http://spreadsheets2.google.com/viewform?hl=en&formkey=dDJXOGUwbzlYaWM4cHN1MERwQS1CSnc6MQ

ext/http-parser/LICENSE-MIT

@@ -1,23 +0,0 @@
-http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright
-Igor Sysoev.
-
-Additional changes are licensed under the same terms as NGINX and
-copyright Joyent, Inc. and other Node contributors. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to
-deal in the Software without restriction, including without limitation the
-rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-sell copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-IN THE SOFTWARE. 

ext/http-parser/README.md

@@ -1,180 +0,0 @@
-HTTP Parser
-===========
-
-[![Build Status](https://travis-ci.org/joyent/http-parser.png?branch=master)](https://travis-ci.org/joyent/http-parser)
-
-This is a parser for HTTP messages written in C. It parses both requests and
-responses. The parser is designed to be used in performance HTTP
-applications. It does not make any syscalls nor allocations, it does not
-buffer data, it can be interrupted at anytime. Depending on your
-architecture, it only requires about 40 bytes of data per message
-stream (in a web server that is per connection).
-
-Features:
-
-  * No dependencies
-  * Handles persistent streams (keep-alive).
-  * Decodes chunked encoding.
-  * Upgrade support
-  * Defends against buffer overflow attacks.
-
-The parser extracts the following information from HTTP messages:
-
-  * Header fields and values
-  * Content-Length
-  * Request method
-  * Response status code
-  * Transfer-Encoding
-  * HTTP version
-  * Request URL
-  * Message body
-
-
-Usage
------
-
-One `http_parser` object is used per TCP connection. Initialize the struct
-using `http_parser_init()` and set the callbacks. That might look something
-like this for a request parser:
-
-    http_parser_settings settings;
-    settings.on_url = my_url_callback;
-    settings.on_header_field = my_header_field_callback;
-    /* ... */
-
-    http_parser *parser = malloc(sizeof(http_parser));
-    http_parser_init(parser, HTTP_REQUEST);
-    parser->data = my_socket;
-
-When data is received on the socket execute the parser and check for errors.
-
-    size_t len = 80*1024, nparsed;
-    char buf[len];
-    ssize_t recved;
-
-    recved = recv(fd, buf, len, 0);
-
-    if (recved < 0) {
-      /* Handle error. */
-    }
-
-    /* Start up / continue the parser.
-     * Note we pass recved==0 to signal that EOF has been recieved.
-     */
-    nparsed = http_parser_execute(parser, &settings, buf, recved);
-
-    if (parser->upgrade) {
-      /* handle new protocol */
-    } else if (nparsed != recved) {
-      /* Handle error. Usually just close the connection. */
-    }
-
-HTTP needs to know where the end of the stream is. For example, sometimes
-servers send responses without Content-Length and expect the client to
-consume input (for the body) until EOF. To tell http_parser about EOF, give
-`0` as the forth parameter to `http_parser_execute()`. Callbacks and errors
-can still be encountered during an EOF, so one must still be prepared
-to receive them.
-
-Scalar valued message information such as `status_code`, `method`, and the
-HTTP version are stored in the parser structure. This data is only
-temporally stored in `http_parser` and gets reset on each new message. If
-this information is needed later, copy it out of the structure during the
-`headers_complete` callback.
-
-The parser decodes the transfer-encoding for both requests and responses
-transparently. That is, a chunked encoding is decoded before being sent to
-the on_body callback.
-
-
-The Special Problem of Upgrade
-------------------------------
-
-HTTP supports upgrading the connection to a different protocol. An
-increasingly common example of this is the Web Socket protocol which sends
-a request like
-
-        GET /demo HTTP/1.1
-        Upgrade: WebSocket
-        Connection: Upgrade
-        Host: example.com
-        Origin: http://example.com
-        WebSocket-Protocol: sample
-
-followed by non-HTTP data.
-
-(See http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 for more
-information the Web Socket protocol.)
-
-To support this, the parser will treat this as a normal HTTP message without a
-body. Issuing both on_headers_complete and on_message_complete callbacks. However
-http_parser_execute() will stop parsing at the end of the headers and return.
-
-The user is expected to check if `parser->upgrade` has been set to 1 after
-`http_parser_execute()` returns. Non-HTTP data begins at the buffer supplied
-offset by the return value of `http_parser_execute()`.
-
-
-Callbacks
----------
-
-During the `http_parser_execute()` call, the callbacks set in
-`http_parser_settings` will be executed. The parser maintains state and
-never looks behind, so buffering the data is not necessary. If you need to
-save certain data for later usage, you can do that from the callbacks.
-
-There are two types of callbacks:
-
-* notification `typedef int (*http_cb) (http_parser*);`
-    Callbacks: on_message_begin, on_headers_complete, on_message_complete.
-* data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);`
-    Callbacks: (requests only) on_uri,
-               (common) on_header_field, on_header_value, on_body;
-
-Callbacks must return 0 on success. Returning a non-zero value indicates
-error to the parser, making it exit immediately.
-
-In case you parse HTTP message in chunks (i.e. `read()` request line
-from socket, parse, read half headers, parse, etc) your data callbacks
-may be called more than once. Http-parser guarantees that data pointer is only
-valid for the lifetime of callback. You can also `read()` into a heap allocated
-buffer to avoid copying memory around if this fits your application.
-
-Reading headers may be a tricky task if you read/parse headers partially.
-Basically, you need to remember whether last header callback was field or value
-and apply following logic:
-
-    (on_header_field and on_header_value shortened to on_h_*)
-     ------------------------ ------------ --------------------------------------------
-    | State (prev. callback) | Callback   | Description/action                         |
-     ------------------------ ------------ --------------------------------------------
-    | nothing (first call)   | on_h_field | Allocate new buffer and copy callback data |
-    |                        |            | into it                                    |
-     ------------------------ ------------ --------------------------------------------
-    | value                  | on_h_field | New header started.                        |
-    |                        |            | Copy current name,value buffers to headers |
-    |                        |            | list and allocate new buffer for new name  |
-     ------------------------ ------------ --------------------------------------------
-    | field                  | on_h_field | Previous name continues. Reallocate name   |
-    |                        |            | buffer and append callback data to it      |
-     ------------------------ ------------ --------------------------------------------
-    | field                  | on_h_value | Value for current header started. Allocate |
-    |                        |            | new buffer and copy callback data to it    |
-     ------------------------ ------------ --------------------------------------------
-    | value                  | on_h_value | Value continues. Reallocate value buffer   |
-    |                        |            | and append callback data to it             |
-     ------------------------ ------------ --------------------------------------------
-
-
-Parsing URLs
-------------
-
-A simplistic zero-copy URL parser is provided as `http_parser_parse_url()`.
-Users of this library may wish to use it to parse URLs constructed from
-consecutive `on_url` callbacks.
-
-See examples of reading in headers:
-
-* [partial example](http://gist.github.com/155877) in C
-* [from http-parser tests](http://github.com/joyent/http-parser/blob/37a0ff8/test.c#L403) in C
-* [from Node library](http://github.com/joyent/node/blob/842eaf4/src/http.js#L284) in Javascript

ext/http-parser/contrib/parsertrace.c

@@ -1,156 +0,0 @@
-/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev
- *
- * Additional changes are licensed under the same terms as NGINX and
- * copyright Joyent, Inc. and other Node contributors. All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-/* Dump what the parser finds to stdout as it happen */
-
-#include "http_parser.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-int on_message_begin(http_parser* _) {
-  (void)_;
-  printf("\n***MESSAGE BEGIN***\n\n");
-  return 0;
-}
-
-int on_headers_complete(http_parser* _) {
-  (void)_;
-  printf("\n***HEADERS COMPLETE***\n\n");
-  return 0;
-}
-
-int on_message_complete(http_parser* _) {
-  (void)_;
-  printf("\n***MESSAGE COMPLETE***\n\n");
-  return 0;
-}
-
-int on_url(http_parser* _, const char* at, size_t length) {
-  (void)_;
-  printf("Url: %.*s\n", (int)length, at);
-  return 0;
-}
-
-int on_header_field(http_parser* _, const char* at, size_t length) {
-  (void)_;
-  printf("Header field: %.*s\n", (int)length, at);
-  return 0;
-}
-
-int on_header_value(http_parser* _, const char* at, size_t length) {
-  (void)_;
-  printf("Header value: %.*s\n", (int)length, at);
-  return 0;
-}
-
-int on_body(http_parser* _, const char* at, size_t length) {
-  (void)_;
-  printf("Body: %.*s\n", (int)length, at);
-  return 0;
-}
-
-void usage(const char* name) {
-  fprintf(stderr,
-          "Usage: %s $type $filename\n"
-          "  type: -x, where x is one of {r,b,q}\n"
-          "  parses file as a Response, reQuest, or Both\n",
-          name);
-  exit(EXIT_FAILURE);
-}
-
-int main(int argc, char* argv[]) {
-  enum http_parser_type file_type;
-
-  if (argc != 3) {
-    usage(argv[0]);
-  }
-
-  char* type = argv[1];
-  if (type[0] != '-') {
-    usage(argv[0]);
-  }
-
-  switch (type[1]) {
-    /* in the case of "-", type[1] will be NUL */
-    case 'r':
-      file_type = HTTP_RESPONSE;
-      break;
-    case 'q':
-      file_type = HTTP_REQUEST;
-      break;
-    case 'b':
-      file_type = HTTP_BOTH;
-      break;
-    default:
-      usage(argv[0]);
-  }
-
-  char* filename = argv[2];
-  FILE* file = fopen(filename, "r");
-  if (file == NULL) {
-    perror("fopen");
-    return EXIT_FAILURE;
-  }
-
-  fseek(file, 0, SEEK_END);
-  long file_length = ftell(file);
-  if (file_length == -1) {
-    perror("ftell");
-    return EXIT_FAILURE;
-  }
-  fseek(file, 0, SEEK_SET);
-
-  char* data = malloc(file_length);
-  if (fread(data, 1, file_length, file) != (size_t)file_length) {
-    fprintf(stderr, "couldn't read entire file\n");
-    free(data);
-    return EXIT_FAILURE;
-  }
-
-  http_parser_settings settings;
-  memset(&settings, 0, sizeof(settings));
-  settings.on_message_begin = on_message_begin;
-  settings.on_url = on_url;
-  settings.on_header_field = on_header_field;
-  settings.on_header_value = on_header_value;
-  settings.on_headers_complete = on_headers_complete;
-  settings.on_body = on_body;
-  settings.on_message_complete = on_message_complete;
-
-  http_parser parser;
-  http_parser_init(&parser, file_type);
-  size_t nparsed = http_parser_execute(&parser, &settings, data, file_length);
-  free(data);
-
-  if (nparsed != (size_t)file_length) {
-    fprintf(stderr,
-            "Error: %s (%s)\n",
-            http_errno_description(HTTP_PARSER_ERRNO(&parser)),
-            http_errno_name(HTTP_PARSER_ERRNO(&parser)));
-    return EXIT_FAILURE;
-  }
-
-  return EXIT_SUCCESS;
-}

ext/http-parser/contrib/url_parser.c

@@ -1,44 +0,0 @@
-#include "http_parser.h"
-#include <stdio.h>
-#include <string.h>
-
-void
-dump_url (const char *url, const struct http_parser_url *u)
-{
-  unsigned int i;
-
-  printf("\tfield_set: 0x%x, port: %u\n", u->field_set, u->port);
-  for (i = 0; i < UF_MAX; i++) {
-    if ((u->field_set & (1 << i)) == 0) {
-      printf("\tfield_data[%u]: unset\n", i);
-      continue;
-    }
-
-    printf("\tfield_data[%u]: off: %u len: %u part: \"%.*s\n",
-           i,
-           u->field_data[i].off,
-           u->field_data[i].len,
-           u->field_data[i].len,
-           url + u->field_data[i].off);
-  }
-}
-
-int main(int argc, char ** argv) {
-  if (argc != 3) {
-    printf("Syntax : %s connect|get url\n", argv[0]);
-    return 1;
-  }
-  struct http_parser_url u;
-  int len = strlen(argv[2]);
-  int connect = strcmp("connect", argv[1]) == 0 ? 1 : 0;
-  printf("Parsing %s, connect %d\n", argv[2], connect);
-
-  int result = http_parser_parse_url(argv[2], len, connect, &u);
-  if (result != 0) {
-    printf("Parse error : %d\n", result);
-    return result;
-  }
-  printf("Parse ok, result : \n");
-  dump_url(argv[2], &u);
-  return 0;
-}

ext/http-parser/http_parser.gyp

@@ -1,111 +0,0 @@
-# This file is used with the GYP meta build system.
-# http://code.google.com/p/gyp/
-# To build try this:
-#   svn co http://gyp.googlecode.com/svn/trunk gyp
-#   ./gyp/gyp -f make --depth=`pwd` http_parser.gyp 
-#   ./out/Debug/test 
-{
-  'target_defaults': {
-    'default_configuration': 'Debug',
-    'configurations': {
-      # TODO: hoist these out and put them somewhere common, because
-      #       RuntimeLibrary MUST MATCH across the entire project
-      'Debug': {
-        'defines': [ 'DEBUG', '_DEBUG' ],
-        'cflags': [ '-Wall', '-Wextra', '-O0', '-g', '-ftrapv' ],
-        'msvs_settings': {
-          'VCCLCompilerTool': {
-            'RuntimeLibrary': 1, # static debug
-          },
-        },
-      },
-      'Release': {
-        'defines': [ 'NDEBUG' ],
-        'cflags': [ '-Wall', '-Wextra', '-O3' ],
-        'msvs_settings': {
-          'VCCLCompilerTool': {
-            'RuntimeLibrary': 0, # static release
-          },
-        },
-      }
-    },
-    'msvs_settings': {
-      'VCCLCompilerTool': {
-      },
-      'VCLibrarianTool': {
-      },
-      'VCLinkerTool': {
-        'GenerateDebugInformation': 'true',
-      },
-    },
-    'conditions': [
-      ['OS == "win"', {
-        'defines': [
-          'WIN32'
-        ],
-      }]
-    ],
-  },
-
-  'targets': [
-    {
-      'target_name': 'http_parser',
-      'type': 'static_library',
-      'include_dirs': [ '.' ],
-      'direct_dependent_settings': {
-        'defines': [ 'HTTP_PARSER_STRICT=0' ],
-        'include_dirs': [ '.' ],
-      },
-      'defines': [ 'HTTP_PARSER_STRICT=0' ],
-      'sources': [ './http_parser.c', ],
-      'conditions': [
-        ['OS=="win"', {
-          'msvs_settings': {
-            'VCCLCompilerTool': {
-              # Compile as C++. http_parser.c is actually C99, but C++ is
-              # close enough in this case.
-              'CompileAs': 2,
-            },
-          },
-        }]
-      ],
-    },
-
-    {
-      'target_name': 'http_parser_strict',
-      'type': 'static_library',
-      'include_dirs': [ '.' ],
-      'direct_dependent_settings': {
-        'defines': [ 'HTTP_PARSER_STRICT=1' ],
-        'include_dirs': [ '.' ],
-      },
-      'defines': [ 'HTTP_PARSER_STRICT=1' ],
-      'sources': [ './http_parser.c', ],
-      'conditions': [
-        ['OS=="win"', {
-          'msvs_settings': {
-            'VCCLCompilerTool': {
-              # Compile as C++. http_parser.c is actually C99, but C++ is
-              # close enough in this case.
-              'CompileAs': 2,
-            },
-          },
-        }]
-      ],
-    },
-
-    {
-      'target_name': 'test-nonstrict',
-      'type': 'executable',
-      'dependencies': [ 'http_parser' ],
-      'sources': [ 'test.c' ]
-    },
-
-    {
-      'target_name': 'test-strict',
-      'type': 'executable',
-      'dependencies': [ 'http_parser_strict' ],
-      'sources': [ 'test.c' ]
-    }
-  ]
-}

ext/http-parser/http_parser.h

@@ -1,304 +0,0 @@
-/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-#ifndef http_parser_h
-#define http_parser_h
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define HTTP_PARSER_VERSION_MAJOR 2
-#define HTTP_PARSER_VERSION_MINOR 0
-
-#include <sys/types.h>
-#if defined(_WIN32) && !defined(__MINGW32__) && (!defined(_MSC_VER) || _MSC_VER<1600)
-#include <BaseTsd.h>
-#include <stddef.h>
-typedef __int8 int8_t;
-typedef unsigned __int8 uint8_t;
-typedef __int16 int16_t;
-typedef unsigned __int16 uint16_t;
-typedef __int32 int32_t;
-typedef unsigned __int32 uint32_t;
-typedef __int64 int64_t;
-typedef unsigned __int64 uint64_t;
-#else
-#include <stdint.h>
-#endif
-
-/* Compile with -DHTTP_PARSER_STRICT=0 to make less checks, but run
- * faster
- */
-#ifndef HTTP_PARSER_STRICT
-# define HTTP_PARSER_STRICT 1
-#endif
-
-/* Maximium header size allowed */
-#define HTTP_MAX_HEADER_SIZE (80*1024)
-
-
-typedef struct http_parser http_parser;
-typedef struct http_parser_settings http_parser_settings;
-
-
-/* Callbacks should return non-zero to indicate an error. The parser will
- * then halt execution.
- *
- * The one exception is on_headers_complete. In a HTTP_RESPONSE parser
- * returning '1' from on_headers_complete will tell the parser that it
- * should not expect a body. This is used when receiving a response to a
- * HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
- * chunked' headers that indicate the presence of a body.
- *
- * http_data_cb does not return data chunks. It will be call arbitrarally
- * many times for each string. E.G. you might get 10 callbacks for "on_url"
- * each providing just a few characters more data.
- */
-typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
-typedef int (*http_cb) (http_parser*);
-
-
-/* Request Methods */
-#define HTTP_METHOD_MAP(XX)         \
-  XX(0,  DELETE,      DELETE)       \
-  XX(1,  GET,         GET)          \
-  XX(2,  HEAD,        HEAD)         \
-  XX(3,  POST,        POST)         \
-  XX(4,  PUT,         PUT)          \
-  /* pathological */                \
-  XX(5,  CONNECT,     CONNECT)      \
-  XX(6,  OPTIONS,     OPTIONS)      \
-  XX(7,  TRACE,       TRACE)        \
-  /* webdav */                      \
-  XX(8,  COPY,        COPY)         \
-  XX(9,  LOCK,        LOCK)         \
-  XX(10, MKCOL,       MKCOL)        \
-  XX(11, MOVE,        MOVE)         \
-  XX(12, PROPFIND,    PROPFIND)     \
-  XX(13, PROPPATCH,   PROPPATCH)    \
-  XX(14, SEARCH,      SEARCH)       \
-  XX(15, UNLOCK,      UNLOCK)       \
-  /* subversion */                  \
-  XX(16, REPORT,      REPORT)       \
-  XX(17, MKACTIVITY,  MKACTIVITY)   \
-  XX(18, CHECKOUT,    CHECKOUT)     \
-  XX(19, MERGE,       MERGE)        \
-  /* upnp */                        \
-  XX(20, MSEARCH,     M-SEARCH)     \
-  XX(21, NOTIFY,      NOTIFY)       \
-  XX(22, SUBSCRIBE,   SUBSCRIBE)    \
-  XX(23, UNSUBSCRIBE, UNSUBSCRIBE)  \
-  /* RFC-5789 */                    \
-  XX(24, PATCH,       PATCH)        \
-  XX(25, PURGE,       PURGE)        \
-
-enum http_method
-  {
-#define XX(num, name, string) HTTP_##name = num,
-  HTTP_METHOD_MAP(XX)
-#undef XX
-  };
-
-
-enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
-
-
-/* Flag values for http_parser.flags field */
-enum flags
-  { F_CHUNKED               = 1 << 0
-  , F_CONNECTION_KEEP_ALIVE = 1 << 1
-  , F_CONNECTION_CLOSE      = 1 << 2
-  , F_TRAILING              = 1 << 3
-  , F_UPGRADE               = 1 << 4
-  , F_SKIPBODY              = 1 << 5
-  };
-
-
-/* Map for errno-related constants
- * 
- * The provided argument should be a macro that takes 2 arguments.
- */
-#define HTTP_ERRNO_MAP(XX)                                           \
-  /* No error */                                                     \
-  XX(OK, "success")                                                  \
-                                                                     \
-  /* Callback-related errors */                                      \
-  XX(CB_message_begin, "the on_message_begin callback failed")       \
-  XX(CB_status_complete, "the on_status_complete callback failed")   \
-  XX(CB_url, "the on_url callback failed")                           \
-  XX(CB_header_field, "the on_header_field callback failed")         \
-  XX(CB_header_value, "the on_header_value callback failed")         \
-  XX(CB_headers_complete, "the on_headers_complete callback failed") \
-  XX(CB_body, "the on_body callback failed")                         \
-  XX(CB_message_complete, "the on_message_complete callback failed") \
-                                                                     \
-  /* Parsing-related errors */                                       \
-  XX(INVALID_EOF_STATE, "stream ended at an unexpected time")        \
-  XX(HEADER_OVERFLOW,                                                \
-     "too many header bytes seen; overflow detected")                \
-  XX(CLOSED_CONNECTION,                                              \
-     "data received after completed connection: close message")      \
-  XX(INVALID_VERSION, "invalid HTTP version")                        \
-  XX(INVALID_STATUS, "invalid HTTP status code")                     \
-  XX(INVALID_METHOD, "invalid HTTP method")                          \
-  XX(INVALID_URL, "invalid URL")                                     \
-  XX(INVALID_HOST, "invalid host")                                   \
-  XX(INVALID_PORT, "invalid port")                                   \
-  XX(INVALID_PATH, "invalid path")                                   \
-  XX(INVALID_QUERY_STRING, "invalid query string")                   \
-  XX(INVALID_FRAGMENT, "invalid fragment")                           \
-  XX(LF_EXPECTED, "LF character expected")                           \
-  XX(INVALID_HEADER_TOKEN, "invalid character in header")            \
-  XX(INVALID_CONTENT_LENGTH,                                         \
-     "invalid character in content-length header")                   \
-  XX(INVALID_CHUNK_SIZE,                                             \
-     "invalid character in chunk size header")                       \
-  XX(INVALID_CONSTANT, "invalid constant string")                    \
-  XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
-  XX(STRICT, "strict mode assertion failed")                         \
-  XX(PAUSED, "parser is paused")                                     \
-  XX(UNKNOWN, "an unknown error occurred")
-
-
-/* Define HPE_* values for each errno value above */
-#define HTTP_ERRNO_GEN(n, s) HPE_##n,
-enum http_errno {
-  HTTP_ERRNO_MAP(HTTP_ERRNO_GEN)
-};
-#undef HTTP_ERRNO_GEN
-
-
-/* Get an http_errno value from an http_parser */
-#define HTTP_PARSER_ERRNO(p)            ((enum http_errno) (p)->http_errno)
-
-
-struct http_parser {
-  /** PRIVATE **/
-  unsigned char type : 2;     /* enum http_parser_type */
-  unsigned char flags : 6;    /* F_* values from 'flags' enum; semi-public */
-  unsigned char state;        /* enum state from http_parser.c */
-  unsigned char header_state; /* enum header_state from http_parser.c */
-  unsigned char index;        /* index into current matcher */
-
-  uint32_t nread;          /* # bytes read in various scenarios */
-  uint64_t content_length; /* # bytes in body (0 if no Content-Length header) */
-
-  /** READ-ONLY **/
-  unsigned short http_major;
-  unsigned short http_minor;
-  unsigned short status_code; /* responses only */
-  unsigned char method;       /* requests only */
-  unsigned char http_errno : 7;
-
-  /* 1 = Upgrade header was present and the parser has exited because of that.
-   * 0 = No upgrade header present.
-   * Should be checked when http_parser_execute() returns in addition to
-   * error checking.
-   */
-  unsigned char upgrade : 1;
-
-  /** PUBLIC **/
-  void *data; /* A pointer to get hook to the "connection" or "socket" object */
-};
-
-
-struct http_parser_settings {
-  http_cb      on_message_begin;
-  http_data_cb on_url;
-  http_cb      on_status_complete;
-  http_data_cb on_header_field;
-  http_data_cb on_header_value;
-  http_cb      on_headers_complete;
-  http_data_cb on_body;
-  http_cb      on_message_complete;
-};
-
-
-enum http_parser_url_fields
-  { UF_SCHEMA           = 0
-  , UF_HOST             = 1
-  , UF_PORT             = 2
-  , UF_PATH             = 3
-  , UF_QUERY            = 4
-  , UF_FRAGMENT         = 5
-  , UF_USERINFO         = 6
-  , UF_MAX              = 7
-  };
-
-
-/* Result structure for http_parser_parse_url().
- *
- * Callers should index into field_data[] with UF_* values iff field_set
- * has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
- * because we probably have padding left over), we convert any port to
- * a uint16_t.
- */
-struct http_parser_url {
-  uint16_t field_set;           /* Bitmask of (1 << UF_*) values */
-  uint16_t port;                /* Converted UF_PORT string */
-
-  struct {
-    uint16_t off;               /* Offset into buffer in which field starts */
-    uint16_t len;               /* Length of run in buffer */
-  } field_data[UF_MAX];
-};
-
-
-void http_parser_init(http_parser *parser, enum http_parser_type type);
-
-
-size_t http_parser_execute(http_parser *parser,
-                           const http_parser_settings *settings,
-                           const char *data,
-                           size_t len);
-
-
-/* If http_should_keep_alive() in the on_headers_complete or
- * on_message_complete callback returns 0, then this should be
- * the last message on the connection.
- * If you are the server, respond with the "Connection: close" header.
- * If you are the client, close the connection.
- */
-int http_should_keep_alive(const http_parser *parser);
-
-/* Returns a string version of the HTTP method. */
-const char *http_method_str(enum http_method m);
-
-/* Return a string name of the given error */
-const char *http_errno_name(enum http_errno err);
-
-/* Return a string description of the given error */
-const char *http_errno_description(enum http_errno err);
-
-/* Parse a URL; return nonzero on failure */
-int http_parser_parse_url(const char *buf, size_t buflen,
-                          int is_connect,
-                          struct http_parser_url *u);
-
-/* Pause or un-pause the parser; a nonzero value pauses */
-void http_parser_pause(http_parser *parser, int paused);
-
-/* Checks if this is the final chunk of the body. */
-int http_body_is_final(const http_parser *parser);
-
-#ifdef __cplusplus
-}
-#endif
-#endif

ext/huffandpuff/huffman.c

@@ -1,252 +0,0 @@
-/*
- * Huffandpuff minimal Huffman coder
- *
- * (c)2013 Adam Ierymenko <adam.ierymenko@zerotier.com>
- * This code is in the public domain and is distributed with NO WARRANTY.
- */
-
-#include "huffman.h"
-
-struct _huffman_node
-{
-	struct _huffman_node *lr[2];
-	struct _huffman_node *qprev,*qnext;
-	double prob;
-	unsigned long c;
-};
-
-struct _huffman_encode_table
-{
-	unsigned long code;
-	unsigned long bits;
-};
-
-static void _huffman_write_tree_and_make_encode_table(unsigned char *out,unsigned long *outbitctr,unsigned long outlen,struct _huffman_encode_table *et,unsigned long code,unsigned int bits,struct _huffman_node *t)
-{
-	struct _huffman_encode_table *eti;
-	unsigned int i;
-	unsigned long byte_index;
-
-	byte_index = (*outbitctr)++ >> 3;
-	byte_index *= (byte_index < outlen);
-	if (t->lr[0]) {
-		out[byte_index] <<= 1;
-		_huffman_write_tree_and_make_encode_table(out,outbitctr,outlen,et,code,bits + 1,t->lr[0]);
-		_huffman_write_tree_and_make_encode_table(out,outbitctr,outlen,et,code | (1 << bits),bits + 1,t->lr[1]);
-	} else {
-		out[byte_index] = (out[byte_index] << 1) | 1;
-		for(i=0;i<9;++i) {
-			byte_index = (*outbitctr)++ >> 3;
-			if (byte_index >= outlen) return;
-			out[byte_index] = (out[byte_index] << 1) | ((unsigned char)((t->c >> i) & 1));
-		}
-		eti = &(et[t->c]);
-		eti->code = code;
-		eti->bits = bits;
-	}
-}
-
-static struct _huffman_node *_huffman_read_tree(const unsigned char *in,unsigned long *inbitctr,unsigned long inlen,unsigned char **heapptr,unsigned char *heapend)
-{
-	struct _huffman_node *n;
-	unsigned int i;
-	unsigned long byte_index;
-
-	n = (struct _huffman_node *)(*heapptr);
-	*heapptr += sizeof(struct _huffman_node);
-	if (*heapptr > heapend) return (struct _huffman_node *)0;
-
-	byte_index = *inbitctr >> 3;
-	byte_index *= (byte_index < inlen);
-	if (((in[byte_index] >> (~((*inbitctr)++) & 7)) & 1)) {
-		n->lr[0] = (struct _huffman_node *)0;
-		n->lr[1] = (struct _huffman_node *)0;
-		n->c = 0;
-		for(i=0;i<9;++i) {
-			byte_index = *inbitctr >> 3;
-			if (byte_index >= inlen) return (struct _huffman_node *)0;
-			n->c |= (((unsigned int)(in[byte_index] >> (~((*inbitctr)++) & 7))) & 1) << i;
-		}
-	} else {
-		n->lr[0] = _huffman_read_tree(in,inbitctr,inlen,heapptr,heapend);
-		n->lr[1] = _huffman_read_tree(in,inbitctr,inlen,heapptr,heapend);
-		if (!((n->lr[0])&&(n->lr[1])))
-			return (struct _huffman_node *)0;
-	}
-
-	return n;
-}
-
-unsigned long huffman_compress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap)
-{
-	struct _huffman_encode_table *et,*eti;
-	struct _huffman_node *t,*n;
-	struct _huffman_node *pair[2];
-	unsigned char *heapptr = (unsigned char *)huffheap;
-	unsigned long i,code,byte_index,outbitctr;
-	unsigned int bits,b;
-	double *counts,lowest_prob,total_symbols;
-
-	counts = (double *)heapptr;
-	heapptr += (sizeof(double) * 257);
-	for(i=0;i<256;++i)
-		counts[i] = 0.0;
-	counts[256] = 1.0; /* one stop code at end */
-	for(i=0;i<inlen;++i)
-		counts[(unsigned long)in[i]] += 1.0;
-
-	t = (struct _huffman_node *)0;
-	total_symbols = (double)(inlen + 1);
-	for(i=0;i<=256;++i) {
-		if (counts[i] > 0.0) {
-			n = (struct _huffman_node *)heapptr;
-			heapptr += sizeof(struct _huffman_node);
-			if (t)
-				t->qprev = n;
-			n->qprev = (struct _huffman_node *)0;
-			n->qnext = t;
-			n->lr[0] = (struct _huffman_node *)0;
-			n->lr[1] = (struct _huffman_node *)0;
-			n->prob = counts[i] / total_symbols;
-			n->c = (unsigned int)i;
-			t = n;
-		}
-	}
-
-	while (t->qnext) {
-		for(i=0;i<2;++i) {
-			lowest_prob = 1.0;
-			pair[i] = (struct _huffman_node *)0;
-			n = t;
-			while (n) {
-				if (n->prob <= lowest_prob) {
-					lowest_prob = n->prob;
-					pair[i] = n;
-				}
-				n = n->qnext;
-			}
-			if (pair[i]->qprev)
-				pair[i]->qprev->qnext = pair[i]->qnext;
-			else t = pair[i]->qnext;
-			if (pair[i]->qnext)
-				pair[i]->qnext->qprev = pair[i]->qprev;
-		}
-		n = (struct _huffman_node *)heapptr;
-		heapptr += sizeof(struct _huffman_node);
-		n->lr[0] = pair[0];
-		n->lr[1] = pair[1];
-		n->prob = pair[0]->prob + pair[1]->prob;
-		if (t)
-			t->qprev = n;
-		n->qprev = (struct _huffman_node *)0;
-		n->qnext = t;
-		t = n;
-	}
-
-	et = (struct _huffman_encode_table *)heapptr;
-	heapptr += (sizeof(struct _huffman_encode_table) * 257);
-	outbitctr = 0;
-	_huffman_write_tree_and_make_encode_table(out,&outbitctr,outlen,et,0,0,t);
-
-	for(i=0;i<inlen;++i) {
-		eti = &(et[(unsigned long)in[i]]);
-		code = eti->code;
-		bits = eti->bits;
-		for(b=0;b<bits;++b) {
-			byte_index = outbitctr++ >> 3;
-			if (byte_index >= outlen) return 0;
-			out[byte_index] = (out[byte_index] << 1) | (unsigned char)(code & 1);
-			code >>= 1;
-		}
-	}
-	code = et[256].code;
-	bits = et[256].bits;
-	for(b=0;b<bits;++b) {
-		byte_index = outbitctr++ >> 3;
-		if (byte_index >= outlen) return 0;
-		out[byte_index] = (out[byte_index] << 1) | (unsigned char)(code & 1);
-		code >>= 1;
-	}
-
-	if (outbitctr > (outlen << 3))
-		return 0;
-	else if ((outbitctr & 7)) {
-		out[i = (outbitctr >> 3)] <<= 8 - (outbitctr & 7);
-		return (i + 1);
-	} else return (outbitctr >> 3);
-}
-
-unsigned long huffman_decompress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap)
-{
-	struct _huffman_node *t,*n;
-	unsigned char *heapptr = (unsigned char *)huffheap;
-	unsigned long inbitctr,outptr,byte_index = 0;
-
-	inbitctr = 0;
-	t = _huffman_read_tree(in,&inbitctr,inlen,&heapptr,heapptr + HUFFHEAP_SIZE);
-	if (!t) return 0;
-	outptr = 0;
-	for(;;) {
-		n = t;
-		while (n->lr[0]) {
-			byte_index = inbitctr >> 3;
-			if (byte_index >= inlen) return 0;
-			n = n->lr[((unsigned long)(in[byte_index] >> (~(inbitctr++) & 7))) & 1];
-		}
-		if (n->c == 256) return outptr;
-		if (outptr == outlen) return 0;
-		out[outptr++] = (unsigned char)n->c;
-	}
-}
-
-#ifdef HUFFANDPUFF_TEST
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-
-#define HUFFANDPUFF_TEST_MAXLEN 1048576
-#define HUFFANDPUFF_TEST_ITER 1024
-
-static unsigned char testin[HUFFANDPUFF_TEST_MAXLEN];
-static unsigned char testout[HUFFANDPUFF_TEST_MAXLEN * 2];
-static unsigned char testver[HUFFANDPUFF_TEST_MAXLEN];
-static unsigned char huffbuf[HUFFHEAP_SIZE];
-
-int main(int argc,char **argv)
-{
-	unsigned long i,k,l,cl,dcl;
-	int v;
-	unsigned char mask;
-
-	srand(time(0));
-
-	for(k=0;k<HUFFANDPUFF_TEST_ITER;++k) {
-		l = (rand() % HUFFANDPUFF_TEST_MAXLEN) + 1;
-		mask = (rand() & 0xff);
-		for(i=0;i<l;++i)
-			testin[i] = (unsigned char)(rand() & 0xff) & mask;
-		cl = huffman_compress(testin,l,testout,sizeof(testout),huffbuf);
-		if (cl) {
-			memset(testver,0,sizeof(testver));
-			dcl = huffman_decompress(testout,cl,testver,sizeof(testver),huffbuf);
-			v = ((dcl)&&(!memcmp(testver,testin,l)));
-			printf("[%d] in: %d, out: %d, verified: %s\n",(int)k,(int)l,(int)cl,(v) ? "OK" : "FAIL");
-		} else printf("[%d] in: %d, out: FAIL\n",(int)k,(int)l);
-	}
-
-	printf("\nFuzzing decompress function...\n");
-	for(;;) {
-		l = (rand() % HUFFANDPUFF_TEST_MAXLEN) + 1;
-		mask = (rand() & 0xff);
-		for(i=0;i<l;++i)
-			testin[i] = (unsigned char)(rand() & 0xff) & mask;
-		huffman_decompress(testin,l,testver,sizeof(testver),huffbuf);
-		printf("."); fflush(stdout);
-	}
-
-	return 0;
-}
-
-#endif

ext/huffandpuff/huffman.h

@@ -1,52 +0,0 @@
-/*
- * Huffandpuff minimal Huffman coder
- *
- * (c)2013 Adam Ierymenko <adam.ierymenko@zerotier.com>
- * This code is in the public domain and is distributed with NO WARRANTY.
- */
-
-#ifndef ____HUFFMAN_H
-#define ____HUFFMAN_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * Required size of huffheap parameter to compress and decompress
- *
- * Note: if you change any of the data types in the _huffman_node
- * or _huffman_encode_table structs in huffman.c, this also must be
- * changed.
- */
-#define HUFFHEAP_SIZE ((sizeof(double) * 257) + (((sizeof(void *) * 4) + sizeof(double) + sizeof(unsigned long)) * (257 * 3)) + ((sizeof(unsigned long) + sizeof(unsigned long)) * 257))
-
-/**
- * Huffman encode a block of data
- *
- * @param in Input data
- * @param inlen Input data length
- * @param out Output buffer
- * @param outlen Output buffer length
- * @param huffheap Heap memory to use for compression (must be HUFFHEAP_SIZE in size)
- * @return Size of encoded result or 0 on out buffer overrun
- */
-extern unsigned long huffman_compress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap);
-
-/**
- * Huffman decode a block of data
- *
- * @param in Input data
- * @param inlen Length of input data
- * @param out Output buffer
- * @param outlen Length of output buffer
- * @param huffheap Heap memory to use for decompression (must be HUFFHEAP_SIZE in size)
- * @return Size of decoded result or 0 on out buffer overrun or corrupt input data
- */
-extern unsigned long huffman_decompress(const unsigned char *in,unsigned long inlen,unsigned char *out,unsigned long outlen,void *huffheap);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

ext/jsoncpp/include/json/autolink.h

@@ -1,24 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef JSON_AUTOLINK_H_INCLUDED
-# define JSON_AUTOLINK_H_INCLUDED
-
-# include "config.h"
-
-# ifdef JSON_IN_CPPTL
-#  include <cpptl/cpptl_autolink.h>
-# endif
-
-# if !defined(JSON_NO_AUTOLINK)  &&  !defined(JSON_DLL_BUILD)  &&  !defined(JSON_IN_CPPTL)
-#  define CPPTL_AUTOLINK_NAME "json"
-#  undef CPPTL_AUTOLINK_DLL
-#  ifdef JSON_DLL
-#   define CPPTL_AUTOLINK_DLL
-#  endif
-#  include "autolink.h"
-# endif
-
-#endif // JSON_AUTOLINK_H_INCLUDED

ext/jsoncpp/include/json/config.h

@@ -1,96 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef JSON_CONFIG_H_INCLUDED
-# define JSON_CONFIG_H_INCLUDED
-
-/// If defined, indicates that json library is embedded in CppTL library.
-//# define JSON_IN_CPPTL 1
-
-/// If defined, indicates that json may leverage CppTL library
-//#  define JSON_USE_CPPTL 1
-/// If defined, indicates that cpptl vector based map should be used instead of std::map
-/// as Value container.
-//#  define JSON_USE_CPPTL_SMALLMAP 1
-/// If defined, indicates that Json specific container should be used
-/// (hash table & simple deque container with customizable allocator).
-/// THIS FEATURE IS STILL EXPERIMENTAL! There is know bugs: See #3177332
-//#  define JSON_VALUE_USE_INTERNAL_MAP 1
-/// Force usage of standard new/malloc based allocator instead of memory pool based allocator.
-/// The memory pools allocator used optimization (initializing Value and ValueInternalLink
-/// as if it was a POD) that may cause some validation tool to report errors.
-/// Only has effects if JSON_VALUE_USE_INTERNAL_MAP is defined.
-//#  define JSON_USE_SIMPLE_INTERNAL_ALLOCATOR 1
-
-/// If defined, indicates that Json use exception to report invalid type manipulation
-/// instead of C assert macro.
-# define JSON_USE_EXCEPTION 1
-
-/// If defined, indicates that the source file is amalgated
-/// to prevent private header inclusion.
-/// Remarks: it is automatically defined in the generated amalgated header.
-// #define JSON_IS_AMALGAMATION
-
-
-# ifdef JSON_IN_CPPTL
-#  include <cpptl/config.h>
-#  ifndef JSON_USE_CPPTL
-#   define JSON_USE_CPPTL 1
-#  endif
-# endif
-
-# ifdef JSON_IN_CPPTL
-#  define JSON_API CPPTL_API
-# elif defined(JSON_DLL_BUILD)
-#  define JSON_API __declspec(dllexport)
-# elif defined(JSON_DLL)
-#  define JSON_API __declspec(dllimport)
-# else
-#  define JSON_API
-# endif
-
-// If JSON_NO_INT64 is defined, then Json only support C++ "int" type for integer
-// Storages, and 64 bits integer support is disabled.
-// #define JSON_NO_INT64 1
-
-#if defined(_MSC_VER)  &&  _MSC_VER <= 1200 // MSVC 6
-// Microsoft Visual Studio 6 only support conversion from __int64 to double
-// (no conversion from unsigned __int64).
-#define JSON_USE_INT64_DOUBLE_CONVERSION 1
-#endif // if defined(_MSC_VER)  &&  _MSC_VER < 1200 // MSVC 6
-
-#if defined(_MSC_VER)  &&  _MSC_VER >= 1500 // MSVC 2008
-/// Indicates that the following function is deprecated.
-# define JSONCPP_DEPRECATED(message) __declspec(deprecated(message))
-#endif
-
-#if !defined(JSONCPP_DEPRECATED)
-# define JSONCPP_DEPRECATED(message)
-#endif // if !defined(JSONCPP_DEPRECATED)
-
-namespace Json {
-   typedef int Int;
-   typedef unsigned int UInt;
-# if defined(JSON_NO_INT64)
-   typedef int LargestInt;
-   typedef unsigned int LargestUInt;
-#  undef JSON_HAS_INT64
-# else // if defined(JSON_NO_INT64)
-   // For Microsoft Visual use specific types as long long is not supported
-#  if defined(_MSC_VER) // Microsoft Visual Studio
-   typedef __int64 Int64;
-   typedef unsigned __int64 UInt64;
-#  else // if defined(_MSC_VER) // Other platforms, use long long
-   typedef long long int Int64;
-   typedef unsigned long long int UInt64;
-#  endif // if defined(_MSC_VER)
-   typedef Int64 LargestInt;
-   typedef UInt64 LargestUInt;
-#  define JSON_HAS_INT64
-# endif // if defined(JSON_NO_INT64)
-} // end namespace Json
-
-
-#endif // JSON_CONFIG_H_INCLUDED

ext/jsoncpp/include/json/features.h

@@ -1,49 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef CPPTL_JSON_FEATURES_H_INCLUDED
-# define CPPTL_JSON_FEATURES_H_INCLUDED
-
-#if !defined(JSON_IS_AMALGAMATION)
-# include "forwards.h"
-#endif // if !defined(JSON_IS_AMALGAMATION)
-
-namespace Json {
-
-   /** \brief Configuration passed to reader and writer.
-    * This configuration object can be used to force the Reader or Writer
-    * to behave in a standard conforming way.
-    */
-   class JSON_API Features
-   {
-   public:
-      /** \brief A configuration that allows all features and assumes all strings are UTF-8.
-       * - C & C++ comments are allowed
-       * - Root object can be any JSON value
-       * - Assumes Value strings are encoded in UTF-8
-       */
-      static Features all();
-
-      /** \brief A configuration that is strictly compatible with the JSON specification.
-       * - Comments are forbidden.
-       * - Root object must be either an array or an object value.
-       * - Assumes Value strings are encoded in UTF-8
-       */
-      static Features strictMode();
-
-      /** \brief Initialize the configuration like JsonConfig::allFeatures;
-       */
-      Features();
-
-      /// \c true if comments are allowed. Default: \c true.
-      bool allowComments_;
-
-      /// \c true if root must be either an array or an object value. Default: \c false.
-      bool strictRoot_;
-   };
-
-} // namespace Json
-
-#endif // CPPTL_JSON_FEATURES_H_INCLUDED

ext/jsoncpp/include/json/forwards.h

@@ -1,44 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef JSON_FORWARDS_H_INCLUDED
-# define JSON_FORWARDS_H_INCLUDED
-
-#if !defined(JSON_IS_AMALGAMATION)
-# include "config.h"
-#endif // if !defined(JSON_IS_AMALGAMATION)
-
-namespace Json {
-
-   // writer.h
-   class FastWriter;
-   class StyledWriter;
-
-   // reader.h
-   class Reader;
-
-   // features.h
-   class Features;
-
-   // value.h
-   typedef unsigned int ArrayIndex;
-   class StaticString;
-   class Path;
-   class PathArgument;
-   class Value;
-   class ValueIteratorBase;
-   class ValueIterator;
-   class ValueConstIterator;
-#ifdef JSON_VALUE_USE_INTERNAL_MAP
-   class ValueMapAllocator;
-   class ValueInternalLink;
-   class ValueInternalArray;
-   class ValueInternalMap;
-#endif // #ifdef JSON_VALUE_USE_INTERNAL_MAP
-
-} // namespace Json
-
-
-#endif // JSON_FORWARDS_H_INCLUDED

ext/jsoncpp/include/json/json.h

@@ -1,15 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef JSON_JSON_H_INCLUDED
-# define JSON_JSON_H_INCLUDED
-
-# include "autolink.h"
-# include "value.h"
-# include "reader.h"
-# include "writer.h"
-# include "features.h"
-
-#endif // JSON_JSON_H_INCLUDED

ext/jsoncpp/include/json/reader.h

@@ -1,214 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef CPPTL_JSON_READER_H_INCLUDED
-# define CPPTL_JSON_READER_H_INCLUDED
-
-#if !defined(JSON_IS_AMALGAMATION)
-# include "features.h"
-# include "value.h"
-#endif // if !defined(JSON_IS_AMALGAMATION)
-# include <deque>
-# include <stack>
-# include <string>
-# include <iostream>
-
-namespace Json {
-
-   /** \brief Unserialize a <a HREF="http://www.json.org">JSON</a> document into a Value.
-    *
-    */
-   class JSON_API Reader
-   {
-   public:
-      typedef char Char;
-      typedef const Char *Location;
-
-      /** \brief Constructs a Reader allowing all features
-       * for parsing.
-       */
-      Reader();
-
-      /** \brief Constructs a Reader allowing the specified feature set
-       * for parsing.
-       */
-      Reader( const Features &features );
-
-      /** \brief Read a Value from a <a HREF="http://www.json.org">JSON</a> document.
-       * \param document UTF-8 encoded string containing the document to read.
-       * \param root [out] Contains the root value of the document if it was
-       *             successfully parsed.
-       * \param collectComments \c true to collect comment and allow writing them back during
-       *                        serialization, \c false to discard comments.
-       *                        This parameter is ignored if Features::allowComments_
-       *                        is \c false.
-       * \return \c true if the document was successfully parsed, \c false if an error occurred.
-       */
-      bool parse( const std::string &document, 
-                  Value &root,
-                  bool collectComments = true );
-
-      /** \brief Read a Value from a <a HREF="http://www.json.org">JSON</a> document.
-       * \param beginDoc Pointer on the beginning of the UTF-8 encoded string of the document to read.
-       * \param endDoc Pointer on the end of the UTF-8 encoded string of the document to read. 
-       \               Must be >= beginDoc.
-       * \param root [out] Contains the root value of the document if it was
-       *             successfully parsed.
-       * \param collectComments \c true to collect comment and allow writing them back during
-       *                        serialization, \c false to discard comments.
-       *                        This parameter is ignored if Features::allowComments_
-       *                        is \c false.
-       * \return \c true if the document was successfully parsed, \c false if an error occurred.
-       */
-      bool parse( const char *beginDoc, const char *endDoc, 
-                  Value &root,
-                  bool collectComments = true );
-
-      /// \brief Parse from input stream.
-      /// \see Json::operator>>(std::istream&, Json::Value&).
-      bool parse( std::istream &is,
-                  Value &root,
-                  bool collectComments = true );
-
-      /** \brief Returns a user friendly string that list errors in the parsed document.
-       * \return Formatted error message with the list of errors with their location in 
-       *         the parsed document. An empty string is returned if no error occurred
-       *         during parsing.
-       * \deprecated Use getFormattedErrorMessages() instead (typo fix).
-       */
-      JSONCPP_DEPRECATED("Use getFormattedErrorMessages instead") 
-      std::string getFormatedErrorMessages() const;
-
-      /** \brief Returns a user friendly string that list errors in the parsed document.
-       * \return Formatted error message with the list of errors with their location in 
-       *         the parsed document. An empty string is returned if no error occurred
-       *         during parsing.
-       */
-      std::string getFormattedErrorMessages() const;
-
-   private:
-      enum TokenType
-      {
-         tokenEndOfStream = 0,
-         tokenObjectBegin,
-         tokenObjectEnd,
-         tokenArrayBegin,
-         tokenArrayEnd,
-         tokenString,
-         tokenNumber,
-         tokenTrue,
-         tokenFalse,
-         tokenNull,
-         tokenArraySeparator,
-         tokenMemberSeparator,
-         tokenComment,
-         tokenError
-      };
-
-      class Token
-      {
-      public:
-         TokenType type_;
-         Location start_;
-         Location end_;
-      };
-
-      class ErrorInfo
-      {
-      public:
-         Token token_;
-         std::string message_;
-         Location extra_;
-      };
-
-      typedef std::deque<ErrorInfo> Errors;
-
-      bool expectToken( TokenType type, Token &token, const char *message );
-      bool readToken( Token &token );
-      void skipSpaces();
-      bool match( Location pattern, 
-                  int patternLength );
-      bool readComment();
-      bool readCStyleComment();
-      bool readCppStyleComment();
-      bool readString();
-      void readNumber();
-      bool readValue();
-      bool readObject( Token &token );
-      bool readArray( Token &token );
-      bool decodeNumber( Token &token );
-      bool decodeString( Token &token );
-      bool decodeString( Token &token, std::string &decoded );
-      bool decodeDouble( Token &token );
-      bool decodeUnicodeCodePoint( Token &token, 
-                                   Location &current, 
-                                   Location end, 
-                                   unsigned int &unicode );
-      bool decodeUnicodeEscapeSequence( Token &token, 
-                                        Location &current, 
-                                        Location end, 
-                                        unsigned int &unicode );
-      bool addError( const std::string &message, 
-                     Token &token,
-                     Location extra = 0 );
-      bool recoverFromError( TokenType skipUntilToken );
-      bool addErrorAndRecover( const std::string &message, 
-                               Token &token,
-                               TokenType skipUntilToken );
-      void skipUntilSpace();
-      Value &currentValue();
-      Char getNextChar();
-      void getLocationLineAndColumn( Location location,
-                                     int &line,
-                                     int &column ) const;
-      std::string getLocationLineAndColumn( Location location ) const;
-      void addComment( Location begin, 
-                       Location end, 
-                       CommentPlacement placement );
-      void skipCommentTokens( Token &token );
-   
-      typedef std::stack<Value *> Nodes;
-      Nodes nodes_;
-      Errors errors_;
-      std::string document_;
-      Location begin_;
-      Location end_;
-      Location current_;
-      Location lastValueEnd_;
-      Value *lastValue_;
-      std::string commentsBefore_;
-      Features features_;
-      bool collectComments_;
-   };
-
-   /** \brief Read from 'sin' into 'root'.
-
-    Always keep comments from the input JSON.
-
-    This can be used to read a file into a particular sub-object.
-    For example:
-    \code
-    Json::Value root;
-    cin >> root["dir"]["file"];
-    cout << root;
-    \endcode
-    Result:
-    \verbatim
-    {
-    "dir": {
-        "file": {
-        // The input stream JSON would be nested here.
-        }
-    }
-    }
-    \endverbatim
-    \throw std::exception on parse error.
-    \see Json::operator<<()
-   */
-   std::istream& operator>>( std::istream&, Value& );
-
-} // namespace Json
-
-#endif // CPPTL_JSON_READER_H_INCLUDED

ext/jsoncpp/include/json/writer.h

@@ -1,185 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef JSON_WRITER_H_INCLUDED
-# define JSON_WRITER_H_INCLUDED
-
-#if !defined(JSON_IS_AMALGAMATION)
-# include "value.h"
-#endif // if !defined(JSON_IS_AMALGAMATION)
-# include <vector>
-# include <string>
-# include <iostream>
-
-namespace Json {
-
-   class Value;
-
-   /** \brief Abstract class for writers.
-    */
-   class JSON_API Writer
-   {
-   public:
-      virtual ~Writer();
-
-      virtual std::string write( const Value &root ) = 0;
-   };
-
-   /** \brief Outputs a Value in <a HREF="http://www.json.org">JSON</a> format without formatting (not human friendly).
-    *
-    * The JSON document is written in a single line. It is not intended for 'human' consumption,
-    * but may be usefull to support feature such as RPC where bandwith is limited.
-    * \sa Reader, Value
-    */
-   class JSON_API FastWriter : public Writer
-   {
-   public:
-      FastWriter();
-      virtual ~FastWriter(){}
-
-      void enableYAMLCompatibility();
-
-   public: // overridden from Writer
-      virtual std::string write( const Value &root );
-
-   private:
-      void writeValue( const Value &value );
-
-      std::string document_;
-      bool yamlCompatiblityEnabled_;
-   };
-
-   /** \brief Writes a Value in <a HREF="http://www.json.org">JSON</a> format in a human friendly way.
-    *
-    * The rules for line break and indent are as follow:
-    * - Object value:
-    *     - if empty then print {} without indent and line break
-    *     - if not empty the print '{', line break & indent, print one value per line
-    *       and then unindent and line break and print '}'.
-    * - Array value:
-    *     - if empty then print [] without indent and line break
-    *     - if the array contains no object value, empty array or some other value types,
-    *       and all the values fit on one lines, then print the array on a single line.
-    *     - otherwise, it the values do not fit on one line, or the array contains
-    *       object or non empty array, then print one value per line.
-    *
-    * If the Value have comments then they are outputed according to their #CommentPlacement.
-    *
-    * \sa Reader, Value, Value::setComment()
-    */
-   class JSON_API StyledWriter: public Writer
-   {
-   public:
-      StyledWriter();
-      virtual ~StyledWriter(){}
-
-   public: // overridden from Writer
-      /** \brief Serialize a Value in <a HREF="http://www.json.org">JSON</a> format.
-       * \param root Value to serialize.
-       * \return String containing the JSON document that represents the root value.
-       */
-      virtual std::string write( const Value &root );
-
-   private:
-      void writeValue( const Value &value );
-      void writeArrayValue( const Value &value );
-      bool isMultineArray( const Value &value );
-      void pushValue( const std::string &value );
-      void writeIndent();
-      void writeWithIndent( const std::string &value );
-      void indent();
-      void unindent();
-      void writeCommentBeforeValue( const Value &root );
-      void writeCommentAfterValueOnSameLine( const Value &root );
-      bool hasCommentForValue( const Value &value );
-      static std::string normalizeEOL( const std::string &text );
-
-      typedef std::vector<std::string> ChildValues;
-
-      ChildValues childValues_;
-      std::string document_;
-      std::string indentString_;
-      int rightMargin_;
-      int indentSize_;
-      bool addChildValues_;
-   };
-
-   /** \brief Writes a Value in <a HREF="http://www.json.org">JSON</a> format in a human friendly way,
-        to a stream rather than to a string.
-    *
-    * The rules for line break and indent are as follow:
-    * - Object value:
-    *     - if empty then print {} without indent and line break
-    *     - if not empty the print '{', line break & indent, print one value per line
-    *       and then unindent and line break and print '}'.
-    * - Array value:
-    *     - if empty then print [] without indent and line break
-    *     - if the array contains no object value, empty array or some other value types,
-    *       and all the values fit on one lines, then print the array on a single line.
-    *     - otherwise, it the values do not fit on one line, or the array contains
-    *       object or non empty array, then print one value per line.
-    *
-    * If the Value have comments then they are outputed according to their #CommentPlacement.
-    *
-    * \param indentation Each level will be indented by this amount extra.
-    * \sa Reader, Value, Value::setComment()
-    */
-   class JSON_API StyledStreamWriter
-   {
-   public:
-      StyledStreamWriter( std::string indentation="\t" );
-      ~StyledStreamWriter(){}
-
-   public:
-      /** \brief Serialize a Value in <a HREF="http://www.json.org">JSON</a> format.
-       * \param out Stream to write to. (Can be ostringstream, e.g.)
-       * \param root Value to serialize.
-       * \note There is no point in deriving from Writer, since write() should not return a value.
-       */
-      void write( std::ostream &out, const Value &root );
-
-   private:
-      void writeValue( const Value &value );
-      void writeArrayValue( const Value &value );
-      bool isMultineArray( const Value &value );
-      void pushValue( const std::string &value );
-      void writeIndent();
-      void writeWithIndent( const std::string &value );
-      void indent();
-      void unindent();
-      void writeCommentBeforeValue( const Value &root );
-      void writeCommentAfterValueOnSameLine( const Value &root );
-      bool hasCommentForValue( const Value &value );
-      static std::string normalizeEOL( const std::string &text );
-
-      typedef std::vector<std::string> ChildValues;
-
-      ChildValues childValues_;
-      std::ostream* document_;
-      std::string indentString_;
-      int rightMargin_;
-      std::string indentation_;
-      bool addChildValues_;
-   };
-
-# if defined(JSON_HAS_INT64)
-   std::string JSON_API valueToString( Int value );
-   std::string JSON_API valueToString( UInt value );
-# endif // if defined(JSON_HAS_INT64)
-   std::string JSON_API valueToString( LargestInt value );
-   std::string JSON_API valueToString( LargestUInt value );
-   std::string JSON_API valueToString( double value );
-   std::string JSON_API valueToString( bool value );
-   std::string JSON_API valueToQuotedString( const char *value );
-
-   /// \brief Output using the StyledStreamWriter.
-   /// \see Json::operator>>()
-   std::ostream& operator<<( std::ostream&, const Value &root );
-
-} // namespace Json
-
-
-
-#endif // JSON_WRITER_H_INCLUDED

ext/jsoncpp/src/json_batchallocator.h

@@ -1,130 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef JSONCPP_BATCHALLOCATOR_H_INCLUDED
-# define JSONCPP_BATCHALLOCATOR_H_INCLUDED
-
-# include <stdlib.h>
-# include <assert.h>
-
-# ifndef JSONCPP_DOC_EXCLUDE_IMPLEMENTATION
-
-namespace Json {
-
-/* Fast memory allocator.
- *
- * This memory allocator allocates memory for a batch of object (specified by
- * the page size, the number of object in each page).
- *
- * It does not allow the destruction of a single object. All the allocated objects
- * can be destroyed at once. The memory can be either released or reused for future
- * allocation.
- * 
- * The in-place new operator must be used to construct the object using the pointer
- * returned by allocate.
- */
-template<typename AllocatedType
-        ,const unsigned int objectPerAllocation>
-class BatchAllocator
-{
-public:
-   typedef AllocatedType Type;
-
-   BatchAllocator( unsigned int objectsPerPage = 255 )
-      : freeHead_( 0 )
-      , objectsPerPage_( objectsPerPage )
-   {
-//      printf( "Size: %d => %s\n", sizeof(AllocatedType), typeid(AllocatedType).name() );
-      assert( sizeof(AllocatedType) * objectPerAllocation >= sizeof(AllocatedType *) ); // We must be able to store a slist in the object free space.
-      assert( objectsPerPage >= 16 );
-      batches_ = allocateBatch( 0 );   // allocated a dummy page
-      currentBatch_ = batches_;
-   }
-
-   ~BatchAllocator()
-   {
-      for ( BatchInfo *batch = batches_; batch;  )
-      {
-         BatchInfo *nextBatch = batch->next_;
-         free( batch );
-         batch = nextBatch;
-      }
-   }
-
-   /// allocate space for an array of objectPerAllocation object.
-   /// @warning it is the responsability of the caller to call objects constructors.
-   AllocatedType *allocate()
-   {
-      if ( freeHead_ ) // returns node from free list.
-      {
-         AllocatedType *object = freeHead_;
-         freeHead_ = *(AllocatedType **)object;
-         return object;
-      }
-      if ( currentBatch_->used_ == currentBatch_->end_ )
-      {
-         currentBatch_ = currentBatch_->next_;
-         while ( currentBatch_  &&  currentBatch_->used_ == currentBatch_->end_ )
-            currentBatch_ = currentBatch_->next_;
-
-         if ( !currentBatch_  ) // no free batch found, allocate a new one
-         { 
-            currentBatch_ = allocateBatch( objectsPerPage_ );
-            currentBatch_->next_ = batches_; // insert at the head of the list
-            batches_ = currentBatch_;
-         }
-      }
-      AllocatedType *allocated = currentBatch_->used_;
-      currentBatch_->used_ += objectPerAllocation;
-      return allocated;
-   }
-
-   /// Release the object.
-   /// @warning it is the responsability of the caller to actually destruct the object.
-   void release( AllocatedType *object )
-   {
-      assert( object != 0 );
-      *(AllocatedType **)object = freeHead_;
-      freeHead_ = object;
-   }
-
-private:
-   struct BatchInfo
-   {
-      BatchInfo *next_;
-      AllocatedType *used_;
-      AllocatedType *end_;
-      AllocatedType buffer_[objectPerAllocation];
-   };
-
-   // disabled copy constructor and assignement operator.
-   BatchAllocator( const BatchAllocator & );
-   void operator =( const BatchAllocator &);
-
-   static BatchInfo *allocateBatch( unsigned int objectsPerPage )
-   {
-      const unsigned int mallocSize = sizeof(BatchInfo) - sizeof(AllocatedType)* objectPerAllocation
-                                + sizeof(AllocatedType) * objectPerAllocation * objectsPerPage;
-      BatchInfo *batch = static_cast<BatchInfo*>( malloc( mallocSize ) );
-      batch->next_ = 0;
-      batch->used_ = batch->buffer_;
-      batch->end_ = batch->buffer_ + objectsPerPage;
-      return batch;
-   }
-
-   BatchInfo *batches_;
-   BatchInfo *currentBatch_;
-   /// Head of a single linked list within the allocated space of freeed object
-   AllocatedType *freeHead_;
-   unsigned int objectsPerPage_;
-};
-
-
-} // namespace Json
-
-# endif // ifndef JSONCPP_DOC_INCLUDE_IMPLEMENTATION
-
-#endif // JSONCPP_BATCHALLOCATOR_H_INCLUDED
-

ext/jsoncpp/src/json_internalarray.inl

@@ -1,456 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-// included by json_value.cpp
-
-namespace Json {
-
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// class ValueInternalArray
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-
-ValueArrayAllocator::~ValueArrayAllocator()
-{
-}
-
-// //////////////////////////////////////////////////////////////////
-// class DefaultValueArrayAllocator
-// //////////////////////////////////////////////////////////////////
-#ifdef JSON_USE_SIMPLE_INTERNAL_ALLOCATOR
-class DefaultValueArrayAllocator : public ValueArrayAllocator
-{
-public: // overridden from ValueArrayAllocator
-   virtual ~DefaultValueArrayAllocator()
-   {
-   }
-
-   virtual ValueInternalArray *newArray()
-   {
-      return new ValueInternalArray();
-   }
-
-   virtual ValueInternalArray *newArrayCopy( const ValueInternalArray &other )
-   {
-      return new ValueInternalArray( other );
-   }
-
-   virtual void destructArray( ValueInternalArray *array )
-   {
-      delete array;
-   }
-
-   virtual void reallocateArrayPageIndex( Value **&indexes, 
-                                          ValueInternalArray::PageIndex &indexCount,
-                                          ValueInternalArray::PageIndex minNewIndexCount )
-   {
-      ValueInternalArray::PageIndex newIndexCount = (indexCount*3)/2 + 1;
-      if ( minNewIndexCount > newIndexCount )
-         newIndexCount = minNewIndexCount;
-      void *newIndexes = realloc( indexes, sizeof(Value*) * newIndexCount );
-      if ( !newIndexes )
-         throw std::bad_alloc();
-      indexCount = newIndexCount;
-      indexes = static_cast<Value **>( newIndexes );
-   }
-   virtual void releaseArrayPageIndex( Value **indexes, 
-                                       ValueInternalArray::PageIndex indexCount )
-   {
-      if ( indexes )
-         free( indexes );
-   }
-
-   virtual Value *allocateArrayPage()
-   {
-      return static_cast<Value *>( malloc( sizeof(Value) * ValueInternalArray::itemsPerPage ) );
-   }
-
-   virtual void releaseArrayPage( Value *value )
-   {
-      if ( value )
-         free( value );
-   }
-};
-
-#else // #ifdef JSON_USE_SIMPLE_INTERNAL_ALLOCATOR
-/// @todo make this thread-safe (lock when accessign batch allocator)
-class DefaultValueArrayAllocator : public ValueArrayAllocator
-{
-public: // overridden from ValueArrayAllocator
-   virtual ~DefaultValueArrayAllocator()
-   {
-   }
-
-   virtual ValueInternalArray *newArray()
-   {
-      ValueInternalArray *array = arraysAllocator_.allocate();
-      new (array) ValueInternalArray(); // placement new
-      return array;
-   }
-
-   virtual ValueInternalArray *newArrayCopy( const ValueInternalArray &other )
-   {
-      ValueInternalArray *array = arraysAllocator_.allocate();
-      new (array) ValueInternalArray( other ); // placement new
-      return array;
-   }
-
-   virtual void destructArray( ValueInternalArray *array )
-   {
-      if ( array )
-      {
-         array->~ValueInternalArray();
-         arraysAllocator_.release( array );
-      }
-   }
-
-   virtual void reallocateArrayPageIndex( Value **&indexes, 
-                                          ValueInternalArray::PageIndex &indexCount,
-                                          ValueInternalArray::PageIndex minNewIndexCount )
-   {
-      ValueInternalArray::PageIndex newIndexCount = (indexCount*3)/2 + 1;
-      if ( minNewIndexCount > newIndexCount )
-         newIndexCount = minNewIndexCount;
-      void *newIndexes = realloc( indexes, sizeof(Value*) * newIndexCount );
-      if ( !newIndexes )
-         throw std::bad_alloc();
-      indexCount = newIndexCount;
-      indexes = static_cast<Value **>( newIndexes );
-   }
-   virtual void releaseArrayPageIndex( Value **indexes, 
-                                       ValueInternalArray::PageIndex indexCount )
-   {
-      if ( indexes )
-         free( indexes );
-   }
-
-   virtual Value *allocateArrayPage()
-   {
-      return static_cast<Value *>( pagesAllocator_.allocate() );
-   }
-
-   virtual void releaseArrayPage( Value *value )
-   {
-      if ( value )
-         pagesAllocator_.release( value );
-   }
-private:
-   BatchAllocator<ValueInternalArray,1> arraysAllocator_;
-   BatchAllocator<Value,ValueInternalArray::itemsPerPage> pagesAllocator_;
-};
-#endif // #ifdef JSON_USE_SIMPLE_INTERNAL_ALLOCATOR
-
-static ValueArrayAllocator *&arrayAllocator()
-{
-   static DefaultValueArrayAllocator defaultAllocator;
-   static ValueArrayAllocator *arrayAllocator = &defaultAllocator;
-   return arrayAllocator;
-}
-
-static struct DummyArrayAllocatorInitializer {
-   DummyArrayAllocatorInitializer() 
-   {
-      arrayAllocator();      // ensure arrayAllocator() statics are initialized before main().
-   }
-} dummyArrayAllocatorInitializer;
-
-// //////////////////////////////////////////////////////////////////
-// class ValueInternalArray
-// //////////////////////////////////////////////////////////////////
-bool 
-ValueInternalArray::equals( const IteratorState &x, 
-                            const IteratorState &other )
-{
-   return x.array_ == other.array_  
-          &&  x.currentItemIndex_ == other.currentItemIndex_  
-          &&  x.currentPageIndex_ == other.currentPageIndex_;
-}
-
-
-void 
-ValueInternalArray::increment( IteratorState &it )
-{
-   JSON_ASSERT_MESSAGE( it.array_  &&
-      (it.currentPageIndex_ - it.array_->pages_)*itemsPerPage + it.currentItemIndex_
-      != it.array_->size_,
-      "ValueInternalArray::increment(): moving iterator beyond end" );
-   ++(it.currentItemIndex_);
-   if ( it.currentItemIndex_ == itemsPerPage )
-   {
-      it.currentItemIndex_ = 0;
-      ++(it.currentPageIndex_);
-   }
-}
-
-
-void 
-ValueInternalArray::decrement( IteratorState &it )
-{
-   JSON_ASSERT_MESSAGE( it.array_  &&  it.currentPageIndex_ == it.array_->pages_ 
-                        &&  it.currentItemIndex_ == 0,
-      "ValueInternalArray::decrement(): moving iterator beyond end" );
-   if ( it.currentItemIndex_ == 0 )
-   {
-      it.currentItemIndex_ = itemsPerPage-1;
-      --(it.currentPageIndex_);
-   }
-   else
-   {
-      --(it.currentItemIndex_);
-   }
-}
-
-
-Value &
-ValueInternalArray::unsafeDereference( const IteratorState &it )
-{
-   return (*(it.currentPageIndex_))[it.currentItemIndex_];
-}
-
-
-Value &
-ValueInternalArray::dereference( const IteratorState &it )
-{
-   JSON_ASSERT_MESSAGE( it.array_  &&
-      (it.currentPageIndex_ - it.array_->pages_)*itemsPerPage + it.currentItemIndex_
-      < it.array_->size_,
-      "ValueInternalArray::dereference(): dereferencing invalid iterator" );
-   return unsafeDereference( it );
-}
-
-void 
-ValueInternalArray::makeBeginIterator( IteratorState &it ) const
-{
-   it.array_ = const_cast<ValueInternalArray *>( this );
-   it.currentItemIndex_ = 0;
-   it.currentPageIndex_ = pages_;
-}
-
-
-void 
-ValueInternalArray::makeIterator( IteratorState &it, ArrayIndex index ) const
-{
-   it.array_ = const_cast<ValueInternalArray *>( this );
-   it.currentItemIndex_ = index % itemsPerPage;
-   it.currentPageIndex_ = pages_ + index / itemsPerPage;
-}
-
-
-void 
-ValueInternalArray::makeEndIterator( IteratorState &it ) const
-{
-   makeIterator( it, size_ );
-}
-
-
-ValueInternalArray::ValueInternalArray()
-   : pages_( 0 )
-   , size_( 0 )
-   , pageCount_( 0 )
-{
-}
-
-
-ValueInternalArray::ValueInternalArray( const ValueInternalArray &other )
-   : pages_( 0 )
-   , pageCount_( 0 )
-   , size_( other.size_ )
-{
-   PageIndex minNewPages = other.size_ / itemsPerPage;
-   arrayAllocator()->reallocateArrayPageIndex( pages_, pageCount_, minNewPages );
-   JSON_ASSERT_MESSAGE( pageCount_ >= minNewPages, 
-                        "ValueInternalArray::reserve(): bad reallocation" );
-   IteratorState itOther;
-   other.makeBeginIterator( itOther );
-   Value *value;
-   for ( ArrayIndex index = 0; index < size_; ++index, increment(itOther) )
-   {
-      if ( index % itemsPerPage == 0 )
-      {
-         PageIndex pageIndex = index / itemsPerPage;
-         value = arrayAllocator()->allocateArrayPage();
-         pages_[pageIndex] = value;
-      }
-      new (value) Value( dereference( itOther ) );
-   }
-}
-
-
-ValueInternalArray &
-ValueInternalArray::operator =( const ValueInternalArray &other )
-{
-   ValueInternalArray temp( other );
-   swap( temp );
-   return *this;
-}
-
-
-ValueInternalArray::~ValueInternalArray()
-{
-   // destroy all constructed items
-   IteratorState it;
-   IteratorState itEnd;
-   makeBeginIterator( it);
-   makeEndIterator( itEnd );
-   for ( ; !equals(it,itEnd); increment(it) )
-   {
-      Value *value = &dereference(it);
-      value->~Value();
-   }
-   // release all pages
-   PageIndex lastPageIndex = size_ / itemsPerPage;
-   for ( PageIndex pageIndex = 0; pageIndex < lastPageIndex; ++pageIndex )
-      arrayAllocator()->releaseArrayPage( pages_[pageIndex] );
-   // release pages index
-   arrayAllocator()->releaseArrayPageIndex( pages_, pageCount_ );
-}
-
-
-void 
-ValueInternalArray::swap( ValueInternalArray &other )
-{
-   Value **tempPages = pages_;
-   pages_ = other.pages_;
-   other.pages_ = tempPages;
-   ArrayIndex tempSize = size_;
-   size_ = other.size_;
-   other.size_ = tempSize;
-   PageIndex tempPageCount = pageCount_;
-   pageCount_ = other.pageCount_;
-   other.pageCount_ = tempPageCount;
-}
-
-void 
-ValueInternalArray::clear()
-{
-   ValueInternalArray dummy;
-   swap( dummy );
-}
-
-
-void 
-ValueInternalArray::resize( ArrayIndex newSize )
-{
-   if ( newSize == 0 )
-      clear();
-   else if ( newSize < size_ )
-   {
-      IteratorState it;
-      IteratorState itEnd;
-      makeIterator( it, newSize );
-      makeIterator( itEnd, size_ );
-      for ( ; !equals(it,itEnd); increment(it) )
-      {
-         Value *value = &dereference(it);
-         value->~Value();
-      }
-      PageIndex pageIndex = (newSize + itemsPerPage - 1) / itemsPerPage;
-      PageIndex lastPageIndex = size_ / itemsPerPage;
-      for ( ; pageIndex < lastPageIndex; ++pageIndex )
-         arrayAllocator()->releaseArrayPage( pages_[pageIndex] );
-      size_ = newSize;
-   }
-   else if ( newSize > size_ )
-      resolveReference( newSize );
-}
-
-
-void 
-ValueInternalArray::makeIndexValid( ArrayIndex index )
-{
-   // Need to enlarge page index ?
-   if ( index >= pageCount_ * itemsPerPage )
-   {
-      PageIndex minNewPages = (index + 1) / itemsPerPage;
-      arrayAllocator()->reallocateArrayPageIndex( pages_, pageCount_, minNewPages );
-      JSON_ASSERT_MESSAGE( pageCount_ >= minNewPages, "ValueInternalArray::reserve(): bad reallocation" );
-   }
-
-   // Need to allocate new pages ?
-   ArrayIndex nextPageIndex = 
-      (size_ % itemsPerPage) != 0 ? size_ - (size_%itemsPerPage) + itemsPerPage
-                                  : size_;
-   if ( nextPageIndex <= index )
-   {
-      PageIndex pageIndex = nextPageIndex / itemsPerPage;
-      PageIndex pageToAllocate = (index - nextPageIndex) / itemsPerPage + 1;
-      for ( ; pageToAllocate-- > 0; ++pageIndex )
-         pages_[pageIndex] = arrayAllocator()->allocateArrayPage();
-   }
-
-   // Initialize all new entries
-   IteratorState it;
-   IteratorState itEnd;
-   makeIterator( it, size_ );
-   size_ = index + 1;
-   makeIterator( itEnd, size_ );
-   for ( ; !equals(it,itEnd); increment(it) )
-   {
-      Value *value = &dereference(it);
-      new (value) Value(); // Construct a default value using placement new
-   }
-}
-
-Value &
-ValueInternalArray::resolveReference( ArrayIndex index )
-{
-   if ( index >= size_ )
-      makeIndexValid( index );
-   return pages_[index/itemsPerPage][index%itemsPerPage];
-}
-
-Value *
-ValueInternalArray::find( ArrayIndex index ) const
-{
-   if ( index >= size_ )
-      return 0;
-   return &(pages_[index/itemsPerPage][index%itemsPerPage]);
-}
-
-ValueInternalArray::ArrayIndex 
-ValueInternalArray::size() const
-{
-   return size_;
-}
-
-int 
-ValueInternalArray::distance( const IteratorState &x, const IteratorState &y )
-{
-   return indexOf(y) - indexOf(x);
-}
-
-
-ValueInternalArray::ArrayIndex 
-ValueInternalArray::indexOf( const IteratorState &iterator )
-{
-   if ( !iterator.array_ )
-      return ArrayIndex(-1);
-   return ArrayIndex(
-      (iterator.currentPageIndex_ - iterator.array_->pages_) * itemsPerPage 
-      + iterator.currentItemIndex_ );
-}
-
-
-int 
-ValueInternalArray::compare( const ValueInternalArray &other ) const
-{
-   int sizeDiff( size_ - other.size_ );
-   if ( sizeDiff != 0 )
-      return sizeDiff;
-   
-   for ( ArrayIndex index =0; index < size_; ++index )
-   {
-      int diff = pages_[index/itemsPerPage][index%itemsPerPage].compare( 
-         other.pages_[index/itemsPerPage][index%itemsPerPage] );
-      if ( diff != 0 )
-         return diff;
-   }
-   return 0;
-}
-
-} // namespace Json

ext/jsoncpp/src/json_internalmap.inl

@@ -1,615 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-// included by json_value.cpp
-
-namespace Json {
-
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// class ValueInternalMap
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-
-/** \internal MUST be safely initialized using memset( this, 0, sizeof(ValueInternalLink) );
-   * This optimization is used by the fast allocator.
-   */
-ValueInternalLink::ValueInternalLink()
-   : previous_( 0 )
-   , next_( 0 )
-{
-}
-
-ValueInternalLink::~ValueInternalLink()
-{ 
-   for ( int index =0; index < itemPerLink; ++index )
-   {
-      if ( !items_[index].isItemAvailable() )
-      {
-         if ( !items_[index].isMemberNameStatic() )
-            free( keys_[index] );
-      }
-      else
-         break;
-   }
-}
-
-
-
-ValueMapAllocator::~ValueMapAllocator()
-{
-}
-
-#ifdef JSON_USE_SIMPLE_INTERNAL_ALLOCATOR
-class DefaultValueMapAllocator : public ValueMapAllocator
-{
-public: // overridden from ValueMapAllocator
-   virtual ValueInternalMap *newMap()
-   {
-      return new ValueInternalMap();
-   }
-
-   virtual ValueInternalMap *newMapCopy( const ValueInternalMap &other )
-   {
-      return new ValueInternalMap( other );
-   }
-
-   virtual void destructMap( ValueInternalMap *map )
-   {
-      delete map;
-   }
-
-   virtual ValueInternalLink *allocateMapBuckets( unsigned int size )
-   {
-      return new ValueInternalLink[size];
-   }
-
-   virtual void releaseMapBuckets( ValueInternalLink *links )
-   {
-      delete [] links;
-   }
-
-   virtual ValueInternalLink *allocateMapLink()
-   {
-      return new ValueInternalLink();
-   }
-
-   virtual void releaseMapLink( ValueInternalLink *link )
-   {
-      delete link;
-   }
-};
-#else
-/// @todo make this thread-safe (lock when accessign batch allocator)
-class DefaultValueMapAllocator : public ValueMapAllocator
-{
-public: // overridden from ValueMapAllocator
-   virtual ValueInternalMap *newMap()
-   {
-      ValueInternalMap *map = mapsAllocator_.allocate();
-      new (map) ValueInternalMap(); // placement new
-      return map;
-   }
-
-   virtual ValueInternalMap *newMapCopy( const ValueInternalMap &other )
-   {
-      ValueInternalMap *map = mapsAllocator_.allocate();
-      new (map) ValueInternalMap( other ); // placement new
-      return map;
-   }
-
-   virtual void destructMap( ValueInternalMap *map )
-   {
-      if ( map )
-      {
-         map->~ValueInternalMap();
-         mapsAllocator_.release( map );
-      }
-   }
-
-   virtual ValueInternalLink *allocateMapBuckets( unsigned int size )
-   {
-      return new ValueInternalLink[size];
-   }
-
-   virtual void releaseMapBuckets( ValueInternalLink *links )
-   {
-      delete [] links;
-   }
-
-   virtual ValueInternalLink *allocateMapLink()
-   {
-      ValueInternalLink *link = linksAllocator_.allocate();
-      memset( link, 0, sizeof(ValueInternalLink) );
-      return link;
-   }
-
-   virtual void releaseMapLink( ValueInternalLink *link )
-   {
-      link->~ValueInternalLink();
-      linksAllocator_.release( link );
-   }
-private:
-   BatchAllocator<ValueInternalMap,1> mapsAllocator_;
-   BatchAllocator<ValueInternalLink,1> linksAllocator_;
-};
-#endif
-
-static ValueMapAllocator *&mapAllocator()
-{
-   static DefaultValueMapAllocator defaultAllocator;
-   static ValueMapAllocator *mapAllocator = &defaultAllocator;
-   return mapAllocator;
-}
-
-static struct DummyMapAllocatorInitializer {
-   DummyMapAllocatorInitializer() 
-   {
-      mapAllocator();      // ensure mapAllocator() statics are initialized before main().
-   }
-} dummyMapAllocatorInitializer;
-
-
-
-// h(K) = value * K >> w ; with w = 32 & K prime w.r.t. 2^32.
-
-/*
-use linked list hash map. 
-buckets array is a container.
-linked list element contains 6 key/values. (memory = (16+4) * 6 + 4 = 124)
-value have extra state: valid, available, deleted
-*/
-
-
-ValueInternalMap::ValueInternalMap()
-   : buckets_( 0 )
-   , tailLink_( 0 )
-   , bucketsSize_( 0 )
-   , itemCount_( 0 )
-{
-}
-
-
-ValueInternalMap::ValueInternalMap( const ValueInternalMap &other )
-   : buckets_( 0 )
-   , tailLink_( 0 )
-   , bucketsSize_( 0 )
-   , itemCount_( 0 )
-{
-   reserve( other.itemCount_ );
-   IteratorState it;
-   IteratorState itEnd;
-   other.makeBeginIterator( it );
-   other.makeEndIterator( itEnd );
-   for ( ; !equals(it,itEnd); increment(it) )
-   {
-      bool isStatic;
-      const char *memberName = key( it, isStatic );
-      const Value &aValue = value( it );
-      resolveReference(memberName, isStatic) = aValue;
-   }
-}
-
-
-ValueInternalMap &
-ValueInternalMap::operator =( const ValueInternalMap &other )
-{
-   ValueInternalMap dummy( other );
-   swap( dummy );
-   return *this;
-}
-
-
-ValueInternalMap::~ValueInternalMap()
-{
-   if ( buckets_ )
-   {
-      for ( BucketIndex bucketIndex =0; bucketIndex < bucketsSize_; ++bucketIndex )
-      {
-         ValueInternalLink *link = buckets_[bucketIndex].next_;
-         while ( link )
-         {
-            ValueInternalLink *linkToRelease = link;
-            link = link->next_;
-            mapAllocator()->releaseMapLink( linkToRelease );
-         }
-      }
-      mapAllocator()->releaseMapBuckets( buckets_ );
-   }
-}
-
-
-void 
-ValueInternalMap::swap( ValueInternalMap &other )
-{
-   ValueInternalLink *tempBuckets = buckets_;
-   buckets_ = other.buckets_;
-   other.buckets_ = tempBuckets;
-   ValueInternalLink *tempTailLink = tailLink_;
-   tailLink_ = other.tailLink_;
-   other.tailLink_ = tempTailLink;
-   BucketIndex tempBucketsSize = bucketsSize_;
-   bucketsSize_ = other.bucketsSize_;
-   other.bucketsSize_ = tempBucketsSize;
-   BucketIndex tempItemCount = itemCount_;
-   itemCount_ = other.itemCount_;
-   other.itemCount_ = tempItemCount;
-}
-
-
-void 
-ValueInternalMap::clear()
-{
-   ValueInternalMap dummy;
-   swap( dummy );
-}
-
-
-ValueInternalMap::BucketIndex 
-ValueInternalMap::size() const
-{
-   return itemCount_;
-}
-
-bool 
-ValueInternalMap::reserveDelta( BucketIndex growth )
-{
-   return reserve( itemCount_ + growth );
-}
-
-bool 
-ValueInternalMap::reserve( BucketIndex newItemCount )
-{
-   if ( !buckets_  &&  newItemCount > 0 )
-   {
-      buckets_ = mapAllocator()->allocateMapBuckets( 1 );
-      bucketsSize_ = 1;
-      tailLink_ = &buckets_[0];
-   }
-//   BucketIndex idealBucketCount = (newItemCount + ValueInternalLink::itemPerLink) / ValueInternalLink::itemPerLink;
-   return true;
-}
-
-
-const Value *
-ValueInternalMap::find( const char *key ) const
-{
-   if ( !bucketsSize_ )
-      return 0;
-   HashKey hashedKey = hash( key );
-   BucketIndex bucketIndex = hashedKey % bucketsSize_;
-   for ( const ValueInternalLink *current = &buckets_[bucketIndex]; 
-         current != 0; 
-         current = current->next_ )
-   {
-      for ( BucketIndex index=0; index < ValueInternalLink::itemPerLink; ++index )
-      {
-         if ( current->items_[index].isItemAvailable() )
-            return 0;
-         if ( strcmp( key, current->keys_[index] ) == 0 )
-            return &current->items_[index];
-      }
-   }
-   return 0;
-}
-
-
-Value *
-ValueInternalMap::find( const char *key )
-{
-   const ValueInternalMap *constThis = this;
-   return const_cast<Value *>( constThis->find( key ) );
-}
-
-
-Value &
-ValueInternalMap::resolveReference( const char *key,
-                                    bool isStatic )
-{
-   HashKey hashedKey = hash( key );
-   if ( bucketsSize_ )
-   {
-      BucketIndex bucketIndex = hashedKey % bucketsSize_;
-      ValueInternalLink **previous = 0;
-      BucketIndex index;
-      for ( ValueInternalLink *current = &buckets_[bucketIndex]; 
-            current != 0; 
-            previous = &current->next_, current = current->next_ )
-      {
-         for ( index=0; index < ValueInternalLink::itemPerLink; ++index )
-         {
-            if ( current->items_[index].isItemAvailable() )
-               return setNewItem( key, isStatic, current, index );
-            if ( strcmp( key, current->keys_[index] ) == 0 )
-               return current->items_[index];
-         }
-      }
-   }
-
-   reserveDelta( 1 );
-   return unsafeAdd( key, isStatic, hashedKey );
-}
-
-
-void 
-ValueInternalMap::remove( const char *key )
-{
-   HashKey hashedKey = hash( key );
-   if ( !bucketsSize_ )
-      return;
-   BucketIndex bucketIndex = hashedKey % bucketsSize_;
-   for ( ValueInternalLink *link = &buckets_[bucketIndex]; 
-         link != 0; 
-         link = link->next_ )
-   {
-      BucketIndex index;
-      for ( index =0; index < ValueInternalLink::itemPerLink; ++index )
-      {
-         if ( link->items_[index].isItemAvailable() )
-            return;
-         if ( strcmp( key, link->keys_[index] ) == 0 )
-         {
-            doActualRemove( link, index, bucketIndex );
-            return;
-         }
-      }
-   }
-}
-
-void 
-ValueInternalMap::doActualRemove( ValueInternalLink *link, 
-                                  BucketIndex index,
-                                  BucketIndex bucketIndex )
-{
-   // find last item of the bucket and swap it with the 'removed' one.
-   // set removed items flags to 'available'.
-   // if last page only contains 'available' items, then desallocate it (it's empty)
-   ValueInternalLink *&lastLink = getLastLinkInBucket( index );
-   BucketIndex lastItemIndex = 1; // a link can never be empty, so start at 1
-   for ( ;   
-         lastItemIndex < ValueInternalLink::itemPerLink; 
-         ++lastItemIndex ) // may be optimized with dicotomic search
-   {
-      if ( lastLink->items_[lastItemIndex].isItemAvailable() )
-         break;
-   }
-   
-   BucketIndex lastUsedIndex = lastItemIndex - 1;
-   Value *valueToDelete = &link->items_[index];
-   Value *valueToPreserve = &lastLink->items_[lastUsedIndex];
-   if ( valueToDelete != valueToPreserve )
-      valueToDelete->swap( *valueToPreserve );
-   if ( lastUsedIndex == 0 )  // page is now empty
-   {  // remove it from bucket linked list and delete it.
-      ValueInternalLink *linkPreviousToLast = lastLink->previous_;
-      if ( linkPreviousToLast != 0 )   // can not deleted bucket link.
-      {
-         mapAllocator()->releaseMapLink( lastLink );
-         linkPreviousToLast->next_ = 0;
-         lastLink = linkPreviousToLast;
-      }
-   }
-   else
-   {
-      Value dummy;
-      valueToPreserve->swap( dummy ); // restore deleted to default Value.
-      valueToPreserve->setItemUsed( false );
-   }
-   --itemCount_;
-}
-
-
-ValueInternalLink *&
-ValueInternalMap::getLastLinkInBucket( BucketIndex bucketIndex )
-{
-   if ( bucketIndex == bucketsSize_ - 1 )
-      return tailLink_;
-   ValueInternalLink *&previous = buckets_[bucketIndex+1].previous_;
-   if ( !previous )
-      previous = &buckets_[bucketIndex];
-   return previous;
-}
-
-
-Value &
-ValueInternalMap::setNewItem( const char *key, 
-                              bool isStatic,
-                              ValueInternalLink *link, 
-                              BucketIndex index )
-{
-   char *duplicatedKey = makeMemberName( key );
-   ++itemCount_;
-   link->keys_[index] = duplicatedKey;
-   link->items_[index].setItemUsed();
-   link->items_[index].setMemberNameIsStatic( isStatic );
-   return link->items_[index]; // items already default constructed.
-}
-
-
-Value &
-ValueInternalMap::unsafeAdd( const char *key, 
-                             bool isStatic, 
-                             HashKey hashedKey )
-{
-   JSON_ASSERT_MESSAGE( bucketsSize_ > 0, "ValueInternalMap::unsafeAdd(): internal logic error." );
-   BucketIndex bucketIndex = hashedKey % bucketsSize_;
-   ValueInternalLink *&previousLink = getLastLinkInBucket( bucketIndex );
-   ValueInternalLink *link = previousLink;
-   BucketIndex index;
-   for ( index =0; index < ValueInternalLink::itemPerLink; ++index )
-   {
-      if ( link->items_[index].isItemAvailable() )
-         break;
-   }
-   if ( index == ValueInternalLink::itemPerLink ) // need to add a new page
-   {
-      ValueInternalLink *newLink = mapAllocator()->allocateMapLink();
-      index = 0;
-      link->next_ = newLink;
-      previousLink = newLink;
-      link = newLink;
-   }
-   return setNewItem( key, isStatic, link, index );
-}
-
-
-ValueInternalMap::HashKey 
-ValueInternalMap::hash( const char *key ) const
-{
-   HashKey hash = 0;
-   while ( *key )
-      hash += *key++ * 37;
-   return hash;
-}
-
-
-int 
-ValueInternalMap::compare( const ValueInternalMap &other ) const
-{
-   int sizeDiff( itemCount_ - other.itemCount_ );
-   if ( sizeDiff != 0 )
-      return sizeDiff;
-   // Strict order guaranty is required. Compare all keys FIRST, then compare values.
-   IteratorState it;
-   IteratorState itEnd;
-   makeBeginIterator( it );
-   makeEndIterator( itEnd );
-   for ( ; !equals(it,itEnd); increment(it) )
-   {
-      if ( !other.find( key( it ) ) )
-         return 1;
-   }
-
-   // All keys are equals, let's compare values
-   makeBeginIterator( it );
-   for ( ; !equals(it,itEnd); increment(it) )
-   {
-      const Value *otherValue = other.find( key( it ) );
-      int valueDiff = value(it).compare( *otherValue );
-      if ( valueDiff != 0 )
-         return valueDiff;
-   }
-   return 0;
-}
-
-
-void 
-ValueInternalMap::makeBeginIterator( IteratorState &it ) const
-{
-   it.map_ = const_cast<ValueInternalMap *>( this );
-   it.bucketIndex_ = 0;
-   it.itemIndex_ = 0;
-   it.link_ = buckets_;
-}
-
-
-void 
-ValueInternalMap::makeEndIterator( IteratorState &it ) const
-{
-   it.map_ = const_cast<ValueInternalMap *>( this );
-   it.bucketIndex_ = bucketsSize_;
-   it.itemIndex_ = 0;
-   it.link_ = 0;
-}
-
-
-bool 
-ValueInternalMap::equals( const IteratorState &x, const IteratorState &other )
-{
-   return x.map_ == other.map_  
-          &&  x.bucketIndex_ == other.bucketIndex_  
-          &&  x.link_ == other.link_
-          &&  x.itemIndex_ == other.itemIndex_;
-}
-
-
-void 
-ValueInternalMap::incrementBucket( IteratorState &iterator )
-{
-   ++iterator.bucketIndex_;
-   JSON_ASSERT_MESSAGE( iterator.bucketIndex_ <= iterator.map_->bucketsSize_,
-      "ValueInternalMap::increment(): attempting to iterate beyond end." );
-   if ( iterator.bucketIndex_ == iterator.map_->bucketsSize_ )
-      iterator.link_ = 0;
-   else
-      iterator.link_ = &(iterator.map_->buckets_[iterator.bucketIndex_]);
-   iterator.itemIndex_ = 0;
-}
-
-
-void 
-ValueInternalMap::increment( IteratorState &iterator )
-{
-   JSON_ASSERT_MESSAGE( iterator.map_, "Attempting to iterator using invalid iterator." );
-   ++iterator.itemIndex_;
-   if ( iterator.itemIndex_ == ValueInternalLink::itemPerLink )
-   {
-      JSON_ASSERT_MESSAGE( iterator.link_ != 0,
-         "ValueInternalMap::increment(): attempting to iterate beyond end." );
-      iterator.link_ = iterator.link_->next_;
-      if ( iterator.link_ == 0 )
-         incrementBucket( iterator );
-   }
-   else if ( iterator.link_->items_[iterator.itemIndex_].isItemAvailable() )
-   {
-      incrementBucket( iterator );
-   }
-}
-
-
-void 
-ValueInternalMap::decrement( IteratorState &iterator )
-{
-   if ( iterator.itemIndex_ == 0 )
-   {
-      JSON_ASSERT_MESSAGE( iterator.map_, "Attempting to iterate using invalid iterator." );
-      if ( iterator.link_ == &iterator.map_->buckets_[iterator.bucketIndex_] )
-      {
-         JSON_ASSERT_MESSAGE( iterator.bucketIndex_ > 0, "Attempting to iterate beyond beginning." );
-         --(iterator.bucketIndex_);
-      }
-      iterator.link_ = iterator.link_->previous_;
-      iterator.itemIndex_ = ValueInternalLink::itemPerLink - 1;
-   }
-}
-
-
-const char *
-ValueInternalMap::key( const IteratorState &iterator )
-{
-   JSON_ASSERT_MESSAGE( iterator.link_, "Attempting to iterate using invalid iterator." );
-   return iterator.link_->keys_[iterator.itemIndex_];
-}
-
-const char *
-ValueInternalMap::key( const IteratorState &iterator, bool &isStatic )
-{
-   JSON_ASSERT_MESSAGE( iterator.link_, "Attempting to iterate using invalid iterator." );
-   isStatic = iterator.link_->items_[iterator.itemIndex_].isMemberNameStatic();
-   return iterator.link_->keys_[iterator.itemIndex_];
-}
-
-
-Value &
-ValueInternalMap::value( const IteratorState &iterator )
-{
-   JSON_ASSERT_MESSAGE( iterator.link_, "Attempting to iterate using invalid iterator." );
-   return iterator.link_->items_[iterator.itemIndex_];
-}
-
-
-int 
-ValueInternalMap::distance( const IteratorState &x, const IteratorState &y )
-{
-   int offset = 0;
-   IteratorState it = x;
-   while ( !equals( it, y ) )
-      increment( it );
-   return offset;
-}
-
-} // namespace Json

ext/jsoncpp/src/json_reader.cpp

@@ -1,880 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#if !defined(JSON_IS_AMALGAMATION)
-# include <json/reader.h>
-# include <json/value.h>
-# include "json_tool.h"
-#endif // if !defined(JSON_IS_AMALGAMATION)
-#include <utility>
-#include <cstdio>
-#include <cassert>
-#include <cstring>
-#include <iostream>
-#include <stdexcept>
-
-#if _MSC_VER >= 1400 // VC++ 8.0
-#pragma warning( disable : 4996 )   // disable warning about strdup being deprecated.
-#endif
-
-namespace Json {
-
-// Implementation of class Features
-// ////////////////////////////////
-
-Features::Features()
-   : allowComments_( true )
-   , strictRoot_( false )
-{
-}
-
-
-Features 
-Features::all()
-{
-   return Features();
-}
-
-
-Features 
-Features::strictMode()
-{
-   Features features;
-   features.allowComments_ = false;
-   features.strictRoot_ = true;
-   return features;
-}
-
-// Implementation of class Reader
-// ////////////////////////////////
-
-
-static inline bool 
-in( Reader::Char c, Reader::Char c1, Reader::Char c2, Reader::Char c3, Reader::Char c4 )
-{
-   return c == c1  ||  c == c2  ||  c == c3  ||  c == c4;
-}
-
-static inline bool 
-in( Reader::Char c, Reader::Char c1, Reader::Char c2, Reader::Char c3, Reader::Char c4, Reader::Char c5 )
-{
-   return c == c1  ||  c == c2  ||  c == c3  ||  c == c4  ||  c == c5;
-}
-
-
-static bool 
-containsNewLine( Reader::Location begin, 
-                 Reader::Location end )
-{
-   for ( ;begin < end; ++begin )
-      if ( *begin == '\n'  ||  *begin == '\r' )
-         return true;
-   return false;
-}
-
-
-// Class Reader
-// //////////////////////////////////////////////////////////////////
-
-Reader::Reader()
-   : features_( Features::all() )
-{
-}
-
-
-Reader::Reader( const Features &features )
-   : features_( features )
-{
-}
-
-
-bool
-Reader::parse( const std::string &document, 
-               Value &root,
-               bool collectComments )
-{
-   document_ = document;
-   const char *begin = document_.c_str();
-   const char *end = begin + document_.length();
-   return parse( begin, end, root, collectComments );
-}
-
-
-bool
-Reader::parse( std::istream& sin,
-               Value &root,
-               bool collectComments )
-{
-   //std::istream_iterator<char> begin(sin);
-   //std::istream_iterator<char> end;
-   // Those would allow streamed input from a file, if parse() were a
-   // template function.
-
-   // Since std::string is reference-counted, this at least does not
-   // create an extra copy.
-   std::string doc;
-   std::getline(sin, doc, (char)EOF);
-   return parse( doc, root, collectComments );
-}
-
-bool 
-Reader::parse( const char *beginDoc, const char *endDoc, 
-               Value &root,
-               bool collectComments )
-{
-   if ( !features_.allowComments_ )
-   {
-      collectComments = false;
-   }
-
-   begin_ = beginDoc;
-   end_ = endDoc;
-   collectComments_ = collectComments;
-   current_ = begin_;
-   lastValueEnd_ = 0;
-   lastValue_ = 0;
-   commentsBefore_ = "";
-   errors_.clear();
-   while ( !nodes_.empty() )
-      nodes_.pop();
-   nodes_.push( &root );
-   
-   bool successful = readValue();
-   Token token;
-   skipCommentTokens( token );
-   if ( collectComments_  &&  !commentsBefore_.empty() )
-      root.setComment( commentsBefore_, commentAfter );
-   if ( features_.strictRoot_ )
-   {
-      if ( !root.isArray()  &&  !root.isObject() )
-      {
-         // Set error location to start of doc, ideally should be first token found in doc
-         token.type_ = tokenError;
-         token.start_ = beginDoc;
-         token.end_ = endDoc;
-         addError( "A valid JSON document must be either an array or an object value.",
-                   token );
-         return false;
-      }
-   }
-   return successful;
-}
-
-
-bool
-Reader::readValue()
-{
-   Token token;
-   skipCommentTokens( token );
-   bool successful = true;
-
-   if ( collectComments_  &&  !commentsBefore_.empty() )
-   {
-      currentValue().setComment( commentsBefore_, commentBefore );
-      commentsBefore_ = "";
-   }
-
-
-   switch ( token.type_ )
-   {
-   case tokenObjectBegin:
-      successful = readObject( token );
-      break;
-   case tokenArrayBegin:
-      successful = readArray( token );
-      break;
-   case tokenNumber:
-      successful = decodeNumber( token );
-      break;
-   case tokenString:
-      successful = decodeString( token );
-      break;
-   case tokenTrue:
-      currentValue() = true;
-      break;
-   case tokenFalse:
-      currentValue() = false;
-      break;
-   case tokenNull:
-      currentValue() = Value();
-      break;
-   default:
-      return addError( "Syntax error: value, object or array expected.", token );
-   }
-
-   if ( collectComments_ )
-   {
-      lastValueEnd_ = current_;
-      lastValue_ = &currentValue();
-   }
-
-   return successful;
-}
-
-
-void 
-Reader::skipCommentTokens( Token &token )
-{
-   if ( features_.allowComments_ )
-   {
-      do
-      {
-         readToken( token );
-      }
-      while ( token.type_ == tokenComment );
-   }
-   else
-   {
-      readToken( token );
-   }
-}
-
-
-bool 
-Reader::expectToken( TokenType type, Token &token, const char *message )
-{
-   readToken( token );
-   if ( token.type_ != type )
-      return addError( message, token );
-   return true;
-}
-
-
-bool 
-Reader::readToken( Token &token )
-{
-   skipSpaces();
-   token.start_ = current_;
-   Char c = getNextChar();
-   bool ok = true;
-   switch ( c )
-   {
-   case '{':
-      token.type_ = tokenObjectBegin;
-      break;
-   case '}':
-      token.type_ = tokenObjectEnd;
-      break;
-   case '[':
-      token.type_ = tokenArrayBegin;
-      break;
-   case ']':
-      token.type_ = tokenArrayEnd;
-      break;
-   case '"':
-      token.type_ = tokenString;
-      ok = readString();
-      break;
-   case '/':
-      token.type_ = tokenComment;
-      ok = readComment();
-      break;
-   case '0':
-   case '1':
-   case '2':
-   case '3':
-   case '4':
-   case '5':
-   case '6':
-   case '7':
-   case '8':
-   case '9':
-   case '-':
-      token.type_ = tokenNumber;
-      readNumber();
-      break;
-   case 't':
-      token.type_ = tokenTrue;
-      ok = match( "rue", 3 );
-      break;
-   case 'f':
-      token.type_ = tokenFalse;
-      ok = match( "alse", 4 );
-      break;
-   case 'n':
-      token.type_ = tokenNull;
-      ok = match( "ull", 3 );
-      break;
-   case ',':
-      token.type_ = tokenArraySeparator;
-      break;
-   case ':':
-      token.type_ = tokenMemberSeparator;
-      break;
-   case 0:
-      token.type_ = tokenEndOfStream;
-      break;
-   default:
-      ok = false;
-      break;
-   }
-   if ( !ok )
-      token.type_ = tokenError;
-   token.end_ = current_;
-   return true;
-}
-
-
-void 
-Reader::skipSpaces()
-{
-   while ( current_ != end_ )
-   {
-      Char c = *current_;
-      if ( c == ' '  ||  c == '\t'  ||  c == '\r'  ||  c == '\n' )
-         ++current_;
-      else
-         break;
-   }
-}
-
-
-bool 
-Reader::match( Location pattern, 
-               int patternLength )
-{
-   if ( end_ - current_ < patternLength )
-      return false;
-   int index = patternLength;
-   while ( index-- )
-      if ( current_[index] != pattern[index] )
-         return false;
-   current_ += patternLength;
-   return true;
-}
-
-
-bool
-Reader::readComment()
-{
-   Location commentBegin = current_ - 1;
-   Char c = getNextChar();
-   bool successful = false;
-   if ( c == '*' )
-      successful = readCStyleComment();
-   else if ( c == '/' )
-      successful = readCppStyleComment();
-   if ( !successful )
-      return false;
-
-   if ( collectComments_ )
-   {
-      CommentPlacement placement = commentBefore;
-      if ( lastValueEnd_  &&  !containsNewLine( lastValueEnd_, commentBegin ) )
-      {
-         if ( c != '*'  ||  !containsNewLine( commentBegin, current_ ) )
-            placement = commentAfterOnSameLine;
-      }
-
-      addComment( commentBegin, current_, placement );
-   }
-   return true;
-}
-
-
-void 
-Reader::addComment( Location begin, 
-                    Location end, 
-                    CommentPlacement placement )
-{
-   assert( collectComments_ );
-   if ( placement == commentAfterOnSameLine )
-   {
-      assert( lastValue_ != 0 );
-      lastValue_->setComment( std::string( begin, end ), placement );
-   }
-   else
-   {
-      if ( !commentsBefore_.empty() )
-         commentsBefore_ += "\n";
-      commentsBefore_ += std::string( begin, end );
-   }
-}
-
-
-bool 
-Reader::readCStyleComment()
-{
-   while ( current_ != end_ )
-   {
-      Char c = getNextChar();
-      if ( c == '*'  &&  *current_ == '/' )
-         break;
-   }
-   return getNextChar() == '/';
-}
-
-
-bool 
-Reader::readCppStyleComment()
-{
-   while ( current_ != end_ )
-   {
-      Char c = getNextChar();
-      if (  c == '\r'  ||  c == '\n' )
-         break;
-   }
-   return true;
-}
-
-
-void 
-Reader::readNumber()
-{
-   while ( current_ != end_ )
-   {
-      if ( !(*current_ >= '0'  &&  *current_ <= '9')  &&
-           !in( *current_, '.', 'e', 'E', '+', '-' ) )
-         break;
-      ++current_;
-   }
-}
-
-bool
-Reader::readString()
-{
-   Char c = 0;
-   while ( current_ != end_ )
-   {
-      c = getNextChar();
-      if ( c == '\\' )
-         getNextChar();
-      else if ( c == '"' )
-         break;
-   }
-   return c == '"';
-}
-
-
-bool 
-Reader::readObject( Token &/*tokenStart*/ )
-{
-   Token tokenName;
-   std::string name;
-   currentValue() = Value( objectValue );
-   while ( readToken( tokenName ) )
-   {
-      bool initialTokenOk = true;
-      while ( tokenName.type_ == tokenComment  &&  initialTokenOk )
-         initialTokenOk = readToken( tokenName );
-      if  ( !initialTokenOk )
-         break;
-      if ( tokenName.type_ == tokenObjectEnd  &&  name.empty() )  // empty object
-         return true;
-      if ( tokenName.type_ != tokenString )
-         break;
-      
-      name = "";
-      if ( !decodeString( tokenName, name ) )
-         return recoverFromError( tokenObjectEnd );
-
-      Token colon;
-      if ( !readToken( colon ) ||  colon.type_ != tokenMemberSeparator )
-      {
-         return addErrorAndRecover( "Missing ':' after object member name", 
-                                    colon, 
-                                    tokenObjectEnd );
-      }
-      Value &value = currentValue()[ name ];
-      nodes_.push( &value );
-      bool ok = readValue();
-      nodes_.pop();
-      if ( !ok ) // error already set
-         return recoverFromError( tokenObjectEnd );
-
-      Token comma;
-      if ( !readToken( comma )
-            ||  ( comma.type_ != tokenObjectEnd  &&  
-                  comma.type_ != tokenArraySeparator &&
-                  comma.type_ != tokenComment ) )
-      {
-         return addErrorAndRecover( "Missing ',' or '}' in object declaration", 
-                                    comma, 
-                                    tokenObjectEnd );
-      }
-      bool finalizeTokenOk = true;
-      while ( comma.type_ == tokenComment &&
-              finalizeTokenOk )
-         finalizeTokenOk = readToken( comma );
-      if ( comma.type_ == tokenObjectEnd )
-         return true;
-   }
-   return addErrorAndRecover( "Missing '}' or object member name", 
-                              tokenName, 
-                              tokenObjectEnd );
-}
-
-
-bool 
-Reader::readArray( Token &/*tokenStart*/ )
-{
-   currentValue() = Value( arrayValue );
-   skipSpaces();
-   if ( *current_ == ']' ) // empty array
-   {
-      Token endArray;
-      readToken( endArray );
-      return true;
-   }
-   int index = 0;
-   for (;;)
-   {
-      Value &value = currentValue()[ index++ ];
-      nodes_.push( &value );
-      bool ok = readValue();
-      nodes_.pop();
-      if ( !ok ) // error already set
-         return recoverFromError( tokenArrayEnd );
-
-      Token token;
-      // Accept Comment after last item in the array.
-      ok = readToken( token );
-      while ( token.type_ == tokenComment  &&  ok )
-      {
-         ok = readToken( token );
-      }
-      bool badTokenType = ( token.type_ != tokenArraySeparator  &&
-                            token.type_ != tokenArrayEnd );
-      if ( !ok  ||  badTokenType )
-      {
-         return addErrorAndRecover( "Missing ',' or ']' in array declaration", 
-                                    token, 
-                                    tokenArrayEnd );
-      }
-      if ( token.type_ == tokenArrayEnd )
-         break;
-   }
-   return true;
-}
-
-
-bool 
-Reader::decodeNumber( Token &token )
-{
-   bool isDouble = false;
-   for ( Location inspect = token.start_; inspect != token.end_; ++inspect )
-   {
-      isDouble = isDouble  
-                 ||  in( *inspect, '.', 'e', 'E', '+' )  
-                 ||  ( *inspect == '-'  &&  inspect != token.start_ );
-   }
-   if ( isDouble )
-      return decodeDouble( token );
-   // Attempts to parse the number as an integer. If the number is
-   // larger than the maximum supported value of an integer then
-   // we decode the number as a double.
-   Location current = token.start_;
-   bool isNegative = *current == '-';
-   if ( isNegative )
-      ++current;
-   Value::LargestUInt maxIntegerValue = isNegative ? Value::LargestUInt(-Value::minLargestInt) 
-                                                   : Value::maxLargestUInt;
-   Value::LargestUInt threshold = maxIntegerValue / 10;
-   Value::UInt lastDigitThreshold = Value::UInt( maxIntegerValue % 10 );
-   assert( lastDigitThreshold >=0  &&  lastDigitThreshold <= 9 );
-   Value::LargestUInt value = 0;
-   while ( current < token.end_ )
-   {
-      Char c = *current++;
-      if ( c < '0'  ||  c > '9' )
-         return addError( "'" + std::string( token.start_, token.end_ ) + "' is not a number.", token );
-      Value::UInt digit(c - '0');
-      if ( value >= threshold )
-      {
-         // If the current digit is not the last one, or if it is
-         // greater than the last digit of the maximum integer value,
-         // the parse the number as a double.
-         if ( current != token.end_  ||  digit > lastDigitThreshold )
-         {
-            return decodeDouble( token );
-         }
-      }
-      value = value * 10 + digit;
-   }
-   if ( isNegative )
-      currentValue() = -Value::LargestInt( value );
-   else if ( value <= Value::LargestUInt(Value::maxInt) )
-      currentValue() = Value::LargestInt( value );
-   else
-      currentValue() = value;
-   return true;
-}
-
-
-bool 
-Reader::decodeDouble( Token &token )
-{
-   double value = 0;
-   const int bufferSize = 32;
-   int count;
-   int length = int(token.end_ - token.start_);
-   if ( length <= bufferSize )
-   {
-      Char buffer[bufferSize+1];
-      memcpy( buffer, token.start_, length );
-      buffer[length] = 0;
-      count = sscanf( buffer, "%lf", &value );
-   }
-   else
-   {
-      std::string buffer( token.start_, token.end_ );
-      count = sscanf( buffer.c_str(), "%lf", &value );
-   }
-
-   if ( count != 1 )
-      return addError( "'" + std::string( token.start_, token.end_ ) + "' is not a number.", token );
-   currentValue() = value;
-   return true;
-}
-
-
-bool 
-Reader::decodeString( Token &token )
-{
-   std::string decoded;
-   if ( !decodeString( token, decoded ) )
-      return false;
-   currentValue() = decoded;
-   return true;
-}
-
-
-bool 
-Reader::decodeString( Token &token, std::string &decoded )
-{
-   decoded.reserve( token.end_ - token.start_ - 2 );
-   Location current = token.start_ + 1; // skip '"'
-   Location end = token.end_ - 1;      // do not include '"'
-   while ( current != end )
-   {
-      Char c = *current++;
-      if ( c == '"' )
-         break;
-      else if ( c == '\\' )
-      {
-         if ( current == end )
-            return addError( "Empty escape sequence in string", token, current );
-         Char escape = *current++;
-         switch ( escape )
-         {
-         case '"': decoded += '"'; break;
-         case '/': decoded += '/'; break;
-         case '\\': decoded += '\\'; break;
-         case 'b': decoded += '\b'; break;
-         case 'f': decoded += '\f'; break;
-         case 'n': decoded += '\n'; break;
-         case 'r': decoded += '\r'; break;
-         case 't': decoded += '\t'; break;
-         case 'u':
-            {
-               unsigned int unicode;
-               if ( !decodeUnicodeCodePoint( token, current, end, unicode ) )
-                  return false;
-               decoded += codePointToUTF8(unicode);
-            }
-            break;
-         default:
-            return addError( "Bad escape sequence in string", token, current );
-         }
-      }
-      else
-      {
-         decoded += c;
-      }
-   }
-   return true;
-}
-
-bool
-Reader::decodeUnicodeCodePoint( Token &token, 
-                                     Location &current, 
-                                     Location end, 
-                                     unsigned int &unicode )
-{
-
-   if ( !decodeUnicodeEscapeSequence( token, current, end, unicode ) )
-      return false;
-   if (unicode >= 0xD800 && unicode <= 0xDBFF)
-   {
-      // surrogate pairs
-      if (end - current < 6)
-         return addError( "additional six characters expected to parse unicode surrogate pair.", token, current );
-      unsigned int surrogatePair;
-      if (*(current++) == '\\' && *(current++)== 'u')
-      {
-         if (decodeUnicodeEscapeSequence( token, current, end, surrogatePair ))
-         {
-            unicode = 0x10000 + ((unicode & 0x3FF) << 10) + (surrogatePair & 0x3FF);
-         } 
-         else
-            return false;
-      } 
-      else
-         return addError( "expecting another \\u token to begin the second half of a unicode surrogate pair", token, current );
-   }
-   return true;
-}
-
-bool 
-Reader::decodeUnicodeEscapeSequence( Token &token, 
-                                     Location &current, 
-                                     Location end, 
-                                     unsigned int &unicode )
-{
-   if ( end - current < 4 )
-      return addError( "Bad unicode escape sequence in string: four digits expected.", token, current );
-   unicode = 0;
-   for ( int index =0; index < 4; ++index )
-   {
-      Char c = *current++;
-      unicode *= 16;
-      if ( c >= '0'  &&  c <= '9' )
-         unicode += c - '0';
-      else if ( c >= 'a'  &&  c <= 'f' )
-         unicode += c - 'a' + 10;
-      else if ( c >= 'A'  &&  c <= 'F' )
-         unicode += c - 'A' + 10;
-      else
-         return addError( "Bad unicode escape sequence in string: hexadecimal digit expected.", token, current );
-   }
-   return true;
-}
-
-
-bool 
-Reader::addError( const std::string &message, 
-                  Token &token,
-                  Location extra )
-{
-   ErrorInfo info;
-   info.token_ = token;
-   info.message_ = message;
-   info.extra_ = extra;
-   errors_.push_back( info );
-   return false;
-}
-
-
-bool 
-Reader::recoverFromError( TokenType skipUntilToken )
-{
-   int errorCount = int(errors_.size());
-   Token skip;
-   for (;;)
-   {
-      if ( !readToken(skip) )
-         errors_.resize( errorCount ); // discard errors caused by recovery
-      if ( skip.type_ == skipUntilToken  ||  skip.type_ == tokenEndOfStream )
-         break;
-   }
-   errors_.resize( errorCount );
-   return false;
-}
-
-
-bool 
-Reader::addErrorAndRecover( const std::string &message, 
-                            Token &token,
-                            TokenType skipUntilToken )
-{
-   addError( message, token );
-   return recoverFromError( skipUntilToken );
-}
-
-
-Value &
-Reader::currentValue()
-{
-   return *(nodes_.top());
-}
-
-
-Reader::Char 
-Reader::getNextChar()
-{
-   if ( current_ == end_ )
-      return 0;
-   return *current_++;
-}
-
-
-void 
-Reader::getLocationLineAndColumn( Location location,
-                                  int &line,
-                                  int &column ) const
-{
-   Location current = begin_;
-   Location lastLineStart = current;
-   line = 0;
-   while ( current < location  &&  current != end_ )
-   {
-      Char c = *current++;
-      if ( c == '\r' )
-      {
-         if ( *current == '\n' )
-            ++current;
-         lastLineStart = current;
-         ++line;
-      }
-      else if ( c == '\n' )
-      {
-         lastLineStart = current;
-         ++line;
-      }
-   }
-   // column & line start at 1
-   column = int(location - lastLineStart) + 1;
-   ++line;
-}
-
-
-std::string
-Reader::getLocationLineAndColumn( Location location ) const
-{
-   int line, column;
-   getLocationLineAndColumn( location, line, column );
-   char buffer[18+16+16+1];
-   sprintf( buffer, "Line %d, Column %d", line, column );
-   return buffer;
-}
-
-
-// Deprecated. Preserved for backward compatibility
-std::string 
-Reader::getFormatedErrorMessages() const
-{
-    return getFormattedErrorMessages();
-}
-
-
-std::string 
-Reader::getFormattedErrorMessages() const
-{
-   std::string formattedMessage;
-   for ( Errors::const_iterator itError = errors_.begin();
-         itError != errors_.end();
-         ++itError )
-   {
-      const ErrorInfo &error = *itError;
-      formattedMessage += "* " + getLocationLineAndColumn( error.token_.start_ ) + "\n";
-      formattedMessage += "  " + error.message_ + "\n";
-      if ( error.extra_ )
-         formattedMessage += "See " + getLocationLineAndColumn( error.extra_ ) + " for detail.\n";
-   }
-   return formattedMessage;
-}
-
-
-std::istream& operator>>( std::istream &sin, Value &root )
-{
-    Json::Reader reader;
-    bool ok = reader.parse(sin, root, true);
-    //JSON_ASSERT( ok );
-    if (!ok) throw std::runtime_error(reader.getFormattedErrorMessages());
-    return sin;
-}
-
-
-} // namespace Json

ext/jsoncpp/src/json_tool.h

@@ -1,93 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#ifndef LIB_JSONCPP_JSON_TOOL_H_INCLUDED
-# define LIB_JSONCPP_JSON_TOOL_H_INCLUDED
-
-/* This header provides common string manipulation support, such as UTF-8,
- * portable conversion from/to string...
- *
- * It is an internal header that must not be exposed.
- */
-
-namespace Json {
-
-/// Converts a unicode code-point to UTF-8.
-static inline std::string 
-codePointToUTF8(unsigned int cp)
-{
-   std::string result;
-   
-   // based on description from http://en.wikipedia.org/wiki/UTF-8
-
-   if (cp <= 0x7f) 
-   {
-      result.resize(1);
-      result[0] = static_cast<char>(cp);
-   } 
-   else if (cp <= 0x7FF) 
-   {
-      result.resize(2);
-      result[1] = static_cast<char>(0x80 | (0x3f & cp));
-      result[0] = static_cast<char>(0xC0 | (0x1f & (cp >> 6)));
-   } 
-   else if (cp <= 0xFFFF) 
-   {
-      result.resize(3);
-      result[2] = static_cast<char>(0x80 | (0x3f & cp));
-      result[1] = 0x80 | static_cast<char>((0x3f & (cp >> 6)));
-      result[0] = 0xE0 | static_cast<char>((0xf & (cp >> 12)));
-   }
-   else if (cp <= 0x10FFFF) 
-   {
-      result.resize(4);
-      result[3] = static_cast<char>(0x80 | (0x3f & cp));
-      result[2] = static_cast<char>(0x80 | (0x3f & (cp >> 6)));
-      result[1] = static_cast<char>(0x80 | (0x3f & (cp >> 12)));
-      result[0] = static_cast<char>(0xF0 | (0x7 & (cp >> 18)));
-   }
-
-   return result;
-}
-
-
-/// Returns true if ch is a control character (in range [0,32[).
-static inline bool 
-isControlCharacter(char ch)
-{
-   return ch > 0 && ch <= 0x1F;
-}
-
-
-enum { 
-   /// Constant that specify the size of the buffer that must be passed to uintToString.
-   uintToStringBufferSize = 3*sizeof(LargestUInt)+1 
-};
-
-// Defines a char buffer for use with uintToString().
-typedef char UIntToStringBuffer[uintToStringBufferSize];
-
-
-/** Converts an unsigned integer to string.
- * @param value Unsigned interger to convert to string
- * @param current Input/Output string buffer. 
- *        Must have at least uintToStringBufferSize chars free.
- */
-static inline void 
-uintToString( LargestUInt value, 
-              char *&current )
-{
-   *--current = 0;
-   do
-   {
-      *--current = char(value % 10) + '0';
-      value /= 10;
-   }
-   while ( value != 0 );
-}
-
-} // namespace Json {
-
-#endif // LIB_JSONCPP_JSON_TOOL_H_INCLUDED

ext/jsoncpp/src/json_valueiterator.inl

@@ -1,299 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-// included by json_value.cpp
-
-namespace Json {
-
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// class ValueIteratorBase
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-
-ValueIteratorBase::ValueIteratorBase()
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   : current_()
-   , isNull_( true )
-{
-}
-#else
-   : isArray_( true )
-   , isNull_( true )
-{
-   iterator_.array_ = ValueInternalArray::IteratorState();
-}
-#endif
-
-
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-ValueIteratorBase::ValueIteratorBase( const Value::ObjectValues::iterator &current )
-   : current_( current )
-   , isNull_( false )
-{
-}
-#else
-ValueIteratorBase::ValueIteratorBase( const ValueInternalArray::IteratorState &state )
-   : isArray_( true )
-{
-   iterator_.array_ = state;
-}
-
-
-ValueIteratorBase::ValueIteratorBase( const ValueInternalMap::IteratorState &state )
-   : isArray_( false )
-{
-   iterator_.map_ = state;
-}
-#endif
-
-Value &
-ValueIteratorBase::deref() const
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   return current_->second;
-#else
-   if ( isArray_ )
-      return ValueInternalArray::dereference( iterator_.array_ );
-   return ValueInternalMap::value( iterator_.map_ );
-#endif
-}
-
-
-void 
-ValueIteratorBase::increment()
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   ++current_;
-#else
-   if ( isArray_ )
-      ValueInternalArray::increment( iterator_.array_ );
-   ValueInternalMap::increment( iterator_.map_ );
-#endif
-}
-
-
-void 
-ValueIteratorBase::decrement()
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   --current_;
-#else
-   if ( isArray_ )
-      ValueInternalArray::decrement( iterator_.array_ );
-   ValueInternalMap::decrement( iterator_.map_ );
-#endif
-}
-
-
-ValueIteratorBase::difference_type 
-ValueIteratorBase::computeDistance( const SelfType &other ) const
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-# ifdef JSON_USE_CPPTL_SMALLMAP
-   return current_ - other.current_;
-# else
-   // Iterator for null value are initialized using the default
-   // constructor, which initialize current_ to the default
-   // std::map::iterator. As begin() and end() are two instance 
-   // of the default std::map::iterator, they can not be compared.
-   // To allow this, we handle this comparison specifically.
-   if ( isNull_  &&  other.isNull_ )
-   {
-      return 0;
-   }
-
-
-   // Usage of std::distance is not portable (does not compile with Sun Studio 12 RogueWave STL,
-   // which is the one used by default).
-   // Using a portable hand-made version for non random iterator instead:
-   //   return difference_type( std::distance( current_, other.current_ ) );
-   difference_type myDistance = 0;
-   for ( Value::ObjectValues::iterator it = current_; it != other.current_; ++it )
-   {
-      ++myDistance;
-   }
-   return myDistance;
-# endif
-#else
-   if ( isArray_ )
-      return ValueInternalArray::distance( iterator_.array_, other.iterator_.array_ );
-   return ValueInternalMap::distance( iterator_.map_, other.iterator_.map_ );
-#endif
-}
-
-
-bool 
-ValueIteratorBase::isEqual( const SelfType &other ) const
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   if ( isNull_ )
-   {
-      return other.isNull_;
-   }
-   return current_ == other.current_;
-#else
-   if ( isArray_ )
-      return ValueInternalArray::equals( iterator_.array_, other.iterator_.array_ );
-   return ValueInternalMap::equals( iterator_.map_, other.iterator_.map_ );
-#endif
-}
-
-
-void 
-ValueIteratorBase::copy( const SelfType &other )
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   current_ = other.current_;
-#else
-   if ( isArray_ )
-      iterator_.array_ = other.iterator_.array_;
-   iterator_.map_ = other.iterator_.map_;
-#endif
-}
-
-
-Value 
-ValueIteratorBase::key() const
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   const Value::CZString czstring = (*current_).first;
-   if ( czstring.c_str() )
-   {
-      if ( czstring.isStaticString() )
-         return Value( StaticString( czstring.c_str() ) );
-      return Value( czstring.c_str() );
-   }
-   return Value( czstring.index() );
-#else
-   if ( isArray_ )
-      return Value( ValueInternalArray::indexOf( iterator_.array_ ) );
-   bool isStatic;
-   const char *memberName = ValueInternalMap::key( iterator_.map_, isStatic );
-   if ( isStatic )
-      return Value( StaticString( memberName ) );
-   return Value( memberName );
-#endif
-}
-
-
-UInt 
-ValueIteratorBase::index() const
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   const Value::CZString czstring = (*current_).first;
-   if ( !czstring.c_str() )
-      return czstring.index();
-   return Value::UInt( -1 );
-#else
-   if ( isArray_ )
-      return Value::UInt( ValueInternalArray::indexOf( iterator_.array_ ) );
-   return Value::UInt( -1 );
-#endif
-}
-
-
-const char *
-ValueIteratorBase::memberName() const
-{
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-   const char *name = (*current_).first.c_str();
-   return name ? name : "";
-#else
-   if ( !isArray_ )
-      return ValueInternalMap::key( iterator_.map_ );
-   return "";
-#endif
-}
-
-
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// class ValueConstIterator
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-
-ValueConstIterator::ValueConstIterator()
-{
-}
-
-
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-ValueConstIterator::ValueConstIterator( const Value::ObjectValues::iterator &current )
-   : ValueIteratorBase( current )
-{
-}
-#else
-ValueConstIterator::ValueConstIterator( const ValueInternalArray::IteratorState &state )
-   : ValueIteratorBase( state )
-{
-}
-
-ValueConstIterator::ValueConstIterator( const ValueInternalMap::IteratorState &state )
-   : ValueIteratorBase( state )
-{
-}
-#endif
-
-ValueConstIterator &
-ValueConstIterator::operator =( const ValueIteratorBase &other )
-{
-   copy( other );
-   return *this;
-}
-
-
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// class ValueIterator
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-// //////////////////////////////////////////////////////////////////
-
-ValueIterator::ValueIterator()
-{
-}
-
-
-#ifndef JSON_VALUE_USE_INTERNAL_MAP
-ValueIterator::ValueIterator( const Value::ObjectValues::iterator &current )
-   : ValueIteratorBase( current )
-{
-}
-#else
-ValueIterator::ValueIterator( const ValueInternalArray::IteratorState &state )
-   : ValueIteratorBase( state )
-{
-}
-
-ValueIterator::ValueIterator( const ValueInternalMap::IteratorState &state )
-   : ValueIteratorBase( state )
-{
-}
-#endif
-
-ValueIterator::ValueIterator( const ValueConstIterator &other )
-   : ValueIteratorBase( other )
-{
-}
-
-ValueIterator::ValueIterator( const ValueIterator &other )
-   : ValueIteratorBase( other )
-{
-}
-
-ValueIterator &
-ValueIterator::operator =( const SelfType &other )
-{
-   copy( other );
-   return *this;
-}
-
-} // namespace Json

ext/jsoncpp/src/json_writer.cpp

@@ -1,838 +0,0 @@
-// Copyright 2007-2010 Baptiste Lepilleur
-// Distributed under MIT license, or public domain if desired and
-// recognized in your jurisdiction.
-// See file LICENSE for detail or copy at http://jsoncpp.sourceforge.net/LICENSE
-
-#if !defined(JSON_IS_AMALGAMATION)
-# include <json/writer.h>
-# include "json_tool.h"
-#endif // if !defined(JSON_IS_AMALGAMATION)
-#include <utility>
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-#include <iostream>
-#include <sstream>
-#include <iomanip>
-
-#if _MSC_VER >= 1400 // VC++ 8.0
-#pragma warning( disable : 4996 )   // disable warning about strdup being deprecated.
-#endif
-
-namespace Json {
-
-static bool containsControlCharacter( const char* str )
-{
-   while ( *str ) 
-   {
-      if ( isControlCharacter( *(str++) ) )
-         return true;
-   }
-   return false;
-}
-
-
-std::string valueToString( LargestInt value )
-{
-   UIntToStringBuffer buffer;
-   char *current = buffer + sizeof(buffer);
-   bool isNegative = value < 0;
-   if ( isNegative )
-      value = -value;
-   uintToString( LargestUInt(value), current );
-   if ( isNegative )
-      *--current = '-';
-   assert( current >= buffer );
-   return current;
-}
-
-
-std::string valueToString( LargestUInt value )
-{
-   UIntToStringBuffer buffer;
-   char *current = buffer + sizeof(buffer);
-   uintToString( value, current );
-   assert( current >= buffer );
-   return current;
-}
-
-#if defined(JSON_HAS_INT64)
-
-std::string valueToString( Int value )
-{
-   return valueToString( LargestInt(value) );
-}
-
-
-std::string valueToString( UInt value )
-{
-   return valueToString( LargestUInt(value) );
-}
-
-#endif // # if defined(JSON_HAS_INT64)
-
-
-std::string valueToString( double value )
-{
-   char buffer[32];
-#if defined(_MSC_VER) && defined(__STDC_SECURE_LIB__) // Use secure version with visual studio 2005 to avoid warning. 
-   sprintf_s(buffer, sizeof(buffer), "%#.16g", value); 
-#else	
-   sprintf(buffer, "%#.16g", value); 
-#endif
-   char* ch = buffer + strlen(buffer) - 1;
-   if (*ch != '0') return buffer; // nothing to truncate, so save time
-   while(ch > buffer && *ch == '0'){
-     --ch;
-   }
-   char* last_nonzero = ch;
-   while(ch >= buffer){
-     switch(*ch){
-     case '0':
-     case '1':
-     case '2':
-     case '3':
-     case '4':
-     case '5':
-     case '6':
-     case '7':
-     case '8':
-     case '9':
-       --ch;
-       continue;
-     case '.':
-       // Truncate zeroes to save bytes in output, but keep one.
-       *(last_nonzero+2) = '\0';
-       return buffer;
-     default:
-       return buffer;
-     }
-   }
-   return buffer;
-}
-
-
-std::string valueToString( bool value )
-{
-   return value ? "true" : "false";
-}
-
-std::string valueToQuotedString( const char *value )
-{
-   // Not sure how to handle unicode...
-   if (strpbrk(value, "\"\\\b\f\n\r\t") == NULL && !containsControlCharacter( value ))
-      return std::string("\"") + value + "\"";
-   // We have to walk value and escape any special characters.
-   // Appending to std::string is not efficient, but this should be rare.
-   // (Note: forward slashes are *not* rare, but I am not escaping them.)
-   std::string::size_type maxsize = strlen(value)*2 + 3; // allescaped+quotes+NULL
-   std::string result;
-   result.reserve(maxsize); // to avoid lots of mallocs
-   result += "\"";
-   for (const char* c=value; *c != 0; ++c)
-   {
-      switch(*c)
-      {
-         case '\"':
-            result += "\\\"";
-            break;
-         case '\\':
-            result += "\\\\";
-            break;
-         case '\b':
-            result += "\\b";
-            break;
-         case '\f':
-            result += "\\f";
-            break;
-         case '\n':
-            result += "\\n";
-            break;
-         case '\r':
-            result += "\\r";
-            break;
-         case '\t':
-            result += "\\t";
-            break;
-         //case '/':
-            // Even though \/ is considered a legal escape in JSON, a bare
-            // slash is also legal, so I see no reason to escape it.
-            // (I hope I am not misunderstanding something.
-            // blep notes: actually escaping \/ may be useful in javascript to avoid </ 
-            // sequence.
-            // Should add a flag to allow this compatibility mode and prevent this 
-            // sequence from occurring.
-         default:
-            if ( isControlCharacter( *c ) )
-            {
-               std::ostringstream oss;
-               oss << "\\u" << std::hex << std::uppercase << std::setfill('0') << std::setw(4) << static_cast<int>(*c);
-               result += oss.str();
-            }
-            else
-            {
-               result += *c;
-            }
-            break;
-      }
-   }
-   result += "\"";
-   return result;
-}
-
-// Class Writer
-// //////////////////////////////////////////////////////////////////
-Writer::~Writer()
-{
-}
-
-
-// Class FastWriter
-// //////////////////////////////////////////////////////////////////
-
-FastWriter::FastWriter()
-   : yamlCompatiblityEnabled_( false )
-{
-}
-
-
-void 
-FastWriter::enableYAMLCompatibility()
-{
-   yamlCompatiblityEnabled_ = true;
-}
-
-
-std::string 
-FastWriter::write( const Value &root )
-{
-   document_ = "";
-   writeValue( root );
-   document_ += "\n";
-   return document_;
-}
-
-
-void 
-FastWriter::writeValue( const Value &value )
-{
-   switch ( value.type() )
-   {
-   case nullValue:
-      document_ += "null";
-      break;
-   case intValue:
-      document_ += valueToString( value.asLargestInt() );
-      break;
-   case uintValue:
-      document_ += valueToString( value.asLargestUInt() );
-      break;
-   case realValue:
-      document_ += valueToString( value.asDouble() );
-      break;
-   case stringValue:
-      document_ += valueToQuotedString( value.asCString() );
-      break;
-   case booleanValue:
-      document_ += valueToString( value.asBool() );
-      break;
-   case arrayValue:
-      {
-         document_ += "[";
-         int size = value.size();
-         for ( int index =0; index < size; ++index )
-         {
-            if ( index > 0 )
-               document_ += ",";
-            writeValue( value[index] );
-         }
-         document_ += "]";
-      }
-      break;
-   case objectValue:
-      {
-         Value::Members members( value.getMemberNames() );
-         document_ += "{";
-         for ( Value::Members::iterator it = members.begin(); 
-               it != members.end(); 
-               ++it )
-         {
-            const std::string &name = *it;
-            if ( it != members.begin() )
-               document_ += ",";
-            document_ += valueToQuotedString( name.c_str() );
-            document_ += yamlCompatiblityEnabled_ ? ": " 
-                                                  : ":";
-            writeValue( value[name] );
-         }
-         document_ += "}";
-      }
-      break;
-   }
-}
-
-
-// Class StyledWriter
-// //////////////////////////////////////////////////////////////////
-
-StyledWriter::StyledWriter()
-   : rightMargin_( 74 )
-   , indentSize_( 3 )
-{
-}
-
-
-std::string 
-StyledWriter::write( const Value &root )
-{
-   document_ = "";
-   addChildValues_ = false;
-   indentString_ = "";
-   writeCommentBeforeValue( root );
-   writeValue( root );
-   writeCommentAfterValueOnSameLine( root );
-   document_ += "\n";
-   return document_;
-}
-
-
-void 
-StyledWriter::writeValue( const Value &value )
-{
-   switch ( value.type() )
-   {
-   case nullValue:
-      pushValue( "null" );
-      break;
-   case intValue:
-      pushValue( valueToString( value.asLargestInt() ) );
-      break;
-   case uintValue:
-      pushValue( valueToString( value.asLargestUInt() ) );
-      break;
-   case realValue:
-      pushValue( valueToString( value.asDouble() ) );
-      break;
-   case stringValue:
-      pushValue( valueToQuotedString( value.asCString() ) );
-      break;
-   case booleanValue:
-      pushValue( valueToString( value.asBool() ) );
-      break;
-   case arrayValue:
-      writeArrayValue( value);
-      break;
-   case objectValue:
-      {
-         Value::Members members( value.getMemberNames() );
-         if ( members.empty() )
-            pushValue( "{}" );
-         else
-         {
-            writeWithIndent( "{" );
-            indent();
-            Value::Members::iterator it = members.begin();
-            for (;;)
-            {
-               const std::string &name = *it;
-               const Value &childValue = value[name];
-               writeCommentBeforeValue( childValue );
-               writeWithIndent( valueToQuotedString( name.c_str() ) );
-               document_ += " : ";
-               writeValue( childValue );
-               if ( ++it == members.end() )
-               {
-                  writeCommentAfterValueOnSameLine( childValue );
-                  break;
-               }
-               document_ += ",";
-               writeCommentAfterValueOnSameLine( childValue );
-            }
-            unindent();
-            writeWithIndent( "}" );
-         }
-      }
-      break;
-   }
-}
-
-
-void 
-StyledWriter::writeArrayValue( const Value &value )
-{
-   unsigned size = value.size();
-   if ( size == 0 )
-      pushValue( "[]" );
-   else
-   {
-      bool isArrayMultiLine = isMultineArray( value );
-      if ( isArrayMultiLine )
-      {
-         writeWithIndent( "[" );
-         indent();
-         bool hasChildValue = !childValues_.empty();
-         unsigned index =0;
-         for (;;)
-         {
-            const Value &childValue = value[index];
-            writeCommentBeforeValue( childValue );
-            if ( hasChildValue )
-               writeWithIndent( childValues_[index] );
-            else
-            {
-               writeIndent();
-               writeValue( childValue );
-            }
-            if ( ++index == size )
-            {
-               writeCommentAfterValueOnSameLine( childValue );
-               break;
-            }
-            document_ += ",";
-            writeCommentAfterValueOnSameLine( childValue );
-         }
-         unindent();
-         writeWithIndent( "]" );
-      }
-      else // output on a single line
-      {
-         assert( childValues_.size() == size );
-         document_ += "[ ";
-         for ( unsigned index =0; index < size; ++index )
-         {
-            if ( index > 0 )
-               document_ += ", ";
-            document_ += childValues_[index];
-         }
-         document_ += " ]";
-      }
-   }
-}
-
-
-bool 
-StyledWriter::isMultineArray( const Value &value )
-{
-   int size = value.size();
-   bool isMultiLine = size*3 >= rightMargin_ ;
-   childValues_.clear();
-   for ( int index =0; index < size  &&  !isMultiLine; ++index )
-   {
-      const Value &childValue = value[index];
-      isMultiLine = isMultiLine  ||
-                     ( (childValue.isArray()  ||  childValue.isObject())  &&  
-                        childValue.size() > 0 );
-   }
-   if ( !isMultiLine ) // check if line length > max line length
-   {
-      childValues_.reserve( size );
-      addChildValues_ = true;
-      int lineLength = 4 + (size-1)*2; // '[ ' + ', '*n + ' ]'
-      for ( int index =0; index < size  &&  !isMultiLine; ++index )
-      {
-         writeValue( value[index] );
-         lineLength += int( childValues_[index].length() );
-         isMultiLine = isMultiLine  &&  hasCommentForValue( value[index] );
-      }
-      addChildValues_ = false;
-      isMultiLine = isMultiLine  ||  lineLength >= rightMargin_;
-   }
-   return isMultiLine;
-}
-
-
-void 
-StyledWriter::pushValue( const std::string &value )
-{
-   if ( addChildValues_ )
-      childValues_.push_back( value );
-   else
-      document_ += value;
-}
-
-
-void 
-StyledWriter::writeIndent()
-{
-   if ( !document_.empty() )
-   {
-      char last = document_[document_.length()-1];
-      if ( last == ' ' )     // already indented
-         return;
-      if ( last != '\n' )    // Comments may add new-line
-         document_ += '\n';
-   }
-   document_ += indentString_;
-}
-
-
-void 
-StyledWriter::writeWithIndent( const std::string &value )
-{
-   writeIndent();
-   document_ += value;
-}
-
-
-void 
-StyledWriter::indent()
-{
-   indentString_ += std::string( indentSize_, ' ' );
-}
-
-
-void 
-StyledWriter::unindent()
-{
-   assert( int(indentString_.size()) >= indentSize_ );
-   indentString_.resize( indentString_.size() - indentSize_ );
-}
-
-
-void 
-StyledWriter::writeCommentBeforeValue( const Value &root )
-{
-   if ( !root.hasComment( commentBefore ) )
-      return;
-   document_ += normalizeEOL( root.getComment( commentBefore ) );
-   document_ += "\n";
-}
-
-
-void 
-StyledWriter::writeCommentAfterValueOnSameLine( const Value &root )
-{
-   if ( root.hasComment( commentAfterOnSameLine ) )
-      document_ += " " + normalizeEOL( root.getComment( commentAfterOnSameLine ) );
-
-   if ( root.hasComment( commentAfter ) )
-   {
-      document_ += "\n";
-      document_ += normalizeEOL( root.getComment( commentAfter ) );
-      document_ += "\n";
-   }
-}
-
-
-bool 
-StyledWriter::hasCommentForValue( const Value &value )
-{
-   return value.hasComment( commentBefore )
-          ||  value.hasComment( commentAfterOnSameLine )
-          ||  value.hasComment( commentAfter );
-}
-
-
-std::string 
-StyledWriter::normalizeEOL( const std::string &text )
-{
-   std::string normalized;
-   normalized.reserve( text.length() );
-   const char *begin = text.c_str();
-   const char *end = begin + text.length();
-   const char *current = begin;
-   while ( current != end )
-   {
-      char c = *current++;
-      if ( c == '\r' ) // mac or dos EOL
-      {
-         if ( *current == '\n' ) // convert dos EOL
-            ++current;
-         normalized += '\n';
-      }
-      else // handle unix EOL & other char
-         normalized += c;
-   }
-   return normalized;
-}
-
-
-// Class StyledStreamWriter
-// //////////////////////////////////////////////////////////////////
-
-StyledStreamWriter::StyledStreamWriter( std::string indentation )
-   : document_(NULL)
-   , rightMargin_( 74 )
-   , indentation_( indentation )
-{
-}
-
-
-void
-StyledStreamWriter::write( std::ostream &out, const Value &root )
-{
-   document_ = &out;
-   addChildValues_ = false;
-   indentString_ = "";
-   writeCommentBeforeValue( root );
-   writeValue( root );
-   writeCommentAfterValueOnSameLine( root );
-   *document_ << "\n";
-   document_ = NULL; // Forget the stream, for safety.
-}
-
-
-void 
-StyledStreamWriter::writeValue( const Value &value )
-{
-   switch ( value.type() )
-   {
-   case nullValue:
-      pushValue( "null" );
-      break;
-   case intValue:
-      pushValue( valueToString( value.asLargestInt() ) );
-      break;
-   case uintValue:
-      pushValue( valueToString( value.asLargestUInt() ) );
-      break;
-   case realValue:
-      pushValue( valueToString( value.asDouble() ) );
-      break;
-   case stringValue:
-      pushValue( valueToQuotedString( value.asCString() ) );
-      break;
-   case booleanValue:
-      pushValue( valueToString( value.asBool() ) );
-      break;
-   case arrayValue:
-      writeArrayValue( value);
-      break;
-   case objectValue:
-      {
-         Value::Members members( value.getMemberNames() );
-         if ( members.empty() )
-            pushValue( "{}" );
-         else
-         {
-            writeWithIndent( "{" );
-            indent();
-            Value::Members::iterator it = members.begin();
-            for (;;)
-            {
-               const std::string &name = *it;
-               const Value &childValue = value[name];
-               writeCommentBeforeValue( childValue );
-               writeWithIndent( valueToQuotedString( name.c_str() ) );
-               *document_ << " : ";
-               writeValue( childValue );
-               if ( ++it == members.end() )
-               {
-                  writeCommentAfterValueOnSameLine( childValue );
-                  break;
-               }
-               *document_ << ",";
-               writeCommentAfterValueOnSameLine( childValue );
-            }
-            unindent();
-            writeWithIndent( "}" );
-         }
-      }
-      break;
-   }
-}
-
-
-void 
-StyledStreamWriter::writeArrayValue( const Value &value )
-{
-   unsigned size = value.size();
-   if ( size == 0 )
-      pushValue( "[]" );
-   else
-   {
-      bool isArrayMultiLine = isMultineArray( value );
-      if ( isArrayMultiLine )
-      {
-         writeWithIndent( "[" );
-         indent();
-         bool hasChildValue = !childValues_.empty();
-         unsigned index =0;
-         for (;;)
-         {
-            const Value &childValue = value[index];
-            writeCommentBeforeValue( childValue );
-            if ( hasChildValue )
-               writeWithIndent( childValues_[index] );
-            else
-            {
-               writeIndent();
-               writeValue( childValue );
-            }
-            if ( ++index == size )
-            {
-               writeCommentAfterValueOnSameLine( childValue );
-               break;
-            }
-            *document_ << ",";
-            writeCommentAfterValueOnSameLine( childValue );
-         }
-         unindent();
-         writeWithIndent( "]" );
-      }
-      else // output on a single line
-      {
-         assert( childValues_.size() == size );
-         *document_ << "[ ";
-         for ( unsigned index =0; index < size; ++index )
-         {
-            if ( index > 0 )
-               *document_ << ", ";
-            *document_ << childValues_[index];
-         }
-         *document_ << " ]";
-      }
-   }
-}
-
-
-bool 
-StyledStreamWriter::isMultineArray( const Value &value )
-{
-   int size = value.size();
-   bool isMultiLine = size*3 >= rightMargin_ ;
-   childValues_.clear();
-   for ( int index =0; index < size  &&  !isMultiLine; ++index )
-   {
-      const Value &childValue = value[index];
-      isMultiLine = isMultiLine  ||
-                     ( (childValue.isArray()  ||  childValue.isObject())  &&  
-                        childValue.size() > 0 );
-   }
-   if ( !isMultiLine ) // check if line length > max line length
-   {
-      childValues_.reserve( size );
-      addChildValues_ = true;
-      int lineLength = 4 + (size-1)*2; // '[ ' + ', '*n + ' ]'
-      for ( int index =0; index < size  &&  !isMultiLine; ++index )
-      {
-         writeValue( value[index] );
-         lineLength += int( childValues_[index].length() );
-         isMultiLine = isMultiLine  &&  hasCommentForValue( value[index] );
-      }
-      addChildValues_ = false;
-      isMultiLine = isMultiLine  ||  lineLength >= rightMargin_;
-   }
-   return isMultiLine;
-}
-
-
-void 
-StyledStreamWriter::pushValue( const std::string &value )
-{
-   if ( addChildValues_ )
-      childValues_.push_back( value );
-   else
-      *document_ << value;
-}
-
-
-void 
-StyledStreamWriter::writeIndent()
-{
-  /*
-    Some comments in this method would have been nice. ;-)
-
-   if ( !document_.empty() )
-   {
-      char last = document_[document_.length()-1];
-      if ( last == ' ' )     // already indented
-         return;
-      if ( last != '\n' )    // Comments may add new-line
-         *document_ << '\n';
-   }
-  */
-   *document_ << '\n' << indentString_;
-}
-
-
-void 
-StyledStreamWriter::writeWithIndent( const std::string &value )
-{
-   writeIndent();
-   *document_ << value;
-}
-
-
-void 
-StyledStreamWriter::indent()
-{
-   indentString_ += indentation_;
-}
-
-
-void 
-StyledStreamWriter::unindent()
-{
-   assert( indentString_.size() >= indentation_.size() );
-   indentString_.resize( indentString_.size() - indentation_.size() );
-}
-
-
-void 
-StyledStreamWriter::writeCommentBeforeValue( const Value &root )
-{
-   if ( !root.hasComment( commentBefore ) )
-      return;
-   *document_ << normalizeEOL( root.getComment( commentBefore ) );
-   *document_ << "\n";
-}
-
-
-void 
-StyledStreamWriter::writeCommentAfterValueOnSameLine( const Value &root )
-{
-   if ( root.hasComment( commentAfterOnSameLine ) )
-      *document_ << " " + normalizeEOL( root.getComment( commentAfterOnSameLine ) );
-
-   if ( root.hasComment( commentAfter ) )
-   {
-      *document_ << "\n";
-      *document_ << normalizeEOL( root.getComment( commentAfter ) );
-      *document_ << "\n";
-   }
-}
-
-
-bool 
-StyledStreamWriter::hasCommentForValue( const Value &value )
-{
-   return value.hasComment( commentBefore )
-          ||  value.hasComment( commentAfterOnSameLine )
-          ||  value.hasComment( commentAfter );
-}
-
-
-std::string 
-StyledStreamWriter::normalizeEOL( const std::string &text )
-{
-   std::string normalized;
-   normalized.reserve( text.length() );
-   const char *begin = text.c_str();
-   const char *end = begin + text.length();
-   const char *current = begin;
-   while ( current != end )
-   {
-      char c = *current++;
-      if ( c == '\r' ) // mac or dos EOL
-      {
-         if ( *current == '\n' ) // convert dos EOL
-            ++current;
-         normalized += '\n';
-      }
-      else // handle unix EOL & other char
-         normalized += c;
-   }
-   return normalized;
-}
-
-
-std::ostream& operator<<( std::ostream &sout, const Value &root )
-{
-   Json::StyledStreamWriter writer;
-   writer.write(sout, root);
-   return sout;
-}
-
-
-} // namespace Json

idtool.cpp

@@ -159,7 +159,7 @@ int main(int argc,char **argv)
 		}
 
 		std::string signature(Utils::base64Decode(argv[4],strlen(argv[4])));
-		if ((signature.length() > ZEROTIER_ADDRESS_LENGTH)&&(id.verifySignature(inf.data(),inf.length(),signature))) {
+		if ((signature.length() > ZT_ADDRESS_LENGTH)&&(id.verifySignature(inf.data(),inf.length(),signature.data(),signature.length()))) {
 			std::cout << argv[3] << " signature valid" << std::endl;
 		} else {
 			std::cerr << argv[3] << " signature check FAILED" << std::endl;

main.cpp

@@ -34,7 +34,9 @@
 #include <stdexcept>
 #include <iostream>
 
-#ifdef _WIN32
+#include "node/Constants.hpp"
+
+#ifdef __WINDOWS__
 #include <Windows.h>
 #else
 #include <unistd.h>
@@ -44,14 +46,45 @@
 #include <signal.h>
 #endif
 
+#include <openssl/rand.h>
+
 #include "node/Node.hpp"
 #include "node/Utils.hpp"
-#include "node/Defaults.hpp"
 
 #include "launcher.h"
 
 using namespace ZeroTier;
 
+// ---------------------------------------------------------------------------
+// Override libcrypto default RAND_ with Utils::getSecureRandom(), which uses
+// a system strong random source. This is because OpenSSL libcrypto's default
+// RAND_ implementation uses uninitialized memory as one of its entropy
+// sources, which plays havoc with all kinds of debuggers and auditing tools.
+
+static void _zeroTier_rand_cleanup() {}
+static void _zeroTier_rand_add(const void *buf, int num, double add_entropy) {}
+static int _zeroTier_rand_status() { return 1; }
+static void _zeroTier_rand_seed(const void *buf, int num) {}
+static int _zeroTier_rand_bytes(unsigned char *buf, int num)
+{
+	Utils::getSecureRandom(buf,num);
+	return 1;
+}
+static RAND_METHOD _zeroTierRandMethod = {
+	_zeroTier_rand_seed,
+	_zeroTier_rand_bytes,
+	_zeroTier_rand_cleanup,
+	_zeroTier_rand_add,
+	_zeroTier_rand_bytes,
+	_zeroTier_rand_status
+};
+static void _initLibCrypto()
+{
+	RAND_set_rand_method(&_zeroTierRandMethod);
+}
+
+// ---------------------------------------------------------------------------
+
 static Node *node = (Node *)0;
 
 static void printHelp(const char *cn,FILE *out)
@@ -67,30 +100,23 @@ static void sighandlerQuit(int sig)
 		n->terminate();
 	else exit(0);
 }
-static void sighandlerUsr(int sig)
-{
-}
-static void sighandlerHup(int sig)
-{
-	Node *n = node;
-	if (n)
-		n->updateStatusNow();
-}
 #endif
 
 int main(int argc,char **argv)
 {
 #ifndef _WIN32
-	signal(SIGHUP,&sighandlerHup);
+	signal(SIGHUP,SIG_IGN);
 	signal(SIGPIPE,SIG_IGN);
-	signal(SIGUSR1,&sighandlerUsr);
-	signal(SIGUSR2,&sighandlerUsr);
+	signal(SIGUSR1,SIG_IGN);
+	signal(SIGUSR2,SIG_IGN);
 	signal(SIGALRM,SIG_IGN);
 	signal(SIGINT,&sighandlerQuit);
 	signal(SIGTERM,&sighandlerQuit);
 	signal(SIGQUIT,&sighandlerQuit);
 #endif
 
+	_initLibCrypto();
+
 	if (argc < 2) {
 		printHelp(argv[0],stderr);
 		return ZT_EXEC_RETURN_VALUE_NORMAL_TERMINATION;
@@ -124,13 +150,16 @@ int main(int argc,char **argv)
 
 	int exitCode = ZT_EXEC_RETURN_VALUE_NORMAL_TERMINATION;
 
-	node = new Node(homeDir,ZT_DEFAULTS.configUrlPrefix.c_str(),ZT_DEFAULTS.configAuthority.c_str());
+	node = new Node(homeDir);
+	const char *termReason = (char *)0;
 	switch(node->run()) {
 		case Node::NODE_RESTART_FOR_RECONFIGURATION:
 			exitCode = ZT_EXEC_RETURN_VALUE_PLEASE_RESTART;
 			break;
 		case Node::NODE_UNRECOVERABLE_ERROR:
 			exitCode = ZT_EXEC_RETURN_VALUE_UNRECOVERABLE_ERROR;
+			termReason = node->reasonForTermination();
+			fprintf(stderr,"%s: abnormal termination: %s\n",argv[0],(termReason) ? termReason : "(unknown reason)");
 			break;
 		case Node::NODE_NEW_VERSION_AVAILABLE:
 			exitCode = ZT_EXEC_RETURN_VALUE_TERMINATED_FOR_UPGRADE;

makekeypair.cpp

@@ -0,0 +1,47 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include <iostream>
+#include <string>
+#include "node/EllipticCurveKeyPair.hpp"
+#include "node/EllipticCurveKey.hpp"
+#include "node/Utils.hpp"
+
+using namespace ZeroTier;
+
+int main(int argc,char **argv)
+{
+	std::cout << "[generating]" << std::endl;
+
+	EllipticCurveKeyPair kp;
+	kp.generate();
+
+	std::cout << "PUBLIC: " << kp.pub().toHex() << std::endl;
+	std::cout << "PRIVATE: " << kp.priv().toHex() << std::endl;
+
+	return 0;
+}

netconf-service/Makefile

@@ -0,0 +1,7 @@
+all:
+	gcc -O6 -c ../ext/lz4/lz4hc.c ../ext/lz4/lz4.c
+	g++ -DZT_OSNAME="linux" -DZT_ARCH="x86_64" -I/usr/include/mysql -I../ext/bin/libcrypto/include -O -pthread -o netconf.service netconf.cpp ../node/Utils.cpp ../node/Identity.cpp ../node/EllipticCurveKeyPair.cpp ../node/Salsa20.cpp ../node/HMAC.cpp lz4.o lz4hc.o ../ext/bin/libcrypto/linux-x86_64/libcrypto.a -lmysqlpp
+	g++ -DZT_OSNAME="linux" -DZT_ARCH="x86_64" -I/usr/include/mysql -I../ext/bin/libcrypto/include -O -pthread -o netconf-test netconf-test.cpp ../node/Utils.cpp ../node/Identity.cpp ../node/EllipticCurveKeyPair.cpp ../node/Salsa20.cpp ../node/HMAC.cpp ../node/Logger.cpp ../node/Service.cpp ../node/Thread.cpp lz4.o lz4hc.o ../ext/bin/libcrypto/linux-x86_64/libcrypto.a
+
+clean:
+	rm -f *.o netconf.service netconf-test

netconf-service/netconf-test.cpp

@@ -0,0 +1,83 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+/* Self-tester that makes both new and repeated requests to netconf */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <sys/time.h>
+
+#include <vector>
+#include <string>
+#include <iostream>
+
+#include "../node/Dictionary.hpp"
+#include "../node/Service.hpp"
+#include "../node/Identity.hpp"
+#include "../node/RuntimeEnvironment.hpp"
+#include "../node/Logger.hpp"
+#include "../node/Thread.hpp"
+
+using namespace ZeroTier;
+
+static void svcHandler(void *arg,Service &svc,const Dictionary &msg)
+{
+	std::cout << msg.toString();
+}
+
+int main(int argc,char **argv)
+{
+	RuntimeEnvironment renv;
+	renv.log = new Logger((const char *)0,(const char *)0,0);
+	Service svc(&renv,"netconf","./netconf.service",&svcHandler,(void *)0);
+
+	srand(time(0));
+
+	std::vector<Identity> population;
+	for(;;) {
+		Identity id;
+		if ((population.empty())||(rand() < (RAND_MAX / 4))) {
+			id.generate();
+			population.push_back(id);
+			std::cout << "Testing with new identity: " << id.address().toString() << std::endl;
+		} else {
+			id = population[rand() % population.size()];
+			Thread::sleep(1000);
+			std::cout << "Testing with existing identity: " << id.address().toString() << std::endl;
+		}
+
+		Dictionary request;
+		request["type"] = "netconf-request";
+		request["peerId"] = id.toString(false);
+		request["nwid"] = "6c92786fee000001";
+		request["requestId"] = "12345";
+
+		svc.send(request);
+	}
+}

netconf-service/netconf.cpp

@@ -0,0 +1,347 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+/*
+ * This is the netconf service. It's currently used only by netconf nodes that
+ * are run by ZeroTier itself. There is nothing to prevent you from running
+ * your own if you wanted to create your own networks outside our system.
+ *
+ * That being said, we'd like to charge for private networks to support
+ * ZeroTier One and future development efforts. So while this software is
+ * open source and we're not going to stop you from sidestepping this, we
+ * do ask -- honor system here -- that you pay for private networks if you
+ * are going to use them for any commercial purpose such as a business VPN
+ * alternative.
+ *
+ * This will at the moment only build on Linux and requires the mysql++
+ * library, which is available here:
+ *
+ * http://tangentsoft.net/mysql++/
+ *
+ * (Packages are available for CentOS via EPEL and for any Debian distro.)
+ *
+ * This program must be built and installed in the services.d subfolder of
+ * the ZeroTier One home folder of the node designated to act as a master
+ * for networks. Doing so will enable the NETWORK_CONFIG_REQUEST protocol
+ * verb.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <arpa/inet.h>
+
+#include <iostream>
+#include <string>
+#include <map>
+#include <list>
+#include <vector>
+#include <algorithm>
+
+#include <mysql++/mysql++.h>
+
+#include "../node/Dictionary.hpp"
+#include "../node/Identity.hpp"
+#include "../node/Utils.hpp"
+#include "../node/Mutex.hpp"
+
+using namespace ZeroTier;
+using namespace mysqlpp;
+
+static Mutex stdoutWriteLock;
+static Connection *dbCon = (Connection *)0;
+static char mysqlHost[64],mysqlPort[64],mysqlDatabase[64],mysqlUser[64],mysqlPassword[64];
+
+static void connectOrReconnect()
+{
+	for(;;) {
+		delete dbCon;
+		try {
+			dbCon = new Connection(mysqlDatabase,mysqlHost,mysqlUser,mysqlPassword,(unsigned int)strtol(mysqlPort,(char **)0,10));
+			if (dbCon->connected()) {
+				fprintf(stderr,"(re?)-connected to mysql server successfully\n");
+				break;
+			} else {
+				fprintf(stderr,"unable to connect to database server (connection closed), trying again in 1s...\n");
+				usleep(1000000);
+			}
+		} catch (std::exception &exc) {
+			fprintf(stderr,"unable to connect to database server (%s), trying again in 1s...\n",exc.what());
+			usleep(1000000);
+		} catch ( ... ) {
+			fprintf(stderr,"unable to connect to database server (unknown exception), trying again in 1s...\n");
+			usleep(1000000);
+		}
+	}
+}
+
+int main(int argc,char **argv)
+{
+	{
+		char *ee = getenv("ZT_NETCONF_MYSQL_HOST");
+		if (!ee) {
+			fprintf(stderr,"missing environment variable: ZT_NETCONF_MYSQL_HOST\n");
+			return -1;
+		}
+		strcpy(mysqlHost,ee);
+		ee = getenv("ZT_NETCONF_MYSQL_PORT");
+		if (!ee)
+			strcpy(mysqlPort,"3306");
+		else strcpy(mysqlPort,ee);
+		ee = getenv("ZT_NETCONF_MYSQL_DATABASE");
+		if (!ee) {
+			fprintf(stderr,"missing environment variable: ZT_NETCONF_MYSQL_DATABASE\n");
+			return -1;
+		}
+		strcpy(mysqlDatabase,ee);
+		ee = getenv("ZT_NETCONF_MYSQL_USER");
+		if (!ee) {
+			fprintf(stderr,"missing environment variable: ZT_NETCONF_MYSQL_USER\n");
+			return -1;
+		}
+		strcpy(mysqlUser,ee);
+		ee = getenv("ZT_NETCONF_MYSQL_PASSWORD");
+		if (!ee) {
+			fprintf(stderr,"missing environment variable: ZT_NETCONF_MYSQL_PASSWORD\n");
+			return -1;
+		}
+		strcpy(mysqlPassword,ee);
+	}
+
+	char buf[131072];
+	std::string dictBuf;
+
+	connectOrReconnect();
+	for(;;) {
+		for(int l=0;l<4;) {
+			int n = (int)read(STDIN_FILENO,buf + l,4 - l);
+			if (n < 0) {
+				fprintf(stderr,"error reading frame size from stdin: %s\n",strerror(errno));
+				return -1;
+			}
+			l += n;
+		}
+		unsigned int fsize = (unsigned int)ntohl(*((const uint32_t *)buf));
+
+		while (dictBuf.length() < fsize) {
+			int n = (int)read(STDIN_FILENO,buf,std::min((int)sizeof(buf),(int)(fsize - dictBuf.length())));
+			if (n < 0) {
+				fprintf(stderr,"error reading frame from stdin: %s\n",strerror(errno));
+				return -1;
+			}
+			for(int i=0;i<n;++i)
+				dictBuf.push_back(buf[i]);
+		}
+		Dictionary request(dictBuf);
+		dictBuf = "";
+
+		if (!dbCon->connected())
+			connectOrReconnect();
+
+		try {
+			const std::string &reqType = request.get("type");
+			if (reqType == "netconf-request") { // NETWORK_CONFIG_REQUEST packet
+				Identity peerIdentity(request.get("peerId"));
+				uint64_t nwid = strtoull(request.get("nwid").c_str(),(char **)0,16);
+				Dictionary meta;
+				if (request.contains("meta"))
+					meta.fromString(request.get("meta"));
+
+				// Do quick signature check / sanity check
+				if (!peerIdentity.locallyValidate(false)) {
+					fprintf(stderr,"identity failed signature check: %s\n",peerIdentity.toString(false).c_str());
+					continue;
+				}
+
+				// Save identity if unknown
+				{
+					Query q = dbCon->query();
+					q << "SELECT identity,identityValidated FROM Node WHERE id = " << peerIdentity.address().toInt();
+					StoreQueryResult rs = q.store();
+					if (rs.num_rows() > 0) {
+						if (rs[0]["identity"] != peerIdentity.toString(false)) {
+							// TODO: handle collisions...
+							continue;
+						} else if ((int)rs[0]["identityValidated"] == 0) {
+							// TODO: launch background validation
+						}
+					} else {
+						q = dbCon->query();
+						q << "INSERT INTO Node (id,creationTime,lastSeen,identity) VALUES (" << peerIdentity.address().toInt() << "," << Utils::now() << ",0," << quote << peerIdentity.toString(false) << ")";
+						if (!q.exec()) {
+							fprintf(stderr,"error inserting Node row for peer %s, aborting netconf request\n",peerIdentity.address().toString().c_str());
+							continue;
+						}
+						// TODO: launch background validation
+					}
+				}
+
+				// Update lastSeen
+				{
+					Query q = dbCon->query();
+					q << "UPDATE Node SET lastSeen = " << Utils::now() << " WHERE id = " << peerIdentity.address().toInt();
+					q.exec();
+				}
+
+				bool isOpen = false;
+				{
+					Query q = dbCon->query();
+					q << "SELECT isOpen FROM Network WHERE id = " << nwid;
+					StoreQueryResult rs = q.store();
+					if (rs.num_rows() > 0)
+						isOpen = ((int)rs[0]["isOpen"] > 0);
+					else {
+						Dictionary response;
+						response["peer"] = peerIdentity.address().toString();
+						response["nwid"] = request.get("nwid");
+						response["type"] = "netconf-response";
+						response["requestId"] = request.get("requestId");
+						response["error"] = "NOT_FOUND";
+						std::string respm = response.toString();
+						uint32_t respml = (uint32_t)htonl((uint32_t)respm.length());
+
+						stdoutWriteLock.lock();
+						write(STDOUT_FILENO,&respml,4);
+						write(STDOUT_FILENO,respm.data(),respm.length());
+						stdoutWriteLock.unlock();
+						continue;
+					}
+				}
+
+				Dictionary netconf;
+
+				netconf["peer"] = peerIdentity.address().toString();
+				sprintf(buf,"%.16llx",(unsigned long long)nwid);
+				netconf["nwid"] = buf;
+				netconf["isOpen"] = (isOpen ? "1" : "0");
+				sprintf(buf,"%llx",(unsigned long long)Utils::now());
+				netconf["ts"] = buf;
+
+				if (!isOpen) {
+					// TODO: handle closed networks, look up private membership,
+					// generate signed cert.
+				}
+
+				std::string ipv4Static,ipv6Static;
+
+				{
+					// Check for IPv4 static assignments
+					Query q = dbCon->query();
+					q << "SELECT INET_NTOA(ip) AS ip,netmaskBits FROM IPv4Static WHERE Node_id = " << peerIdentity.address().toInt() << " AND Network_id = " << nwid;
+					StoreQueryResult rs = q.store();
+					if (rs.num_rows() > 0) {
+						for(int i=0;i<rs.num_rows();++i) {
+							if (ipv4Static.length())
+								ipv4Static.push_back(',');
+							ipv4Static.append(rs[i]["ip"].c_str());
+							ipv4Static.push_back('/');
+							ipv4Static.append(rs[i]["netmaskBits"].c_str());
+						}
+					}
+
+					// Try to auto-assign if there's any auto-assign networks with space
+					// available.
+					if (!ipv4Static.length()) {
+						unsigned char addressBytes[5];
+						peerIdentity.address().copyTo(addressBytes,5);
+
+						q = dbCon->query();
+						q << "SELECT ipNet,netmaskBits FROM IPv4AutoAssign WHERE Network_id = " << nwid;
+						rs = q.store();
+						if (rs.num_rows() > 0) {
+							for(int aaRow=0;aaRow<rs.num_rows();++aaRow) {
+								uint32_t ipNet = (uint32_t)((unsigned long)rs[aaRow]["ipNet"]);
+								unsigned int netmaskBits = (unsigned int)rs[aaRow]["netmaskBits"];
+
+								uint32_t tryIp = (((uint32_t)addressBytes[1]) << 24) |
+								                 (((uint32_t)addressBytes[2]) << 16) |
+								                 (((uint32_t)addressBytes[3]) << 8) |
+								                 ((((uint32_t)addressBytes[4]) % 254) + 1);
+								tryIp &= (0xffffffff >> netmaskBits);
+								tryIp |= ipNet;
+
+								for(int k=0;k<100000;++k) {
+									Query q2 = dbCon->query();
+									q2 << "INSERT INTO IPv4Static (Network_id,Node_id,ip,netmaskBits) VALUES (" << nwid << "," << peerIdentity.address().toInt() << "," << tryIp << "," << netmaskBits << ")";
+									if (q2.exec()) {
+										sprintf(buf,"%u.%u.%u.%u",(unsigned int)((tryIp >> 24) & 0xff),(unsigned int)((tryIp >> 16) & 0xff),(unsigned int)((tryIp >> 8) & 0xff),(unsigned int)(tryIp & 0xff));
+										if (ipv4Static.length())
+											ipv4Static.push_back(',');
+										ipv4Static.append(buf);
+										ipv4Static.push_back('/');
+										sprintf(buf,"%u",netmaskBits);
+										ipv4Static.append(buf);
+										break;
+									} else { // insert will fail if IP is in use due to uniqueness constraints in DB
+										++tryIp;
+										if ((tryIp & 0xff) == 0)
+											tryIp |= 1;
+										tryIp &= (0xffffffff >> netmaskBits);
+										tryIp |= ipNet;
+									}
+								}
+
+								if (ipv4Static.length())
+									break;
+							}
+						}
+					}
+				}
+
+				if (ipv4Static.length())
+					netconf["ipv4Static"] = ipv4Static;
+				if (ipv6Static.length())
+					netconf["ipv6Static"] = ipv6Static;
+
+				{
+					Dictionary response;
+					response["peer"] = peerIdentity.address().toString();
+					response["nwid"] = request.get("nwid");
+					response["type"] = "netconf-response";
+					response["requestId"] = request.get("requestId");
+					response["netconf"] = netconf.toString();
+					std::string respm = response.toString();
+					uint32_t respml = (uint32_t)htonl((uint32_t)respm.length());
+
+					stdoutWriteLock.lock();
+					write(STDOUT_FILENO,&respml,4);
+					write(STDOUT_FILENO,respm.data(),respm.length());
+					stdoutWriteLock.unlock();
+				}
+			}
+		} catch (std::exception &exc) {
+			fprintf(stderr,"unexpected exception handling message: %s\n",exc.what());
+		} catch ( ... ) {
+			fprintf(stderr,"unexpected exception handling message: unknown exception\n");
+		}
+	}
+}

node/Address.hpp

@@ -28,72 +28,156 @@
 #ifndef _ZT_ADDRESS_HPP
 #define _ZT_ADDRESS_HPP
 
+#include <stdio.h>
+#include <stdlib.h>
 #include <stdint.h>
+#include <string.h>
 #include <string>
 #include "Utils.hpp"
 #include "MAC.hpp"
 #include "Constants.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
 /**
- * ZeroTier address, which doubles as the last 5 octets of the MAC on taps
- *
- * Natural sort order will differ on big vs. little endian machines, but that
- * won't matter when it's used as a local map/set key.
+ * A ZeroTier address
  */
 class Address
 {
-private:
-	union {
-		unsigned char o[ZT_ADDRESS_LENGTH];
-		uint64_t v;
-	} _a;
-
 public:
 	Address()
-		throw()
+		throw() :
+		_a(0)
 	{
-		_a.v = 0;
 	}
 
 	Address(const Address &a)
+		throw() :
+		_a(a._a)
+	{
+	}
+
+	Address(uint64_t a)
+		throw() :
+		_a(a & 0xffffffffffULL)
+	{
+	}
+
+	Address(const char *s)
 		throw()
 	{
-		_a.v = a._a.v;
+		unsigned char foo[ZT_ADDRESS_LENGTH];
+		setTo(foo,Utils::unhex(s,foo,ZT_ADDRESS_LENGTH));
+	}
+
+	Address(const std::string &s)
+		throw()
+	{
+		unsigned char foo[ZT_ADDRESS_LENGTH];
+		setTo(foo,Utils::unhex(s.c_str(),foo,ZT_ADDRESS_LENGTH));
 	}
 
 	/**
-	 * Create from a ZeroTier MAC
-	 *
-	 * @param m MAC (assumed to be a ZeroTier MAC)
+	 * @param bits Raw address -- 5 bytes, big-endian byte order
+	 * @param len Length of array
 	 */
-	Address(const MAC &m)
+	Address(const void *bits,unsigned int len)
 		throw()
 	{
-		_a.v = 0;
-		for(int i=0;i<ZT_ADDRESS_LENGTH;++i)
-			_a.o[i] = m.data[i + 1];
-	}
-
-	/**
-	 * @param bits Raw address -- 5 bytes in length
-	 */
-	Address(const void *bits)
-		throw()
-	{
-		_a.v = 0;
-		for(int i=0;i<ZT_ADDRESS_LENGTH;++i)
-			_a.o[i] = ((const unsigned char *)bits)[i];
+		setTo(bits,len);
 	}
 
 	inline Address &operator=(const Address &a)
 		throw()
 	{
-		_a.v = a._a.v;
+		_a = a._a;
 		return *this;
 	}
 
+	inline Address &operator=(const uint64_t a)
+		throw()
+	{
+		_a = (a & 0xffffffffffULL);
+		return *this;
+	}
+
+	/**
+	 * @param bits Raw address -- 5 bytes, big-endian byte order
+	 * @param len Length of array
+	 */
+	inline void setTo(const void *bits,unsigned int len)
+		throw()
+	{
+		if (len < ZT_ADDRESS_LENGTH) {
+			_a = 0;
+			return;
+		}
+		const unsigned char *b = (const unsigned char *)bits;
+		uint64_t a = ((uint64_t)*b++) << 32;
+		a |= ((uint64_t)*b++) << 24;
+		a |= ((uint64_t)*b++) << 16;
+		a |= ((uint64_t)*b++) << 8;
+		a |= ((uint64_t)*b);
+		_a = a;
+	}
+
+	/**
+	 * @param bits Buffer to hold 5-byte address in big-endian byte order
+	 * @param len Length of array
+	 */
+	inline void copyTo(void *bits,unsigned int len) const
+		throw()
+	{
+		if (len < ZT_ADDRESS_LENGTH)
+			return;
+		unsigned char *b = (unsigned char *)bits;
+		*(b++) = (unsigned char)((_a >> 32) & 0xff);
+		*(b++) = (unsigned char)((_a >> 24) & 0xff);
+		*(b++) = (unsigned char)((_a >> 16) & 0xff);
+		*(b++) = (unsigned char)((_a >> 8) & 0xff);
+		*b = (unsigned char)(_a & 0xff);
+	}
+
+	/**
+	 * Append to a buffer in big-endian byte order
+	 *
+	 * @param b Buffer to append to
+	 */
+	template<unsigned int C>
+	inline void appendTo(Buffer<C> &b) const
+		throw(std::out_of_range)
+	{
+		b.append((unsigned char)((_a >> 32) & 0xff));
+		b.append((unsigned char)((_a >> 24) & 0xff));
+		b.append((unsigned char)((_a >> 16) & 0xff));
+		b.append((unsigned char)((_a >> 8) & 0xff));
+		b.append((unsigned char)(_a & 0xff));
+	}
+
+	/**
+	 * @return String containing address as 5 binary bytes
+	 */
+	inline std::string toBinaryString() const
+	{
+		std::string b;
+		b.push_back((char)((_a >> 32) & 0xff));
+		b.push_back((char)((_a >> 24) & 0xff));
+		b.push_back((char)((_a >> 16) & 0xff));
+		b.push_back((char)((_a >> 8) & 0xff));
+		b.push_back((char)(_a & 0xff));
+		return b;
+	}
+
+	/**
+	 * @return Integer containing address (0 to 2^40)
+	 */
+	inline uint64_t toInt() const
+		throw()
+	{
+		return _a;
+	}
+
 	/**
 	 * Derive a MAC whose first octet is the ZeroTier LAN standard
 	 * 
@@ -104,8 +188,7 @@ public:
 	{
 		MAC m;
 		m.data[0] = ZT_MAC_FIRST_OCTET;
-		for(int i=1;i<6;++i)
-			m.data[i] = _a.o[i - 1];
+		copyTo(m.data + 1,ZT_ADDRESS_LENGTH);
 		return m;
 	}
 
@@ -114,18 +197,15 @@ public:
 	 */
 	inline std::string toString() const
 	{
-		return Utils::hex(_a.o,ZT_ADDRESS_LENGTH);
+		char buf[16];
+		sprintf(buf,"%.10llx",(unsigned long long)_a);
+		return std::string(buf);
 	};
 
-	/**
-	 * Set address to zero
-	 */
-	inline void zero() throw() { _a.v = 0; }
-
 	/**
 	 * @return True if this address is not zero
 	 */
-	inline operator bool() const throw() { return (_a.v); }
+	inline operator bool() const throw() { return (_a); }
 
 	/**
 	 * @return Sum of all bytes in address
@@ -133,10 +213,7 @@ public:
 	inline unsigned int sum() const
 		throw()
 	{
-		unsigned int s = 0;
-		for(unsigned int i=0;i<ZT_ADDRESS_LENGTH;++i)
-			s += _a.o[i];
-		return s;
+		return (unsigned int)(((_a >> 32) & 0xff) + ((_a >> 24) & 0xff) + ((_a >> 16) & 0xff) + ((_a >> 8) & 0xff) + (_a & 0xff));
 	}
 
 	/**
@@ -151,23 +228,24 @@ public:
 	inline bool isReserved() const
 		throw()
 	{
-		return ((!_a.v)||(_a.o[0] == ZT_ADDRESS_RESERVED_PREFIX));
+		return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX));
 	}
 
-	inline unsigned char *data() throw() { return _a.o; }
-	inline const unsigned char *data() const throw() { return _a.o; }
+	/**
+	 * @param i Value from 0 to 4 (inclusive)
+	 * @return Byte at said position (address interpreted in big-endian order)
+	 */
+	inline unsigned char operator[](unsigned int i) const throw() { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); }
 
-	inline unsigned int size() const throw() { return ZT_ADDRESS_LENGTH; }
+	inline bool operator==(const Address &a) const throw() { return (_a == a._a); }
+	inline bool operator!=(const Address &a) const throw() { return (_a != a._a); }
+	inline bool operator>(const Address &a) const throw() { return (_a > a._a); }
+	inline bool operator<(const Address &a) const throw() { return (_a < a._a); }
+	inline bool operator>=(const Address &a) const throw() { return (_a >= a._a); }
+	inline bool operator<=(const Address &a) const throw() { return (_a <= a._a); }
 
-	inline unsigned char &operator[](unsigned int i) throw() { return _a.o[i]; }
-	inline unsigned char operator[](unsigned int i) const throw() { return _a.o[i]; }
-
-	inline bool operator==(const Address &a) const throw() { return (_a.v == a._a.v); }
-	inline bool operator!=(const Address &a) const throw() { return (_a.v != a._a.v); }
-	inline bool operator<(const Address &a) const throw() { return (_a.v < a._a.v); }
-	inline bool operator>(const Address &a) const throw() { return (_a.v > a._a.v); }
-	inline bool operator<=(const Address &a) const throw() { return (_a.v <= a._a.v); }
-	inline bool operator>=(const Address &a) const throw() { return (_a.v >= a._a.v); }
+private:
+	uint64_t _a;
 };
 
 } // namespace ZeroTier

node/BlobArray.hpp

@@ -1,94 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef _ZT_BLOBARRAY_HPP
-#define _ZT_BLOBARRAY_HPP
-
-#include <vector>
-#include <string>
-#include <algorithm>
-
-namespace ZeroTier {
-
-/**
- * A vector of binary strings serializable in a packed format
- *
- * The format uses variable-length integers to indicate the length of each
- * field. Each byte of the length has another byte with seven more significant
- * bits if its 8th bit is set. Fields can be up to 2^28 in length.
- */
-class BlobArray : public std::vector<std::string>
-{
-public:
-	inline std::string serialize() const
-	{
-		std::string r;
-		for(BlobArray::const_iterator i=begin();i!=end();++i) {
-			unsigned int flen = (unsigned int)i->length();
-			do {
-				unsigned char flenb = (unsigned char)(flen & 0x7f);
-				flen >>= 7;
-				flenb |= (flen) ? 0x80 : 0;
-				r.push_back((char)flenb);
-			} while (flen);
-			r.append(*i);
-		}
-		return r;
-	}
-
-	/**
-	 * Deserialize, replacing the current contents of this array
-	 *
-	 * @param data Serialized binary data
-	 * @param len Length of serialized data
-	 */
-	inline void deserialize(const void *data,unsigned int len)
-	{
-		clear();
-		for(unsigned int i=0;i<len;) {
-			unsigned int flen = 0;
-			unsigned int chunk = 0;
-			while (i < len) {
-				flen |= ((unsigned int)(((const unsigned char *)data)[i] & 0x7f)) << (7 * chunk++);
-				if (!(((const unsigned char *)data)[i++] & 0x80))
-					break;
-			}
-			flen = std::min(flen,len - i);
-			push_back(std::string(((const char *)data) + i,flen));
-			i += flen;
-		}
-	}
-	inline void deserialize(const std::string &data)
-	{
-		deserialize(data.data(),(unsigned int)data.length());
-	}
-};
-
-} // namespace ZeroTier
-
-#endif
-

node/BloomFilter.hpp

@@ -0,0 +1,133 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_BLOOMFILTER_HPP
+#define _ZT_BLOOMFILTER_HPP
+
+#include <string.h>
+#include "Utils.hpp"
+
+namespace ZeroTier {
+
+/**
+ * A simple bit field bloom filter
+ *
+ * This actually isn't a total filter, in that it does not contain a hashing
+ * algorithm. It's up to the caller to hash/sum the items being remembered
+ * so that the distribution of 'n' is random.
+ *
+ * @tparam B Size in BITS, must be a multiple of 8
+ */
+template<unsigned int B>
+class BloomFilter
+{
+public:
+	/**
+	 * Construct an empty filter
+	 */
+	BloomFilter()
+		throw()
+	{
+		memset(_field,0,sizeof(_field));
+	}
+
+	/**
+	 * Construct from a raw filter
+	 *
+	 * @param b Raw filter bits, must be exactly bytes() in length, or NULL to construct empty
+	 */
+	BloomFilter(const void *b)
+		throw()
+	{
+		if (b)
+			memcpy(_field,b,sizeof(_field));
+		else memset(_field,0,sizeof(_field));
+	}
+
+	/**
+	 * @return Size of filter in bits
+	 */
+	static inline unsigned int bits() throw() { return B; }
+
+	/**
+	 * @return Size of filter in bytes
+	 */
+	static inline unsigned int bytes() throw() { return (B / 8); }
+
+	/**
+	 * @return Pointer to portable array of bytes of bytes() length representing filter
+	 */
+	inline const unsigned char *data() const throw() { return _field; }
+
+	/**
+	 * Clear all bits in filter
+	 */
+	inline void clear()
+		throw()
+	{
+		memset(_field,0,sizeof(_field));
+	}
+
+	/**
+	 * @param n Value to set
+	 */
+	inline void set(unsigned int n)
+		throw()
+	{
+		n %= B;
+		_field[n / 8] |= (1 << (n % 8));
+	}
+
+	/**
+	 * @param n Value to test
+	 * @return True if value is present is set
+	 */
+	inline bool contains(unsigned int n)
+		throw()
+	{
+		n %= B;
+		return (_field[n / 8] & (1 << (n % 8)));
+	}
+
+	/**
+	 * Clear a random bit in this bloom filter
+	 *
+	 * @param rn Random number
+	 */
+	inline void decay(unsigned int rn)
+		throw()
+	{
+		_field[(rn >> 3) % (B / 8)] &= ~((unsigned char)(1 << (rn & 7)));
+	}
+
+private:
+	unsigned char _field[B / 8];
+};
+
+} // namespace ZeroTier
+
+#endif

node/Buffer.hpp

@@ -124,7 +124,7 @@ public:
 	{
 		if (b._l > C)
 			throw std::out_of_range("Buffer: assignment from buffer larger than capacity");
-		memcpy(this,&b,sizeof(_l) + b._l); // one memcpy for all fields
+		memcpy(_b,b._b,_l = b._l);
 		return *this;
 	}
 
@@ -240,6 +240,22 @@ public:
 		_l += sizeof(T);
 	}
 
+	/**
+	 * Append a run of bytes
+	 *
+	 * @param c Character value to append
+	 * @param n Number of times to append
+	 * @throws std::out_of_range Attempt to append beyond capacity
+	 */
+	inline void append(unsigned char c,unsigned int n)
+		throw(std::out_of_range)
+	{
+		if ((_l + n) > C)
+			throw std::out_of_range("Buffer: append beyond capacity");
+		for(unsigned int i=0;i<n;++i)
+			_b[_l++] = (char)c;
+	}
+
 	/**
 	 * Append a C-array of bytes
 	 * 
@@ -341,6 +357,15 @@ public:
 		memset(_b + _l,0,C - _l);
 	}
 
+	/**
+	 * Unconditionally zero buffer's underlying memory
+	 */
+	inline void zeroAll()
+		throw()
+	{
+		memset(_b,0,sizeof(_b));
+	}
+
 	/**
 	 * @return Size of data in buffer
 	 */
@@ -388,7 +413,7 @@ public:
 		return !(*this < b);
 	}
 
-protected:
+private:
 	unsigned int _l;
 	char ZT_VAR_MAY_ALIAS _b[C];
 };

node/CMWC4096.hpp

@@ -0,0 +1,91 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_CMWC4096_HPP
+#define _ZT_CMWC4096_HPP
+
+#include <stdint.h>
+#include "Utils.hpp"
+
+namespace ZeroTier {
+
+/** 
+ * Complement Multiply With Carry random number generator
+ *
+ * Based on original code posted to Usenet in the public domain by
+ * George Marsaglia. Period is approximately 2^131086.
+ *
+ * This is not used for cryptographic purposes but for a very fast
+ * and high-quality PRNG elsewhere in the code.
+ */
+class CMWC4096
+{
+public:
+	/**
+	 * Construct and initialize from secure random source
+	 */
+	CMWC4096()
+		throw()
+	{
+		Utils::getSecureRandom(Q,sizeof(Q));
+		Utils::getSecureRandom(&c,sizeof(c));
+		c %= 809430660;
+		i = 4095;
+	}
+
+	inline uint32_t next32()
+		throw()
+	{
+		uint32_t __i = ++i & 4095;
+		const uint64_t t = (18782ULL * (uint64_t)Q[__i]) + (uint64_t)c;
+		c = (uint32_t)(t >> 32);
+		uint32_t x = c + (uint32_t)t;
+		const uint32_t p = (uint32_t)(x < c); x += p; c += p;
+		return (Q[__i] = 0xfffffffe - x);
+	}
+
+	inline uint64_t next64()
+		throw()
+	{
+		return ((((uint64_t)next32()) << 32) ^ (uint64_t)next32());
+	}
+
+	inline double nextDouble()
+		throw()
+	{
+		return ((double)(next32()) / 4294967296.0);
+	}
+
+private:
+	uint32_t Q[4096];
+	uint32_t c;
+	uint32_t i;
+};
+
+} // namespace ZeroTier
+
+#endif

node/Constants.hpp

@@ -28,10 +28,55 @@
 #ifndef _ZT_CONSTANTS_HPP
 #define _ZT_CONSTANTS_HPP
 
-// Assume these are little-endian, since we don't support old PPC MACs
-// and all newer Mac or Windows systems are either x86_32, x86_64, or
-// ARM in little-endian mode.
-#if defined(__APPLE__) || defined(_WIN32)
+//
+// This include file also auto-detects and canonicalizes some environment
+// information defines:
+//
+// __LINUX__
+// __APPLE__
+// __UNIX_LIKE__ - any "unix like" OS (BSD, posix, etc.)
+// __WINDOWS__
+//
+// Also makes sure __BYTE_ORDER is defined reasonably.
+//
+
+// Canonicalize Linux... is this necessary? Do it anyway to be defensive.
+#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
+#ifndef __LINUX__
+#define __LINUX__
+#ifndef __UNIX_LIKE__
+#define __UNIX_LIKE__
+#endif
+#endif
+#endif
+
+// TODO: Android is what? Linux technically, but does it define it?
+
+// OSX and iOS are unix-like OSes far as we're concerned
+#ifdef __APPLE__
+#ifndef __UNIX_LIKE__
+#define __UNIX_LIKE__
+#endif
+#endif
+
+// Linux has endian.h
+#ifdef __LINUX__
+#include <endian.h>
+#endif
+
+#if defined(_WIN32) || defined(_WIN64)
+#ifndef __WINDOWS__
+#define __WINDOWS__
+#endif
+#undef __UNIX_LIKE__
+#define ZT_PATH_SEPARATOR '\\'
+#define ZT_PATH_SEPARATOR_S "\\"
+#define ZT_EOL_S "\r\n"
+#endif
+
+// Assume these are little-endian. PPC is not supported for OSX, and ARM
+// runs in little-endian mode for these OS families.
+#if defined(__APPLE__) || defined(__WINDOWS__)
 #undef __BYTE_ORDER
 #undef __LITTLE_ENDIAN
 #undef __BIG_ENDIAN
@@ -40,33 +85,23 @@
 #define __BYTE_ORDER 1234
 #endif
 
-// Linux has endian.h, which should tell us
-#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-#include <endian.h>
-#endif
-
-#ifndef __BYTE_ORDER
-error_no_byte_order_defined
-#endif
-
-#ifndef ZT_OSNAME
-error_no_ZT_OSNAME
-#endif
-
-#ifndef ZT_ARCH
-error_no_ZT_ARCH
-#endif
-
-#ifdef _WIN32
-#define ZT_PATH_SEPARATOR '\\'
-#define ZT_PATH_SEPARATOR_S "\\"
-#define ZT_EOL_S "\r\n"
-#else
+#ifdef __UNIX_LIKE__
 #define ZT_PATH_SEPARATOR '/'
 #define ZT_PATH_SEPARATOR_S "/"
 #define ZT_EOL_S "\n"
 #endif
 
+// Error out if required symbols are missing
+#ifndef __BYTE_ORDER
+error_no_byte_order_defined;
+#endif
+#ifndef ZT_OSNAME
+error_no_ZT_OSNAME_defined;
+#endif
+#ifndef ZT_ARCH
+error_no_ZT_ARCH_defined;
+#endif
+
 /**
  * Length of a ZeroTier address in bytes
  */
@@ -82,6 +117,11 @@ error_no_ZT_ARCH
  */
 #define ZT_DEFAULT_UDP_PORT 8993
 
+/**
+ * Local control port, also used for multiple invocation check
+ */
+#define ZT_CONTROL_UDP_PORT 39393
+
 /**
  * Default payload MTU for UDP packets
  *
@@ -116,13 +156,6 @@ error_no_ZT_ARCH
  */
 #define ZT_IF_MTU 2800
 
-/**
- * Maximum number of networks we can be a member of
- *
- * This is a safe value that's within the tap device limit on all known OSes.
- */
-#define ZT_MAX_NETWORK_MEMBERSHIPS 16
-
 /**
  * Maximum number of packet fragments we'll support
  * 
@@ -150,9 +183,9 @@ error_no_ZT_ARCH
 #define ZT_MAC_FIRST_OCTET 0x32
 
 /**
- * How often Topology::clean() is called in ms
+ * How often Topology::clean() and Network::clean() are called in ms
  */
-#define ZT_TOPOLOGY_CLEAN_PERIOD 300000
+#define ZT_DB_CLEAN_PERIOD 300000
 
 /**
  * Delay between WHOIS retries in ms
@@ -193,19 +226,26 @@ error_no_ZT_ARCH
  * the sum of BREADTH^i where I is from 1 to DEPTH. This ignores the effect
  * of the rate limiting algorithm or bloom filter collisions.
  *
- * 7 results in a max of 21844 recipients for a given multicast.
+ * 5 results in a max of 1364 recipients for a given multicast. With a limit
+ * of 50 bytes/sec (average) for multicast, this results in a worst case of
+ * around 68kb/sec of multicast traffic. FYI the average multicast traffic
+ * from a Mac seems to be about ~25bytes/sec. Windows measurements are TBD.
+ * Linux is quieter than Mac.
+ *
+ * This are eventually going to become per-network tunable parameters, along
+ * with per-network peer multicast rate limits.
  */
-#define ZT_MULTICAST_PROPAGATION_DEPTH 7
+#define ZT_MULTICAST_PROPAGATION_DEPTH 5
 
 /**
- * Length of circular ring buffer history of multicast packets
+ * Length of ring buffer history of recent multicast packets
  */
-#define ZT_MULTICAST_DEDUP_HISTORY_LENGTH 4096
+#define ZT_MULTICAST_DEDUP_HISTORY_LENGTH 1024
 
 /**
- * Expiration time in ms for multicast history items
+ * Expiration time in ms for multicast deduplication history items
  */
-#define ZT_MULTICAST_DEDUP_HISTORY_EXPIRE 8000
+#define ZT_MULTICAST_DEDUP_HISTORY_EXPIRE 4000
 
 /**
  * Period between announcements of all multicast 'likes' in ms
@@ -226,6 +266,26 @@ error_no_ZT_ARCH
  */
 #define ZT_MULTICAST_LOCAL_POLL_PERIOD 10000
 
+/**
+ * Default bytes per second limit for multicasts per peer on a network
+ */
+#define ZT_MULTICAST_DEFAULT_BYTES_PER_SECOND 50.0
+
+/**
+ * Default balance preload for multicast rate limiters on a network
+ */
+#define ZT_MULTICAST_DEFAULT_RATE_PRELOAD 20000.0
+
+/**
+ * Default maximum balance for multicast rate limiters
+ */
+#define ZT_MULTICAST_DEFAULT_RATE_MAX_BALANCE 20000.0
+
+/**
+ * Default minimum balance for multicast rate limiters (max debt)
+ */
+#define ZT_MULTICAST_DEFAULT_RATE_MIN_BALANCE -10000.0
+
 /**
  * Delay between scans of the topology active peer DB for peers that need ping
  */
@@ -242,21 +302,22 @@ error_no_ZT_ARCH
 #define ZT_PEER_DIRECT_PING_DELAY 120000
 
 /**
- * Period between rechecks of autoconfigure URL
+ * Delay in ms between firewall opener packets to direct links
  *
- * This is in the absence of an external message ordering a recheck.
+ * This should be lower than the UDP conversation entry timeout in most
+ * stateful firewalls.
  */
-#define ZT_AUTOCONFIGURE_INTERVAL 3600000
+#define ZT_FIREWALL_OPENER_DELAY 50000
 
 /**
- * Period between autoconfigure attempts if no successful autoconfig
+ * Delay between requests for updated network autoconf information
  */
-#define ZT_AUTOCONFIGURE_CHECK_DELAY 15000
+#define ZT_NETWORK_AUTOCONF_DELAY 120000
 
 /**
- * Delay between updates of status file in home directory
+ * Delay in core loop between checks of network autoconf newness
  */
-#define ZT_STATUS_OUTPUT_PERIOD 120000
+#define ZT_NETWORK_AUTOCONF_CHECK_DELAY 7000
 
 /**
  * Minimum delay in Node service loop
@@ -272,14 +333,6 @@ error_no_ZT_ARCH
  */
 #define ZT_PEER_LINK_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 2) + 1000)
 
-/**
- * Delay in ms between firewall opener packets to direct links
- *
- * This should be lower than the UDP conversation entry timeout in most
- * stateful firewalls.
- */
-#define ZT_FIREWALL_OPENER_DELAY 50000
-
 /**
  * IP hops (a.k.a. TTL) to set for firewall opener packets
  *
@@ -308,9 +361,4 @@ error_no_ZT_ARCH
  */
 #define ZT_RENDEZVOUS_NAT_T_DELAY 500
 
-/**
- * Generate a new ownership verify secret on launch if older than this
- */
-#define ZT_OVS_GENERATE_NEW_IF_OLDER_THAN 86400000
-
 #endif

node/Defaults.cpp

@@ -68,9 +68,7 @@ static inline std::map< Identity,std::vector<InetAddress> > _mkSupernodeMap()
 
 Defaults::Defaults()
 	throw(std::runtime_error) :
-	supernodes(_mkSupernodeMap()),
-	configUrlPrefix("http://api.zerotier.com/one/nc/"),
-	configAuthority("f9f34184ac:1:AwGgrWjb8dARXzruqxiy1+Qf+gz4iM5IMfQTCWrJXkwERdvbvxTPZvtIyitw4gS90TGIxW+e7uJxweg9Vyq5lZJBrg==:QeEQLm9ymLC3EcnIw2OUqufUwb2wgHSAg6wQOXKyhT779p/8Hz5485PZLJCbr/aVHjwzop8APJk9B45Zm0Mb/LEhQTBMH2jvc7qqoYnMCNCO9jpADeMJwMW5e1VFgIObWl9uNjhRbf5/m8dZcn0pKKGwjSoP1QTeVWOC8GkZhE25bUWj")
+	supernodes(_mkSupernodeMap())
 {
 }
 

node/Defaults.hpp

@@ -55,16 +55,6 @@ public:
 	 * Supernodes on the ZeroTier network
 	 */
 	const std::map< Identity,std::vector<InetAddress> > supernodes;
-
-	/**
-	 * URL prefix for autoconfiguration
-	 */
-	const std::string configUrlPrefix;
-
-	/**
-	 * Identity used to encrypt and authenticate configuration from URL
-	 */
-	const std::string configAuthority;
 };
 
 extern const Defaults ZT_DEFAULTS;

node/Demarc.cpp

@@ -100,7 +100,7 @@ bool Demarc::bindLocalUdp(unsigned int localPort)
 		DemarcPortObj *v4r = &(_ports[(Port)v4p]);
 		v4r->port = (Port)v4p;
 		v4r->parent = this;
-		v4r->obj = v4 = new UdpSocket(localPort,false,&Demarc::_CBudpSocketPacketHandler,v4r);
+		v4r->obj = v4 = new UdpSocket(false,localPort,false,&Demarc::_CBudpSocketPacketHandler,v4r);
 		v4r->type = PORT_TYPE_UDP_SOCKET_V4;
 	} catch ( ... ) {
 		_ports.erase((Port)v4p);
@@ -112,7 +112,7 @@ bool Demarc::bindLocalUdp(unsigned int localPort)
 		DemarcPortObj *v6r = &(_ports[(Port)v6p]);
 		v6r->port = (Port)v6p;
 		v6r->parent = this;
-		v6r->obj = v6 = new UdpSocket(localPort,true,&Demarc::_CBudpSocketPacketHandler,v6r);
+		v6r->obj = v6 = new UdpSocket(false,localPort,true,&Demarc::_CBudpSocketPacketHandler,v6r);
 		v6r->type = PORT_TYPE_UDP_SOCKET_V6;
 	} catch ( ... ) {
 		_ports.erase((Port)v6p);
@@ -143,7 +143,7 @@ Demarc::Port Demarc::pick(const InetAddress &to) const
 			}
 		}
 		if (possibilities.size())
-			return possibilities[Utils::randomInt<unsigned int>() % possibilities.size()]->first;
+			return possibilities[_r->prng->next32() % possibilities.size()]->first;
 		else return NULL_PORT;
 	} catch ( ... ) {
 		return NULL_PORT;
@@ -174,7 +174,7 @@ Demarc::Port Demarc::send(Demarc::Port fromPort,const InetAddress &to,const void
 				}
 			}
 			if (possibilities.size())
-				pe = possibilities[Utils::randomInt<unsigned int>() % possibilities.size()];
+				pe = possibilities[_r->prng->next32() % possibilities.size()];
 			else {
 				_ports_m.unlock();
 				return NULL_PORT;

node/Demarc.hpp

@@ -49,6 +49,9 @@ class UdpSocket;
  * about what they actually are.
  *
  * All ports are closed when this class is destroyed.
+ *
+ * Its name "demarcation point" comes from the telco/cable terminology for
+ * the box where wires terminate at a customer's property.
  */
 class Demarc
 {

node/Dictionary.hpp

@@ -0,0 +1,207 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_DICTIONARY_HPP
+#define _ZT_DICTIONARY_HPP
+
+#include <string>
+#include <map>
+#include <stdexcept>
+#include "Constants.hpp"
+
+namespace ZeroTier {
+
+/**
+ * Simple key/value dictionary with string serialization
+ *
+ * The serialization format is a flat key=value with backslash escape.
+ * It does not support comments or other syntactic complexities. It is
+ * human-readable if the keys and values in the dictionary are also
+ * human-readable. Otherwise it might contain unprintable characters.
+ */
+class Dictionary : public std::map<std::string,std::string>
+{
+public:
+	Dictionary()
+	{
+	}
+
+	/**
+	 * @param s String-serialized dictionary
+	 */
+	Dictionary(const char *s)
+	{
+		fromString(s);
+	}
+
+	/**
+	 * @param s String-serialized dictionary
+	 */
+	Dictionary(const std::string &s)
+	{
+		fromString(s.c_str());
+	}
+
+	/**
+	 * Get a key, throwing an exception if it is not present
+	 *
+	 * @param key Key to look up
+	 * @return Reference to value
+	 * @throws std::invalid_argument Key not found
+	 */
+	inline const std::string &get(const std::string &key) const
+		throw(std::invalid_argument)
+	{
+		const_iterator e(find(key));
+		if (e == end())
+			throw std::invalid_argument(std::string("missing required field: ")+key);
+		return e->second;
+	}
+
+	/**
+	 * Get a key, returning a default if not present
+	 *
+	 * @param key Key to look up
+	 * @param dfl Default if not present
+	 * @return Value or default
+	 */
+	inline const std::string &get(const std::string &key,const std::string &dfl) const
+	{
+		const_iterator e(find(key));
+		if (e == end())
+			return dfl;
+		return e->second;
+	}
+
+	/**
+	 * @param key Key to check
+	 * @return True if dictionary contains key
+	 */
+	inline bool contains(const std::string &key) const
+	{
+		return (find(key) != end());
+	}
+
+	/**
+	 * @return String-serialized dictionary
+	 */
+	inline std::string toString() const
+	{
+		std::string s;
+
+		for(const_iterator kv(begin());kv!=end();++kv) {
+			_appendEsc(kv->first.data(),kv->first.length(),s);
+			s.push_back('=');
+			_appendEsc(kv->second.data(),kv->second.length(),s);
+			s.append(ZT_EOL_S);
+		}
+
+		return s;
+	}
+
+	/**
+	 * Clear and initialize from a string
+	 *
+	 * @param s String-serialized dictionary
+	 */
+	inline void fromString(const char *s)
+	{
+		clear();
+		bool escapeState = false;
+		std::string keyBuf;
+		std::string *element = &keyBuf;
+		while (*s) {
+			if (escapeState) {
+				escapeState = false;
+				switch(*s) {
+					case '0':
+						element->push_back((char)0);
+						break;
+					case 'r':
+						element->push_back('\r');
+						break;
+					case 'n':
+						element->push_back('\n');
+						break;
+					default:
+						element->push_back(*s);
+						break;
+				}
+			} else {
+				if (*s == '\\') {
+					escapeState = true;
+				} else if (*s == '=') {
+					if (element == &keyBuf)
+						element = &((*this)[keyBuf]);
+				} else if ((*s == '\r')||(*s == '\n')) {
+					if ((element == &keyBuf)&&(keyBuf.length() > 0))
+						(*this)[keyBuf];
+					keyBuf = "";
+					element = &keyBuf;
+				} else element->push_back(*s);
+			}
+			++s;
+		}
+		if ((element == &keyBuf)&&(keyBuf.length() > 0))
+			(*this)[keyBuf];
+	}
+	inline void fromString(const std::string &s)
+	{
+		fromString(s.c_str());
+	}
+
+private:
+	static inline void _appendEsc(const char *data,unsigned int len,std::string &to)
+	{
+		for(unsigned int i=0;i<len;++i) {
+			switch(data[i]) {
+				case 0:
+					to.append("\\0");
+					break;
+				case '\r':
+					to.append("\\r");
+					break;
+				case '\n':
+					to.append("\\n");
+					break;
+				case '\\':
+					to.append("\\\\");
+					break;
+				case '=':
+					to.append("\\=");
+					break;
+				default:
+					to.push_back(data[i]);
+					break;
+			}
+		}
+	}
+};
+
+} // namespace ZeroTier
+
+#endif

node/EllipticCurveKey.hpp

@@ -30,6 +30,7 @@
 
 #include <string>
 #include <algorithm>
+#include <stdexcept>
 #include <string.h>
 #include "Utils.hpp"
 
@@ -65,39 +66,28 @@ public:
 		throw() :
 		_bytes(0)
 	{
+		memset(_key,0,sizeof(_key));
 	}
 
 	EllipticCurveKey(const void *data,unsigned int len)
-		throw()
+		throw(std::out_of_range)
 	{
-		if (len <= ZT_EC_MAX_BYTES) {
-			_bytes = len;
-			memcpy(_key,data,len);
-		} else _bytes = 0;
+		set(data,len);
 	}
 
-	EllipticCurveKey(const EllipticCurveKey &k)
-		throw()
+	EllipticCurveKey(const std::string &data)
+		throw(std::out_of_range)
 	{
-		_bytes = k._bytes;
-		memcpy(_key,k._key,_bytes);
-	}
-
-	inline EllipticCurveKey &operator=(const EllipticCurveKey &k)
-		throw()
-	{
-		_bytes = k._bytes;
-		memcpy(_key,k._key,_bytes);
-		return *this;
+		set(data.data(),data.length());
 	}
 
 	inline void set(const void *data,unsigned int len)
-		throw()
+		throw(std::out_of_range)
 	{
 		if (len <= ZT_EC_MAX_BYTES) {
 			_bytes = len;
 			memcpy(_key,data,len);
-		} else _bytes = 0;
+		} else throw std::out_of_range("key too large");
 	}
 
 	inline const unsigned char *data() const throw() { return _key; }

node/EllipticCurveKeyPair.cpp

@@ -55,7 +55,20 @@ public:
 };
 static _EC_Group ZT_EC_GROUP;
 
-/* Key derivation function */
+/**
+ * Key derivation function
+ *
+ * TODO:
+ * If/when we document the protocol, this will have to be documented as
+ * well. It's a fairly standard KDF that uses SHA-256 to transform the
+ * raw EC key. It's generally considered good crypto practice to do this
+ * to eliminate the possibility of leaking information from EC exchange to
+ * downstream algorithms.
+ *
+ * In our code it is used to produce a two 32-bit keys. One key is used
+ * for Salsa20 and the other for HMAC-SHA-256. They are generated together
+ * as a single 64-bit key.
+ */
 static void *_zt_EC_KDF(const void *in,size_t inlen,void *out,size_t *outlen)
 {
 	SHA256_CTX sha;
@@ -130,9 +143,8 @@ bool EllipticCurveKeyPair::generate()
 			fread(tmp,sizeof(tmp),1,rf);
 			fclose(rf);
 		} else {
-			fprintf(stderr,"WARNING: cannot open /dev/urandom\n");
-			for(unsigned int i=0;i<sizeof(tmp);++i)
-				tmp[i] = (unsigned char)(rand() >> 3);
+			fprintf(stderr,"FATAL: could not open /dev/urandom\n");
+			exit(-1);
 		}
 		RAND_seed(tmp,sizeof(tmp));
 #else

node/EllipticCurveKeyPair.hpp

@@ -35,6 +35,8 @@ namespace ZeroTier {
 
 /**
  * An elliptic curve key pair supporting generation and key agreement
+ *
+ * This is basically OpenSSL libcrypto glue.
  */
 class EllipticCurveKeyPair
 {

node/EthernetTap.cpp

@@ -30,11 +30,17 @@
 #include "EthernetTap.hpp"
 #include "Logger.hpp"
 #include "RuntimeEnvironment.hpp"
+#include "Utils.hpp"
 #include "Mutex.hpp"
 
-/* ======================================================================== */
-#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-/* ======================================================================== */
+// ff:ff:ff:ff:ff:ff with no ADI
+static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
+
+//
+// TAP implementation for *nix OSes, with some specialization for different flavors
+//
+
+#ifdef __UNIX_LIKE__ /////////////////////////////////////////////////////////
 
 #include <stdint.h>
 #include <stdio.h>
@@ -48,10 +54,13 @@
 #include <sys/stat.h>
 #include <sys/ioctl.h>
 #include <sys/wait.h>
+#include <sys/select.h>
 #include <netinet/in.h>
 #include <net/if_arp.h>
 #include <arpa/inet.h>
 
+#ifdef __LINUX__
+
 #include <linux/if.h>
 #include <linux/if_tun.h>
 #include <linux/if_addr.h>
@@ -60,26 +69,52 @@
 #define ZT_ETHERTAP_IP_COMMAND "/sbin/ip"
 #define ZT_ETHERTAP_SYSCTL_COMMAND "/sbin/sysctl"
 
+#endif // __LINUX__
+
+#ifdef __APPLE__
+
+#include <sys/uio.h>
+#include <sys/param.h>
+#include <sys/sysctl.h>
+#include <net/route.h>
+#include <net/if_dl.h>
+#include <ifaddrs.h>
+
+#define ZT_ETHERTAP_IFCONFIG "/sbin/ifconfig"
+#define ZT_MAC_KEXTLOAD "/sbin/kextload"
+#define ZT_MAC_IPCONFIG "/usr/sbin/ipconfig"
+
+#endif // __APPLE__
+
 namespace ZeroTier {
 
+// Only permit one tap to be opened concurrently across the entire process
 static Mutex __tapCreateLock;
 
-EthernetTap::EthernetTap(const RuntimeEnvironment *renv,const MAC &mac,unsigned int mtu) 
+#ifdef __LINUX__
+EthernetTap::EthernetTap(
+	const RuntimeEnvironment *renv,
+	const MAC &mac,
+	unsigned int mtu,
+	void (*handler)(void *,const MAC &,const MAC &,unsigned int,const Buffer<4096> &),
+	void *arg)
 	throw(std::runtime_error) :
 	_mac(mac),
 	_mtu(mtu),
 	_r(renv),
-	_putBuf((unsigned char *)0),
-	_getBuf((unsigned char *)0),
-	_fd(0),
-	_isReading(false)
+	_handler(handler),
+	_arg(arg),
+	_fd(0)
 {
 	char procpath[128];
 	Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally
 
+	if (mtu > 4096)
+		throw std::runtime_error("max tap MTU is 4096");
+
 	_fd = ::open("/dev/net/tun",O_RDWR);
 	if (_fd <= 0)
-		throw std::runtime_error("could not open TUN/TAP device");
+		throw std::runtime_error(std::string("could not open TUN/TAP device: ") + strerror(errno));
 
 	struct ifreq ifr;
 	memset(&ifr,0,sizeof(ifr));
@@ -148,31 +183,135 @@ EthernetTap::EthernetTap(const RuntimeEnvironment *renv,const MAC &mac,unsigned
 
 	::close(sock);
 
-	_putBuf = new unsigned char[((mtu + 16) * 2)];
-	_getBuf = _putBuf + (mtu + 16);
+	::pipe(_shutdownSignalPipe);
 
 	TRACE("tap %s created",_dev);
+
+	_thread = Thread::start(this);
 }
+#endif // __LINUX__
+
+#ifdef __APPLE__
+EthernetTap::EthernetTap(
+	const RuntimeEnvironment *renv,
+	const MAC &mac,
+	unsigned int mtu,
+	void (*handler)(void *,const MAC &,const MAC &,unsigned int,const Buffer<4096> &),
+	void *arg)
+	throw(std::runtime_error) :
+	_mac(mac),
+	_mtu(mtu),
+	_r(renv),
+	_handler(handler),
+	_arg(arg),
+	_fd(0)
+{
+	char devpath[64],ethaddr[64],mtustr[16];
+	struct stat tmp;
+	Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally
+
+	if (mtu > 4096)
+		throw std::runtime_error("max tap MTU is 4096");
+
+	// Check for existence of ZT tap devices, try to load module if not there
+	if (stat("/dev/zt0",&tmp)) {
+		int kextpid;
+		char tmp[4096];
+		strcpy(tmp,_r->homePath.c_str());
+		if ((kextpid = (int)vfork()) == 0) {
+			chdir(tmp);
+			execl(ZT_MAC_KEXTLOAD,ZT_MAC_KEXTLOAD,"-q","-repository",tmp,"tap.kext",(const char *)0);
+			exit(-1);
+		} else {
+			int exitcode = -1;
+			waitpid(kextpid,&exitcode,0);
+			usleep(500);
+		}
+	}
+	if (stat("/dev/zt0",&tmp))
+		throw std::runtime_error("/dev/zt# tap devices do not exist and unable to load kernel extension");
+
+	// Open the first available device (ones in use will fail with resource busy)
+	for(int i=0;i<256;++i) {
+		sprintf(devpath,"/dev/zt%d",i);
+		if (stat(devpath,&tmp))
+			throw std::runtime_error("no more TAP devices available");
+		_fd = ::open(devpath,O_RDWR);
+		if (_fd > 0) {
+			sprintf(_dev,"zt%d",i);
+			break;
+		}
+	}
+	if (_fd <= 0)
+		throw std::runtime_error("unable to open TAP device or no more devices available");
+
+	if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) {
+		::close(_fd);
+		throw std::runtime_error("unable to set flags on file descriptor for TAP device");
+	}
+
+	sprintf(ethaddr,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]);
+	sprintf(mtustr,"%u",mtu);
+
+	// Configure MAC address and MTU, bring interface up
+	long cpid;
+	if ((cpid = (long)vfork()) == 0) {
+		execl(ZT_ETHERTAP_IFCONFIG,ZT_ETHERTAP_IFCONFIG,_dev,"lladdr",ethaddr,"mtu",mtustr,"up",(const char *)0);
+		exit(-1);
+	} else {
+		int exitcode = -1;
+		waitpid(cpid,&exitcode,0);
+		if (exitcode) {
+			::close(_fd);
+			throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface");
+		}
+	}
+
+	whack(); // turns on IPv6 on OSX
+
+	::pipe(_shutdownSignalPipe);
+
+	_thread = Thread::start(this);
+}
+#endif // __APPLE__
 
 EthernetTap::~EthernetTap()
 {
-	this->close();
-	delete [] _putBuf;
+	::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit
+	Thread::join(_thread);
+	::close(_fd);
 }
 
-static bool ___removeIp(const char *_dev,std::set<InetAddress> &_ips,const InetAddress &ip)
+#ifdef __APPLE__
+void EthernetTap::whack()
 {
-	long cpid;
-	if ((cpid = (long)fork()) == 0) {
+	long cpid = (long)vfork();
+	if (cpid == 0) {
+		execl(ZT_MAC_IPCONFIG,ZT_MAC_IPCONFIG,"set",_dev,"AUTOMATIC-V6",(const char *)0);
+		exit(-1);
+	} else {
+		int exitcode = -1;
+		waitpid(cpid,&exitcode,0);
+		if (exitcode) {
+			LOG("%s: ipconfig set AUTOMATIC-V6 failed",_dev);
+		}
+	}
+}
+#else
+void EthernetTap::whack() {}
+#endif // __APPLE__ / !__APPLE__
+
+#ifdef __LINUX__
+static bool ___removeIp(const char *_dev,const InetAddress &ip)
+{
+	long cpid = (long)vfork();
+	if (cpid == 0) {
 		execl(ZT_ETHERTAP_IP_COMMAND,ZT_ETHERTAP_IP_COMMAND,"addr","del",ip.toString().c_str(),"dev",_dev,(const char *)0);
 		exit(1); /* not reached unless exec fails */
 	} else {
 		int exitcode = 1;
 		waitpid(cpid,&exitcode,0);
-		if (exitcode == 0) {
-			_ips.erase(ip);
-			return true;
-		} else return false;
+		return (exitcode == 0);
 	}
 }
 
@@ -188,13 +327,17 @@ bool EthernetTap::addIP(const InetAddress &ip)
 	// Remove and reconfigure if address is the same but netmask is different
 	for(std::set<InetAddress>::iterator i(_ips.begin());i!=_ips.end();++i) {
 		if (i->ipsEqual(ip)) {
-			___removeIp(_dev,_ips,*i);
-			break;
+			if (___removeIp(_dev,*i)) {
+				_ips.erase(i);
+				break;
+			} else {
+				LOG("WARNING: failed to remove old IP/netmask %s to replace with %s",i->toString().c_str(),ip.toString().c_str());
+			}
 		}
 	}
 
-	int cpid;
-	if ((cpid = (int)fork()) == 0) {
+	long cpid;
+	if ((cpid = (long)vfork()) == 0) {
 		execl(ZT_ETHERTAP_IP_COMMAND,ZT_ETHERTAP_IP_COMMAND,"addr","add",ip.toString().c_str(),"dev",_dev,(const char *)0);
 		exit(-1);
 	} else {
@@ -208,97 +351,101 @@ bool EthernetTap::addIP(const InetAddress &ip)
 
 	return false;
 }
+#endif // __LINUX__
+
+#ifdef __APPLE__
+static bool ___removeIp(const char *_dev,const InetAddress &ip)
+{
+	int cpid;
+	if ((cpid = (int)vfork()) == 0) {
+		execl(ZT_ETHERTAP_IFCONFIG,ZT_ETHERTAP_IFCONFIG,_dev,"inet",ip.toIpString().c_str(),"-alias",(const char *)0);
+		exit(-1);
+	} else {
+		int exitcode = -1;
+		waitpid(cpid,&exitcode,0);
+		return (exitcode == 0);
+	}
+	return false; // never reached, make compiler shut up about return value
+}
+
+bool EthernetTap::addIP(const InetAddress &ip)
+{
+	Mutex::Lock _l(_ips_m);
+
+	if (!ip)
+		return false;
+	if (_ips.count(ip) > 0)
+		return true; // IP/netmask already assigned
+
+	// Remove and reconfigure if address is the same but netmask is different
+	for(std::set<InetAddress>::iterator i(_ips.begin());i!=_ips.end();++i) {
+		if ((i->ipsEqual(ip))&&(i->netmaskBits() != ip.netmaskBits())) {
+			if (___removeIp(_dev,*i)) {
+				_ips.erase(i);
+				break;
+			} else {
+				LOG("WARNING: failed to remove old IP/netmask %s to replace with %s",i->toString().c_str(),ip.toString().c_str());
+			}
+		}
+	}
+
+	int cpid;
+	if ((cpid = (int)vfork()) == 0) {
+		execl(ZT_ETHERTAP_IFCONFIG,ZT_ETHERTAP_IFCONFIG,_dev,ip.isV4() ? "inet" : "inet6",ip.toString().c_str(),"alias",(const char *)0);
+		exit(-1);
+	} else {
+		int exitcode = -1;
+		waitpid(cpid,&exitcode,0);
+		if (exitcode == 0) {
+			_ips.insert(ip);
+			return true;
+		}
+	}
+
+	return false;
+}
+#endif // __APPLE__
 
 bool EthernetTap::removeIP(const InetAddress &ip)
 {
 	Mutex::Lock _l(_ips_m);
-	if (_ips.count(ip) > 0)
-		return ___removeIp(_dev,_ips,ip);
+	if (_ips.count(ip) > 0) {
+		if (___removeIp(_dev,ip)) {
+			_ips.erase(ip);
+			return true;
+		}
+	}
 	return false;
 }
 
 void EthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
 {
+	char putBuf[4096 + 14];
 	if ((_fd > 0)&&(len <= _mtu)) {
 		for(int i=0;i<6;++i)
-			_putBuf[i] = to.data[i];
+			putBuf[i] = to.data[i];
 		for(int i=0;i<6;++i)
-			_putBuf[i+6] = from.data[i];
-		*((uint16_t *)(_putBuf + 12)) = htons((uint16_t)etherType);
-		memcpy(_putBuf + 14,data,len);
-		::write(_fd,_putBuf,len + 14);
-	}
-}
-
-unsigned int EthernetTap::get(MAC &from,MAC &to,unsigned int &etherType,void *buf)
-{
-	for(;;) {
-		if (_fd > 0) {
-			_isReading_m.lock();
-			_isReading = true;
-			_isReadingThreadId = pthread_self();
-			_isReading_m.unlock();
-
-			int n = (int)::read(_fd,_getBuf,_mtu + 14);
-
-			_isReading_m.lock();
-			_isReading = false;
-			_isReading_m.unlock();
-
-			if (n > 14) {
-				for(int i=0;i<6;++i)
-					to.data[i] = _getBuf[i];
-				for(int i=0;i<6;++i)
-					from.data[i] = _getBuf[i + 6];
-				etherType = ntohs(((uint16_t *)_getBuf)[6]);
-				n -= 14;
-				memcpy(buf,_getBuf + 14,n);
-				return (unsigned int)n;
-			} else if (n < 0) {
-				if (_fd <= 0)
-					break;
-				else if ((errno == EINTR)||(errno == ETIMEDOUT))
-					continue;
-				else {
-					TRACE("unexpected error reading from tap: %s",strerror(errno));
-					::close(_fd);
-					_fd = 0;
-					break;
-				}
-			} else {
-				TRACE("incomplete read from tap: %d bytes",n);
-				continue;
-			}
+			putBuf[i+6] = from.data[i];
+		*((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType);
+		memcpy(putBuf + 14,data,len);
+		len += 14;
+		int n = ::write(_fd,putBuf,len);
+		if (n <= 0) {
+			LOG("error writing packet to Ethernet tap device: %s",strerror(errno));
+		} else if (n != (int)len) {
+			// Saw this gremlin once, so log it if we see it again... OSX tap
+			// or something seems to have goofy issues with certain MTUs.
+			LOG("ERROR: write underrun: %s tap write() wrote %d of %u bytes of frame",_dev,n,len);
 		}
 	}
-	return 0;
 }
 
-std::string EthernetTap::deviceName()
+std::string EthernetTap::deviceName() const
 {
 	return std::string(_dev);
 }
 
-bool EthernetTap::open() const
-{
-	return (_fd > 0);
-}
-
-void EthernetTap::close()
-{
-	Mutex::Lock _l(__tapCreateLock); // also prevent create during close()
-	if (_fd > 0) {
-		int f = _fd;
-		_fd = 0;
-		::close(f);
-
-		_isReading_m.lock();
-		if (_isReading)
-			pthread_kill(_isReadingThreadId,SIGUSR2);
-		_isReading_m.unlock();
-	}
-}
-
+#ifdef __LINUX__
 bool EthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 {
 	char *ptr,*ptr2;
@@ -337,6 +484,8 @@ bool EthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 
 	bool changed = false;
 
+	newGroups.insert(_blindWildcardMulticastGroup); // always join this
+
 	for(std::set<MulticastGroup>::iterator mg(newGroups.begin());mg!=newGroups.end();++mg) {
 		if (!groups.count(*mg)) {
 			groups.insert(*mg);
@@ -352,286 +501,9 @@ bool EthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 
 	return changed;
 }
+#endif // __LINUX__
 
-} // namespace ZeroTier
-
-/* ======================================================================== */
-#elif defined(__APPLE__) /* ----------------------------------------------- */
-/* ======================================================================== */
-
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <signal.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/uio.h>
-#include <sys/param.h>
-#include <sys/sysctl.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <net/route.h>
-#include <net/if_dl.h>
-#include <ifaddrs.h>
-
-#define ZT_ETHERTAP_IFCONFIG "/sbin/ifconfig"
-#define ZT_MAC_KEXTLOAD "/sbin/kextload"
-#define ZT_MAC_IPCONFIG "/usr/sbin/ipconfig"
-
-namespace ZeroTier {
-
-static Mutex __tapCreateLock;
-
-EthernetTap::EthernetTap(const RuntimeEnvironment *renv,const MAC &mac,unsigned int mtu)
-	throw(std::runtime_error) :
-	_mac(mac),
-	_mtu(mtu),
-	_r(renv),
-	_putBuf((unsigned char *)0),
-	_getBuf((unsigned char *)0),
-	_fd(0),
-	_isReading(false)
-{
-	char devpath[64],ethaddr[64],mtustr[16];
-	struct stat tmp;
-	Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally
-
-	// Check for existence of ZT tap devices, try to load module if not there
-	if (stat("/dev/zt0",&tmp)) {
-		int kextpid;
-		char tmp[4096];
-		strcpy(tmp,_r->homePath.c_str());
-		if ((kextpid = (int)fork()) == 0) {
-			chdir(tmp);
-			execl(ZT_MAC_KEXTLOAD,ZT_MAC_KEXTLOAD,"-q","-repository",tmp,"tap.kext",(const char *)0);
-			exit(-1);
-		} else {
-			int exitcode = -1;
-			waitpid(kextpid,&exitcode,0);
-			usleep(500);
-		}
-	}
-	if (stat("/dev/zt0",&tmp))
-		throw std::runtime_error("/dev/zt# tap devices do not exist and unable to load kernel extension");
-
-	// Open the first available device (ones in use will fail with resource busy)
-	for(int i=0;i<256;++i) {
-		sprintf(devpath,"/dev/zt%d",i);
-		if (stat(devpath,&tmp))
-			throw std::runtime_error("no more TAP devices available");
-		_fd = ::open(devpath,O_RDWR);
-		if (_fd > 0) {
-			sprintf(_dev,"zt%d",i);
-			break;
-		}
-	}
-	if (_fd <= 0)
-		throw std::runtime_error("unable to open TAP device or no more devices available");
-
-	if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) {
-		::close(_fd);
-		throw std::runtime_error("unable to set flags on file descriptor for TAP device");
-	}
-
-	sprintf(ethaddr,"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]);
-	sprintf(mtustr,"%u",mtu);
-
-	// Configure MAC address and MTU, bring interface up
-	int cpid;
-	if ((cpid = (int)fork()) == 0) {
-		execl(ZT_ETHERTAP_IFCONFIG,ZT_ETHERTAP_IFCONFIG,_dev,"lladdr",ethaddr,"mtu",mtustr,"up",(const char *)0);
-		exit(-1);
-	} else {
-		int exitcode = -1;
-		waitpid(cpid,&exitcode,0);
-		if (exitcode) {
-			::close(_fd);
-			throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface");
-		}
-	}
-
-	// OSX seems to require that IPv6 be turned on on tap devices
-	if ((cpid = (int)fork()) == 0) {
-		execl(ZT_MAC_IPCONFIG,ZT_MAC_IPCONFIG,"set",_dev,"AUTOMATIC-V6",(const char *)0);
-		exit(-1);
-	} else {
-		int exitcode = -1;
-		waitpid(cpid,&exitcode,0);
-		if (exitcode) {
-			::close(_fd);
-			throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface");
-		}
-	}
-
-	_putBuf = new unsigned char[((mtu + 14) * 2)];
-	_getBuf = _putBuf + (mtu + 14);
-}
-
-EthernetTap::~EthernetTap()
-{
-	this->close();
-	delete [] _putBuf;
-}
-
-// Helper function to actually remove IP from network device, execs ifconfig
-static bool ___removeIp(const char *_dev,const InetAddress &ip)
-{
-	int cpid;
-	if ((cpid = (int)fork()) == 0) {
-		execl(ZT_ETHERTAP_IFCONFIG,ZT_ETHERTAP_IFCONFIG,_dev,"inet",ip.toIpString().c_str(),"-alias",(const char *)0);
-		exit(-1);
-	} else {
-		int exitcode = -1;
-		waitpid(cpid,&exitcode,0);
-		return (exitcode == 0);
-	}
-	return false; // never reached, make compiler shut up about return value
-}
-
-bool EthernetTap::addIP(const InetAddress &ip)
-{
-	Mutex::Lock _l(_ips_m);
-
-	if (!ip)
-		return false;
-	if (_ips.count(ip) > 0)
-		return true; // IP/netmask already assigned
-
-	// Remove and reconfigure if address is the same but netmask is different
-	for(std::set<InetAddress>::iterator i(_ips.begin());i!=_ips.end();++i) {
-		if ((i->ipsEqual(ip))&&(i->netmaskBits() != ip.netmaskBits())) {
-			if (___removeIp(_dev,*i)) {
-				_ips.erase(i);
-				break;
-			} else {
-				LOG("WARNING: failed to remove old IP/netmask %s to replace with %s",i->toString().c_str(),ip.toString().c_str());
-			}
-		}
-	}
-
-	int cpid;
-	if ((cpid = (int)fork()) == 0) {
-		execl(ZT_ETHERTAP_IFCONFIG,ZT_ETHERTAP_IFCONFIG,_dev,ip.isV4() ? "inet" : "inet6",ip.toString().c_str(),"alias",(const char *)0);
-		exit(-1);
-	} else {
-		int exitcode = -1;
-		waitpid(cpid,&exitcode,0);
-		if (exitcode == 0) {
-			_ips.insert(ip);
-			return true;
-		}
-	}
-
-	return false;
-}
-
-bool EthernetTap::removeIP(const InetAddress &ip)
-{
-	Mutex::Lock _l(_ips_m);
-	if (_ips.count(ip) > 0) {
-		if (___removeIp(_dev,ip)) {
-			_ips.erase(ip);
-			return true;
-		}
-	}
-	return false;
-}
-
-void EthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
-{
-	if ((_fd > 0)&&(len <= _mtu)) {
-		for(int i=0;i<6;++i)
-			_putBuf[i] = to.data[i];
-		for(int i=0;i<6;++i)
-			_putBuf[i+6] = from.data[i];
-		*((uint16_t *)(_putBuf + 12)) = htons((uint16_t)etherType);
-		memcpy(_putBuf + 14,data,len);
-		len += 14;
-		int n = (int)::write(_fd,_putBuf,len);
-		if (n <= 0) {
-			LOG("error writing packet to Ethernet tap device: %s",strerror(errno));
-		} else if (n != (int)len) {
-			// Saw this gremlin once, so log it if we see it again... OSX tap
-			// or something seems to have goofy issues with certain MTUs.
-			LOG("WARNING: Apple gremlin: tap write() wrote %d of %u bytes of frame",n,len);
-		}
-	}
-}
-
-unsigned int EthernetTap::get(MAC &from,MAC &to,unsigned int &etherType,void *buf)
-{
-	for(;;) {
-		if (_fd > 0) {
-			_isReading_m.lock();
-			_isReading = true;
-			_isReadingThreadId = pthread_self();
-			_isReading_m.unlock();
-
-			int n = (int)::read(_fd,_getBuf,_mtu + 14);
-
-			_isReading_m.lock();
-			_isReading = false;
-			_isReading_m.unlock();
-
-			if (n > 14) {
-				for(int i=0;i<6;++i)
-					to.data[i] = _getBuf[i];
-				for(int i=0;i<6;++i)
-					from.data[i] = _getBuf[i + 6];
-				etherType = ntohs(((uint16_t *)_getBuf)[6]);
-				n -= 14;
-				memcpy(buf,_getBuf + 14,n);
-				return (unsigned int)n;
-			} else if (n < 0) {
-				if (_fd <= 0)
-					break;
-				else if ((errno == EINTR)||(errno == ETIMEDOUT))
-					continue;
-				else {
-					TRACE("unexpected error reading from tap: %s",strerror(errno));
-					::close(_fd);
-					_fd = 0;
-					break;
-				}
-			} else {
-				TRACE("incomplete read from tap: %d bytes",n);
-				continue;
-			}
-		}
-	}
-	return 0;
-}
-
-std::string EthernetTap::deviceName()
-{
-	return std::string(_dev);
-}
-
-bool EthernetTap::open() const
-{
-	return (_fd > 0);
-}
-
-void EthernetTap::close()
-{
-	Mutex::Lock _l(__tapCreateLock); // also prevent create during close()
-	if (_fd > 0) {
-		int f = _fd;
-		_fd = 0;
-		::close(f);
-
-		_isReading_m.lock();
-		if (_isReading)
-			pthread_kill(_isReadingThreadId,SIGUSR2);
-		_isReading_m.unlock();
-	}
-}
-
+#ifdef __APPLE__
 bool EthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 {
 	std::set<MulticastGroup> newGroups;
@@ -658,6 +530,8 @@ bool EthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 
 	bool changed = false;
 
+	newGroups.insert(_blindWildcardMulticastGroup); // always join this
+
 	for(std::set<MulticastGroup>::iterator mg(newGroups.begin());mg!=newGroups.end();++mg) {
 		if (!groups.count(*mg)) {
 			groups.insert(*mg);
@@ -673,13 +547,58 @@ bool EthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 
 	return changed;
 }
+#endif // __APPLE__
+
+void EthernetTap::threadMain()
+	throw()
+{
+	fd_set readfds,nullfds;
+	MAC to,from;
+	char getBuf[4096 + 14];
+	Buffer<4096> data;
+
+	// Wait for a moment after startup -- wait for Network to finish
+	// constructing itself.
+	Thread::sleep(500);
+
+	FD_ZERO(&readfds);
+	FD_ZERO(&nullfds);
+	int nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
+
+	for(;;) {
+		FD_SET(_shutdownSignalPipe[0],&readfds);
+		FD_SET(_fd,&readfds);
+		select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
+
+		if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
+			break;
+
+		if (FD_ISSET(_fd,&readfds)) {
+			int n = (int)::read(_fd,getBuf,_mtu + 14);
+
+			if (n > 14) {
+				for(int i=0;i<6;++i)
+					to.data[i] = (unsigned char)getBuf[i];
+				for(int i=0;i<6;++i)
+					from.data[i] = (unsigned char)getBuf[i + 6];
+				data.copyFrom(getBuf + 14,(unsigned int)n - 14);
+				_handler(_arg,from,to,ntohs(((const uint16_t *)getBuf)[6]),data);
+			} else if (n < 0) {
+				if ((errno != EINTR)&&(errno != ETIMEDOUT)) {
+					TRACE("unexpected error reading from tap: %s",strerror(errno));
+					break;
+				}
+			}
+		}
+	}
+}
 
 } // namespace ZeroTier
 
-/* ======================================================================== */
-#elif defined(_WIN32) /* -------------------------------------------------- */
-/* ======================================================================== */
+#endif // __UNIX_LIKE__ //////////////////////////////////////////////////////
 
-/* ======================================================================== */
-#endif
-/* ======================================================================== */
+#ifdef __WINDOWS__
+
+// TODO
+
+#endif // __WINDOWS__

node/EthernetTap.hpp

@@ -36,14 +36,13 @@
 #include <set>
 #include <string>
 #include <stdexcept>
-#include "Array.hpp"
-#include "Utils.hpp"
-#include "InetAddress.hpp"
-#include "NonCopyable.hpp"
-#include "MAC.hpp"
 #include "Constants.hpp"
+#include "InetAddress.hpp"
+#include "MAC.hpp"
 #include "Mutex.hpp"
 #include "MulticastGroup.hpp"
+#include "Thread.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
@@ -52,22 +51,41 @@ class RuntimeEnvironment;
 /**
  * System ethernet tap device
  */
-class EthernetTap : NonCopyable
+class EthernetTap
 {
 public:
 	/**
 	 * Construct a new TAP device
 	 *
+	 * Handler arguments: arg,from,to,etherType,data
+	 * 
 	 * @param renv Runtime environment
 	 * @param mac MAC address of device
 	 * @param mtu MTU of device
+	 * @param handler Handler function to be called when data is received from the tap
+	 * @param arg First argument to handler function
 	 * @throws std::runtime_error Unable to allocate device
 	 */
-	EthernetTap(const RuntimeEnvironment *renv,const MAC &mac,unsigned int mtu)
+	EthernetTap(
+		const RuntimeEnvironment *renv,
+		const MAC &mac,
+		unsigned int mtu,
+		void (*handler)(void *,const MAC &,const MAC &,unsigned int,const Buffer<4096> &),
+		void *arg)
 		throw(std::runtime_error);
 
+	/**
+	 * Close tap and shut down thread
+	 *
+	 * This may block for a few seconds while thread exits.
+	 */
 	~EthernetTap();
 
+	/**
+	 * Perform OS dependent actions on network configuration change detection
+	 */
+	void whack();
+
 	/**
 	 * @return MAC address of this interface
 	 */
@@ -132,31 +150,10 @@ public:
 	 */
 	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
 
-	/**
-	 * Get the next packet from the interface, blocking if none is available.
-	 *
-	 * @param from Filled with MAC address of source (normally our own)
-	 * @param to Filled with MAC address of destination
-	 * @param etherType Filled with Ethernet frame type
-	 * @param buf Buffer to fill (must have room for MTU bytes)
-	 * @return Number of bytes read or 0 if none
-	 */
-	unsigned int get(MAC &from,MAC &to,unsigned int &etherType,void *buf);
-
 	/**
 	 * @return OS-specific device or connection name
 	 */
-	std::string deviceName();
-
-	/**
-	 * @return True if tap is open
-	 */
-	bool open() const;
-
-	/**
-	 * Close this tap, invalidating the object and causing get() to abort
-	 */
-	void close();
+	std::string deviceName() const;
 
 	/**
 	 * Fill or modify a set to contain multicast groups for this device
@@ -164,34 +161,38 @@ public:
 	 * This populates a set or, if already populated, modifies it to contain
 	 * only multicast groups in which this device is interested.
 	 *
+	 * This should always include the blind wildcard MulticastGroup (MAC of
+	 * ff:ff:ff:ff:ff:ff and 0 ADI field).
+	 *
 	 * @param groups Set to modify in place
 	 * @return True if set was changed since last call
 	 */
 	bool updateMulticastGroups(std::set<MulticastGroup> &groups);
 
+	/**
+	 * Thread main method; do not call elsewhere
+	 */
+	void threadMain()
+		throw();
+
 private:
 	const MAC _mac;
 	const unsigned int _mtu;
 
 	const RuntimeEnvironment *_r;
+	Thread _thread;
 
 	std::set<InetAddress> _ips;
 	Mutex _ips_m;
 
-#if defined(__APPLE__) || defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
+	void (*_handler)(void *,const MAC &,const MAC &,unsigned int,const Buffer<4096> &);
+	void *_arg;
 
+#ifdef __UNIX_LIKE__
 	char _dev[16];
-	unsigned char *_putBuf;
-	unsigned char *_getBuf;
 	int _fd;
-
-	bool _isReading;
-	pthread_t _isReadingThreadId;
-	Mutex _isReading_m;
-
-#elif defined(_WIN32) /* -------------------------------------------------- */
-
-#endif /* ----------------------------------------------------------------- */
+	int _shutdownSignalPipe[2];
+#endif
 };
 
 } // namespace ZeroTier

node/Filter.cpp

@@ -0,0 +1,372 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "RuntimeEnvironment.hpp"
+#include "Logger.hpp"
+#include "Filter.hpp"
+#include "Utils.hpp"
+
+namespace ZeroTier {
+
+const char *const Filter::UNKNOWN_NAME = "(unknown)";
+const Range<unsigned int> Filter::ANY;
+
+bool Filter::Rule::operator()(unsigned int etype,const void *data,unsigned int len) const
+	throw(std::invalid_argument)
+{
+	if ((!_etherType)||(_etherType(etype))) { // ethertype is ANY, or matches
+		// Ethertype determines meaning of protocol and port
+		switch(etype) {
+			case ZT_ETHERTYPE_IPV4:
+				if (len > 20) {
+					if ((!_protocol)||(_protocol(((const uint8_t *)data)[9]))) { // protocol is ANY or match
+						if (!_port) // port is ANY
+							return true;
+
+						// Don't match on fragments beyond fragment 0. If we've blocked
+						// fragment 0, further fragments will fall on deaf ears anyway.
+						if ((Utils::ntoh(((const uint16_t *)data)[3]) & 0x1fff))
+							return false;
+
+						// Internet header length determines where data begins, in multiples of 32 bits
+						unsigned int ihl = 4 * (((const uint8_t *)data)[0] & 0x0f);
+
+						switch(((const uint8_t *)data)[9]) { // port's meaning depends on IP protocol
+							case ZT_IPPROTO_ICMP:
+								// For ICMP, port is ICMP type
+								return _port(((const uint8_t *)data)[ihl]);
+							case ZT_IPPROTO_TCP:
+							case ZT_IPPROTO_UDP:
+							case ZT_IPPROTO_SCTP:
+							case ZT_IPPROTO_UDPLITE:
+								// For these, port is destination port. Protocol designers were
+								// nice enough to put the field in the same place.
+								return _port(((const uint16_t *)data)[(ihl / 2) + 1]);
+							default:
+								// port has no meaning for other IP types, so ignore it
+								return true;
+						}
+
+						return false; // no match on port
+					}
+				} else throw std::invalid_argument("undersized IPv4 packet");
+				break;
+
+			case ZT_ETHERTYPE_IPV6:
+				if (len > 40) {
+					int nextHeader = ((const uint8_t *)data)[6];
+					unsigned int pos = 40;
+					while ((pos < len)&&(nextHeader >= 0)&&(nextHeader != 59)) { // 59 == no next header
+						fprintf(stderr,"[rule] V6: start header parse, header %.2x pos %d\n",nextHeader,pos);
+
+						switch(nextHeader) {
+							case 0: // hop-by-hop options
+							case 60: // destination options
+							case 43: // routing
+							case 135: // mobility (mobile IPv6 options)
+								if (_protocol((unsigned int)nextHeader))
+									return true; // match if our goal was to match any of these
+								nextHeader = ((const uint8_t *)data)[pos];
+								pos += 8 + (8 * ((const uint8_t *)data)[pos + 1]);
+								break;
+							case 44: // fragment
+								if (_protocol(44))
+									return true; // match if our goal was to match fragments
+								nextHeader = ((const uint8_t *)data)[pos];
+								pos += 8;
+								break;
+							case ZT_IPPROTO_AH: // AH
+								return _protocol(ZT_IPPROTO_AH); // true if AH is matched protocol, otherwise false since packet will be IPsec
+							case ZT_IPPROTO_ESP: // ESP
+								return _protocol(ZT_IPPROTO_ESP); // true if ESP is matched protocol, otherwise false since packet will be IPsec
+							case ZT_IPPROTO_ICMPV6:
+								// Only match ICMPv6 if we've selected it specifically
+								if (_protocol(ZT_IPPROTO_ICMPV6)) {
+									// Port is interpreted as ICMPv6 type
+									if ((!_port)||(_port(((const uint8_t *)data)[pos])))
+										return true;
+								}
+								break;
+							case ZT_IPPROTO_TCP:
+							case ZT_IPPROTO_UDP:
+							case ZT_IPPROTO_SCTP:
+							case ZT_IPPROTO_UDPLITE:
+								// If we encounter any of these, match if protocol matches or is wildcard as
+								// we'll consider these the "real payload" if present.
+								if ((!_protocol)||(_protocol(nextHeader))) {
+									if ((!_port)||(_port(((const uint16_t *)data)[(pos / 2) + 1])))
+										return true; // protocol matches or is ANY, port is ANY or matches
+								}
+								break;
+							default: {
+								char foo[128];
+								sprintf(foo,"unrecognized IPv6 header type %d",(int)nextHeader);
+								throw std::invalid_argument(foo);
+							}
+						}
+
+						fprintf(stderr,"[rule] V6: end header parse, next header %.2x, new pos %d\n",nextHeader,pos);
+					}
+				} else throw std::invalid_argument("undersized IPv6 packet");
+				break;
+
+			default:
+				// For other ethertypes, protocol and port are ignored. What would they mean?
+				return true;
+		}
+	}
+
+	return false;
+}
+
+std::string Filter::Rule::toString() const
+{
+	char buf[128];
+	std::string s;
+
+	switch(_etherType.magnitude()) {
+		case 0:
+			s.push_back('*');
+			break;
+		case 1:
+			sprintf(buf,"%u",_etherType.start);
+			s.append(buf);
+			break;
+		default:
+			sprintf(buf,"%u-%u",_etherType.start,_etherType.end);
+			s.append(buf);
+			break;
+	}
+	s.push_back('/');
+	switch(_protocol.magnitude()) {
+		case 0:
+			s.push_back('*');
+			break;
+		case 1:
+			sprintf(buf,"%u",_protocol.start);
+			s.append(buf);
+			break;
+		default:
+			sprintf(buf,"%u-%u",_protocol.start,_protocol.end);
+			s.append(buf);
+			break;
+	}
+	s.push_back('/');
+	switch(_port.magnitude()) {
+		case 0:
+			s.push_back('*');
+			break;
+		case 1:
+			sprintf(buf,"%u",_port.start);
+			s.append(buf);
+			break;
+		default:
+			sprintf(buf,"%u-%u",_port.start,_port.end);
+			s.append(buf);
+			break;
+	}
+
+	return s;
+}
+
+void Filter::add(const Rule &r,const Action &a)
+{
+	Mutex::Lock _l(_chain_m);
+	for(std::vector<Entry>::iterator i(_chain.begin());i!=_chain.end();++i) {
+		if (i->rule == r) {
+			_chain.erase(i);
+			break;
+		}
+	}
+	_chain.push_back(Entry(r,a));
+}
+
+std::string Filter::toString(const char *sep) const
+{
+	if (!sep)
+		sep = ",";
+
+	std::string s;
+
+	bool first = true;
+	Mutex::Lock _l(_chain_m);
+	for(std::vector<Entry>::const_iterator i(_chain.begin());i!=_chain.end();++i) {
+		s.append(i->rule.toString());
+		if (first)
+			first = false;
+		else s.append(sep);
+	}
+
+	return s;
+}
+
+const char *Filter::etherTypeName(const unsigned int etherType)
+	throw()
+{
+	switch(etherType) {
+		case ZT_ETHERTYPE_IPV4:  return "ETHERTYPE_IPV4";
+		case ZT_ETHERTYPE_ARP:   return "ETHERTYPE_ARP";
+		case ZT_ETHERTYPE_RARP:  return "ETHERTYPE_RARP";
+		case ZT_ETHERTYPE_ATALK: return "ETHERTYPE_ATALK";
+		case ZT_ETHERTYPE_AARP:  return "ETHERTYPE_AARP";
+		case ZT_ETHERTYPE_IPX_A: return "ETHERTYPE_IPX_A";
+		case ZT_ETHERTYPE_IPX_B: return "ETHERTYPE_IPX_B";
+		case ZT_ETHERTYPE_IPV6:  return "ETHERTYPE_IPV6";
+	}
+	return UNKNOWN_NAME;
+}
+
+const char *Filter::ipProtocolName(const unsigned int ipp)
+	throw()
+{
+	switch(ipp) {
+		case ZT_IPPROTO_ICMP:    return "IPPROTO_ICMP";
+		case ZT_IPPROTO_IGMP:    return "IPPROTO_IGMP";
+		case ZT_IPPROTO_TCP:     return "IPPROTO_TCP";
+		case ZT_IPPROTO_UDP:     return "IPPROTO_UDP";
+		case ZT_IPPROTO_GRE:     return "IPPROTO_GRE";
+		case ZT_IPPROTO_ESP:     return "IPPROTO_ESP";
+		case ZT_IPPROTO_AH:      return "IPPROTO_AH";
+		case ZT_IPPROTO_ICMPV6:  return "IPPROTO_ICMPV6";
+		case ZT_IPPROTO_OSPF:    return "IPPROTO_OSPF";
+		case ZT_IPPROTO_IPIP:    return "IPPROTO_IPIP";
+		case ZT_IPPROTO_IPCOMP:  return "IPPROTO_IPCOMP";
+		case ZT_IPPROTO_L2TP:    return "IPPROTO_L2TP";
+		case ZT_IPPROTO_SCTP:    return "IPPROTO_SCTP";
+		case ZT_IPPROTO_FC:      return "IPPROTO_FC";
+		case ZT_IPPROTO_UDPLITE: return "IPPROTO_UDPLITE";
+		case ZT_IPPROTO_HIP:     return "IPPROTO_HIP";
+	}
+	return UNKNOWN_NAME;
+}
+
+const char *Filter::icmpTypeName(const unsigned int icmpType)
+	throw()
+{
+	switch(icmpType) {
+		case ZT_ICMP_ECHO_REPLY:                  return "ICMP_ECHO_REPLY";
+		case ZT_ICMP_DESTINATION_UNREACHABLE:     return "ICMP_DESTINATION_UNREACHABLE";
+		case ZT_ICMP_SOURCE_QUENCH:               return "ICMP_SOURCE_QUENCH";
+		case ZT_ICMP_REDIRECT:                    return "ICMP_REDIRECT";
+		case ZT_ICMP_ALTERNATE_HOST_ADDRESS:      return "ICMP_ALTERNATE_HOST_ADDRESS";
+		case ZT_ICMP_ECHO_REQUEST:                return "ICMP_ECHO_REQUEST";
+		case ZT_ICMP_ROUTER_ADVERTISEMENT:        return "ICMP_ROUTER_ADVERTISEMENT";
+		case ZT_ICMP_ROUTER_SOLICITATION:         return "ICMP_ROUTER_SOLICITATION";
+		case ZT_ICMP_TIME_EXCEEDED:               return "ICMP_TIME_EXCEEDED";
+		case ZT_ICMP_BAD_IP_HEADER:               return "ICMP_BAD_IP_HEADER";
+		case ZT_ICMP_TIMESTAMP:                   return "ICMP_TIMESTAMP";
+		case ZT_ICMP_TIMESTAMP_REPLY:             return "ICMP_TIMESTAMP_REPLY";
+		case ZT_ICMP_INFORMATION_REQUEST:         return "ICMP_INFORMATION_REQUEST";
+		case ZT_ICMP_INFORMATION_REPLY:           return "ICMP_INFORMATION_REPLY";
+		case ZT_ICMP_ADDRESS_MASK_REQUEST:        return "ICMP_ADDRESS_MASK_REQUEST";
+		case ZT_ICMP_ADDRESS_MASK_REPLY:          return "ICMP_ADDRESS_MASK_REPLY";
+		case ZT_ICMP_TRACEROUTE:                  return "ICMP_TRACEROUTE";
+		case ZT_ICMP_MOBILE_HOST_REDIRECT:        return "ICMP_MOBILE_HOST_REDIRECT";
+		case ZT_ICMP_MOBILE_REGISTRATION_REQUEST: return "ICMP_MOBILE_REGISTRATION_REQUEST";
+		case ZT_ICMP_MOBILE_REGISTRATION_REPLY:   return "ICMP_MOBILE_REGISTRATION_REPLY";
+	}
+	return UNKNOWN_NAME;
+}
+
+const char *Filter::icmp6TypeName(const unsigned int icmp6Type)
+	throw()
+{
+	switch(icmp6Type) {
+		case ZT_ICMP6_DESTINATION_UNREACHABLE:              return "ICMP6_DESTINATION_UNREACHABLE";
+		case ZT_ICMP6_PACKET_TOO_BIG:                       return "ICMP6_PACKET_TOO_BIG";
+		case ZT_ICMP6_TIME_EXCEEDED:                        return "ICMP6_TIME_EXCEEDED";
+		case ZT_ICMP6_PARAMETER_PROBLEM:                    return "ICMP6_PARAMETER_PROBLEM";
+		case ZT_ICMP6_ECHO_REQUEST:                         return "ICMP6_ECHO_REQUEST";
+		case ZT_ICMP6_ECHO_REPLY:                           return "ICMP6_ECHO_REPLY";
+		case ZT_ICMP6_MULTICAST_LISTENER_QUERY:             return "ICMP6_MULTICAST_LISTENER_QUERY";
+		case ZT_ICMP6_MULTICAST_LISTENER_REPORT:            return "ICMP6_MULTICAST_LISTENER_REPORT";
+		case ZT_ICMP6_MULTICAST_LISTENER_DONE:              return "ICMP6_MULTICAST_LISTENER_DONE";
+		case ZT_ICMP6_ROUTER_SOLICITATION:                  return "ICMP6_ROUTER_SOLICITATION";
+		case ZT_ICMP6_ROUTER_ADVERTISEMENT:                 return "ICMP6_ROUTER_ADVERTISEMENT";
+		case ZT_ICMP6_NEIGHBOR_SOLICITATION:                return "ICMP6_NEIGHBOR_SOLICITATION";
+		case ZT_ICMP6_NEIGHBOR_ADVERTISEMENT:               return "ICMP6_NEIGHBOR_ADVERTISEMENT";
+		case ZT_ICMP6_REDIRECT_MESSAGE:                     return "ICMP6_REDIRECT_MESSAGE";
+		case ZT_ICMP6_ROUTER_RENUMBERING:                   return "ICMP6_ROUTER_RENUMBERING";
+		case ZT_ICMP6_NODE_INFORMATION_QUERY:               return "ICMP6_NODE_INFORMATION_QUERY";
+		case ZT_ICMP6_NODE_INFORMATION_RESPONSE:            return "ICMP6_NODE_INFORMATION_RESPONSE";
+		case ZT_ICMP6_INV_NEIGHBOR_SOLICITATION:            return "ICMP6_INV_NEIGHBOR_SOLICITATION";
+		case ZT_ICMP6_INV_NEIGHBOR_ADVERTISEMENT:           return "ICMP6_INV_NEIGHBOR_ADVERTISEMENT";
+		case ZT_ICMP6_MLDV2:                                return "ICMP6_MLDV2";
+		case ZT_ICMP6_HOME_AGENT_ADDRESS_DISCOVERY_REQUEST: return "ICMP6_HOME_AGENT_ADDRESS_DISCOVERY_REQUEST";
+		case ZT_ICMP6_HOME_AGENT_ADDRESS_DISCOVERY_REPLY:   return "ICMP6_HOME_AGENT_ADDRESS_DISCOVERY_REPLY";
+		case ZT_ICMP6_MOBILE_PREFIX_SOLICITATION:           return "ICMP6_MOBILE_PREFIX_SOLICITATION";
+		case ZT_ICMP6_MOBILE_PREFIX_ADVERTISEMENT:          return "ICMP6_MOBILE_PREFIX_ADVERTISEMENT";
+		case ZT_ICMP6_CERTIFICATION_PATH_SOLICITATION:      return "ICMP6_CERTIFICATION_PATH_SOLICITATION";
+		case ZT_ICMP6_CERTIFICATION_PATH_ADVERTISEMENT:     return "ICMP6_CERTIFICATION_PATH_ADVERTISEMENT";
+		case ZT_ICMP6_MULTICAST_ROUTER_ADVERTISEMENT:       return "ICMP6_MULTICAST_ROUTER_ADVERTISEMENT";
+		case ZT_ICMP6_MULTICAST_ROUTER_SOLICITATION:        return "ICMP6_MULTICAST_ROUTER_SOLICITATION";
+		case ZT_ICMP6_MULTICAST_ROUTER_TERMINATION:         return "ICMP6_MULTICAST_ROUTER_TERMINATION";
+		case ZT_ICMP6_RPL_CONTROL_MESSAGE:                  return "ICMP6_RPL_CONTROL_MESSAGE";
+	}
+	return UNKNOWN_NAME;
+}
+
+Filter::Action Filter::operator()(const RuntimeEnvironment *_r,unsigned int etherType,const void *frame,unsigned int len) const
+{
+	Mutex::Lock _l(_chain_m);
+
+	TRACE("starting match against %d rules",(int)_chain.size());
+
+	int ruleNo = 0;
+	for(std::vector<Entry>::const_iterator r(_chain.begin());r!=_chain.end();++r,++ruleNo) {
+		try {
+			if (r->rule(etherType,frame,len)) {
+				TRACE("match: %s",r->rule.toString().c_str());
+
+				switch(r->action) {
+					case ACTION_ALLOW:
+					case ACTION_DENY:
+						return r->action;
+					default:
+						break;
+				}
+			} else {
+				TRACE("no match: %s",r->rule.toString().c_str());
+			}
+		} catch (std::invalid_argument &exc) {
+			LOG("filter: unable to parse packet on rule %s (%d): %s",r->rule.toString().c_str(),ruleNo,exc.what());
+			return ACTION_UNPARSEABLE;
+		} catch ( ... ) {
+			LOG("filter: unable to parse packet on rule %s (%d): unknown exception",r->rule.toString().c_str(),ruleNo);
+			return ACTION_UNPARSEABLE;
+		}
+	}
+
+	return ACTION_ALLOW;
+}
+
+} // namespace ZeroTier

node/Filter.hpp

@@ -0,0 +1,351 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_FILTER_HPP
+#define _ZT_FILTER_HPP
+
+#include <stdio.h>
+#include <string.h>
+#include <string>
+#include <vector>
+#include <utility>
+#include <stdexcept>
+
+#include "Mutex.hpp"
+#include "Range.hpp"
+
+/* Ethernet frame types that might be relevant to us */
+#define ZT_ETHERTYPE_IPV4 0x0800
+#define ZT_ETHERTYPE_ARP 0x0806
+#define ZT_ETHERTYPE_RARP 0x8035
+#define ZT_ETHERTYPE_ATALK 0x809b
+#define ZT_ETHERTYPE_AARP 0x80f3
+#define ZT_ETHERTYPE_IPX_A 0x8137
+#define ZT_ETHERTYPE_IPX_B 0x8138
+#define ZT_ETHERTYPE_IPV6 0x86dd
+
+/* IP protocols we might care about */
+#define ZT_IPPROTO_ICMP 0x01
+#define ZT_IPPROTO_IGMP 0x02
+#define ZT_IPPROTO_TCP 0x06
+#define ZT_IPPROTO_UDP 0x11
+#define ZT_IPPROTO_GRE 0x2f
+#define ZT_IPPROTO_ESP 0x32
+#define ZT_IPPROTO_AH 0x33
+#define ZT_IPPROTO_ICMPV6 0x3a
+#define ZT_IPPROTO_OSPF 0x59
+#define ZT_IPPROTO_IPIP 0x5e
+#define ZT_IPPROTO_IPCOMP 0x6c
+#define ZT_IPPROTO_L2TP 0x73
+#define ZT_IPPROTO_SCTP 0x84
+#define ZT_IPPROTO_FC 0x85
+#define ZT_IPPROTO_UDPLITE 0x88
+#define ZT_IPPROTO_HIP 0x8b
+
+/* IPv4 ICMP types */
+#define ZT_ICMP_ECHO_REPLY 0
+#define ZT_ICMP_DESTINATION_UNREACHABLE 3
+#define ZT_ICMP_SOURCE_QUENCH 4
+#define ZT_ICMP_REDIRECT 5
+#define ZT_ICMP_ALTERNATE_HOST_ADDRESS 6
+#define ZT_ICMP_ECHO_REQUEST 8
+#define ZT_ICMP_ROUTER_ADVERTISEMENT 9
+#define ZT_ICMP_ROUTER_SOLICITATION 10
+#define ZT_ICMP_TIME_EXCEEDED 11
+#define ZT_ICMP_BAD_IP_HEADER 12
+#define ZT_ICMP_TIMESTAMP 13
+#define ZT_ICMP_TIMESTAMP_REPLY 14
+#define ZT_ICMP_INFORMATION_REQUEST 15
+#define ZT_ICMP_INFORMATION_REPLY 16
+#define ZT_ICMP_ADDRESS_MASK_REQUEST 17
+#define ZT_ICMP_ADDRESS_MASK_REPLY 18
+#define ZT_ICMP_TRACEROUTE 30
+#define ZT_ICMP_MOBILE_HOST_REDIRECT 32
+#define ZT_ICMP_MOBILE_REGISTRATION_REQUEST 35
+#define ZT_ICMP_MOBILE_REGISTRATION_REPLY 36
+
+/* IPv6 ICMP types */
+#define ZT_ICMP6_DESTINATION_UNREACHABLE 1
+#define ZT_ICMP6_PACKET_TOO_BIG 2
+#define ZT_ICMP6_TIME_EXCEEDED 3
+#define ZT_ICMP6_PARAMETER_PROBLEM 4
+#define ZT_ICMP6_ECHO_REQUEST 128
+#define ZT_ICMP6_ECHO_REPLY 129
+#define ZT_ICMP6_MULTICAST_LISTENER_QUERY 130
+#define ZT_ICMP6_MULTICAST_LISTENER_REPORT 131
+#define ZT_ICMP6_MULTICAST_LISTENER_DONE 132
+#define ZT_ICMP6_ROUTER_SOLICITATION 133
+#define ZT_ICMP6_ROUTER_ADVERTISEMENT 134
+#define ZT_ICMP6_NEIGHBOR_SOLICITATION 135
+#define ZT_ICMP6_NEIGHBOR_ADVERTISEMENT 136
+#define ZT_ICMP6_REDIRECT_MESSAGE 137
+#define ZT_ICMP6_ROUTER_RENUMBERING 138
+#define ZT_ICMP6_NODE_INFORMATION_QUERY 139
+#define ZT_ICMP6_NODE_INFORMATION_RESPONSE 140
+#define ZT_ICMP6_INV_NEIGHBOR_SOLICITATION 141
+#define ZT_ICMP6_INV_NEIGHBOR_ADVERTISEMENT 142
+#define ZT_ICMP6_MLDV2 143
+#define ZT_ICMP6_HOME_AGENT_ADDRESS_DISCOVERY_REQUEST 144
+#define ZT_ICMP6_HOME_AGENT_ADDRESS_DISCOVERY_REPLY 145
+#define ZT_ICMP6_MOBILE_PREFIX_SOLICITATION 146
+#define ZT_ICMP6_MOBILE_PREFIX_ADVERTISEMENT 147
+#define ZT_ICMP6_CERTIFICATION_PATH_SOLICITATION 148
+#define ZT_ICMP6_CERTIFICATION_PATH_ADVERTISEMENT 149
+#define ZT_ICMP6_MULTICAST_ROUTER_ADVERTISEMENT 151
+#define ZT_ICMP6_MULTICAST_ROUTER_SOLICITATION 152
+#define ZT_ICMP6_MULTICAST_ROUTER_TERMINATION 153
+#define ZT_ICMP6_RPL_CONTROL_MESSAGE 155
+
+namespace ZeroTier {
+
+class RuntimeEnvironment;
+
+/**
+ * A simple Ethernet frame level filter supporting basic IP port DENY
+ */
+class Filter
+{
+public:
+	/**
+	 * Value returned by etherTypeName, etc. on unknown
+	 *
+	 * These static methods return precisely this, so a pointer equality
+	 * check will work.
+	 */
+	static const char *const UNKNOWN_NAME;
+
+	/**
+	 * An empty range as a more idiomatic way of specifying a wildcard match
+	 */
+	static const Range<unsigned int> ANY;
+
+	/**
+	 * A filter rule
+	 *
+	 * This behaves as an immutable value object.
+	 */
+	class Rule
+	{
+	public:
+		Rule()
+			throw() :
+			_etherType(),
+			_protocol(),
+			_port()
+		{
+		}
+
+		/**
+		 * Construct a new rule
+		 *
+		 * @param etype Ethernet type or empty range for ANY
+		 * @param prot Protocol or empty range for ANY (meaning depends on ethertype, e.g. IP protocol numbers)
+		 * @param prt Port or empty range for ANY (only applies to some protocols)
+		 */
+		Rule(const Range<unsigned int> &etype,const Range<unsigned int> &prot,const Range<unsigned int> &prt)
+			throw() :
+			_etherType(etype),
+			_protocol(prot),
+			_port(prt)
+		{
+		}
+
+		inline const Range<unsigned int> &etherType() const throw() { return _etherType; }
+		inline const Range<unsigned int> &protocol() const throw() { return _protocol; }
+		inline const Range<unsigned int> &port() const throw() { return _port; }
+
+		/**
+		 * Test this rule against a frame
+		 *
+		 * @param etype Type of ethernet frame
+		 * @param data Ethernet frame data
+		 * @param len Length of ethernet frame
+		 * @return True if rule matches
+		 * @throws std::invalid_argument Frame invalid or not parseable
+		 */
+		bool operator()(unsigned int etype,const void *data,unsigned int len) const
+			throw(std::invalid_argument);
+
+		/**
+		 * @return Human readable representation of rule
+		 */
+		std::string toString() const;
+
+		inline bool operator==(const Rule &r) const throw() { return ((_etherType == r._etherType)&&(_protocol == r._protocol)&&(_port == r._port)); }
+		inline bool operator!=(const Rule &r) const throw() { return !(*this == r); }
+		inline bool operator<(const Rule &r) const
+			throw()
+		{
+			if (_etherType < r._etherType)
+				return true;
+			else if (_etherType == r._etherType) {
+				if (_protocol < r._protocol)
+					return true;
+				else if (_protocol == r._protocol) {
+					if (_port < r._port)
+						return true;
+				}
+			}
+			return false;
+		}
+		inline bool operator>(const Rule &r) const throw() { return (r < *this); }
+		inline bool operator<=(const Rule &r) const throw() { return !(r < *this); }
+		inline bool operator>=(const Rule &r) const throw() { return !(*this < r); }
+
+	private:
+		Range<unsigned int> _etherType;
+		Range<unsigned int> _protocol;
+		Range<unsigned int> _port;
+	};
+
+	/**
+	 * Action if a rule matches
+	 */
+	enum Action
+	{
+		ACTION_DENY = 0,
+		ACTION_ALLOW = 1,
+		ACTION_UNPARSEABLE = 2
+	};
+
+	/**
+	 * Entry in filter chain
+	 */
+	struct Entry
+	{
+		Entry() {}
+		Entry(const Rule &r,const Action &a) :
+			rule(r),
+			action(a)
+		{
+		}
+
+		Rule rule;
+		Action action;
+	};
+
+	Filter() :
+		_chain(),
+		_chain_m()
+	{
+	}
+
+	Filter(const Filter &f) :
+		_chain(),
+		_chain_m()
+	{
+		Mutex::Lock _l(f._chain_m);
+		_chain = f._chain;
+	}
+
+	inline Filter &operator=(const Filter &f)
+	{
+		Mutex::Lock _l1(_chain_m);
+		Mutex::Lock _l2(f._chain_m);
+		_chain = f._chain;
+		return *this;
+	}
+
+	/**
+	 * Remove all filter entries
+	 */
+	inline void clear()
+	{
+		Mutex::Lock _l(_chain_m);
+		_chain.clear();
+	}
+
+	/**
+	 * Append a rule/action pair to this chain
+	 *
+	 * If an identical rule already exists it is removed and a new entry is
+	 * added to the end with the new action. (Two identical rules with the
+	 * same action wouldn't make sense.)
+	 *
+	 * @param r Rule to add
+	 * @param a Action if rule matches
+	 */
+	void add(const Rule &r,const Action &a);
+
+	/**
+	 * @return Number of rules in filter chain
+	 */
+	inline unsigned int length() const
+		throw()
+	{
+		Mutex::Lock _l(_chain_m);
+		return _chain.size();
+	}
+
+	/**
+	 * @return Entry in filter chain or null entry if out of bounds
+	 */
+	inline Entry operator[](const unsigned int i) const
+		throw()
+	{
+		Mutex::Lock _l(_chain_m);
+		if (i < _chain.size())
+			return _chain[i];
+		return Entry();
+	}
+
+	/**
+	 * Get a string representation of this filter
+	 *
+	 * @param sep Separator between filter rules, or NULL for comma (default)
+	 * @return Human-readable string
+	 */
+	std::string toString(const char *sep = (const char *)0) const;
+
+	static const char *etherTypeName(const unsigned int etherType)
+		throw();
+	static const char *ipProtocolName(const unsigned int ipp)
+		throw();
+	static const char *icmpTypeName(const unsigned int icmpType)
+		throw();
+	static const char *icmp6TypeName(const unsigned int icmp6Type)
+		throw();
+
+	/**
+	 * Match against an Ethernet frame
+	 *
+	 * @param _r Runtime environment
+	 * @param etherType Ethernet frame type
+	 * @param frame Ethernet frame data
+	 * @param len Length of frame in bytes
+	 * @return Action if matched or ACTION_ALLOW if not matched
+	 */
+	Action operator()(const RuntimeEnvironment *_r,unsigned int etherType,const void *frame,unsigned int len) const;
+
+private:
+	std::vector<Entry> _chain;
+	Mutex _chain_m;
+};
+
+} // namespace ZeroTier
+
+#endif

node/Http.cpp

@@ -1,323 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <vector>
-#include <set>
-#include <list>
-
-#ifndef _WIN32
-#include <unistd.h>
-#endif
-
-#include "Http.hpp"
-#include "Utils.hpp"
-#include "InetAddress.hpp"
-
-static http_parser_settings _http_parser_settings;
-
-namespace ZeroTier {
-
-static bool _sendAll(int fd,const void *buf,unsigned int len)
-{
-	for(;;) {
-		int n = (int)::send(fd,buf,len,0);
-		if ((n < 0)&&(errno == EINTR))
-			continue;
-		return (n == (int)len);
-	}
-}
-
-const std::map<std::string,std::string> Http::EMPTY_HEADERS;
-
-Http::Request::Request(
-	Http::Method m,
-	const std::string &url,
-	const std::map<std::string,std::string> &rh,
-	const std::string &rb,
-	bool (*handler)(Request *,void *,const std::string &,int,const std::map<std::string,std::string> &,const std::string &),
-	void *arg) :
-	_url(url),
-	_requestHeaders(rh),
-	_handler(handler),
-	_arg(arg),
-	_method(m),
-	_fd(0)
-{
-	_http_parser_settings.on_message_begin = &Http::Request::_http_on_message_begin;
-	_http_parser_settings.on_url = &Http::Request::_http_on_url;
-	_http_parser_settings.on_status_complete = &Http::Request::_http_on_status_complete;
-	_http_parser_settings.on_header_field = &Http::Request::_http_on_header_field;
-	_http_parser_settings.on_header_value = &Http::Request::_http_on_header_value;
-	_http_parser_settings.on_headers_complete = &Http::Request::_http_on_headers_complete;
-	_http_parser_settings.on_body = &Http::Request::_http_on_body;
-	_http_parser_settings.on_message_complete = &Http::Request::_http_on_message_complete;
-
-	start();
-}
-
-Http::Request::~Request()
-{
-	if (_fd > 0)
-		::close(_fd);
-	join();
-}
-
-void Http::Request::main()
-	throw()
-{
-	char buf[131072];
-
-	try {
-		http_parser_init(&_parser,HTTP_RESPONSE);
-		_parser.data = this;
-
-		http_parser_url urlParsed;
-		if (http_parser_parse_url(_url.c_str(),_url.length(),0,&urlParsed)) {
-			suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"URL parse error");
-			return;
-		}
-		if (!(urlParsed.field_set & (1 << UF_SCHEMA))) {
-			suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"URL specifies no schema");
-			return;
-		}
-
-		std::string schema(_url.substr(urlParsed.field_data[UF_SCHEMA].off,urlParsed.field_data[UF_SCHEMA].len));
-
-		if (schema == "file") {
-			const std::string filePath(_url.substr(urlParsed.field_data[UF_PATH].off,urlParsed.field_data[UF_PATH].len));
-
-			uint64_t lm = Utils::getLastModified(filePath.c_str());
-			if (lm) {
-				const std::map<std::string,std::string>::const_iterator ifModSince(_requestHeaders.find("If-Modified-Since"));
-				if ((ifModSince != _requestHeaders.end())&&(ifModSince->second.length())) {
-					uint64_t t64 = Utils::fromRfc1123(ifModSince->second);
-					if ((t64)&&(lm > t64)) {
-						suicidalThread = !_handler(this,_arg,_url,304,_responseHeaders,"");
-						return;
-					}
-				}
-
-				if (Utils::readFile(filePath.c_str(),_responseBody)) {
-					_responseHeaders["Last-Modified"] = Utils::toRfc1123(lm);
-					suicidalThread = !_handler(this,_arg,_url,200,_responseHeaders,_responseBody);
-					return;
-				}
-			}
-
-			suicidalThread = !_handler(this,_arg,_url,404,_responseHeaders,"file not found or not readable");
-			return;
-		} else if (schema == "http") {
-			if (!(urlParsed.field_set & (1 << UF_HOST))) {
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"URL contains no host");
-				return;
-			}
-			std::string host(_url.substr(urlParsed.field_data[UF_HOST].off,urlParsed.field_data[UF_HOST].len));
-
-			std::list<InetAddress> v4,v6;
-			{
-				struct addrinfo *res = (struct addrinfo *)0;
-				if (!getaddrinfo(host.c_str(),(const char *)0,(const struct addrinfo *)0,&res)) {
-					struct addrinfo *p = res;
-					do {
-						if (p->ai_family == AF_INET)
-							v4.push_back(InetAddress(p->ai_addr));
-						else if (p->ai_family == AF_INET6)
-							v6.push_back(InetAddress(p->ai_addr));
-					} while ((p = p->ai_next));
-					freeaddrinfo(res);
-				}
-			}
-
-			std::list<InetAddress> *addrList;
-			if (v4.empty()&&v6.empty()) {
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"could not find address for host in URL");
-				return;
-			} else if (v4.empty()) {
-				addrList = &v6;
-			} else {
-				addrList = &v4;
-			}
-			InetAddress *addr;
-			{
-				addrList->sort();
-				addrList->unique();
-				unsigned int i = 0,k = 0;
-				k = Utils::randomInt<unsigned int>() % addrList->size();
-				std::list<InetAddress>::iterator a(addrList->begin());
-				while (i++ != k) ++a;
-				addr = &(*a);
-			}
-
-			int remotePort = ((urlParsed.field_set & (1 << UF_PORT))&&(urlParsed.port)) ? (int)urlParsed.port : (int)80;
-			if ((remotePort <= 0)||(remotePort > 0xffff)) {
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"URL port out of range");
-				return;
-			}
-			addr->setPort(remotePort);
-
-			_fd = socket(addr->isV6() ? AF_INET6 : AF_INET,SOCK_STREAM,0);
-			if (_fd <= 0) {
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"could not open socket");
-				return;
-			}
-
-			for(;;) {
-				if (connect(_fd,addr->saddr(),addr->saddrLen())) {
-					if (errno == EINTR)
-						continue;
-					::close(_fd); _fd = 0;
-					suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"connection failed to remote host");
-					return;
-				} else break;
-			}
-
-			const char *mstr = "GET";
-			switch(_method) {
-				case HTTP_METHOD_HEAD: mstr = "HEAD"; break;
-				default: break;
-			}
-			int mlen = (int)snprintf(buf,sizeof(buf),"%s %s HTTP/1.1\r\nAccept-Encoding: \r\nHost: %s\r\n",mstr,_url.substr(urlParsed.field_data[UF_PATH].off,urlParsed.field_data[UF_PATH].len).c_str(),host.c_str());
-			if (mlen >= (int)sizeof(buf)) {
-				::close(_fd); _fd = 0;
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"URL too long");
-				return;
-			}
-			if (!_sendAll(_fd,buf,mlen)) {
-				::close(_fd); _fd = 0;
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"write error");
-				return;
-			}
-
-			for(std::map<std::string,std::string>::const_iterator rh(_requestHeaders.begin());rh!=_requestHeaders.end();++rh) {
-				mlen = (int)snprintf(buf,sizeof(buf),"%s: %s\r\n",rh->first.c_str(),rh->second.c_str());
-				if (mlen >= (int)sizeof(buf)) {
-					::close(_fd); _fd = 0;
-					suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"header too long");
-					return;
-				}
-				if (!_sendAll(_fd,buf,mlen)) {
-					::close(_fd); _fd = 0;
-					suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"write error");
-					return;
-				}
-			}
-
-			if (!_sendAll(_fd,"\r\n",2)) {
-				::close(_fd); _fd = 0;
-				suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"write error");
-				return;
-			}
-
-			_responseStatusCode = 0;
-			_messageComplete = false;
-			for(;;) {
-				mlen = (int)::recv(_fd,buf,sizeof(buf),0);
-				if (mlen < 0) {
-					if (errno != EINTR)
-						break;
-					else continue;
-				}
-				if (((int)http_parser_execute(&_parser,&_http_parser_settings,buf,mlen) != mlen)||(_parser.upgrade)) {
-					::close(_fd); _fd = 0;
-					suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"invalid HTTP response from server");
-					return;
-				}
-				if (_messageComplete) {
-					::close(_fd); _fd = 0;
-					suicidalThread = !_handler(this,_arg,_url,_responseStatusCode,_responseHeaders,_responseBody);
-					return;
-				}
-			}
-
-			::close(_fd); _fd = 0;
-			suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"empty HTTP response from server");
-			return;
-		} else {
-			suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"only 'file' and 'http' methods are supported");
-			return;
-		}
-	} catch ( ... ) {
-		suicidalThread = !_handler(this,_arg,_url,0,_responseHeaders,"unexpected exception retrieving URL");
-		return;
-	}
-}
-
-int Http::Request::_http_on_message_begin(http_parser *parser)
-{
-	return 0;
-}
-int Http::Request::_http_on_url(http_parser *parser,const char *data,size_t length)
-{
-	return 0;
-}
-int Http::Request::_http_on_status_complete(http_parser *parser)
-{
-	Http::Request *r = (Http::Request *)parser->data;
-	r->_responseStatusCode = parser->status_code;
-	return 0;
-}
-int Http::Request::_http_on_header_field(http_parser *parser,const char *data,size_t length)
-{
-	Http::Request *r = (Http::Request *)parser->data;
-	if ((r->_currentHeaderField.length())&&(r->_responseHeaders.find(r->_currentHeaderField) != r->_responseHeaders.end()))
-		r->_currentHeaderField.assign("");
-	r->_currentHeaderField.append(data,length);
-	return 0;
-}
-int Http::Request::_http_on_header_value(http_parser *parser,const char *data,size_t length)
-{
-	Http::Request *r = (Http::Request *)parser->data;
-	if (r->_currentHeaderField.length())
-		r->_responseHeaders[r->_currentHeaderField].append(data,length);
-	return 0;
-}
-int Http::Request::_http_on_headers_complete(http_parser *parser)
-{
-	Http::Request *r = (Http::Request *)parser->data;
-	return ((r->_method == Http::HTTP_METHOD_HEAD) ? 1 : 0);
-}
-int Http::Request::_http_on_body(http_parser *parser,const char *data,size_t length)
-{
-	Http::Request *r = (Http::Request *)parser->data;
-	r->_responseBody.append(data,length);
-	return 0;
-}
-int Http::Request::_http_on_message_complete(http_parser *parser)
-{
-	Http::Request *r = (Http::Request *)parser->data;
-	r->_messageComplete = true;
-	return 0;
-}
-
-} // namespace ZeroTier

node/Http.hpp

@@ -1,129 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef _ZT_HTTP_HPP
-#define _ZT_HTTP_HPP
-
-#include <map>
-#include <string>
-#include <stdexcept>
-#include "Thread.hpp"
-
-#include "../ext/http-parser/http_parser.h"
-
-namespace ZeroTier {
-
-class Http
-{
-public:
-	/**
-	 * HTTP request methods
-	 */
-	enum Method
-	{
-		HTTP_METHOD_GET,
-		HTTP_METHOD_HEAD
-	};
-
-	/**
-	 * An empty headers map for convenience
-	 */
-	static const std::map<std::string,std::string> EMPTY_HEADERS;
-
-	/**
-	 * HTTP request
-	 */
-	class Request : protected Thread
-	{
-	public:
-		/**
-		 * Create and issue an HTTP request
-		 *
-		 * The supplied handler is called when the request is
-		 * complete or if an error occurs. A code of zero indicates
-		 * that the server could not be reached, and a description
-		 * of the error will be in 'body'. If the handler returns
-		 * false the Request object deletes itself. Otherwise the
-		 * object must be deleted by other code.
-		 *
-		 * @param m Request method
-		 * @param url Destination URL
-		 * @param rh Request headers
-		 * @param rb Request body or empty string for none (currently unused)
-		 * @param handler Request handler function
-		 * @param arg First argument to request handler
-		 */
-		Request(
-			Http::Method m,
-			const std::string &url,
-			const std::map<std::string,std::string> &rh,
-			const std::string &rb,
-			bool (*handler)(Request *,void *,const std::string &,int,const std::map<std::string,std::string> &,const std::string &),
-			void *arg);
-
-		/**
-		 * Destruction cancels any in-progress request
-		 */
-		virtual ~Request();
-
-	protected:
-		virtual void main()
-			throw();
-
-	private:
-		// HTTP parser handlers
-		static int _http_on_message_begin(http_parser *parser);
-		static int _http_on_url(http_parser *parser,const char *data,size_t length);
-		static int _http_on_status_complete(http_parser *parser);
-		static int _http_on_header_field(http_parser *parser,const char *data,size_t length);
-		static int _http_on_header_value(http_parser *parser,const char *data,size_t length);
-		static int _http_on_headers_complete(http_parser *parser);
-		static int _http_on_body(http_parser *parser,const char *data,size_t length);
-		static int _http_on_message_complete(http_parser *parser);
-
-		http_parser _parser;
-		std::string _url;
-
-		std::map<std::string,std::string> _requestHeaders;
-		std::map<std::string,std::string> _responseHeaders;
-
-		std::string _currentHeaderField;
-		std::string _responseBody;
-
-		bool (*_handler)(Request *,void *,const std::string &,int,const std::map<std::string,std::string> &,const std::string &);
-		void *_arg;
-
-		Http::Method _method;
-		int _responseStatusCode;
-		bool _messageComplete;
-		volatile int _fd;
-	};
-};
-
-} // namespace ZeroTier
-
-#endif

node/Identity.cpp

@@ -57,11 +57,13 @@ void Identity::generate()
 	// the address of an identity will be detected as its signature will be
 	// invalid. Of course, deep verification of address/key relationship is
 	// required to cover the more elaborate address claim jump attempt case.
+	unsigned char atmp[ZT_ADDRESS_LENGTH];
+	_address.copyTo(atmp,ZT_ADDRESS_LENGTH);
 	SHA256_CTX sha;
 	unsigned char dig[32];
 	unsigned char idtype = IDENTITY_TYPE_NIST_P_521,zero = 0;
 	SHA256_Init(&sha);
-	SHA256_Update(&sha,_address.data(),ZT_ADDRESS_LENGTH);
+	SHA256_Update(&sha,atmp,ZT_ADDRESS_LENGTH);
 	SHA256_Update(&sha,&zero,1);
 	SHA256_Update(&sha,&idtype,1);
 	SHA256_Update(&sha,&zero,1);
@@ -73,11 +75,13 @@ void Identity::generate()
 
 bool Identity::locallyValidate(bool doAddressDerivationCheck) const
 {
+	unsigned char atmp[ZT_ADDRESS_LENGTH];
+	_address.copyTo(atmp,ZT_ADDRESS_LENGTH);
 	SHA256_CTX sha;
 	unsigned char dig[32];
 	unsigned char idtype = IDENTITY_TYPE_NIST_P_521,zero = 0;
 	SHA256_Init(&sha);
-	SHA256_Update(&sha,_address.data(),ZT_ADDRESS_LENGTH);
+	SHA256_Update(&sha,atmp,ZT_ADDRESS_LENGTH);
 	SHA256_Update(&sha,&zero,1);
 	SHA256_Update(&sha,&idtype,1);
 	SHA256_Update(&sha,&zero,1);
@@ -119,7 +123,7 @@ bool Identity::fromString(const char *str)
 	std::string b(Utils::unhex(fields[0]));
 	if (b.length() != ZT_ADDRESS_LENGTH)
 		return false;
-	_address = b.data();
+	_address.setTo(b.data(),ZT_ADDRESS_LENGTH);
 
 	b = Utils::base64Decode(fields[2]);
 	if ((!b.length())||(b.length() > ZT_EC_MAX_BYTES))
@@ -218,83 +222,7 @@ Address Identity::deriveAddress(const void *keyBytes,unsigned int keyLen)
 
 	delete [] ram;
 
-	return Address(dig); // first 5 bytes of dig[]
-}
-
-std::string Identity::encrypt(const Identity &to,const void *data,unsigned int len) const
-{
-	unsigned char key[64];
-	unsigned char mac[32];
-	unsigned char iv[8];
-
-	if (!agree(to,key,sizeof(key)))
-		return std::string();
-	Utils::getSecureRandom(iv,8);
-	for(int i=0;i<8;++i)
-		key[i + 32] ^= iv[i]; // perturb HMAC key with IV so IV is effectively included in HMAC
-	Salsa20 s20(key,256,iv);
-
-	std::string compressed;
-	compressed.reserve(len);
-	Utils::compress((const char *)data,(const char *)data + len,Utils::StringAppendOutput(compressed));
-	if (!compressed.length())
-		return std::string();
-
-	char *encrypted = new char[compressed.length() + 16];
-	try {
-		s20.encrypt(compressed.data(),encrypted + 16,(unsigned int)compressed.length());
-		HMAC::sha256(key + 32,32,encrypted + 16,(unsigned int)compressed.length(),mac);
-		for(int i=0;i<8;++i)
-			encrypted[i] = iv[i];
-		for(int i=0;i<8;++i)
-			encrypted[i + 8] = mac[i];
-
-		std::string s(encrypted,compressed.length() + 16);
-		delete [] encrypted;
-		return s;
-	} catch ( ... ) {
-		delete [] encrypted;
-		return std::string();
-	}
-}
-
-std::string Identity::decrypt(const Identity &from,const void *cdata,unsigned int len) const
-{
-	unsigned char key[64];
-	unsigned char mac[32];
-
-	if (len < 16)
-		return std::string();
-
-	if (!agree(from,key,sizeof(key)))
-		return std::string();
-
-	for(int i=0;i<8;++i)
-		key[i + 32] ^= ((const unsigned char *)cdata)[i]; // apply IV to HMAC key
-	HMAC::sha256(key + 32,32,((const char *)cdata) + 16,(unsigned int)(len - 16),mac);
-	for(int i=0;i<8;++i) {
-		if (((const unsigned char *)cdata)[i + 8] != mac[i])
-			return std::string();
-	}
-
-	char *decbuf = new char[len - 16];
-	try {
-		Salsa20 s20(key,256,cdata); // first 8 bytes are IV
-		len -= 16;
-		s20.decrypt((const char *)cdata + 16,decbuf,len);
-
-		std::string decompressed;
-		if (Utils::decompress((const char *)decbuf,(const char *)decbuf + len,Utils::StringAppendOutput(decompressed))) {
-			delete [] decbuf;
-			return decompressed;
-		} else {
-			delete [] decbuf;
-			return std::string();
-		}
-	} catch ( ... ) {
-		delete [] decbuf;
-		return std::string();
-	}
+	return Address(dig,ZT_ADDRESS_LENGTH); // first 5 bytes of dig[]
 }
 
 } // namespace ZeroTier

node/Identity.hpp

@@ -104,7 +104,7 @@ public:
 		_keyPair((EllipticCurveKeyPair *)0)
 	{
 		if (!fromString(str))
-			throw std::invalid_argument("invalid string-serialized identity");
+			throw std::invalid_argument(std::string("invalid string-serialized identity: ") + str);
 	}
 
 	Identity(const std::string &str)
@@ -112,7 +112,7 @@ public:
 		_keyPair((EllipticCurveKeyPair *)0)
 	{
 		if (!fromString(str))
-			throw std::invalid_argument("invalid string-serialized identity");
+			throw std::invalid_argument(std::string("invalid string-serialized identity: ") + str);
 	}
 
 	template<unsigned int C>
@@ -175,40 +175,6 @@ public:
 	 */
 	inline bool hasPrivate() const throw() { return (_keyPair); }
 
-	/**
-	 * Encrypt a block of data to send to another identity
-	 *
-	 * This identity must have a secret key.
-	 *
-	 * The encrypted data format is:
-	 *   <[8] Salsa20 initialization vector>
-	 *   <[8] first 8 bytes of HMAC-SHA-256 of ciphertext>
-	 *   <[...] encrypted compressed data>
-	 *
-	 * Keying is accomplished using agree() (KDF function is in the
-	 * EllipticCurveKeyPair.cpp source) to generate 64 bytes of key. The first
-	 * 32 bytes are used as the Salsa20 key, and the last 32 bytes are used
-	 * as the HMAC key.
-	 *
-	 * @param to Identity of recipient of encrypted message
-	 * @param data Data to encrypt
-	 * @param len Length of data
-	 * @return Encrypted data or empty string on failure
-	 */
-	std::string encrypt(const Identity &to,const void *data,unsigned int len) const;
-
-	/**
-	 * Decrypt a message encrypted with encrypt()
-	 *
-	 * This identity must have a secret key.
-	 *
-	 * @param from Identity of sender of encrypted message
-	 * @param cdata Encrypted message
-	 * @param len Length of encrypted message
-	 * @return Decrypted data or empty string on failure
-	 */
-	std::string decrypt(const Identity &from,const void *cdata,unsigned int len) const;
-
 	/**
 	 * Shortcut method to perform key agreement with another identity
 	 *
@@ -307,7 +273,7 @@ public:
 	inline void serialize(Buffer<C> &b,bool includePrivate = false) const
 		throw(std::out_of_range)
 	{
-		b.append(_address.data(),ZT_ADDRESS_LENGTH);
+		_address.appendTo(b);
 		b.append((unsigned char)IDENTITY_TYPE_NIST_P_521);
 		b.append((unsigned char)(_publicKey.size() & 0xff));
 		b.append(_publicKey.data(),_publicKey.size());
@@ -340,7 +306,7 @@ public:
 
 		unsigned int p = startAt;
 
-		_address = b.field(p,ZT_ADDRESS_LENGTH);
+		_address.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
 		p += ZT_ADDRESS_LENGTH;
 
 		if (b[p++] != IDENTITY_TYPE_NIST_P_521)

node/Multicaster.hpp

@@ -0,0 +1,341 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_MULTICASTER_HPP
+#define _ZT_MULTICASTER_HPP
+
+#include <stdint.h>
+#include <string.h>
+
+#include <openssl/sha.h>
+
+#include <utility>
+#include <algorithm>
+#include <map>
+#include <set>
+#include <vector>
+#include <string>
+
+#include "Constants.hpp"
+#include "Buffer.hpp"
+#include "Packet.hpp"
+#include "MulticastGroup.hpp"
+#include "Utils.hpp"
+#include "MAC.hpp"
+#include "Address.hpp"
+#include "SharedPtr.hpp"
+#include "BloomFilter.hpp"
+#include "Identity.hpp"
+#include "CMWC4096.hpp"
+
+// Maximum sample size to pick during choice of multicast propagation peers
+#define ZT_MULTICAST_PICK_MAX_SAMPLE_SIZE (ZT_MULTICAST_PROPAGATION_BREADTH * 8)
+
+namespace ZeroTier {
+
+/**
+ * Multicast propagation engine
+ *
+ * This is written as a generic class so that it can be mocked and tested
+ * in simulation. It also always takes 'now' as an argument, permitting
+ * running in simulated time.
+ *
+ * This does not handle network permission or rate limiting, only the
+ * propagation algorithm.
+ */
+class Multicaster
+{
+public:
+	/**
+	 * Simple bit field bloom filter included with multicast frame packets
+	 */
+	typedef BloomFilter<ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BITS> MulticastBloomFilter;
+
+	Multicaster()
+		throw()
+	{
+		memset(_multicastHistory,0,sizeof(_multicastHistory));
+		_multicastHistoryPtr = 0;
+	}
+
+	/**
+	 * Generate a signature of a multicast packet using an identity
+	 *
+	 * @param id Identity to sign with (must have secret key portion)
+	 * @param nwid Network ID
+	 * @param from MAC address of sender
+	 * @param to Multicast group
+	 * @param etherType 16-bit ethernet type
+	 * @param data Ethernet frame data
+	 * @param len Length of frame
+	 * @return ECDSA signature
+	 */
+	static inline std::string signMulticastPacket(const Identity &id,uint64_t nwid,const MAC &from,const MulticastGroup &to,unsigned int etherType,const void *data,unsigned int len)
+	{
+		unsigned char digest[32];
+		_hashMulticastPacketForSig(nwid,from,to,etherType,data,len,digest);
+		return id.sign(digest);
+	}
+
+	/**
+	 * Verify a signature from a multicast packet
+	 *
+	 * @param id Identity of original signer
+	 * @param nwid Network ID
+	 * @param from MAC address of sender
+	 * @param to Multicast group
+	 * @param etherType 16-bit ethernet type
+	 * @param data Ethernet frame data
+	 * @param len Length of frame
+	 * @param signature ECDSA signature
+	 * @param siglen Length of signature in bytes
+	 * @return ECDSA signature
+	 */
+	static bool verifyMulticastPacket(const Identity &id,uint64_t nwid,const MAC &from,const MulticastGroup &to,unsigned int etherType,const void *data,unsigned int len,const void *signature,unsigned int siglen)
+	{
+		unsigned char digest[32];
+		_hashMulticastPacketForSig(nwid,from,to,etherType,data,len,digest);
+		return id.verifySignature(digest,signature,siglen);
+	}
+
+	/**
+	 * Compute the CRC64 code for multicast deduplication
+	 *
+	 * @param nwid Network ID
+	 * @param from Sender MAC
+	 * @param to Destination multicast group
+	 * @param etherType Ethernet frame type
+	 * @param payload Multicast frame data
+	 * @param len Length of frame
+	 */
+	static inline uint64_t computeMulticastDedupCrc(
+		uint64_t nwid,
+		const MAC &from,
+		const MulticastGroup &to,
+		unsigned int etherType,
+		const void *payload,
+		unsigned int len)
+		throw()
+	{
+		// This CRC is only used locally, so byte order issues and
+		// such don't matter. It can also be changed without protocol
+		// impact.
+		uint64_t crc = Utils::crc64(0,from.data,6);
+		crc = Utils::crc64(crc,to.mac().data,6);
+		crc ^= (uint64_t)to.adi();
+		crc ^= (uint64_t)etherType;
+		crc = Utils::crc64(crc,payload,len);
+		crc ^= nwid; // also include network ID in CRC
+		return crc;
+	}
+
+	/**
+	 * Check multicast history to see if this is a duplicate
+	 *
+	 * @param crc Multicast CRC
+	 * @param now Current time
+	 * @return True if this appears to be a duplicate to within history expiration time
+	 */
+	inline bool checkDuplicate(uint64_t crc,uint64_t now) const
+		throw()
+	{
+		for(unsigned int i=0;i<ZT_MULTICAST_DEDUP_HISTORY_LENGTH;++i) {
+			if ((_multicastHistory[i][0] == crc)&&((now - _multicastHistory[i][1]) < ZT_MULTICAST_DEDUP_HISTORY_EXPIRE))
+				return true;
+		}
+		return false;
+	}
+
+	/**
+	 * Add a multicast CRC to the multicast deduplication history
+	 *
+	 * @param crc Multicast CRC
+	 * @param now Current time
+	 */
+	inline void addToDedupHistory(uint64_t crc,uint64_t now)
+		throw()
+	{
+		unsigned int mhi = ++_multicastHistoryPtr % ZT_MULTICAST_DEDUP_HISTORY_LENGTH;
+		_multicastHistory[mhi][0] = crc;
+		_multicastHistory[mhi][1] = now;
+	}
+
+	/**
+	 * Update the most recent LIKE time for an address in a given multicast group on a given network
+	 *
+	 * @param nwid Network ID
+	 * @param mg Multicast group
+	 * @param addr Address that likes group on given network
+	 * @param now Current timestamp
+	 */
+	inline void likesMulticastGroup(const uint64_t nwid,const MulticastGroup &mg,const Address &addr,const uint64_t now)
+	{
+		Mutex::Lock _l(_multicastMemberships_m);
+		std::vector<MulticastMembership> &memberships = _multicastMemberships[MulticastChannel(nwid,mg)];
+		for(std::vector<MulticastMembership>::iterator mm(memberships.begin());mm!=memberships.end();++mm) {
+			if (mm->first == addr) {
+				mm->second = now;
+				return;
+			}
+		}
+		memberships.push_back(MulticastMembership(addr,now));
+	}
+
+	/**
+	 * Choose peers to send a propagating multicast to
+	 *
+	 * @param topology Topology object or mock thereof
+	 * @param nwid Network ID
+	 * @param mg Multicast group
+	 * @param originalSubmitter Original submitter of multicast message to network
+	 * @param upstream Address from which message originated, or null (0) address if none
+	 * @param bf Bloom filter, updated in place with sums of addresses in chosen peers and/or decay
+	 * @param max Maximum number of peers to pick
+	 * @param peers Array of objects of type P to fill with up to [max] peers
+	 * @param now Current timestamp
+	 * @return Number of peers actually stored in peers array
+	 * @tparam T Type of topology, which is Topology in running code or a mock in simulation
+	 * @tparam P Type of peers, which is SharedPtr<Peer> in running code or a mock in simulation (mock must behave like a pointer type)
+	 */
+	template<typename T,typename P>
+	inline unsigned int pickNextPropagationPeers(
+		CMWC4096 &prng,
+		T &topology,
+		uint64_t nwid,
+		const MulticastGroup &mg,
+		const Address &originalSubmitter,
+		const Address &upstream,
+		MulticastBloomFilter &bf,
+		unsigned int max,
+		P *peers,
+		uint64_t now)
+	{
+		typename std::set< P,_PeerPropagationPrioritySortOrder<P> > toConsider;
+
+		// Pick up to ZT_MULTICAST_PICK_MAX_SAMPLE_SIZE peers that have
+		// subscribed to this channel and that are not in bloom filter.
+		// Pick randomly from subscribers, but place into a set that is
+		// sorted in descending order of time of most recent unicast
+		// frame transfer. (Implicit social ordering.) Also ignore original
+		// submitter and upstream, since we know these have seen this
+		// message.
+		{
+			Mutex::Lock _l(_multicastMemberships_m);
+			std::map< MulticastChannel,std::vector<MulticastMembership> >::iterator mm(_multicastMemberships.find(MulticastChannel(nwid,mg)));
+			if ((mm != _multicastMemberships.end())&&(!mm->second.empty())) {
+				for(unsigned int stries=0;stries<ZT_MULTICAST_PICK_MAX_SAMPLE_SIZE;++stries) {
+					MulticastMembership &m = mm->second[prng.next32() % mm->second.size()];
+					if (((now - m.second) < ZT_MULTICAST_LIKE_EXPIRE)&&(!bf.contains(m.first.sum()))&&(m.first != originalSubmitter)&&(m.first != upstream)) {
+						P peer(topology.getPeer(m.first));
+						if (peer)
+							toConsider.insert(peer);
+					}
+				}
+			}
+		}
+
+		// The first peers in toConsider will be the 'best'
+		unsigned int chosen = 0;
+		for(typename std::set< P,_PeerPropagationPrioritySortOrder<P> >::iterator i(toConsider.begin());((i!=toConsider.end())&&(chosen < max));++i)
+			bf.set((peers[chosen++] = *i)->address().sum());
+
+		// Add a supernode if there are fewer than the desired
+		// number of recipients. Note that we do not use the bloom
+		// filter to track visits to supernodes, intentionally
+		// allowing multicasts to ping pong between supernodes.
+		// Supernodes propagate even messages they've already seen,
+		// while regular nodes do not. Thus this ping-ponging will
+		// cause the supernodes to pick new starting points for
+		// peer to peer graph traversal multiple times. It's a
+		// simple, stateless way to increase supernode-driven
+		// propagation of a multicast in the event that peer to
+		// peer connectivity for its group is sparse.
+		if (chosen < max) {
+			Address avoid[2];
+			avoid[0] = originalSubmitter;
+			avoid[1] = upstream;
+			P peer = topology.getBestSupernode(avoid,2,true);
+			if (peer)
+				peers[chosen++] = peer;
+		}
+
+		return chosen;
+	}
+
+private:
+	// Sort order for chosen propagation peers
+	template<typename P>
+	struct _PeerPropagationPrioritySortOrder
+	{
+		inline bool operator()(const P &p1,const P &p2) const
+		{
+			return (p1->lastUnicastFrame() > p2->lastUnicastFrame());
+		}
+	};
+
+	static inline void _hashMulticastPacketForSig(uint64_t nwid,const MAC &from,const MulticastGroup &to,unsigned int etherType,const void *data,unsigned int len,unsigned char *digest)
+		throw()
+	{
+		unsigned char zero = 0;
+		SHA256_CTX sha;
+		SHA256_Init(&sha);
+		uint64_t _nwid = Utils::hton(nwid);
+		SHA256_Update(&sha,(unsigned char *)&_nwid,sizeof(_nwid));
+		SHA256_Update(&sha,&zero,1);
+		SHA256_Update(&sha,(unsigned char *)from.data,6);
+		SHA256_Update(&sha,&zero,1);
+		SHA256_Update(&sha,(unsigned char *)to.mac().data,6);
+		SHA256_Update(&sha,&zero,1);
+		uint32_t _adi = Utils::hton(to.adi());
+		SHA256_Update(&sha,(unsigned char *)&_adi,sizeof(_adi));
+		SHA256_Update(&sha,&zero,1);
+		uint16_t _etype = Utils::hton((uint16_t)etherType);
+		SHA256_Update(&sha,(unsigned char *)&_etype,sizeof(_etype));
+		SHA256_Update(&sha,&zero,1);
+		SHA256_Update(&sha,(unsigned char *)data,len);
+		SHA256_Final(digest,&sha);
+	}
+
+	// ring buffer: [0] - CRC, [1] - timestamp
+	uint64_t _multicastHistory[ZT_MULTICAST_DEDUP_HISTORY_LENGTH][2];
+	volatile unsigned int _multicastHistoryPtr;
+
+	// A multicast channel, essentially a pub/sub channel. It consists of a
+	// network ID and a multicast group within that network.
+	typedef std::pair<uint64_t,MulticastGroup> MulticastChannel;
+
+	// Address and time of last LIKE
+	typedef std::pair<Address,uint64_t> MulticastMembership;
+
+	// Network : MulticastGroup -> vector<Address : time of last LIKE>
+	std::map< MulticastChannel,std::vector<MulticastMembership> > _multicastMemberships;
+	Mutex _multicastMemberships_m;
+};
+
+} // namespace ZeroTier
+
+#endif

node/Network.cpp

@@ -25,56 +25,282 @@
  * LLC. Start here: http://www.zerotier.com/
  */
 
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+
+#include <openssl/sha.h>
+
+#include "RuntimeEnvironment.hpp"
+#include "NodeConfig.hpp"
 #include "Network.hpp"
 #include "Switch.hpp"
+#include "Packet.hpp"
+#include "Utils.hpp"
 
 namespace ZeroTier {
 
-Network::Network(const RuntimeEnvironment *renv,uint64_t id)
-	throw(std::runtime_error) :
-	Thread(),
-	_r(renv),
-	_id(id),
-	_tap(renv,renv->identity.address().toMAC(),ZT_IF_MTU),
-	_members(),
-	_open(false),
-	_lock()
+void Network::Certificate::_shaForSignature(unsigned char *dig) const
 {
-	TRACE("new network %llu created, TAP device: %s",id,_tap.deviceName().c_str());
-	start();
+	SHA256_CTX sha;
+	SHA256_Init(&sha);
+	unsigned char zero = 0;
+	for(const_iterator i(begin());i!=end();++i) {
+		SHA256_Update(&sha,&zero,1);
+		SHA256_Update(&sha,(const unsigned char *)i->first.data(),i->first.length());
+		SHA256_Update(&sha,&zero,1);
+		SHA256_Update(&sha,(const unsigned char *)i->second.data(),i->second.length());
+		SHA256_Update(&sha,&zero,1);
+	}
+	SHA256_Final(dig,&sha);
+}
+
+static const std::string _DELTA_PREFIX("~");
+bool Network::Certificate::qualifyMembership(const Network::Certificate &mc) const
+{
+	// Note: optimization probably needed here, probably via some kind of
+	// memoization / dynamic programming. But make it work first, then make
+	// it fast.
+
+	for(const_iterator myField(begin());myField!=end();++myField) {
+		if (!((myField->first.length() > 1)&&(myField->first[0] == '~'))) { // ~fields are max delta range specs
+			// If they lack the same field, comparison fails.
+			const_iterator theirField(mc.find(myField->first));
+			if (theirField == mc.end())
+				return false;
+
+			const_iterator deltaField(find(_DELTA_PREFIX + myField->first));
+			if (deltaField == end()) {
+				// If there is no delta, compare on simple equality
+				if (myField->second != theirField->second)
+					return false;
+			} else {
+				// Otherwise compare range with max delta. Presence of a dot in delta
+				// indicates a floating point comparison. Otherwise an integer
+				// comparison occurs.
+				if (deltaField->second.find('.') != std::string::npos) {
+					double my = strtod(myField->second.c_str(),(char **)0);
+					double their = strtod(theirField->second.c_str(),(char **)0);
+					double delta = strtod(deltaField->second.c_str(),(char **)0);
+					if (fabs(my - their) > delta)
+						return false;
+				} else {
+					int64_t my = strtoll(myField->second.c_str(),(char **)0,10);
+					int64_t their = strtoll(theirField->second.c_str(),(char **)0,10);
+					int64_t delta = strtoll(deltaField->second.c_str(),(char **)0,10);
+					if (my > their) {
+						if ((my - their) > delta)
+							return false;
+					} else {
+						if ((their - my) > delta)
+							return false;
+					}
+				}
+			}
+		}
+	}
+
+	return true;
+}
+
+const char *Network::statusString(const Status s)
+	throw()
+{
+	switch(s) {
+		case NETWORK_WAITING_FOR_FIRST_AUTOCONF: return "WAITING_FOR_FIRST_AUTOCONF";
+		case NETWORK_OK: return "OK";
+		case NETWORK_ACCESS_DENIED: return "ACCESS_DENIED";
+	}
+	return "(invalid)";
 }
 
 Network::~Network()
 {
-	_tap.close();
-	join();
-	TRACE("network %llu (%s) closed",_id,_tap.deviceName().c_str());
+	delete _tap;
+
+	if (_destroyOnDelete) {
+		std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".conf");
+		std::string mcdbPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".mcerts");
+		Utils::rm(confPath);
+		Utils::rm(mcdbPath);
+	} else {
+		// Causes flush of membership certs to disk
+		clean();
+	}
 }
 
-void Network::main()
-	throw()
+SharedPtr<Network> Network::newInstance(const RuntimeEnvironment *renv,uint64_t id)
+	throw(std::runtime_error)
 {
-	Buffer<4096> buf;
-	MAC from,to;
-	unsigned int etherType = 0;
+	// We construct Network via a static method to ensure that it is immediately
+	// wrapped in a SharedPtr<>. Otherwise if there is traffic on the Ethernet
+	// tap device, a SharedPtr<> wrap can occur in the Ethernet frame handler
+	// that then causes the Network instance to be deleted before it is finished
+	// being constructed. C++ edge cases, how I love thee.
+	SharedPtr<Network> nw(new Network());
+	nw->_ready = false; // disable handling of Ethernet frames during construct
+	nw->_r = renv;
+	nw->_rlLimit.bytesPerSecond = ZT_MULTICAST_DEFAULT_BYTES_PER_SECOND;
+	nw->_rlLimit.maxBalance = ZT_MULTICAST_DEFAULT_RATE_MAX_BALANCE;
+	nw->_rlLimit.minBalance = ZT_MULTICAST_DEFAULT_RATE_MIN_BALANCE;
+	nw->_tap = new EthernetTap(renv,renv->identity.address().toMAC(),ZT_IF_MTU,&_CBhandleTapData,nw.ptr());
+	nw->_id = id;
+	nw->_lastConfigUpdate = 0;
+	nw->_destroyOnDelete = false;
+	if (nw->controller() == renv->identity.address()) // sanity check, this isn't supported for now
+		throw std::runtime_error("cannot add a network for which I am the netconf master");
+	nw->_restoreState();
+	nw->_ready = true; // enable handling of Ethernet frames
+	nw->requestConfiguration();
+	return nw;
+}
 
-	while (_tap.open()) {
-		unsigned int len = _tap.get(from,to,etherType,buf.data());
-		if (len) {
-			buf.setSize(len);
-			try {
-				if (!*__refCount)
-					break; // sanity check
-				_r->sw->onLocalEthernet(SharedPtr<Network>(this),from,to,etherType,buf);
-			} catch (std::exception &exc) {
-				TRACE("unexpected exception handling local packet: %s",exc.what());
-			} catch ( ... ) {
-				TRACE("unexpected exception handling local packet");
-			}
-		} else break;
+void Network::setConfiguration(const Network::Config &conf)
+{
+	Mutex::Lock _l(_lock);
+	if ((conf.networkId() == _id)&&(conf.peerAddress() == _r->identity.address())) { // sanity check
+		//TRACE("network %.16llx got netconf:\n%s",(unsigned long long)_id,conf.toString().c_str());
+		_configuration = conf;
+		_myCertificate = conf.certificateOfMembership();
+		_lastConfigUpdate = Utils::now();
+
+		_tap->setIps(conf.staticAddresses());
+
+		std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".conf");
+		if (!Utils::writeFile(confPath.c_str(),conf.toString())) {
+			LOG("error: unable to write network configuration file at: %s",confPath.c_str());
+		}
 	}
+}
 
-	TRACE("network %llu thread terminating",_id);
+void Network::requestConfiguration()
+{
+	if (controller() == _r->identity.address()) {
+		LOG("unable to request network configuration for network %.16llx: I am the network master, cannot query self",(unsigned long long)_id);
+		return;
+	}
+	TRACE("requesting netconf for network %.16llx from netconf master %s",(unsigned long long)_id,controller().toString().c_str());
+	Packet outp(controller(),_r->identity.address(),Packet::VERB_NETWORK_CONFIG_REQUEST);
+	outp.append((uint64_t)_id);
+	outp.append((uint16_t)0); // no meta-data
+	_r->sw->send(outp,true);
+}
+
+void Network::addMembershipCertificate(const Address &peer,const Certificate &cert)
+{
+	Mutex::Lock _l(_lock);
+	if (!_configuration.isOpen())
+		_membershipCertificates[peer] = cert;
+}
+
+bool Network::isAllowed(const Address &peer) const
+{
+	// Exceptions can occur if we do not yet have *our* configuration.
+	try {
+		Mutex::Lock _l(_lock);
+		if (_configuration.isOpen())
+			return true;
+		std::map<Address,Certificate>::const_iterator pc(_membershipCertificates.find(peer));
+		if (pc == _membershipCertificates.end())
+			return false;
+		return _myCertificate.qualifyMembership(pc->second);
+	} catch (std::exception &exc) {
+		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer.toString().c_str());
+	}
+	return false;
+}
+
+void Network::clean()
+{
+	std::string mcdbPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".mcerts");
+
+	Mutex::Lock _l(_lock);
+
+	if (_configuration.isOpen()) {
+		_membershipCertificates.clear();
+		Utils::rm(mcdbPath);
+	} else {
+		FILE *mcdb = fopen(mcdbPath.c_str(),"wb");
+		bool writeError = false;
+		if (!mcdb) {
+			LOG("error: unable to open membership cert database at: %s",mcdbPath.c_str());
+		} else {
+			if ((writeError)||(fwrite("MCDB0",5,1,mcdb) != 1)) // version
+				writeError = true;
+		}
+
+		for(std::map<Address,Certificate>::iterator i=(_membershipCertificates.begin());i!=_membershipCertificates.end();) {
+			if (_myCertificate.qualifyMembership(i->second)) {
+				if ((!writeError)&&(mcdb)) {
+					char tmp[ZT_ADDRESS_LENGTH];
+					i->first.copyTo(tmp,ZT_ADDRESS_LENGTH);
+					if ((writeError)||(fwrite(tmp,ZT_ADDRESS_LENGTH,1,mcdb) != 1))
+						writeError = true;
+					std::string c(i->second.toString());
+					uint32_t cl = Utils::hton((uint32_t)c.length());
+					if ((writeError)||(fwrite(&cl,sizeof(cl),1,mcdb) != 1))
+						writeError = true;
+					if ((writeError)||(fwrite(c.data(),c.length(),1,mcdb) != 1))
+						writeError = true;
+				}
+				++i;
+			} else _membershipCertificates.erase(i++);
+		}
+
+		if (mcdb)
+			fclose(mcdb);
+		if (writeError) {
+			Utils::rm(mcdbPath);
+			LOG("error: unable to write to membership cert database at: %s",mcdbPath.c_str());
+		}
+	}
+}
+
+Network::Status Network::status() const
+{
+	Mutex::Lock _l(_lock);
+	if (_configuration.containsAllFields())
+		return NETWORK_OK;
+	return NETWORK_WAITING_FOR_FIRST_AUTOCONF;
+}
+
+void Network::_CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data)
+{
+	if (!((Network *)arg)->_ready)
+		return;
+	const RuntimeEnvironment *_r = ((Network *)arg)->_r;
+	try {
+		_r->sw->onLocalEthernet(SharedPtr<Network>((Network *)arg),from,to,etherType,data);
+	} catch (std::exception &exc) {
+		TRACE("unexpected exception handling local packet: %s",exc.what());
+	} catch ( ... ) {
+		TRACE("unexpected exception handling local packet");
+	}
+}
+
+void Network::_restoreState()
+{
+	std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + toString() + ".conf");
+	std::string confs;
+	if (Utils::readFile(confPath.c_str(),confs)) {
+		try {
+			if (confs.length()) {
+				Config conf(confs);
+				if (conf.containsAllFields())
+					setConfiguration(conf);
+			}
+		} catch ( ... ) {} // ignore invalid config on disk, we will re-request
+	} else {
+		// If the conf file isn't present, "touch" it so we'll remember
+		// the existence of this network.
+		FILE *tmp = fopen(confPath.c_str(),"w");
+		if (tmp)
+			fclose(tmp);
+	}
+	// TODO: restore membership certs
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -30,37 +30,293 @@
 
 #include <string>
 #include <set>
+#include <map>
 #include <vector>
 #include <stdexcept>
+
+#include "Constants.hpp"
+#include "Utils.hpp"
 #include "EthernetTap.hpp"
 #include "Address.hpp"
 #include "Mutex.hpp"
-#include "InetAddress.hpp"
-#include "Constants.hpp"
 #include "SharedPtr.hpp"
 #include "AtomicCounter.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "Thread.hpp"
 #include "MulticastGroup.hpp"
+#include "NonCopyable.hpp"
+#include "MAC.hpp"
+#include "Dictionary.hpp"
+#include "Identity.hpp"
+#include "InetAddress.hpp"
+#include "RateLimiter.hpp"
 
 namespace ZeroTier {
 
+class RuntimeEnvironment;
 class NodeConfig;
 
 /**
- * Local network endpoint
+ * A virtual LAN
+ *
+ * Networks can be open or closed. Each network has an ID whose most
+ * significant 40 bits are the ZeroTier address of the node that should
+ * be contacted for network configuration. The least significant 24
+ * bits are arbitrary, allowing up to 2^24 networks per managing
+ * node.
+ *
+ * Open networks do not track membership. Anyone is allowed to communicate
+ * over them.
+ *
+ * Closed networks track membership by way of timestamped signatures. When
+ * the network requests its configuration, one of the fields returned is
+ * a signature for the identity of the peer on the network. This signature
+ * includes a timestamp. When a peer communicates with other peers on a
+ * closed network, it periodically (and pre-emptively) propagates this
+ * signature to the peers with which it is communicating. Peers reject
+ * packets with an error if no recent signature is on file.
  */
-class Network : protected Thread
+class Network : NonCopyable
 {
 	friend class SharedPtr<Network>;
 	friend class NodeConfig;
 
-private:
-	virtual ~Network();
+public:
+	/**
+	 * A certificate of network membership for private network participation
+	 */
+	class Certificate : private Dictionary
+	{
+	public:
+		Certificate()
+		{
+		}
 
-	Network(const RuntimeEnvironment *renv,uint64_t id)
+		Certificate(const char *s) :
+			Dictionary(s)
+		{
+		}
+
+		Certificate(const std::string &s) :
+			Dictionary(s)
+		{
+		}
+
+		inline std::string toString() const
+		{
+			return Dictionary::toString();
+		}
+
+		inline void setNetworkId(uint64_t id)
+		{
+			char buf[32];
+			sprintf(buf,"%.16llx",(unsigned long long)id);
+			(*this)["nwid"] = buf;
+		}
+
+		inline uint64_t networkId() const
+			throw(std::invalid_argument)
+		{
+			return strtoull(get("nwid").c_str(),(char **)0,16);
+		}
+
+		inline void setPeerAddress(Address &a)
+		{
+			(*this)["peer"] = a.toString();
+		}
+
+		inline Address peerAddress() const
+			throw(std::invalid_argument)
+		{
+			return Address(get("peer"));
+		}
+
+		/**
+		 * Set the timestamp and timestamp max-delta
+		 *
+		 * @param ts Timestamp in ms since epoch
+		 * @param maxDelta Maximum difference between two peers on the same network
+		 */
+		inline void setTimestamp(uint64_t ts,uint64_t maxDelta)
+		{
+			char foo[32];
+			sprintf(foo,"%llu",(unsigned long long)ts);
+			(*this)["ts"] = foo;
+			sprintf(foo,"%llu",(unsigned long long)maxDelta);
+			(*this)["~ts"] = foo;
+		}
+
+		/**
+		 * Sign this certificate
+		 *
+		 * @param with Signing identity -- the identity of this network's controller
+		 * @return Signature or empty string on failure
+		 */
+		inline std::string sign(const Identity &with) const
+		{
+			unsigned char dig[32];
+			_shaForSignature(dig);
+			return with.sign(dig);
+		}
+
+		/**
+		 * Verify this certificate's signature
+		 *
+		 * @param with Signing identity -- the identity of this network's controller
+		 * @param sig Signature
+		 * @param siglen Length of signature in bytes
+		 */
+		inline bool verify(const Identity &with,const void *sig,unsigned int siglen) const
+		{
+			unsigned char dig[32];
+			_shaForSignature(dig);
+			return with.verifySignature(dig,sig,siglen);
+		}
+
+		/**
+		 * Check if another peer is indeed a current member of this network
+		 *
+		 * Fields with companion ~fields are compared with the defined maximum
+		 * delta in this certificate. Fields without ~fields are compared for
+		 * equality.
+		 *
+		 * This does not verify the certificate's signature!
+		 * 
+		 * @param mc Peer membership certificate
+		 * @return True if mc's membership in this network is current
+		 */
+		bool qualifyMembership(const Certificate &mc) const;
+
+	private:
+		void _shaForSignature(unsigned char *dig) const;
+	};
+
+	/**
+	 * A network configuration for a given node
+	 */
+	class Config : private Dictionary
+	{
+	public:
+		Config()
+		{
+		}
+
+		Config(const char *s) :
+			Dictionary(s)
+		{
+		}
+
+		Config(const std::string &s) :
+			Dictionary(s)
+		{
+		}
+
+		inline bool containsAllFields() const
+		{
+			return (contains("nwid")&&contains("peer"));
+		}
+
+		inline std::string toString() const
+		{
+			return Dictionary::toString();
+		}
+
+		inline uint64_t networkId() const
+			throw(std::invalid_argument)
+		{
+			return strtoull(get("nwid").c_str(),(char **)0,16);
+		}
+
+		inline Address peerAddress() const
+			throw(std::invalid_argument)
+		{
+			return Address(get("peer"));
+		}
+
+		/**
+		 * @return Certificate of membership for this network, or empty cert if none
+		 */
+		inline Certificate certificateOfMembership() const
+		{
+			const_iterator cm(find("com"));
+			if (cm == end())
+				return Certificate();
+			else return Certificate(cm->second);
+		}
+
+		/**
+		 * @return True if this is an open non-access-controlled network
+		 */
+		inline bool isOpen() const
+		{
+			return (get("isOpen","0") == "1");
+		}
+
+		/**
+		 * @return All static addresses / netmasks, IPv4 or IPv6
+		 */
+		inline std::set<InetAddress> staticAddresses() const
+		{
+			std::set<InetAddress> sa;
+			std::vector<std::string> ips(Utils::split(get("ipv4Static","").c_str(),",","",""));
+			for(std::vector<std::string>::const_iterator i(ips.begin());i!=ips.end();++i)
+				sa.insert(InetAddress(*i));
+			ips = Utils::split(get("ipv6Static","").c_str(),",","","");
+			for(std::vector<std::string>::const_iterator i(ips.begin());i!=ips.end();++i)
+				sa.insert(InetAddress(*i));
+			return sa;
+		}
+	};
+
+	/**
+	 * Status for networks
+	 */
+	enum Status
+	{
+		NETWORK_WAITING_FOR_FIRST_AUTOCONF,
+		NETWORK_OK,
+		NETWORK_ACCESS_DENIED
+	};
+
+	/**
+	 * @param s Status
+	 * @return String description
+	 */
+	static const char *statusString(const Status s)
+		throw();
+
+private:
+	// Only NodeConfig can create, only SharedPtr can delete
+
+	// Actual construction happens in newInstance()
+	Network()
+		throw() :
+		_tap((EthernetTap *)0)
+	{
+	}
+
+	~Network();
+
+	/**
+	 * Create a new Network instance and restore any saved state
+	 *
+	 * If there is no saved state, a dummy .conf is created on disk to remember
+	 * this network across restarts.
+	 *
+	 * @param renv Runtime environment
+	 * @param id Network ID
+	 * @return Reference counted pointer to new network
+	 * @throws std::runtime_error Unable to create tap device or other fatal error
+	 */
+	static SharedPtr<Network> newInstance(const RuntimeEnvironment *renv,uint64_t id)
 		throw(std::runtime_error);
 
+	/**
+	 * Causes all persistent disk presence to be erased on delete
+	 */
+	inline void destroyOnDelete()
+	{
+		_destroyOnDelete = true;
+	}
+
 public:
 	/**
 	 * @return Network ID
@@ -70,70 +326,50 @@ public:
 	/**
 	 * @return Ethernet tap
 	 */
-	inline EthernetTap &tap() throw() { return _tap; }
+	inline EthernetTap &tap() throw() { return *_tap; }
 
 	/**
-	 * Get this network's members
-	 * 
-	 * If this is an open network, membership isn't relevant and this doesn't
-	 * mean much. If it's a closed network, frames will only be exchanged to/from
-	 * members.
-	 * 
-	 * @return Members of this network
+	 * @return Address of network's controlling node
 	 */
-	inline std::set<Address> members() const
-	{
-		Mutex::Lock _l(_lock);
-		return _members;
-	}
+	inline Address controller() throw() { return Address(_id >> 24); }
 
 	/**
-	 * @param addr Address to check
-	 * @return True if address is a member
+	 * @return Network ID in hexadecimal form
 	 */
-	inline bool isMember(const Address &addr) const
-		throw()
+	inline std::string toString()
 	{
-		Mutex::Lock _l(_lock);
-		return (_members.count(addr) > 0);
-	}
-
-	/**
-	 * Shortcut to check open() and then isMember()
-	 * 
-	 * @param addr Address to check
-	 * @return True if network is open or if address is a member
-	 */
-	inline bool isAllowed(const Address &addr) const
-		throw()
-	{
-		Mutex::Lock _l(_lock);
-		return ((_open)||(_members.count(addr) > 0));
+		char buf[64];
+		sprintf(buf,"%.16llx",(unsigned long long)_id);
+		return std::string(buf);
 	}
 
 	/**
 	 * @return True if network is open (no membership required)
 	 */
-	inline bool open() const
+	inline bool isOpen() const
 		throw()
 	{
-		Mutex::Lock _l(_lock);
-		return _open;
+		try {
+			Mutex::Lock _l(_lock);
+			return _configuration.isOpen();
+		} catch ( ... ) {
+			return false;
+		}
 	}
 
 	/**
-	 * Update internal multicast group set and return true if changed
+	 * Update multicast groups for this network's tap
 	 *
 	 * @return True if internal multicast group set has changed
 	 */
 	inline bool updateMulticastGroups()
 	{
 		Mutex::Lock _l(_lock);
-		return _tap.updateMulticastGroups(_multicastGroups);
+		return _tap->updateMulticastGroups(_multicastGroups);
 	}
 
 	/**
-	 * @return Latest set of multicast groups
+	 * @return Latest set of multicast groups for this network's tap
 	 */
 	inline std::set<MulticastGroup> multicastGroups() const
 	{
@@ -141,17 +377,103 @@ public:
 		return _multicastGroups;
 	}
 
-protected:
-	virtual void main()
-		throw();
+	/**
+	 * Set or update this network's configuration
+	 *
+	 * This is called by PacketDecoder when an update comes over the wire, or
+	 * internally when an old config is reloaded from disk.
+	 *
+	 * @param conf Configuration in key/value dictionary form
+	 */
+	void setConfiguration(const Config &conf);
+
+	/**
+	 * Causes this network to request an updated configuration from its master node now
+	 */
+	void requestConfiguration();
+
+	/**
+	 * Add or update a peer's membership certificate
+	 *
+	 * The certificate must already have been validated via signature checking.
+	 *
+	 * @param peer Peer that owns certificate
+	 * @param cert Certificate itself
+	 */
+	void addMembershipCertificate(const Address &peer,const Certificate &cert);
+
+	/**
+	 * @param peer Peer address to check
+	 * @return True if peer is allowed to communicate on this network
+	 */
+	bool isAllowed(const Address &peer) const;
+
+	/**
+	 * Perform cleanup and possibly save state
+	 */
+	void clean();
+
+	/**
+	 * @return Time of last updated configuration or 0 if none
+	 */
+	inline uint64_t lastConfigUpdate() const
+		throw()
+	{
+		return _lastConfigUpdate;
+	}
+
+	/**
+	 * @return Status of this network
+	 */
+	Status status() const;
+
+	/**
+	 * Invoke multicast rate limiter gate for a given address
+	 *
+	 * @param addr Address to check
+	 * @param bytes Bytes address wishes to send us / propagate
+	 * @return True if allowed, false if overshot rate limit
+	 */
+	inline bool multicastRateGate(const Address &addr,unsigned int bytes)
+	{
+		Mutex::Lock _l(_lock);
+		std::map<Address,RateLimiter>::iterator rl(_multicastRateLimiters.find(addr));
+		if (rl == _multicastRateLimiters.end()) {
+			RateLimiter &newrl = _multicastRateLimiters[addr];
+			newrl.init(ZT_MULTICAST_DEFAULT_RATE_PRELOAD);
+			return newrl.gate(_rlLimit,(double)bytes);
+		}
+		return rl->second.gate(_rlLimit,(double)bytes);
+	}
 
 private:
+	static void _CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data);
+	void _restoreState();
+
 	const RuntimeEnvironment *_r;
-	uint64_t _id;
-	EthernetTap _tap;
-	std::set<Address> _members;
+
+	// Rate limits for this network
+	RateLimiter::Limit _rlLimit;
+
+	// Tap and tap multicast memberships
+	EthernetTap *_tap;
 	std::set<MulticastGroup> _multicastGroups;
-	bool _open;
+
+	// Membership certificates supplied by peers
+	std::map<Address,Certificate> _membershipCertificates;
+
+	// Rate limiters for each multicasting peer
+	std::map<Address,RateLimiter> _multicastRateLimiters;
+
+	// Configuration from network master node
+	Config _configuration;
+	Certificate _myCertificate;
+
+	uint64_t _id;
+	volatile uint64_t _lastConfigUpdate;
+	volatile bool _destroyOnDelete;
+	volatile bool _ready;
+
 	Mutex _lock;
 
 	AtomicCounter __refCount;

node/Node.cpp

@@ -37,26 +37,19 @@
 #include <vector>
 #include <string>
 
-#ifndef _WIN32
-#include <fcntl.h>
-#include <unistd.h>
-#include <signal.h>
-#include <sys/file.h>
-#endif
-
-#include <openssl/sha.h>
-
 #include "Condition.hpp"
 #include "Node.hpp"
 #include "Topology.hpp"
 #include "Demarc.hpp"
+#include "Packet.hpp"
 #include "Switch.hpp"
 #include "Utils.hpp"
 #include "EthernetTap.hpp"
 #include "Logger.hpp"
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "Pack.hpp"
+#include "Salsa20.hpp"
+#include "HMAC.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "NodeConfig.hpp"
 #include "Defaults.hpp"
@@ -64,11 +57,121 @@
 #include "Network.hpp"
 #include "MulticastGroup.hpp"
 #include "Mutex.hpp"
+#include "Multicaster.hpp"
+#include "CMWC4096.hpp"
+#include "Service.hpp"
+
+#ifdef __WINDOWS__
+#include <Windows.h>
+#else
+#include <fcntl.h>
+#include <unistd.h>
+#include <signal.h>
+#include <sys/file.h>
+#include <sys/stat.h>
+#endif
 
 #include "../version.h"
 
 namespace ZeroTier {
 
+struct _LocalClientImpl
+{
+	unsigned char key[32];
+	UdpSocket *sock;
+	void (*resultHandler)(void *,unsigned long,const char *);
+	void *arg;
+	InetAddress localDestAddr;
+	Mutex inUseLock;
+};
+
+static void _CBlocalClientHandler(UdpSocket *sock,void *arg,const InetAddress &remoteAddr,const void *data,unsigned int len)
+{
+	_LocalClientImpl *impl = (_LocalClientImpl *)arg;
+	if (!impl)
+		return;
+	if (!impl->resultHandler)
+		return; // sanity check
+	Mutex::Lock _l(impl->inUseLock);
+
+	try {
+		unsigned long convId = 0;
+		std::vector<std::string> results;
+		if (!NodeConfig::decodeControlMessagePacket(impl->key,data,len,convId,results))
+			return;
+		for(std::vector<std::string>::iterator r(results.begin());r!=results.end();++r)
+			impl->resultHandler(impl->arg,convId,r->c_str());
+	} catch ( ... ) {}
+}
+
+Node::LocalClient::LocalClient(const char *authToken,void (*resultHandler)(void *,unsigned long,const char *),void *arg)
+	throw() :
+	_impl((void *)0)
+{
+	_LocalClientImpl *impl = new _LocalClientImpl;
+
+	UdpSocket *sock = (UdpSocket *)0;
+	for(unsigned int i=0;i<5000;++i) {
+		try {
+			sock = new UdpSocket(true,32768 + (rand() % 20000),false,&_CBlocalClientHandler,impl);
+			break;
+		} catch ( ... ) {
+			sock = (UdpSocket *)0;
+		}
+	}
+
+	// If socket fails to bind, there's a big problem like missing IPv4 stack
+	if (sock) {
+		SHA256_CTX sha;
+		SHA256_Init(&sha);
+		SHA256_Update(&sha,authToken,strlen(authToken));
+		SHA256_Final(impl->key,&sha);
+
+		impl->sock = sock;
+		impl->resultHandler = resultHandler;
+		impl->arg = arg;
+		impl->localDestAddr = InetAddress::LO4;
+		impl->localDestAddr.setPort(ZT_CONTROL_UDP_PORT);
+		_impl = impl;
+	} else delete impl;
+}
+
+Node::LocalClient::~LocalClient()
+{
+	if (_impl) {
+		((_LocalClientImpl *)_impl)->inUseLock.lock();
+		delete ((_LocalClientImpl *)_impl)->sock;
+		((_LocalClientImpl *)_impl)->inUseLock.unlock();
+		delete ((_LocalClientImpl *)_impl);
+	}
+}
+
+unsigned long Node::LocalClient::send(const char *command)
+	throw()
+{
+	if (!_impl)
+		return 0;
+	_LocalClientImpl *impl = (_LocalClientImpl *)_impl;
+	Mutex::Lock _l(impl->inUseLock);
+
+	try {
+		uint32_t convId = (uint32_t)rand();
+		if (!convId)
+			convId = 1;
+
+		std::vector<std::string> tmp;
+		tmp.push_back(std::string(command));
+		std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> > packets(NodeConfig::encodeControlMessage(impl->key,convId,tmp));
+
+		for(std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> >::iterator p(packets.begin());p!=packets.end();++p)
+			impl->sock->send(impl->localDestAddr,p->data(),p->size(),-1);
+
+		return convId;
+	} catch ( ... ) {
+		return 0;
+	}
+}
+
 struct _NodeImpl
 {
 	RuntimeEnvironment renv;
@@ -76,7 +179,6 @@ struct _NodeImpl
 	Node::ReasonForTermination reasonForTermination;
 	volatile bool started;
 	volatile bool running;
-	volatile bool updateStatusNow;
 	volatile bool terminateNow;
 
 	// Helper used to rapidly terminate from run()
@@ -92,20 +194,68 @@ struct _NodeImpl
 	}
 };
 
-Node::Node(const char *hp,const char *urlPrefix,const char *configAuthorityIdentity)
+#ifndef __WINDOWS__
+static void _netconfServiceMessageHandler(void *renv,Service &svc,const Dictionary &msg)
+{
+	if (!renv)
+		return; // sanity check
+	const RuntimeEnvironment *_r = (const RuntimeEnvironment *)renv;
+
+	try {
+		//TRACE("from netconf:\n%s",msg.toString().c_str());
+		const std::string &type = msg.get("type");
+		if (type == "netconf-response") {
+			uint64_t inRePacketId = strtoull(msg.get("requestId").c_str(),(char **)0,16);
+			uint64_t nwid = strtoull(msg.get("nwid").c_str(),(char **)0,16);
+			Address peerAddress(msg.get("peer").c_str());
+
+			if (peerAddress) {
+				if (msg.contains("error")) {
+					Packet::ErrorCode errCode = Packet::ERROR_INVALID_REQUEST;
+					const std::string &err = msg.get("error");
+					if (err == "NOT_FOUND")
+						errCode = Packet::ERROR_NOT_FOUND;
+
+					Packet outp(peerAddress,_r->identity.address(),Packet::VERB_ERROR);
+					outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
+					outp.append(inRePacketId);
+					outp.append((unsigned char)errCode);
+					outp.append(nwid);
+					_r->sw->send(outp,true);
+				} else if (msg.contains("netconf")) {
+					const std::string &netconf = msg.get("netconf");
+					if (netconf.length() < 2048) { // sanity check
+						Packet outp(peerAddress,_r->identity.address(),Packet::VERB_OK);
+						outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
+						outp.append(inRePacketId);
+						outp.append(nwid);
+						outp.append((uint16_t)netconf.length());
+						outp.append(netconf.data(),netconf.length());
+						outp.compress();
+						_r->sw->send(outp,true);
+					}
+				}
+			}
+		}
+	} catch (std::exception &exc) {
+		LOG("unexpected exception parsing response from netconf service: %s",exc.what());
+	} catch ( ... ) {
+		LOG("unexpected exception parsing response from netconf service: unknown exception");
+	}
+}
+#endif // !__WINDOWS__
+
+Node::Node(const char *hp)
 	throw() :
 	_impl(new _NodeImpl)
 {
 	_NodeImpl *impl = (_NodeImpl *)_impl;
 
 	impl->renv.homePath = hp;
-	impl->renv.autoconfUrlPrefix = urlPrefix;
-	impl->renv.configAuthorityIdentityStr = configAuthorityIdentity;
 
 	impl->reasonForTermination = Node::NODE_RUNNING;
 	impl->started = false;
 	impl->running = false;
-	impl->updateStatusNow = false;
 	impl->terminateNow = false;
 }
 
@@ -113,11 +263,17 @@ Node::~Node()
 {
 	_NodeImpl *impl = (_NodeImpl *)_impl;
 
+#ifndef __WINDOWS__
+	delete impl->renv.netconfService;
+#endif
+
+	delete impl->renv.nc;
 	delete impl->renv.sysEnv;
 	delete impl->renv.topology;
 	delete impl->renv.sw;
+	delete impl->renv.multicaster;
 	delete impl->renv.demarc;
-	delete impl->renv.nc;
+	delete impl->renv.prng;
 	delete impl->renv.log;
 
 	delete impl;
@@ -151,8 +307,8 @@ Node::ReasonForTermination Node::run()
 
 		TRACE("initializing...");
 
-		if (!_r->configAuthority.fromString(_r->configAuthorityIdentityStr))
-			return impl->terminateBecause(Node::NODE_UNRECOVERABLE_ERROR,"configuration authority identity is not valid");
+		// Create non-crypto PRNG right away in case other code in init wants to use it
+		_r->prng = new CMWC4096();
 
 		bool gotId = false;
 		std::string identitySecretPath(_r->homePath + ZT_PATH_SEPARATOR_S + "identity.secret");
@@ -182,38 +338,43 @@ Node::ReasonForTermination Node::run()
 		}
 		Utils::lockDownFile(identitySecretPath.c_str(),false);
 
-		// Generate ownership verification secret, which can be presented to
-		// a controlling web site (like ours) to prove ownership of a node and
-		// permit its configuration to be centrally modified. When ZeroTier One
-		// requests its config it sends a hash of this secret, and so the
-		// config server can verify this hash to determine if the secret the
-		// user presents is correct.
-		std::string ovsPath(_r->homePath + ZT_PATH_SEPARATOR_S + "thisdeviceismine");
-		if (((Utils::now() - Utils::getLastModified(ovsPath.c_str())) >= ZT_OVS_GENERATE_NEW_IF_OLDER_THAN)||(!Utils::readFile(ovsPath.c_str(),_r->ownershipVerificationSecret))) {
-			_r->ownershipVerificationSecret = "";
-			for(unsigned int i=0;i<24;++i)
-				_r->ownershipVerificationSecret.push_back("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"[Utils::randomInt<unsigned int>() % 62]);
-			_r->ownershipVerificationSecret.append(ZT_EOL_S);
-			if (!Utils::writeFile(ovsPath.c_str(),_r->ownershipVerificationSecret))
-				return impl->terminateBecause(Node::NODE_UNRECOVERABLE_ERROR,"could not write 'thisdeviceismine' (home path not writable?)");
+		// Clean up some obsolete files if present -- this will be removed later
+		Utils::rm((_r->homePath + ZT_PATH_SEPARATOR_S + "status"));
+		Utils::rm((_r->homePath + ZT_PATH_SEPARATOR_S + "thisdeviceismine"));
+
+		// Make sure networks.d exists
+		mkdir((_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str(),0700);
+
+		// Load or generate config authentication secret
+		std::string configAuthTokenPath(_r->homePath + ZT_PATH_SEPARATOR_S + "authtoken.secret");
+		std::string configAuthToken;
+		if (!Utils::readFile(configAuthTokenPath.c_str(),configAuthToken)) {
+			configAuthToken = "";
+			unsigned int sr = 0;
+			for(unsigned int i=0;i<24;++i) {
+				Utils::getSecureRandom(&sr,sizeof(sr));
+				configAuthToken.push_back("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"[sr % 62]);
+			}
+			if (!Utils::writeFile(configAuthTokenPath.c_str(),configAuthToken))
+				return impl->terminateBecause(Node::NODE_UNRECOVERABLE_ERROR,"could not write authtoken.secret (home path not writable?)");
 		}
-		Utils::lockDownFile(ovsPath.c_str(),false);
-		_r->ownershipVerificationSecret = Utils::trim(_r->ownershipVerificationSecret); // trim off CR file is saved with
-		unsigned char ovsDig[32];
-		SHA256_CTX sha;
-		SHA256_Init(&sha);
-		SHA256_Update(&sha,_r->ownershipVerificationSecret.data(),_r->ownershipVerificationSecret.length());
-		SHA256_Final(ovsDig,&sha);
-		_r->ownershipVerificationSecretHash = Utils::base64Encode(ovsDig,32);
+		Utils::lockDownFile(configAuthTokenPath.c_str(),false);
 
 		// Create the core objects in RuntimeEnvironment: node config, demarcation
 		// point, switch, network topology database, and system environment
 		// watcher.
-		_r->nc = new NodeConfig(_r,_r->autoconfUrlPrefix + _r->identity.address().toString());
 		_r->demarc = new Demarc(_r);
+		_r->multicaster = new Multicaster();
 		_r->sw = new Switch(_r);
 		_r->topology = new Topology(_r,(_r->homePath + ZT_PATH_SEPARATOR_S + "peer.db").c_str());
 		_r->sysEnv = new SysEnv(_r);
+		try {
+			_r->nc = new NodeConfig(_r,configAuthToken.c_str());
+		} catch ( ... ) {
+			// An exception here currently means that another instance of ZeroTier
+			// One is running.
+			return impl->terminateBecause(Node::NODE_UNRECOVERABLE_ERROR,"another instance of ZeroTier One appears to be running, or local control UDP port cannot be bound");
+		}
 
 		// TODO: make configurable
 		bool boundPort = false;
@@ -237,65 +398,71 @@ Node::ReasonForTermination Node::run()
 		return impl->terminateBecause(Node::NODE_UNRECOVERABLE_ERROR,"unknown exception during initialization");
 	}
 
+#ifndef __WINDOWS__
 	try {
-		std::string statusPath(_r->homePath + ZT_PATH_SEPARATOR_S + "status");
+		std::string netconfServicePath(_r->homePath + ZT_PATH_SEPARATOR_S + "services.d" + ZT_PATH_SEPARATOR_S + "netconf.service");
+		if (Utils::fileExists(netconfServicePath.c_str())) {
+			LOG("netconf.d/netconfi.service appears to exist, starting...");
+			_r->netconfService = new Service(_r,"netconf",netconfServicePath.c_str(),&_netconfServiceMessageHandler,_r);
+		}
+	} catch ( ... ) {
+		LOG("unexpected exception attempting to start services");
+	}
+#endif
 
+	try {
+		uint64_t lastNetworkAutoconfCheck = 0;
 		uint64_t lastPingCheck = 0;
-		uint64_t lastTopologyClean = Utils::now(); // don't need to do this immediately
+		uint64_t lastClean = Utils::now(); // don't need to do this immediately
 		uint64_t lastNetworkFingerprintCheck = 0;
-		uint64_t lastAutoconfigureCheck = 0;
 		uint64_t networkConfigurationFingerprint = _r->sysEnv->getNetworkConfigurationFingerprint();
 		uint64_t lastMulticastCheck = 0;
 		uint64_t lastMulticastAnnounceAll = 0;
-		uint64_t lastStatusUpdate = 0;
 		long lastDelayDelta = 0;
 
 		LOG("%s starting version %s",_r->identity.address().toString().c_str(),versionString());
 
 		while (!impl->terminateNow) {
 			uint64_t now = Utils::now();
-			bool pingAll = false; // set to true to force a ping of *all* known direct links
+			bool resynchronize = false;
 
 			// Detect sleep/wake by looking for delay loop pauses that are longer
 			// than we intended to pause.
 			if (lastDelayDelta >= ZT_SLEEP_WAKE_DETECTION_THRESHOLD) {
-				lastNetworkFingerprintCheck = 0; // force network environment check
-				lastMulticastCheck = 0; // force multicast group check on taps
-				pingAll = true;
-
+				resynchronize = true;
 				LOG("probable suspend/resume detected, pausing a moment for things to settle...");
 				Thread::sleep(ZT_SLEEP_WAKE_SETTLE_TIME);
 			}
 
 			// Periodically check our network environment, sending pings out to all
 			// our direct links if things look like we got a different address.
-			if ((now - lastNetworkFingerprintCheck) >= ZT_NETWORK_FINGERPRINT_CHECK_DELAY) {
+			if ((resynchronize)||((now - lastNetworkFingerprintCheck) >= ZT_NETWORK_FINGERPRINT_CHECK_DELAY)) {
 				lastNetworkFingerprintCheck = now;
 				uint64_t fp = _r->sysEnv->getNetworkConfigurationFingerprint();
 				if (fp != networkConfigurationFingerprint) {
-					LOG("netconf fingerprint change: %.16llx != %.16llx, pinging all peers",networkConfigurationFingerprint,fp);
+					LOG("netconf fingerprint change: %.16llx != %.16llx, resyncing with network",networkConfigurationFingerprint,fp);
 					networkConfigurationFingerprint = fp;
-					pingAll = true;
-					lastAutoconfigureCheck = 0; // check autoconf after network config change
-					lastMulticastCheck = 0; // check multicast group membership after network config change
+					resynchronize = true;
+					_r->nc->whackAllTaps(); // call whack() on all tap devices -- hack, might go away
 				}
 			}
 
-			if ((now - lastAutoconfigureCheck) >= ZT_AUTOCONFIGURE_CHECK_DELAY) {
-				// It seems odd to only do this simple check every so often, but the purpose is to
-				// delay between calls to refreshConfiguration() enough that the previous attempt
-				// has time to either succeed or fail. Otherwise we'll block the whole loop, since
-				// config update is guarded by a Mutex.
-				lastAutoconfigureCheck = now;
-				if ((now - _r->nc->lastAutoconfigure()) >= ZT_AUTOCONFIGURE_INTERVAL)
-					_r->nc->refreshConfiguration(); // happens in background
+			// Request configuration for unconfigured nets, or nets with out of date
+			// configuration information.
+			if ((resynchronize)||((now - lastNetworkAutoconfCheck) >= ZT_NETWORK_AUTOCONF_CHECK_DELAY)) {
+				lastNetworkAutoconfCheck = now;
+				std::vector< SharedPtr<Network> > nets(_r->nc->networks());
+				for(std::vector< SharedPtr<Network> >::iterator n(nets.begin());n!=nets.end();++n) {
+					if ((now - (*n)->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)
+						(*n)->requestConfiguration();
+				}
 			}
 
 			// Periodically check for changes in our local multicast subscriptions and broadcast
 			// those changes to peers.
-			if ((now - lastMulticastCheck) >= ZT_MULTICAST_LOCAL_POLL_PERIOD) {
+			if ((resynchronize)||((now - lastMulticastCheck) >= ZT_MULTICAST_LOCAL_POLL_PERIOD)) {
 				lastMulticastCheck = now;
-				bool announceAll = ((now - lastMulticastAnnounceAll) >= ZT_MULTICAST_LIKE_ANNOUNCE_ALL_PERIOD);
+				bool announceAll = ((resynchronize)||((now - lastMulticastAnnounceAll) >= ZT_MULTICAST_LIKE_ANNOUNCE_ALL_PERIOD));
 				try {
 					std::map< SharedPtr<Network>,std::set<MulticastGroup> > toAnnounce;
 					{
@@ -323,14 +490,13 @@ Node::ReasonForTermination Node::run()
 				}
 			}
 
-			if ((now - lastPingCheck) >= ZT_PING_CHECK_DELAY) {
+			if ((resynchronize)||((now - lastPingCheck) >= ZT_PING_CHECK_DELAY)) {
 				lastPingCheck = now;
 				try {
-					if (_r->topology->isSupernode(_r->identity.address())) {
-						// The only difference in how supernodes behave is here: they only
-						// actively ping each other and only passively listen for pings
-						// from anyone else. They also don't send firewall openers, since
-						// they're never firewalled.
+					if (_r->topology->amSupernode()) {
+						// Supernodes are so super they don't even have to ping out, since
+						// all nodes ping them. They're also never firewalled so they
+						// don't need firewall openers. They just ping each other.
 						std::vector< SharedPtr<Peer> > sns(_r->topology->supernodePeers());
 						for(std::vector< SharedPtr<Peer> >::const_iterator p(sns.begin());p!=sns.end();++p) {
 							if ((now - (*p)->lastDirectSend()) > ZT_PEER_DIRECT_PING_DELAY)
@@ -339,11 +505,10 @@ Node::ReasonForTermination Node::run()
 					} else {
 						std::vector< SharedPtr<Peer> > needPing,needFirewallOpener;
 
-						if (pingAll) {
-							_r->topology->eachPeer(Topology::CollectPeersWithActiveDirectPath(needPing));
+						if (resynchronize) {
+							_r->topology->eachPeer(Topology::CollectPeersWithDirectPath(needPing));
 						} else {
 							_r->topology->eachPeer(Topology::CollectPeersThatNeedPing(needPing));
-							_r->topology->eachPeer(Topology::CollectPeersThatNeedFirewallOpener(needFirewallOpener));
 						}
 
 						for(std::vector< SharedPtr<Peer> >::iterator p(needPing.begin());p!=needPing.end();++p) {
@@ -356,6 +521,7 @@ Node::ReasonForTermination Node::run()
 							}
 						}
 
+						_r->topology->eachPeer(Topology::CollectPeersThatNeedFirewallOpener(needFirewallOpener));
 						for(std::vector< SharedPtr<Peer> >::iterator p(needFirewallOpener.begin());p!=needFirewallOpener.end();++p) {
 							try {
 								(*p)->sendFirewallOpener(_r,now);
@@ -373,30 +539,17 @@ Node::ReasonForTermination Node::run()
 				}
 			}
 
-			if ((now - lastTopologyClean) >= ZT_TOPOLOGY_CLEAN_PERIOD) {
-				lastTopologyClean = now;
-				_r->topology->clean(); // happens in background
-			}
-
-			if (((now - lastStatusUpdate) >= ZT_STATUS_OUTPUT_PERIOD)||(impl->updateStatusNow)) {
-				lastStatusUpdate = now;
-				impl->updateStatusNow = false;
-				FILE *statusf = ::fopen(statusPath.c_str(),"w");
-				if (statusf) {
-					try {
-						_r->topology->eachPeer(Topology::DumpPeerStatistics(statusf));
-					} catch ( ... ) {
-						TRACE("unexpected exception updating status dump");
-					}
-					::fclose(statusf);
-				}
+			if ((now - lastClean) >= ZT_DB_CLEAN_PERIOD) {
+				lastClean = now;
+				_r->topology->clean();
+				_r->nc->clean();
 			}
 
 			try {
 				unsigned long delay = std::min((unsigned long)ZT_MIN_SERVICE_LOOP_INTERVAL,_r->sw->doTimerTasks());
 				uint64_t start = Utils::now();
-				Thread::sleep(delay);
-				lastDelayDelta = (long)(Utils::now() - start) - (long)delay;
+				_r->mainLoopWaitCondition.wait(delay);
+				lastDelayDelta = (long)(Utils::now() - start) - (long)delay; // used to detect sleep/wake
 			} catch (std::exception &exc) {
 				LOG("unexpected exception running Switch doTimerTasks: %s",exc.what());
 			} catch ( ... ) {
@@ -422,12 +575,7 @@ void Node::terminate()
 	throw()
 {
 	((_NodeImpl *)_impl)->terminateNow = true;
-}
-
-void Node::updateStatusNow()
-	throw()
-{
-	((_NodeImpl *)_impl)->updateStatusNow = true;
+	((_NodeImpl *)_impl)->renv.mainLoopWaitCondition.signal();
 }
 
 class _VersionStringMaker

node/Node.hpp

@@ -40,6 +40,44 @@ namespace ZeroTier {
 class Node
 {
 public:
+	/**
+	 * Client for controlling a local ZeroTier One node
+	 */
+	class LocalClient
+	{
+	public:
+		/**
+		 * Create a new node config client
+		 *
+		 * @param authToken Authentication token
+		 * @param resultHandler Function to call when commands provide results
+		 */
+		LocalClient(const char *authToken,void (*resultHandler)(void *,unsigned long,const char *),void *arg)
+			throw();
+
+		~LocalClient();
+
+		/**
+		 * Send a command to the local node
+		 *
+		 * Note that the returned conversation ID will never be 0. A return value
+		 * of 0 indicates a fatal error such as failure to bind to any local UDP
+		 * port.
+		 *
+		 * @param command
+		 * @return Conversation ID that will be provided to result handler when/if results are sent back
+		 */
+		unsigned long send(const char *command)
+			throw();
+
+	private:
+		// LocalClient is not copyable
+		LocalClient(const LocalClient&);
+		const LocalClient& operator=(const LocalClient&);
+
+		void *_impl;
+	};
+
 	/**
 	 * Returned by node main if/when it terminates
 	 */
@@ -58,11 +96,8 @@ public:
 	 * The node is not executed until run() is called.
 	 *
 	 * @param hp Home directory path
-	 * @param url URL prefix for autoconfiguration (http and file permitted)
-	 * @param configAuthorityIdentity Public identity used to encrypt/authenticate configuration from this URL (ASCII string format)
-	 * @throws std::invalid_argument Invalid argument supplied to constructor
 	 */
-	Node(const char *hp,const char *urlPrefix,const char *configAuthorityIdentity)
+	Node(const char *hp)
 		throw();
 
 	~Node();
@@ -98,12 +133,6 @@ public:
 	void terminate()
 		throw();
 
-	/**
-	 * Update the status file in the home directory on next service loop
-	 */
-	void updateStatusNow()
-		throw();
-
 	/**
 	 * Get the ZeroTier version in major.minor.revision string format
 	 * 
@@ -117,6 +146,10 @@ public:
 	static unsigned int versionRevision() throw();
 
 private:
+	// Nodes are not copyable
+	Node(const Node&);
+	const Node& operator=(const Node&);
+
 	void *const _impl; // private implementation
 };
 

node/NodeConfig.cpp

@@ -27,180 +27,311 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+
 #include <memory>
 #include <string>
+#include <map>
+#include <set>
 
-#include <json/json.h>
+#include <openssl/sha.h>
 
 #include "NodeConfig.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "Defaults.hpp"
 #include "Utils.hpp"
 #include "Logger.hpp"
+#include "Topology.hpp"
+#include "Demarc.hpp"
+#include "InetAddress.hpp"
+#include "Peer.hpp"
+#include "Salsa20.hpp"
+#include "HMAC.hpp"
 
 namespace ZeroTier {
 
-NodeConfig::NodeConfig(const RuntimeEnvironment *renv,const std::string &url) :
+NodeConfig::NodeConfig(const RuntimeEnvironment *renv,const char *authToken)
+	throw(std::runtime_error) :
 	_r(renv),
-	_lastAutoconfigure(0),
-	_lastAutoconfigureLastModified(),
-	_url(url),
-	_autoconfigureLock(),
-	_networks(),
-	_networks_m()
+	_controlSocket(true,ZT_CONTROL_UDP_PORT,false,&_CBcontrolPacketHandler,this)
 {
+	SHA256_CTX sha;
+	SHA256_Init(&sha);
+	SHA256_Update(&sha,authToken,strlen(authToken));
+	SHA256_Final(_controlSocketKey,&sha);
+
+	std::map<std::string,bool> networksDotD(Utils::listDirectory((_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str()));
+	std::set<uint64_t> nwids;
+	for(std::map<std::string,bool>::iterator d(networksDotD.begin());d!=networksDotD.end();++d) {
+		if (!d->second) {
+			std::string::size_type dot = d->first.rfind(".conf");
+			if (dot != std::string::npos) {
+				uint64_t nwid = strtoull(d->first.substr(0,dot).c_str(),(char **)0,16);
+				if (nwid > 0)
+					nwids.insert(nwid);
+			}
+		}
+	}
+
+	// TODO:
+	// This might go away eventually. This causes the LAN called Earth to be
+	// automatically joined if there are no other networks. This is for backward
+	// compatibility with the expectations of previous alpha users.
+	if (nwids.empty())
+		nwids.insert(0x6c92786fee000001ULL);
+
+	for(std::set<uint64_t>::iterator nwid(nwids.begin());nwid!=nwids.end();++nwid) {
+		try {
+			SharedPtr<Network> nw(Network::newInstance(_r,*nwid));
+			_networks[*nwid] = nw;
+		} catch (std::exception &exc) {
+			LOG("unable to create network %.16llx: %s",(unsigned long long)*nwid,exc.what());
+		} catch ( ... ) {
+			LOG("unable to create network %.16llx: (unknown exception)",(unsigned long long)*nwid);
+		}
+	}
 }
 
 NodeConfig::~NodeConfig()
 {
-	_autoconfigureLock.lock(); // wait for any autoconfs to finish
-	_autoconfigureLock.unlock();
 }
 
-void NodeConfig::refreshConfiguration()
+void NodeConfig::whackAllTaps()
 {
-	_autoconfigureLock.lock(); // unlocked when handler gets called
-
-	TRACE("refreshing autoconfigure URL %s (if modified since: '%s')",_url.c_str(),_lastAutoconfigureLastModified.c_str());
-
-	std::map<std::string,std::string> reqHeaders;
-	reqHeaders["X-ZT-ID"] = _r->identity.toString(false);
-	reqHeaders["X-ZT-OVSH"] = _r->ownershipVerificationSecretHash;
-	if (_lastAutoconfigureLastModified.length())
-		reqHeaders["If-Modified-Since"] = _lastAutoconfigureLastModified;
-
-	new Http::Request(Http::HTTP_METHOD_GET,_url,reqHeaders,std::string(),&NodeConfig::_CBautoconfHandler,this);
+	std::vector< SharedPtr<Network> > nwlist;
+	Mutex::Lock _l(_networks_m);
+	for(std::map< uint64_t,SharedPtr<Network> >::const_iterator n(_networks.begin());n!=_networks.end();++n)
+		n->second->tap().whack();
 }
 
-void NodeConfig::__CBautoconfHandler(const std::string &lastModified,const std::string &body)
+void NodeConfig::clean()
 {
-	try {
-		Json::Value root;
-		Json::Reader reader;
+	Mutex::Lock _l(_networks_m);
+	for(std::map< uint64_t,SharedPtr<Network> >::const_iterator n(_networks.begin());n!=_networks.end();++n)
+		n->second->clean();
+}
 
-		std::string dec(_r->identity.decrypt(_r->configAuthority,body.data(),body.length()));
-		if (!dec.length()) {
-			LOG("autoconfigure from %s failed: data did not decrypt as from config authority %s",_url.c_str(),_r->configAuthority.address().toString().c_str());
-			return;
-		}
-		TRACE("decrypted autoconf: %s",dec.c_str());
+// Macro used in execute()
+#undef _P
+#define _P(f,...) { r.push_back(std::string()); Utils::stdsprintf(r.back(),(f),##__VA_ARGS__); }
 
-		if (!reader.parse(dec,root,false)) {
-			LOG("autoconfigure from %s failed: JSON parse error: %s",_url.c_str(),reader.getFormattedErrorMessages().c_str());
-			return;
-		}
+// Used with Topology::eachPeer to dump peer stats
+class _DumpPeerStatistics
+{
+public:
+	_DumpPeerStatistics(std::vector<std::string> &out) :
+		r(out),
+		_now(Utils::now())
+	{
+	}
 
-		if (!root.isObject()) {
-			LOG("autoconfigure from %s failed: not a JSON object",_url.c_str());
-			return;
-		}
+	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
+	{
+		InetAddress v4(p->ipv4ActivePath(_now));
+		InetAddress v6(p->ipv6ActivePath(_now));
+		_P("200 listpeers %s %s %s %u %s",
+			p->address().toString().c_str(),
+			((v4) ? v4.toString().c_str() : "-"),
+			((v6) ? v6.toString().c_str() : "-"),
+			(((v4)||(v6)) ? p->latency() : 0),
+			p->remoteVersion().c_str());
+	}
 
-		// Configure networks
-		const Json::Value &networks = root["_networks"];
-		if (networks.isArray()) {
-			Mutex::Lock _l(_networks_m);
-			for(unsigned int ni=0;ni<networks.size();++ni) {
-				if (networks[ni].isObject()) {
-					const Json::Value &nwid_ = networks[ni]["id"];
-					uint64_t nwid = nwid_.isNumeric() ? (uint64_t)nwid_.asUInt64() : (uint64_t)strtoull(networks[ni]["id"].asString().c_str(),(char **)0,10);
+private:
+	std::vector<std::string> &r;
+	uint64_t _now;
+};
 
-					if (nwid) {
-						SharedPtr<Network> nw;
-						std::map< uint64_t,SharedPtr<Network> >::iterator nwent(_networks.find(nwid));
-						if (nwent != _networks.end())
-							nw = nwent->second;
-						else {
-							try {
-								nw = SharedPtr<Network>(new Network(_r,nwid));
-								_networks[nwid] = nw;
-							} catch (std::exception &exc) {
-								LOG("unable to create network %llu: %s",nwid,exc.what());
-							} catch ( ... ) {
-								LOG("unable to create network %llu: unknown exception",nwid);
-							}
-						}
+std::vector<std::string> NodeConfig::execute(const char *command)
+{
+	std::vector<std::string> r;
+	std::vector<std::string> cmd(Utils::split(command,"\r\n \t","\\","'"));
 
-						if (nw) {
-							Mutex::Lock _l2(nw->_lock);
-							nw->_open = networks[ni]["isOpen"].asBool();
+	//
+	// Not coincidentally, response type codes correspond with HTTP
+	// status codes.
+	//
 
-							// Ensure that TAP device has all the right IP addresses
-							// TODO: IPv6 might work a tad differently
-							std::set<InetAddress> allIps;
-							const Json::Value &addresses = networks[ni]["_addresses"];
-							if (addresses.isArray()) {
-								for(unsigned int ai=0;ai<addresses.size();++ai) {
-									if (addresses[ai].isString()) {
-										InetAddress addr(addresses[ai].asString());
-										if (addr) {
-											TRACE("network %llu IP/netmask: %s",nwid,addr.toString().c_str());
-											allIps.insert(addr);
-										}
-									}
-								}
-							}
-							nw->_tap.setIps(allIps);
-
-							// NOTE: the _members field is optional for open networks,
-							// since members of open nets do not need to check membership
-							// of packet senders and mutlicasters.
-							const Json::Value &members = networks[ni]["_members"];
-							nw->_members.clear();
-							if (members.isArray()) {
-								for(unsigned int mi=0;mi<members.size();++mi) {
-									std::string rawAddr(Utils::unhex(members[mi].asString()));
-									if (rawAddr.length() == ZT_ADDRESS_LENGTH) {
-										Address addr(rawAddr.data());
-										if ((addr)&&(!addr.isReserved())) {
-											TRACE("network %llu member: %s",nwid,addr.toString().c_str());
-											nw->_members.insert(addr);
-										}
-									}
-								}
-							}
-						}
-					} else {
-						TRACE("ignored networks[%u], 'id' field missing");
-					}
-				} else {
-					TRACE("ignored networks[%u], not a JSON object",ni);
-				}
+	if ((cmd.empty())||(cmd[0] == "help")) {
+		_P("200 help help");
+		_P("200 help listpeers");
+		_P("200 help listnetworks");
+		_P("200 help join <network ID>");
+		_P("200 help leave <network ID>");
+	} else if (cmd[0] == "listpeers") {
+		_P("200 listpeers <ztaddr> <ipv4> <ipv6> <latency> <version>");
+		_r->topology->eachPeer(_DumpPeerStatistics(r));
+	} else if (cmd[0] == "listnetworks") {
+		Mutex::Lock _l(_networks_m);
+		_P("200 listnetworks <nwid> <status> <type> <dev> <ips>");
+		for(std::map< uint64_t,SharedPtr<Network> >::const_iterator nw(_networks.begin());nw!=_networks.end();++nw) {
+			std::string tmp;
+			std::set<InetAddress> ips(nw->second->tap().ips());
+			for(std::set<InetAddress>::iterator i(ips.begin());i!=ips.end();++i) {
+				if (tmp.length())
+					tmp.push_back(',');
+				tmp.append(i->toString());
 			}
+			// TODO: display network status, such as "permission denied to closed
+			// network" or "waiting".
+			_P("200 listnetworks %.16llx %s %s %s %s",
+				(unsigned long long)nw->first,
+				Network::statusString(nw->second->status()),
+				(nw->second->isOpen() ? "open" : "private"),
+				nw->second->tap().deviceName().c_str(),
+				((tmp.length() > 0) ? tmp.c_str() : "-"));
+		}
+	} else if (cmd[0] == "join") {
+		if (cmd.size() > 1) {
+			uint64_t nwid = strtoull(cmd[1].c_str(),(char **)0,16);
+			if (nwid > 0) {
+				Mutex::Lock _l(_networks_m);
+				if (_networks.count(nwid)) {
+					_P("400 already a member of %.16llx",(unsigned long long)nwid);
+				} else {
+					try {
+						SharedPtr<Network> nw(Network::newInstance(_r,nwid));
+						_networks[nwid] = nw;
+						_P("200 join %.16llx OK",(unsigned long long)nwid);
+					} catch (std::exception &exc) {
+						_P("500 join %.16llx ERROR: %s",(unsigned long long)nwid,exc.what());
+					} catch ( ... ) {
+						_P("500 join %.16llx ERROR: (unknown exception)",(unsigned long long)nwid);
+					}
+				}
+			} else {
+				_P("400 join requires a network ID (>0) in hexadecimal format");
+			}
+		} else {
+			_P("400 join requires a network ID (>0) in hexadecimal format");
+		}
+	} else if (cmd[0] == "leave") {
+		if (cmd.size() > 1) {
+			Mutex::Lock _l(_networks_m);
+			uint64_t nwid = strtoull(cmd[1].c_str(),(char **)0,16);
+			std::map< uint64_t,SharedPtr<Network> >::iterator nw(_networks.find(nwid));
+			if (nw == _networks.end()) {
+				_P("404 leave %.16llx ERROR: not a member of that network",(unsigned long long)nwid);
+			} else {
+				nw->second->destroyOnDelete();
+				_networks.erase(nw);
+			}
+		} else {
+			_P("400 leave requires a network ID (>0) in hexadecimal format");
+		}
+	} else {
+		_P("404 %s No such command. Use 'help' for help.",cmd[0].c_str());
+	}
+
+	r.push_back(std::string()); // terminate with empty line
+
+	return r;
+}
+
+std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> > NodeConfig::encodeControlMessage(const void *key,unsigned long conversationId,const std::vector<std::string> &payload)
+	throw(std::out_of_range)
+{
+	char hmac[32];
+	char keytmp[32];
+	std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> > packets;
+	Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> packet;
+
+	packet.setSize(16); // room for HMAC and IV
+	packet.append((uint32_t)(conversationId & 0xffffffff));
+
+	for(unsigned int i=0;i<payload.size();++i) {
+		packet.append(payload[i]); // will throw if too big
+		packet.append((unsigned char)0);
+
+		if (((i + 1) >= payload.size())||((packet.size() + payload[i + 1].length() + 1) >= packet.capacity())) {
+			Utils::getSecureRandom(packet.field(8,8),8);
+
+			Salsa20 s20(key,256,packet.field(8,8));
+			s20.encrypt(packet.field(16,packet.size() - 16),packet.field(16,packet.size() - 16),packet.size() - 16);
+
+			memcpy(keytmp,key,32);
+			for(unsigned int i=0;i<32;++i)
+				keytmp[i] ^= 0x77; // use a different permutation of key for HMAC than for Salsa20
+			HMAC::sha256(keytmp,32,packet.field(16,packet.size() - 16),packet.size() - 16,hmac);
+			memcpy(packet.field(0,8),hmac,8);
+
+			packets.push_back(packet);
+
+			packet.setSize(16); // room for HMAC and IV
+			packet.append((uint32_t)(conversationId & 0xffffffff));
+		}
+	}
+
+	return packets;
+}
+
+bool NodeConfig::decodeControlMessagePacket(const void *key,const void *data,unsigned int len,unsigned long &conversationId,std::vector<std::string> &payload)
+{
+	char hmac[32];
+	char keytmp[32];
+
+	try {
+		if (len < 20)
+			return false;
+
+		Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> packet(data,len);
+
+		memcpy(keytmp,key,32);
+		for(unsigned int i=0;i<32;++i)
+			keytmp[i] ^= 0x77; // use a different permutation of key for HMAC than for Salsa20
+		HMAC::sha256(keytmp,32,packet.field(16,packet.size() - 16),packet.size() - 16,hmac);
+		if (memcmp(packet.field(0,8),hmac,8))
+			return false;
+
+		Salsa20 s20(key,256,packet.field(8,8));
+		s20.decrypt(packet.field(16,packet.size() - 16),packet.field(16,packet.size() - 16),packet.size() - 16);
+
+		conversationId = packet.at<uint32_t>(16);
+
+		const char *pl = ((const char *)packet.data()) + 20;
+		unsigned int pll = packet.size() - 20;
+		for(unsigned int i=0;i<pll;) {
+			unsigned int eos = i;
+			while ((eos < pll)&&(pl[eos]))
+				++eos;
+			if (eos >= i) {
+				payload.push_back(std::string(pl + i,eos - i));
+				i = eos + 1;
+			} else break;
 		}
 
-		_lastAutoconfigure = Utils::now();
-		_lastAutoconfigureLastModified = lastModified;
-	} catch (std::exception &exc) {
-		TRACE("exception parsing autoconf URL response: %s",exc.what());
+		return true;
 	} catch ( ... ) {
-		TRACE("unexpected exception parsing autoconf URL response");
+		return false;
 	}
 }
 
-bool NodeConfig::_CBautoconfHandler(Http::Request *req,void *arg,const std::string &url,int code,const std::map<std::string,std::string> &headers,const std::string &body)
+void NodeConfig::_CBcontrolPacketHandler(UdpSocket *sock,void *arg,const InetAddress &remoteAddr,const void *data,unsigned int len)
 {
-#ifdef ZT_TRACE
-	const RuntimeEnvironment *_r = ((NodeConfig *)arg)->_r;
-#endif
+	NodeConfig *nc = (NodeConfig *)arg;
+	const RuntimeEnvironment *_r = nc->_r;
 
-	if (code == 200) {
-		TRACE("200 got autoconfigure response from %s: %u bytes",url.c_str(),(unsigned int)body.length());
+	try {
+		unsigned long convId = 0;
+		std::vector<std::string> commands;
 
-		std::map<std::string,std::string>::const_iterator lm(headers.find("Last-Modified"));
-		if (lm != headers.end())
-			((NodeConfig *)arg)->__CBautoconfHandler(lm->second,body);
-		else ((NodeConfig *)arg)->__CBautoconfHandler(std::string(),body);
-	} else if (code == 304) {
-		TRACE("304 autoconfigure deferred, remote URL %s not modified",url.c_str());
-		((NodeConfig *)arg)->_lastAutoconfigure = Utils::now(); // still considered a success
-	} else if (code == 409) { // conflict, ID address in use by another ID
-		TRACE("%d autoconfigure failed from %s",code,url.c_str());
-	} else {
-		TRACE("%d autoconfigure failed from %s",code,url.c_str());
+		if (!decodeControlMessagePacket(nc->_controlSocketKey,data,len,convId,commands)) {
+			TRACE("control bus packet from %s failed decode, discarded",remoteAddr.toString().c_str());
+			return;
+		}
+		TRACE("control bus packet from %s, contains %d commands",remoteAddr.toString().c_str(),(int)commands.size());
+
+		for(std::vector<std::string>::iterator c(commands.begin());c!=commands.end();++c) {
+			std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> > resultPackets(encodeControlMessage(nc->_controlSocketKey,convId,nc->execute(c->c_str())));
+			for(std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> >::iterator p(resultPackets.begin());p!=resultPackets.end();++p)
+				sock->send(remoteAddr,p->data(),p->size(),-1);
+		}
+	} catch (std::exception &exc) {
+		TRACE("exception handling control bus packet from %s: %s",remoteAddr.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("exception handling control bus packet from %s: (unknown)",remoteAddr.toString().c_str());
 	}
-
-	((NodeConfig *)arg)->_autoconfigureLock.unlock();
-	return false; // causes Request to delete itself
 }
 
 } // namespace ZeroTier

node/NodeConfig.hpp

@@ -31,27 +31,38 @@
 #include <map>
 #include <set>
 #include <string>
+#include <vector>
+#include <stdexcept>
+
 #include <stdint.h>
 #include "SharedPtr.hpp"
 #include "Network.hpp"
 #include "Utils.hpp"
-#include "Http.hpp"
+#include "UdpSocket.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
 class RuntimeEnvironment;
 
 /**
- * Node configuration holder and fetcher
+ * Maximum size of a packet for node configuration
+ */
+#define ZT_NODECONFIG_MAX_PACKET_SIZE 4096
+
+/**
+ * Node configuration endpoint
  */
 class NodeConfig
 {
 public:
 	/**
 	 * @param renv Runtime environment
-	 * @param url Autoconfiguration URL (http:// or file://)
+	 * @param authToken Configuration authentication token
+	 * @throws std::runtime_error Unable to bind to local control port
 	 */
-	NodeConfig(const RuntimeEnvironment *renv,const std::string &url);
+	NodeConfig(const RuntimeEnvironment *renv,const char *authToken)
+		throw(std::runtime_error);
 
 	~NodeConfig();
 
@@ -78,6 +89,16 @@ public:
 		return nwlist;
 	}
 
+	/**
+	 * Call whack() on all networks' tap devices
+	 */
+	void whackAllTaps();
+
+	/**
+	 * Perform cleanup and possibly update saved state
+	 */
+	void clean();
+
 	/**
 	 * @param nwid Network ID
 	 * @return True if this network exists
@@ -101,32 +122,49 @@ public:
 	}
 
 	/**
-	 * @return Time of last successful autoconfigure or refresh
+	 * Execute a command
+	 *
+	 * @param command Command and arguments separated by whitespace (must already be trimmed of CR+LF, etc.)
+	 * @return One or more command results (lines of output)
 	 */
-	inline uint64_t lastAutoconfigure() const { return _lastAutoconfigure; }
+	std::vector<std::string> execute(const char *command);
 
 	/**
-	 * @return Autoconfiguration URL
+	 * Armor payload for control bus
+	 *
+	 * Note that no single element of payload can be longer than the max packet
+	 * size. If this occurs out_of_range is thrown.
+	 *
+	 * @param key 32 byte key
+	 * @param conversationId 32-bit conversation ID (bits beyond 32 are ignored)
+	 * @param payload One or more strings to encode in packet
+	 * @return One or more transport armored packets (if payload too big)
+	 * @throws std::out_of_range An element of payload is too big
 	 */
-	inline const std::string &url() const { return _url; }
+	static std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> > encodeControlMessage(const void *key,unsigned long conversationId,const std::vector<std::string> &payload)
+		throw(std::out_of_range);
 
 	/**
-	 * Refresh configuration from autoconf URL
+	 * Decode a packet from the control bus
+	 *
+	 * Note that 'payload' is appended to. Existing data is not cleared.
+	 *
+	 * @param key 32 byte key
+	 * @param data Packet data
+	 * @param len Packet length
+	 * @param conversationId Result parameter filled with conversation ID on success
+	 * @param payload Result parameter to which results are appended
+	 * @return True on success, false on invalid packet or packet that failed authentication
 	 */
-	void refreshConfiguration();
+	static bool decodeControlMessagePacket(const void *key,const void *data,unsigned int len,unsigned long &conversationId,std::vector<std::string> &payload);
 
 private:
-	void __CBautoconfHandler(const std::string &lastModified,const std::string &body);
-	static bool _CBautoconfHandler(Http::Request *req,void *arg,const std::string &url,int code,const std::map<std::string,std::string> &headers,const std::string &body);
+	static void _CBcontrolPacketHandler(UdpSocket *sock,void *arg,const InetAddress &remoteAddr,const void *data,unsigned int len);
 
 	const RuntimeEnvironment *_r;
 
-	volatile uint64_t _lastAutoconfigure;
-
-	std::string _lastAutoconfigureLastModified;
-	std::string _url;
-	Mutex _autoconfigureLock;
-
+	unsigned char _controlSocketKey[32];
+	UdpSocket _controlSocket;
 	std::map< uint64_t,SharedPtr<Network> > _networks;
 	Mutex _networks_m;
 };

node/Pack.cpp

@@ -1,159 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include <iostream>
-#include <string.h>
-#include <stdlib.h>
-#include "Pack.hpp"
-#include "BlobArray.hpp"
-#include "Utils.hpp"
-
-#include <openssl/sha.h>
-
-namespace ZeroTier {
-
-std::vector<const Pack::Entry *> Pack::getAll() const
-{
-	std::vector<const Entry *> v;
-	for(std::map<std::string,Entry>::const_iterator e=_entries.begin();e!=_entries.end();++e)
-		v.push_back(&(e->second));
-	return v;
-}
-
-const Pack::Entry *Pack::get(const std::string &name) const
-{
-	std::map<std::string,Entry>::const_iterator e(_entries.find(name));
-	return ((e == _entries.end()) ? (const Entry *)0 : &(e->second));
-}
-
-const Pack::Entry *Pack::put(const std::string &name,const std::string &content)
-{
-	SHA256_CTX sha;
-
-	Pack::Entry &e = _entries[name];
-	e.name = name;
-	e.content = content;
-
-	SHA256_Init(&sha);
-	SHA256_Update(&sha,content.data(),content.length());
-	SHA256_Final(e.sha256,&sha);
-
-	e.signedBy.zero();
-	e.signature.assign((const char *)0,0);
-
-	return &e;
-}
-
-void Pack::clear()
-{
-	_entries.clear();
-}
-
-std::string Pack::serialize() const
-{
-	BlobArray archive;
-	for(std::map<std::string,Entry>::const_iterator e=_entries.begin();e!=_entries.end();++e) {
-		BlobArray entry;
-		entry.push_back(e->second.name);
-		entry.push_back(e->second.content);
-		entry.push_back(std::string((const char *)e->second.sha256,sizeof(e->second.sha256)));
-		entry.push_back(std::string((const char *)e->second.signedBy.data(),e->second.signedBy.size()));
-		entry.push_back(e->second.signature);
-		archive.push_back(entry.serialize());
-	}
-
-	std::string ser(archive.serialize());
-	std::string comp;
-	Utils::compress(ser.begin(),ser.end(),Utils::StringAppendOutput(comp));
-	return comp;
-}
-
-bool Pack::deserialize(const void *sd,unsigned int sdlen)
-{
-	unsigned char dig[32];
-	SHA256_CTX sha;
-
-	std::string decomp;
-	if (!Utils::decompress(((const char *)sd),((const char *)sd) + sdlen,Utils::StringAppendOutput(decomp)))
-		return false;
-
-	BlobArray archive;
-	archive.deserialize(decomp.data(),decomp.length());
-	clear();
-	for(BlobArray::const_iterator i=archive.begin();i!=archive.end();++i) {
-		BlobArray entry;
-		entry.deserialize(i->data(),i->length());
-
-		if (entry.size() != 5) return false;
-		if (entry[2].length() != 32) return false; // SHA-256
-		if (entry[3].length() != ZT_ADDRESS_LENGTH) return false; // Address
-
-		Pack::Entry &e = _entries[entry[0]];
-		e.name = entry[0];
-		e.content = entry[1];
-
-		SHA256_Init(&sha);
-		SHA256_Update(&sha,e.content.data(),e.content.length());
-		SHA256_Final(dig,&sha);
-		if (memcmp(dig,entry[2].data(),32)) return false; // integrity check failed
-		memcpy(e.sha256,dig,32);
-
-		if (entry[3].length() == ZT_ADDRESS_LENGTH)
-			e.signedBy = entry[3].data();
-		else e.signedBy.zero();
-		e.signature = entry[4];
-	}
-	return true;
-}
-
-bool Pack::signAll(const Identity &id)
-{
-	for(std::map<std::string,Entry>::iterator e=_entries.begin();e!=_entries.end();++e) {
-		e->second.signedBy = id.address();
-		e->second.signature = id.sign(e->second.sha256);
-		if (!e->second.signature.length())
-			return false;
-	}
-	return true;
-}
-
-std::vector<const Pack::Entry *> Pack::verifyAll(const Identity &id,bool mandatory) const
-{
-	std::vector<const Entry *> bad;
-	for(std::map<std::string,Entry>::const_iterator e=_entries.begin();e!=_entries.end();++e) {
-		if ((e->second.signedBy)&&(e->second.signature.length())) {
-			if (id.address() != e->second.signedBy)
-				bad.push_back(&(e->second));
-			else if (!id.verifySignature(e->second.sha256,e->second.signature.data(),e->second.signature.length()))
-				bad.push_back(&(e->second));
-		} else if (mandatory)
-			bad.push_back(&(e->second));
-	}
-	return bad;
-}
-
-} // namespace ZeroTier

node/Pack.hpp

@@ -1,141 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef _ZT_PACK_HPP
-#define _ZT_PACK_HPP
-
-#include <string>
-#include <map>
-#include <list>
-#include <stdexcept>
-#include "Address.hpp"
-#include "Identity.hpp"
-
-namespace ZeroTier {
-
-/**
- * A very simple archive format for distributing packs of files or resources
- * 
- * This is used for things like the auto-updater. It's not suitable for huge
- * files, since at present it must work in memory. Packs support signing with
- * identities and signature verification.
- */
-class Pack
-{
-public:
-	/**
-	 * Pack entry structure for looking up deserialized entries
-	 */
-	struct Entry
-	{
-		std::string name;
-		std::string content;
-		unsigned char sha256[32];
-		Address signedBy;
-		std::string signature;
-	};
-
-	Pack() {}
-	~Pack() {}
-
-	/**
-	 * @return Vector of all entries
-	 */
-	std::vector<const Entry *> getAll() const;
-
-	/**
-	 * Look up an entry
-	 * 
-	 * @param name Name to look up
-	 * @return Pointer to entry if it exists or NULL if not found
-	 */
-	const Entry *get(const std::string &name) const;
-
-	/**
-	 * Add an entry to this pack
-	 * 
-	 * @param name Entry to add
-	 * @param content Entry's contents
-	 * @return The new entry
-	 */
-	const Entry *put(const std::string &name,const std::string &content);
-
-	/**
-	 * Remove all entries
-	 */
-	void clear();
-
-	/**
-	 * @return Number of entries in pack
-	 */
-	inline unsigned int numEntries() const { return (unsigned int)_entries.size(); }
-
-	/**
-	 * Serialize this pack
-	 * 
-	 * @return Serialized form (compressed with LZ4)
-	 */
-	std::string serialize() const;
-
-	/**
-	 * Deserialize this pack
-	 * 
-	 * Any current contents are lost. This does not verify signatures,
-	 * but does check SHA256 hashes for entry integrity. If the return
-	 * value is false, the pack's contents are undefined.
-	 * 
-	 * @param sd Serialized data
-	 * @param sdlen Length of serialized data
-	 * @return True on success, false on deserialization error
-	 */
-	bool deserialize(const void *sd,unsigned int sdlen);
-	inline bool deserialize(const std::string &sd) { return deserialize(sd.data(),sd.length()); }
-
-	/**
-	 * Sign all entries in this pack with a given identity
-	 * 
-	 * @param id Identity to sign with
-	 * @return True on signature success, false if error
-	 */
-	bool signAll(const Identity &id);
-
-	/**
-	 * Verify all signed entries
-	 * 
-	 * @param id Identity to verify against
-	 * @param mandatory If true, require that all entries be signed and fail if no signature
-	 * @return Vector of entries that failed verification or empty vector if all passed
-	 */
-	std::vector<const Entry *> verifyAll(const Identity &id,bool mandatory) const;
-
-private:
-	std::map<std::string,Entry> _entries;
-};
-
-} // namespace ZeroTier
-
-#endif

node/Packet.cpp

@@ -42,6 +42,9 @@ const char *Packet::verbString(Verb v)
 		case VERB_FRAME: return "FRAME";
 		case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME";
 		case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE";
+		case VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return "NETWORK_MEMBERSHIP_CERTIFICATE";
+		case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST";
+		case VERB_NETWORK_CONFIG_REFRESH: return "NETWORK_CONFIG_REFRESH";
 	}
 	return "(unknown)";
 }
@@ -57,6 +60,7 @@ const char *Packet::errorString(ErrorCode e)
 		case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION";
 		case ERROR_IDENTITY_INVALID: return "IDENTITY_INVALID";
 		case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION";
+		case ERROR_NO_MEMBER_CERTIFICATE: return "NO_MEMBER_CERTIFICATE";
 	}
 	return "(unknown)";
 }

node/Packet.hpp

@@ -45,8 +45,13 @@
 
 /**
  * Protocol version
+ *
+ * 1 - 0.2.0 ... 0.2.5
+ * 2 - 0.3.0 ...
+ *   * Added signature and originating peer to multicast frame
+ *   * Double size of multicast frame bloom filter
  */
-#define ZT_PROTO_VERSION 1
+#define ZT_PROTO_VERSION 2
 
 /**
  * Maximum hop count allowed by packet structure (3 bits, 0-7)
@@ -122,45 +127,70 @@
  */
 #define ZT_PROTO_MIN_FRAGMENT_LENGTH ZT_PACKET_FRAGMENT_IDX_PAYLOAD
 
-#define ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE 32
+// Size of bloom filter used in multicast propagation graph exploration
+#define ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BITS 512
+#define ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BYTES 64
+
+// Field incides for parsing verbs -------------------------------------------
 
-// Field incides for parsing verbs
 #define ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION (ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION + 1)
 #define ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION (ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION + 1)
 #define ZT_PROTO_VERB_HELLO_IDX_REVISION (ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION + 1)
 #define ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP (ZT_PROTO_VERB_HELLO_IDX_REVISION + 2)
 #define ZT_PROTO_VERB_HELLO_IDX_IDENTITY (ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP + 8)
+
 #define ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID (ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB + 1)
 #define ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE (ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID + 8)
 #define ZT_PROTO_VERB_ERROR_IDX_PAYLOAD (ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE + 1)
+
 #define ZT_PROTO_VERB_OK_IDX_IN_RE_VERB (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID (ZT_PROTO_VERB_OK_IDX_IN_RE_VERB + 1)
 #define ZT_PROTO_VERB_OK_IDX_PAYLOAD (ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID + 8)
+
 #define ZT_PROTO_VERB_WHOIS_IDX_ZTADDRESS (ZT_PACKET_IDX_PAYLOAD)
+
 #define ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT (ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS + 5)
 #define ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN (ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT + 2)
 #define ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS (ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN + 1)
+
 #define ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID + 8)
 #define ZT_PROTO_VERB_FRAME_IDX_PAYLOAD (ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE + 2)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_MULTICAST_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID + 8)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ADI (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_MULTICAST_MAC + 6)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_BLOOM (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ADI + 4)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_HOPS (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_BLOOM + ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_LOAD_FACTOR (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_HOPS + 1)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FROM_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_LOAD_FACTOR + 2)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FROM_MAC + 6)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + 2)
 
-// Field indices for parsing OK and ERROR payloads of replies
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS (ZT_PACKET_IDX_PAYLOAD)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SUBMITTER_ADDRESS (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID + 8)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SUBMITTER_ADDRESS + 5)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DESTINATION_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC + 6)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ADI (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DESTINATION_MAC + 6)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_BLOOM_FILTER (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ADI + 4)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_HOP_COUNT (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_BLOOM_FILTER + 64)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_HOP_COUNT + 1)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD_LENGTH (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + 2)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SIGNATURE_LENGTH (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD_LENGTH + 2)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SIGNATURE_LENGTH + 2)
+
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID + 8)
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN + 2)
+
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REFRESH_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
+
 #define ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP (ZT_PROTO_VERB_OK_IDX_PAYLOAD)
+
 #define ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY (ZT_PROTO_VERB_OK_IDX_PAYLOAD)
+
 #define ZT_PROTO_VERB_WHOIS__ERROR__IDX_ZTADDRESS (ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)
 
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID (ZT_PROTO_VERB_OK_IDX_PAYLOAD)
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID + 8)
+#define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN + 2)
+
+// ---------------------------------------------------------------------------
+
 namespace ZeroTier {
 
 /**
@@ -263,13 +293,13 @@ public:
 			setSize(fragLen + ZT_PROTO_MIN_FRAGMENT_LENGTH);
 
 			// NOTE: this copies both the IV/packet ID and the destination address.
-			memcpy(_b + ZT_PACKET_FRAGMENT_IDX_PACKET_ID,p.data() + ZT_PACKET_IDX_IV,13);
+			memcpy(field(ZT_PACKET_FRAGMENT_IDX_PACKET_ID,13),p.data() + ZT_PACKET_IDX_IV,13);
 
-			_b[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] = ZT_PACKET_FRAGMENT_INDICATOR;
-			_b[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO] = (char)(((fragTotal & 0xf) << 4) | (fragNo & 0xf));
-			_b[ZT_PACKET_FRAGMENT_IDX_HOPS] = 0;
+			(*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] = ZT_PACKET_FRAGMENT_INDICATOR;
+			(*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO] = (char)(((fragTotal & 0xf) << 4) | (fragNo & 0xf));
+			(*this)[ZT_PACKET_FRAGMENT_IDX_HOPS] = 0;
 
-			memcpy(_b + ZT_PACKET_FRAGMENT_IDX_PAYLOAD,p.data() + fragStart,fragLen);
+			memcpy(field(ZT_PACKET_FRAGMENT_IDX_PAYLOAD,fragLen),p.data() + fragStart,fragLen);
 		}
 
 		/**
@@ -277,12 +307,12 @@ public:
 		 * 
 		 * @return Destination ZT address
 		 */
-		inline Address destination() const { return Address(_b + ZT_PACKET_FRAGMENT_IDX_DEST); }
+		inline Address destination() const { return Address(field(ZT_PACKET_FRAGMENT_IDX_DEST,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
 
 		/**
 		 * @return True if fragment is of a valid length
 		 */
-		inline bool lengthValid() const { return (_l >= ZT_PACKET_FRAGMENT_IDX_PAYLOAD); }
+		inline bool lengthValid() const { return (size() >= ZT_PACKET_FRAGMENT_IDX_PAYLOAD); }
 
 		/**
 		 * @return ID of packet this is a fragment of
@@ -292,36 +322,38 @@ public:
 		/**
 		 * @return Total number of fragments in packet
 		 */
-		inline unsigned int totalFragments() const { return (((unsigned int)_b[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO] >> 4) & 0xf); }
+		inline unsigned int totalFragments() const { return (((unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO]) >> 4) & 0xf); }
 
 		/**
 		 * @return Fragment number of this fragment
 		 */
-		inline unsigned int fragmentNumber() const { return ((unsigned int)_b[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO] & 0xf); }
+		inline unsigned int fragmentNumber() const { return ((unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_NO]) & 0xf); }
 
 		/**
 		 * @return Fragment ZT hop count
 		 */
-		inline unsigned int hops() const { return (unsigned int)_b[ZT_PACKET_FRAGMENT_IDX_HOPS]; }
+		inline unsigned int hops() const { return (unsigned int)((*this)[ZT_PACKET_FRAGMENT_IDX_HOPS]); }
 
 		/**
 		 * Increment this packet's hop count
 		 */
 		inline void incrementHops()
 		{
-			_b[ZT_PACKET_FRAGMENT_IDX_HOPS] = (_b[ZT_PACKET_FRAGMENT_IDX_HOPS] + 1) & ZT_PROTO_MAX_HOPS;
+			(*this)[ZT_PACKET_FRAGMENT_IDX_HOPS] = (((*this)[ZT_PACKET_FRAGMENT_IDX_HOPS]) + 1) & ZT_PROTO_MAX_HOPS;
 		}
 
-		/**
-		 * @return Fragment payload
-		 */
-		inline unsigned char *payload() { return (unsigned char *)(_b + ZT_PACKET_FRAGMENT_IDX_PAYLOAD); }
-		inline const unsigned char *payload() const { return (const unsigned char *)(_b + ZT_PACKET_FRAGMENT_IDX_PAYLOAD); }
-
 		/**
 		 * @return Length of payload in bytes
 		 */
-		inline unsigned int payloadLength() const { return ((_l > ZT_PACKET_FRAGMENT_IDX_PAYLOAD) ? (_l - ZT_PACKET_FRAGMENT_IDX_PAYLOAD) : 0); }
+		inline unsigned int payloadLength() const { return ((size() > ZT_PACKET_FRAGMENT_IDX_PAYLOAD) ? (size() - ZT_PACKET_FRAGMENT_IDX_PAYLOAD) : 0); }
+
+		/**
+		 * @return Raw packet payload
+		 */
+		inline const unsigned char *payload() const
+		{
+			return field(ZT_PACKET_FRAGMENT_IDX_PAYLOAD,size() - ZT_PACKET_FRAGMENT_IDX_PAYLOAD);
+		}
 	};
 
 	/**
@@ -411,20 +443,8 @@ public:
 		 */
 		VERB_FRAME = 6,
 
-		/* A multicast frame:
-		 *   <[8] 64-bit network ID>
-		 *   <[6] destination multicast Ethernet address>
-		 *   <[4] multicast additional distinguishing information (ADI)>
-		 *   <[32] multicast propagation bloom filter>
-		 *   <[1] 8-bit strict propagation hop count>
-		 *   <[2] 16-bit average peer multicast bandwidth load>
-		 *   <[6] source Ethernet address>
-		 *   <[2] 16-bit ethertype>
-		 *   <[...] ethernet payload>
-		 *
-		 * No OK or ERROR is generated.
-		 */
-		VERB_MULTICAST_FRAME = 7,
+		/* 7 - old VERB_MULTICAST_FRAME, might be reused once all old 0.2
+		 * clients are off the net. */
 
 		/* Announce interest in multicast group(s):
 		 *   <[8] 64-bit network ID>
@@ -434,7 +454,86 @@ public:
 		 *
 		 * OK is generated on successful receipt.
 		 */
-		VERB_MULTICAST_LIKE = 8
+		VERB_MULTICAST_LIKE = 8,
+
+		/* A multicast frame:
+		 *   <[1] flags, currently unused and must be 0>
+		 *   <[8] 64-bit network ID>
+		 *   <[5] ZeroTier address of original submitter of this multicast>
+		 *   <[6] source MAC address>
+		 *   <[6] destination multicast Ethernet address>
+		 *   <[4] multicast additional distinguishing information (ADI)>
+		 *   <[64] multicast propagation bloom filter>
+		 *   <[1] 8-bit propagation hop count>
+		 *   <[2] 16-bit ethertype>
+		 *   <[2] 16-bit length of payload>
+		 *   <[2] 16-bit length of signature>
+		 *   <[...] ethernet payload>
+		 *   <[...] ECDSA signature of SHA-256 hash (see below)>
+		 *
+		 * The signature is made using the key of the original submitter, and
+		 * can be used to authenticate the submitter for security and rate
+		 * control purposes. Fields in the signature are: network ID, source
+		 * MAC, destination MAC, multicast ADI, ethertype, and payload. All
+		 * integers are hashed in big-endian byte order. A zero byte is added
+		 * to the hash between each field.
+		 *
+		 * In the future flags could indicate additional fields appended to the
+		 * end or a different signature algorithm.
+		 *
+		 * No OK or ERROR is generated.
+		 */
+		VERB_MULTICAST_FRAME = 9,
+
+		/* Network member certificate for sending peer:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of certificate>
+		 *   <[2] 16-bit length of signature>
+		 *   <[...] string-serialized certificate dictionary>
+		 *   <[...] ECDSA signature of certificate>
+		 *
+		 * OK is generated on acceptance. ERROR is returned on failure. In both
+		 * cases the payload is the network ID.
+		 */
+		VERB_NETWORK_MEMBERSHIP_CERTIFICATE = 10,
+
+		/* Network configuration request:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of request meta-data dictionary>
+		 *   <[...] string-serialized request meta-data>
+		 *
+		 * This message requests network configuration from a node capable of
+		 * providing it. Such nodes run the netconf service, which must be
+		 * installed into the ZeroTier home directory.
+		 *
+		 * OK response payload:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of network configuration dictionary>
+		 *   <[...] network configuration dictionary>
+		 *
+		 * OK returns a Dictionary (string serialized) containing the network's
+		 * configuration and IP address assignment information for the querying
+		 * node. It also contains a membership certificate that the querying
+		 * node can push to other peers to demonstrate its right to speak on
+		 * a given network.
+		 *
+		 * ERROR may be NOT_FOUND if no such network is known, or
+		 * UNSUPPORTED_OPERATION if the netconf service isn't available. The
+		 * payload will be the network ID.
+		 */
+		VERB_NETWORK_CONFIG_REQUEST = 11,
+
+		/* Network configuration refresh request:
+		 *   <[8] 64-bit network ID>
+		 *
+		 * This message can be sent by the network configuration master node
+		 * to request that nodes refresh their network configuration. It can
+		 * thus be used to "push" updates.
+		 *
+		 * It does not generate an OK or ERROR message, and is treated only as
+		 * a hint to refresh now.
+		 */
+		VERB_NETWORK_CONFIG_REFRESH = 12
 	};
 
 	/**
@@ -461,7 +560,10 @@ public:
 		ERROR_IDENTITY_INVALID = 5,
 
 		/* Verb or use case not supported/enabled by this node */
-		ERROR_UNSUPPORTED_OPERATION = 6
+		ERROR_UNSUPPORTED_OPERATION = 6,
+
+		/* Message to private network rejected -- no unexpired certificate on file */
+		ERROR_NO_MEMBER_CERTIFICATE = 7
 	};
 
 	/**
@@ -495,8 +597,8 @@ public:
 	Packet() :
 		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(ZT_PROTO_MIN_PACKET_LENGTH)
 	{
-		Utils::getSecureRandom(_b + ZT_PACKET_IDX_IV,8);
-		_b[ZT_PACKET_IDX_FLAGS] = 0; // zero flags and hops
+		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV,8),8);
+		(*this)[ZT_PACKET_IDX_FLAGS] = 0; // zero flags and hops
 	}
 
 	/**
@@ -509,10 +611,10 @@ public:
 	Packet(const Address &dest,const Address &source,const Verb v) :
 		Buffer<ZT_PROTO_MAX_PACKET_LENGTH>(ZT_PROTO_MIN_PACKET_LENGTH)
 	{
-		Utils::getSecureRandom(_b + ZT_PACKET_IDX_IV,8);
+		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV,8),8);
 		setDestination(dest);
 		setSource(source);
-		_b[ZT_PACKET_IDX_FLAGS] = 0; // zero flags and hops
+		(*this)[ZT_PACKET_IDX_FLAGS] = 0; // zero flags and hops
 		setVerb(v);
 	}
 
@@ -526,13 +628,25 @@ public:
 	inline void reset(const Address &dest,const Address &source,const Verb v)
 	{
 		setSize(ZT_PROTO_MIN_PACKET_LENGTH);
-		Utils::getSecureRandom(_b + ZT_PACKET_IDX_IV,8);
+		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV,8),8);
 		setDestination(dest);
 		setSource(source);
-		_b[ZT_PACKET_IDX_FLAGS] = 0; // zero flags and hops
+		(*this)[ZT_PACKET_IDX_FLAGS] = 0; // zero flags and hops
 		setVerb(v);
 	}
 
+	/**
+	 * Generate a new IV / packet ID in place
+	 *
+	 * This can be used to re-use a packet buffer multiple times to send
+	 * technically different but otherwise identical copies of the same
+	 * packet.
+	 */
+	inline void newInitializationVector()
+	{
+		Utils::getSecureRandom(field(ZT_PACKET_IDX_IV,8),8);
+	}
+
 	/**
 	 * Set this packet's destination
 	 * 
@@ -540,8 +654,9 @@ public:
 	 */
 	inline void setDestination(const Address &dest)
 	{
+		unsigned char *d = field(ZT_PACKET_IDX_DEST,ZT_ADDRESS_LENGTH);
 		for(unsigned int i=0;i<ZT_ADDRESS_LENGTH;++i)
-			_b[i + ZT_PACKET_IDX_DEST] = dest[i];
+			d[i] = dest[i];
 	}
 
 	/**
@@ -551,8 +666,9 @@ public:
 	 */
 	inline void setSource(const Address &source)
 	{
+		unsigned char *s = field(ZT_PACKET_IDX_SOURCE,ZT_ADDRESS_LENGTH);
 		for(unsigned int i=0;i<ZT_ADDRESS_LENGTH;++i)
-			_b[i + ZT_PACKET_IDX_SOURCE] = source[i];
+			s[i] = source[i];
 	}
 
 	/**
@@ -560,29 +676,29 @@ public:
 	 * 
 	 * @return Destination ZT address
 	 */
-	inline Address destination() const { return Address(_b + ZT_PACKET_IDX_DEST); }
+	inline Address destination() const { return Address(field(ZT_PACKET_IDX_DEST,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
 
 	/**
 	 * Get this packet's source
 	 * 
 	 * @return Source ZT address
 	 */
-	inline Address source() const { return Address(_b + ZT_PACKET_IDX_SOURCE); }
+	inline Address source() const { return Address(field(ZT_PACKET_IDX_SOURCE,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); }
 
 	/**
 	 * @return True if packet is of valid length
 	 */
-	inline bool lengthValid() const { return (_l >= ZT_PROTO_MIN_PACKET_LENGTH); }
+	inline bool lengthValid() const { return (size() >= ZT_PROTO_MIN_PACKET_LENGTH); }
 
 	/**
 	 * @return True if packet is encrypted
 	 */
-	inline bool encrypted() const { return (((unsigned char)_b[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_ENCRYPTED)); }
+	inline bool encrypted() const { return (((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_ENCRYPTED)); }
 
 	/**
 	 * @return True if packet is fragmented (expect fragments)
 	 */
-	inline bool fragmented() const { return (((unsigned char)_b[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED)); }
+	inline bool fragmented() const { return (((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] & ZT_PROTO_FLAG_FRAGMENTED)); }
 
 	/**
 	 * Set this packet's fragmented flag
@@ -592,26 +708,26 @@ public:
 	inline void setFragmented(bool f)
 	{
 		if (f)
-			_b[ZT_PACKET_IDX_FLAGS] |= (char)ZT_PROTO_FLAG_FRAGMENTED;
-		else _b[ZT_PACKET_IDX_FLAGS] &= (char)(~ZT_PROTO_FLAG_FRAGMENTED);
+			(*this)[ZT_PACKET_IDX_FLAGS] |= (char)ZT_PROTO_FLAG_FRAGMENTED;
+		else (*this)[ZT_PACKET_IDX_FLAGS] &= (char)(~ZT_PROTO_FLAG_FRAGMENTED);
 	}
 
 	/**
 	 * @return True if compressed (result only valid if unencrypted)
 	 */
-	inline bool compressed() const { return (((unsigned char)_b[ZT_PACKET_IDX_VERB] & ZT_PROTO_VERB_FLAG_COMPRESSED)); }
+	inline bool compressed() const { return (((unsigned char)(*this)[ZT_PACKET_IDX_VERB] & ZT_PROTO_VERB_FLAG_COMPRESSED)); }
 
 	/**
 	 * @return ZeroTier forwarding hops (0 to 7)
 	 */
-	inline unsigned int hops() const { return ((unsigned int)_b[ZT_PACKET_IDX_FLAGS] & 0x07); }
+	inline unsigned int hops() const { return ((unsigned int)(*this)[ZT_PACKET_IDX_FLAGS] & 0x07); }
 
 	/**
 	 * Increment this packet's hop count
 	 */
 	inline void incrementHops()
 	{
-		_b[ZT_PACKET_IDX_FLAGS] = (char)((unsigned char)_b[ZT_PACKET_IDX_FLAGS] & 0xf8) | (((unsigned char)_b[ZT_PACKET_IDX_FLAGS] + 1) & 0x07);
+		(*this)[ZT_PACKET_IDX_FLAGS] = (char)((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] & 0xf8) | (((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] + 1) & 0x07);
 	}
 
 	/**
@@ -629,23 +745,25 @@ public:
 	 * 
 	 * @param v New packet verb
 	 */
-	inline void setVerb(Verb v) { _b[ZT_PACKET_IDX_VERB] = (char)v; }
+	inline void setVerb(Verb v) { (*this)[ZT_PACKET_IDX_VERB] = (char)v; }
 
 	/**
 	 * @return Packet verb (not including flag bits)
 	 */
-	inline Verb verb() const { return (Verb)(_b[ZT_PACKET_IDX_VERB] & 0x1f); }
+	inline Verb verb() const { return (Verb)((*this)[ZT_PACKET_IDX_VERB] & 0x1f); }
 
 	/**
 	 * @return Length of packet payload
 	 */
-	inline unsigned int payloadLength() const throw() { return ((_l < ZT_PROTO_MIN_PACKET_LENGTH) ? 0 : (_l - ZT_PROTO_MIN_PACKET_LENGTH)); }
+	inline unsigned int payloadLength() const { return ((size() < ZT_PROTO_MIN_PACKET_LENGTH) ? 0 : (size() - ZT_PROTO_MIN_PACKET_LENGTH)); }
 
 	/**
-	 * @return Packet payload
+	 * @return Raw packet payload
 	 */
-	inline unsigned char *payload() throw() { return (unsigned char *)(_b + ZT_PACKET_IDX_PAYLOAD); }
-	inline const unsigned char *payload() const throw() { return (const unsigned char *)(_b + ZT_PACKET_IDX_PAYLOAD); }
+	inline const unsigned char *payload() const
+	{
+		return field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD);
+	}
 
 	/**
 	 * Compute the HMAC of this packet's payload and set HMAC field
@@ -655,13 +773,13 @@ public:
 	 * @param key 256-bit (32 byte) key
 	 */
 	inline void hmacSet(const void *key)
-		throw()
 	{
 		unsigned char mac[32];
 		unsigned char key2[32];
 		_mangleKey((const unsigned char *)key,key2);
-		HMAC::sha256(key2,sizeof(key2),_b + ZT_PACKET_IDX_VERB,(_l >= ZT_PACKET_IDX_VERB) ? (_l - ZT_PACKET_IDX_VERB) : 0,mac);
-		memcpy(_b + ZT_PACKET_IDX_HMAC,mac,8);
+		unsigned int hmacLen = (size() >= ZT_PACKET_IDX_VERB) ? (size() - ZT_PACKET_IDX_VERB) : 0;
+		HMAC::sha256(key2,sizeof(key2),field(ZT_PACKET_IDX_VERB,hmacLen),hmacLen,mac);
+		memcpy(field(ZT_PACKET_IDX_HMAC,8),mac,8);
 	}
 
 	/**
@@ -672,15 +790,15 @@ public:
 	 * @param key 256-bit (32 byte) key
 	 */
 	inline bool hmacVerify(const void *key) const
-		throw()
 	{
 		unsigned char mac[32];
 		unsigned char key2[32];
-		if (_l < ZT_PACKET_IDX_VERB)
+		if (size() < ZT_PACKET_IDX_VERB)
 			return false; // incomplete packets fail
 		_mangleKey((const unsigned char *)key,key2);
-		HMAC::sha256(key2,sizeof(key2),_b + ZT_PACKET_IDX_VERB,_l - ZT_PACKET_IDX_VERB,mac);
-		return (!memcmp(_b + ZT_PACKET_IDX_HMAC,mac,8));
+		unsigned int hmacLen = size() - ZT_PACKET_IDX_VERB;
+		HMAC::sha256(key2,sizeof(key2),field(ZT_PACKET_IDX_VERB,hmacLen),hmacLen,mac);
+		return (!memcmp(field(ZT_PACKET_IDX_HMAC,8),mac,8));
 	}
 
 	/**
@@ -689,13 +807,16 @@ public:
 	 * @param key 256-bit (32 byte) key
 	 */
 	inline void encrypt(const void *key)
-		throw()
 	{
-		_b[ZT_PACKET_IDX_FLAGS] |= ZT_PROTO_FLAG_ENCRYPTED;
+		(*this)[ZT_PACKET_IDX_FLAGS] |= ZT_PROTO_FLAG_ENCRYPTED;
 		unsigned char key2[32];
-		_mangleKey((const unsigned char *)key,key2);
-		Salsa20 s20(key2,256,_b + ZT_PACKET_IDX_IV);
-		s20.encrypt(_b + ZT_PACKET_IDX_VERB,_b + ZT_PACKET_IDX_VERB,(_l >= ZT_PACKET_IDX_VERB) ? (_l - ZT_PACKET_IDX_VERB) : 0);
+		if (size() >= ZT_PACKET_IDX_VERB) {
+			_mangleKey((const unsigned char *)key,key2);
+			Salsa20 s20(key2,256,field(ZT_PACKET_IDX_IV,8));
+			unsigned int encLen = size() - ZT_PACKET_IDX_VERB;
+			unsigned char *const encBuf = field(ZT_PACKET_IDX_VERB,encLen);
+			s20.encrypt(encBuf,encBuf,encLen);
+		}
 	}
 
 	/**
@@ -704,13 +825,16 @@ public:
 	 * @param key 256-bit (32 byte) key
 	 */
 	inline void decrypt(const void *key)
-		throw()
 	{
 		unsigned char key2[32];
-		_mangleKey((const unsigned char *)key,key2);
-		Salsa20 s20(key2,256,_b + ZT_PACKET_IDX_IV);
-		s20.decrypt(_b + ZT_PACKET_IDX_VERB,_b + ZT_PACKET_IDX_VERB,(_l >= ZT_PACKET_IDX_VERB) ? (_l - ZT_PACKET_IDX_VERB) : 0);
-		_b[ZT_PACKET_IDX_FLAGS] &= (char)(~ZT_PROTO_FLAG_ENCRYPTED);
+		if (size() >= ZT_PACKET_IDX_VERB) {
+			_mangleKey((const unsigned char *)key,key2);
+			Salsa20 s20(key2,256,field(ZT_PACKET_IDX_IV,8));
+			unsigned int decLen = size() - ZT_PACKET_IDX_VERB;
+			unsigned char *const decBuf = field(ZT_PACKET_IDX_VERB,decLen);
+			s20.decrypt(decBuf,decBuf,decLen);
+		}
+		(*this)[ZT_PACKET_IDX_FLAGS] &= (char)(~ZT_PROTO_FLAG_ENCRYPTED);
 	}
 
 	/**
@@ -724,20 +848,19 @@ public:
 	 * @return True if compression occurred
 	 */
 	inline bool compress()
-		throw()
 	{
 		unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH * 2];
-		if ((!compressed())&&(_l > (ZT_PACKET_IDX_PAYLOAD + 32))) {
-			int pl = (int)(_l - ZT_PACKET_IDX_PAYLOAD);
-			int cl = LZ4_compress((const char *)(_b + ZT_PACKET_IDX_PAYLOAD),(char *)buf,pl);
+		if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 32))) {
+			int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD);
+			int cl = LZ4_compress((const char *)field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)pl),(char *)buf,pl);
 			if ((cl > 0)&&(cl < pl)) {
-				_b[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED;
-				memcpy(_b + ZT_PACKET_IDX_PAYLOAD,buf,cl);
-				_l = (unsigned int)cl + ZT_PACKET_IDX_PAYLOAD;
+				(*this)[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED;
+				setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD);
+				memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)cl),buf,cl);
 				return true;
 			}
 		}
-		_b[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED);
+		(*this)[ZT_PACKET_IDX_VERB] &= (char)(~ZT_PROTO_VERB_FLAG_COMPRESSED);
 		return false;
 	}
 
@@ -750,18 +873,18 @@ public:
 	 * @return True if data is now decompressed and valid, false on error
 	 */
 	inline bool uncompress()
-		throw()
 	{
 		unsigned char buf[ZT_PROTO_MAX_PACKET_LENGTH];
-		if ((compressed())&&(_l >= ZT_PROTO_MIN_PACKET_LENGTH)) {
-			if (_l > ZT_PACKET_IDX_PAYLOAD) {
-				int ucl = LZ4_uncompress_unknownOutputSize((const char *)(_b + ZT_PACKET_IDX_PAYLOAD),(char *)buf,_l - ZT_PACKET_IDX_PAYLOAD,sizeof(buf));
+		if ((compressed())&&(size() >= ZT_PROTO_MIN_PACKET_LENGTH)) {
+			if (size() > ZT_PACKET_IDX_PAYLOAD) {
+				unsigned int compLen = size() - ZT_PACKET_IDX_PAYLOAD;
+				int ucl = LZ4_uncompress_unknownOutputSize((const char *)field(ZT_PACKET_IDX_PAYLOAD,compLen),(char *)buf,compLen,sizeof(buf));
 				if ((ucl > 0)&&(ucl <= (int)(capacity() - ZT_PACKET_IDX_PAYLOAD))) {
-					memcpy(_b + ZT_PACKET_IDX_PAYLOAD,buf,ucl);
-					_l = (unsigned int)ucl + ZT_PACKET_IDX_PAYLOAD;
+					setSize((unsigned int)ucl + ZT_PACKET_IDX_PAYLOAD);
+					memcpy(field(ZT_PACKET_IDX_PAYLOAD,(unsigned int)ucl),buf,ucl);
 				} else return false;
 			}
-			_b[ZT_PACKET_IDX_VERB] &= ~ZT_PROTO_VERB_FLAG_COMPRESSED;
+			(*this)[ZT_PACKET_IDX_VERB] &= ~ZT_PROTO_VERB_FLAG_COMPRESSED;
 		}
 		return true;
 	}
@@ -788,19 +911,18 @@ private:
 	 * @param out Output buffer (32 bytes)
 	 */
 	inline void _mangleKey(const unsigned char *in,unsigned char *out) const
-		throw()
 	{
 		// Random IV (Salsa20 also uses the IV natively, but HMAC doesn't), and
 		// destination and source addresses. Using dest and source addresses
 		// gives us a (likely) different key space for a->b vs b->a.
 		for(unsigned int i=0;i<18;++i) // 8 + (ZT_ADDRESS_LENGTH * 2) == 18
-			out[i] = in[i] ^ (unsigned char)_b[i];
+			out[i] = in[i] ^ (unsigned char)(*this)[i];
 		// Flags, but masking off hop count which is altered by forwarding nodes
-		out[18] = in[18] ^ ((unsigned char)_b[ZT_PACKET_IDX_FLAGS] & 0xf8);
+		out[18] = in[18] ^ ((unsigned char)(*this)[ZT_PACKET_IDX_FLAGS] & 0xf8);
 		// Raw packet size in bytes -- each raw packet size defines a possibly
 		// different space of keys.
-		out[19] = in[19] ^ (unsigned char)(_l & 0xff);
-		out[20] = in[20] ^ (unsigned char)((_l >> 8) & 0xff); // little endian
+		out[19] = in[19] ^ (unsigned char)(size() & 0xff);
+		out[20] = in[20] ^ (unsigned char)((size() >> 8) & 0xff); // little endian
 		// Rest of raw key is used unchanged
 		for(unsigned int i=21;i<32;++i)
 			out[i] = in[i];

node/PacketDecoder.cpp

@@ -0,0 +1,669 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include "Constants.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Topology.hpp"
+#include "PacketDecoder.hpp"
+#include "Switch.hpp"
+#include "Peer.hpp"
+#include "NodeConfig.hpp"
+#include "Filter.hpp"
+#include "Service.hpp"
+
+/*
+ * The big picture:
+ *
+ * tryDecode() gets called for a given fully-assembled packet until it returns
+ * true or the packet's time to live has been exceeded. The state machine must
+ * therefore be re-entrant if it ever returns false. Take care here!
+ *
+ * Stylistic note:
+ *
+ * There's a lot of unnecessary if nesting. It's mostly to allow TRACE to
+ * print informative messages on every possible reason something gets
+ * rejected or fails.
+ */
+
+namespace ZeroTier {
+
+bool PacketDecoder::tryDecode(const RuntimeEnvironment *_r)
+	throw(std::out_of_range,std::runtime_error)
+{
+	if ((!encrypted())&&(verb() == Packet::VERB_HELLO)) {
+		// Unencrypted HELLOs are handled here since they are used to
+		// populate our identity cache in the first place. Thus we might get
+		// a HELLO for someone for whom we don't have a Peer record.
+		TRACE("HELLO from %s(%s)",source().toString().c_str(),_remoteAddress.toString().c_str());
+		return _doHELLO(_r);
+	}
+
+	SharedPtr<Peer> peer = _r->topology->getPeer(source());
+	if (peer) {
+		// Resume saved state?
+		if (_step == DECODE_WAITING_FOR_MULTICAST_FRAME_ORIGINAL_SENDER_LOOKUP) {
+			// In this state we have already authenticated and decrypted the
+			// packet and are waiting for the lookup of the original sender
+			// for a multicast frame. So check to see if we've got it.
+			return _doMULTICAST_FRAME(_r,peer);
+		}
+
+		// No saved state? Verify MAC before we proceed.
+		if (!hmacVerify(peer->macKey())) {
+			TRACE("dropped packet from %s(%s), HMAC authentication failed (size: %u)",source().toString().c_str(),_remoteAddress.toString().c_str(),size());
+			return true;
+		}
+
+		// If MAC authentication passed, decrypt and uncompress
+		if (encrypted()) {
+			decrypt(peer->cryptKey());
+		} else {
+			// Unencrypted is tolerated in case we want to run this on
+			// devices where squeezing out cycles matters. HMAC is
+			// what's really important. But log it in debug to catch any
+			// packets being mistakenly sent in the clear.
+			TRACE("ODD: %s from %s(%s) wasn't encrypted",Packet::verbString(verb()),source().toString().c_str(),_remoteAddress.toString().c_str());
+		}
+		if (!uncompress()) {
+			TRACE("dropped packet from %s(%s), compressed data invalid",source().toString().c_str(),_remoteAddress.toString().c_str());
+			return true;
+		}
+
+		Packet::Verb v = verb();
+
+		// Once a packet is determined to be basically valid, it can be used
+		// to passively learn a new network path to the sending peer. It
+		// also results in statistics updates.
+		peer->onReceive(_r,_localPort,_remoteAddress,hops(),v,Utils::now());
+
+		switch(v) {
+			case Packet::VERB_NOP:
+				TRACE("NOP from %s(%s)",source().toString().c_str(),_remoteAddress.toString().c_str());
+				return true;
+			case Packet::VERB_HELLO:
+				return _doHELLO(_r); // encrypted HELLO is technically allowed, but kind of pointless... :)
+			case Packet::VERB_ERROR:
+				return _doERROR(_r,peer);
+			case Packet::VERB_OK:
+				return _doOK(_r,peer);
+			case Packet::VERB_WHOIS:
+				return _doWHOIS(_r,peer);
+			case Packet::VERB_RENDEZVOUS:
+				return _doRENDEZVOUS(_r,peer);
+			case Packet::VERB_FRAME:
+				return _doFRAME(_r,peer);
+			case Packet::VERB_MULTICAST_LIKE:
+				return _doMULTICAST_LIKE(_r,peer);
+			case Packet::VERB_MULTICAST_FRAME:
+				return _doMULTICAST_FRAME(_r,peer);
+			case Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE:
+				return _doNETWORK_MEMBERSHIP_CERTIFICATE(_r,peer);
+			case Packet::VERB_NETWORK_CONFIG_REQUEST:
+				return _doNETWORK_CONFIG_REQUEST(_r,peer);
+			case Packet::VERB_NETWORK_CONFIG_REFRESH:
+				return _doNETWORK_CONFIG_REFRESH(_r,peer);
+			default:
+				// This might be something from a new or old version of the protocol.
+				// Technically it passed HMAC so the packet is still valid, but we
+				// ignore it.
+				TRACE("ignored unrecognized verb %.2x from %s(%s)",(unsigned int)v,source().toString().c_str(),_remoteAddress.toString().c_str());
+				return true;
+		}
+	} else {
+		_step = DECODE_WAITING_FOR_SENDER_LOOKUP; // should already be this...
+		_r->sw->requestWhois(source());
+		return false;
+	}
+}
+
+void PacketDecoder::_CBaddPeerFromHello(void *arg,const SharedPtr<Peer> &p,Topology::PeerVerifyResult result)
+{
+	_CBaddPeerFromHello_Data *req = (_CBaddPeerFromHello_Data *)arg;
+	const RuntimeEnvironment *_r = req->renv;
+
+	try {
+		switch(result) {
+			case Topology::PEER_VERIFY_ACCEPTED_NEW:
+			case Topology::PEER_VERIFY_ACCEPTED_ALREADY_HAVE:
+			case Topology::PEER_VERIFY_ACCEPTED_DISPLACED_INVALID_ADDRESS: {
+				_r->sw->doAnythingWaitingForPeer(p);
+
+				Packet outp(req->source,_r->identity.address(),Packet::VERB_OK);
+				outp.append((unsigned char)Packet::VERB_HELLO);
+				outp.append(req->helloPacketId);
+				outp.append(req->helloTimestamp);
+				outp.encrypt(p->cryptKey());
+				outp.hmacSet(p->macKey());
+				_r->demarc->send(req->localPort,req->remoteAddress,outp.data(),outp.size(),-1);
+			}	break;
+
+			case Topology::PEER_VERIFY_REJECTED_INVALID_IDENTITY: {
+				Packet outp(req->source,_r->identity.address(),Packet::VERB_ERROR);
+				outp.append((unsigned char)Packet::VERB_HELLO);
+				outp.append(req->helloPacketId);
+				outp.append((unsigned char)Packet::ERROR_IDENTITY_INVALID);
+				outp.encrypt(p->cryptKey());
+				outp.hmacSet(p->macKey());
+				_r->demarc->send(req->localPort,req->remoteAddress,outp.data(),outp.size(),-1);
+			}	break;
+
+			case Topology::PEER_VERIFY_REJECTED_DUPLICATE:
+			case Topology::PEER_VERIFY_REJECTED_DUPLICATE_TRIAGED: {
+				Packet outp(req->source,_r->identity.address(),Packet::VERB_ERROR);
+				outp.append((unsigned char)Packet::VERB_HELLO);
+				outp.append(req->helloPacketId);
+				outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
+				outp.encrypt(p->cryptKey());
+				outp.hmacSet(p->macKey());
+				_r->demarc->send(req->localPort,req->remoteAddress,outp.data(),outp.size(),-1);
+			}	break;
+		}
+	} catch ( ... ) {
+		TRACE("unexpected exception in addPeer() result callback for peer received via HELLO");
+	}
+
+	delete req;
+}
+
+void PacketDecoder::_CBaddPeerFromWhois(void *arg,const SharedPtr<Peer> &p,Topology::PeerVerifyResult result)
+{
+	const RuntimeEnvironment *_r = (const RuntimeEnvironment *)arg;
+	try {
+		switch(result) {
+			case Topology::PEER_VERIFY_ACCEPTED_NEW:
+			case Topology::PEER_VERIFY_ACCEPTED_ALREADY_HAVE:
+			case Topology::PEER_VERIFY_ACCEPTED_DISPLACED_INVALID_ADDRESS:
+				_r->sw->doAnythingWaitingForPeer(p);
+				break;
+			default:
+				break;
+		}
+	} catch ( ... ) {
+		TRACE("unexpected exception in addPeer() result callback for peer received via OK(WHOIS)");
+	}
+}
+
+bool PacketDecoder::_doERROR(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+#ifdef ZT_TRACE
+		Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_ERROR_IDX_IN_RE_VERB];
+		Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE];
+		TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+#endif
+		// TODO (sorta):
+		// The fact is that the protocol works fine without error handling.
+		// The only error that really needs to be handled here is duplicate
+		// identity collision, which if it comes from a supernode should cause
+		// us to restart and regenerate a new identity.
+	} catch (std::exception &ex) {
+		TRACE("dropped ERROR from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped ERROR from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
+{
+	try {
+		//unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION];
+		unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MAJOR_VERSION];
+		unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO_IDX_MINOR_VERSION];
+		unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO_IDX_REVISION);
+		uint64_t timestamp = at<uint64_t>(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP);
+		Identity id(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
+
+		// Create a new candidate peer that we might decide to add to our
+		// database. We create it now since we want its keys to send replies
+		// even in the error case, and the code for keying is in Peer.
+		SharedPtr<Peer> candidate(new Peer(_r->identity,id));
+		candidate->setPathAddress(_remoteAddress,false);
+
+		// The initial sniff test... is the identity valid, and is it
+		// the sender's identity?
+		if ((id.address().isReserved())||(id.address() != source())) {
+#ifdef ZT_TRACE
+			if (id.address().isReserved()) {
+				TRACE("rejected HELLO from %s(%s): identity has reserved address",source().toString().c_str(),_remoteAddress.toString().c_str());
+			} else {
+				TRACE("rejected HELLO from %s(%s): identity is not for sender of packet (HELLO is a self-announcement)",source().toString().c_str(),_remoteAddress.toString().c_str());
+			}
+#endif
+			Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+			outp.append((unsigned char)Packet::VERB_HELLO);
+			outp.append(packetId());
+			outp.append((unsigned char)((id.address().isReserved()) ? Packet::ERROR_IDENTITY_INVALID : Packet::ERROR_INVALID_REQUEST));
+			outp.encrypt(candidate->cryptKey());
+			outp.hmacSet(candidate->macKey());
+			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+			return true;
+		}
+
+		// Is this a HELLO for a peer we already know? If so just update its
+		// packet receive stats and send an OK.
+		SharedPtr<Peer> existingPeer(_r->topology->getPeer(id.address()));
+		if ((existingPeer)&&(existingPeer->identity() == id)) {
+			existingPeer->onReceive(_r,_localPort,_remoteAddress,hops(),Packet::VERB_HELLO,Utils::now());
+			existingPeer->setRemoteVersion(vMajor,vMinor,vRevision);
+
+			Packet outp(source(),_r->identity.address(),Packet::VERB_OK);
+			outp.append((unsigned char)Packet::VERB_HELLO);
+			outp.append(packetId());
+			outp.append(timestamp);
+			outp.encrypt(existingPeer->cryptKey());
+			outp.hmacSet(existingPeer->macKey());
+			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+			return true;
+		}
+
+		// Otherwise we call addPeer() and set up a callback to handle the verdict.
+		// Topology evaluates the peer in the background, possibly doing the entire
+		// expensive analysis before determining whether to add it to the database.
+		_CBaddPeerFromHello_Data *arg = new _CBaddPeerFromHello_Data;
+		arg->renv = _r;
+		arg->source = source();
+		arg->remoteAddress = _remoteAddress;
+		arg->localPort = _localPort;
+		arg->vMajor = vMajor;
+		arg->vMinor = vMinor;
+		arg->vRevision = vRevision;
+		arg->helloPacketId = packetId();
+		arg->helloTimestamp = timestamp;
+		_r->topology->addPeer(candidate,&PacketDecoder::_CBaddPeerFromHello,arg);
+	} catch (std::exception &ex) {
+		TRACE("dropped HELLO from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
+		switch(inReVerb) {
+			case Packet::VERB_HELLO: {
+				// OK from HELLO permits computation of latency.
+				unsigned int latency = std::min((unsigned int)(Utils::now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)),(unsigned int)0xffff);
+				TRACE("%s(%s): OK(HELLO), latency: %u",source().toString().c_str(),_remoteAddress.toString().c_str(),latency);
+				peer->setLatency(_remoteAddress,latency);
+			}	break;
+			case Packet::VERB_WHOIS: {
+				TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+				if (_r->topology->isSupernode(source())) {
+					// Right now, only supernodes are queried for WHOIS so we only
+					// accept OK(WHOIS) from supernodes. Otherwise peers could
+					// potentially cache-poison. A more elegant but memory-intensive
+					// solution would be to remember packet IDs of WHOIS requests.
+					_r->topology->addPeer(SharedPtr<Peer>(new Peer(_r->identity,Identity(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY))),&PacketDecoder::_CBaddPeerFromWhois,const_cast<void *>((const void *)_r));
+				}
+			} break;
+			case Packet::VERB_NETWORK_CONFIG_REQUEST: {
+				SharedPtr<Network> nw(_r->nc->network(at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID)));
+				if ((nw)&&(nw->controller() == source())) {
+					// Only accept OK(NETWORK_CONFIG_REQUEST) from masters for
+					// networks we have.
+					unsigned int dictlen = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN);
+					std::string dict((const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,dictlen),dictlen);
+					if (dict.length()) {
+						Network::Config netconf(dict);
+						if ((netconf.networkId() == nw->id())&&(netconf.peerAddress() == _r->identity.address())) { // sanity check
+							LOG("got network configuration for network %.16llx from %s",(unsigned long long)nw->id(),source().toString().c_str());
+							nw->setConfiguration(netconf);
+						}
+					}
+				}
+			}	break;
+			default:
+				//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+				break;
+		}
+	} catch (std::exception &ex) {
+		TRACE("dropped OK from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped OK from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doWHOIS(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	if (payloadLength() == ZT_ADDRESS_LENGTH) {
+		SharedPtr<Peer> p(_r->topology->getPeer(Address(payload(),ZT_ADDRESS_LENGTH)));
+		if (p) {
+			Packet outp(source(),_r->identity.address(),Packet::VERB_OK);
+			outp.append((unsigned char)Packet::VERB_WHOIS);
+			outp.append(packetId());
+			p->identity().serialize(outp,false);
+			outp.encrypt(peer->cryptKey());
+			outp.hmacSet(peer->macKey());
+			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+			TRACE("sent WHOIS response to %s for %s",source().toString().c_str(),Address(payload(),ZT_ADDRESS_LENGTH).toString().c_str());
+		} else {
+			Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+			outp.append((unsigned char)Packet::VERB_WHOIS);
+			outp.append(packetId());
+			outp.append((unsigned char)Packet::ERROR_NOT_FOUND);
+			outp.append(payload(),ZT_ADDRESS_LENGTH);
+			outp.encrypt(peer->cryptKey());
+			outp.hmacSet(peer->macKey());
+			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+			TRACE("sent WHOIS ERROR to %s for %s (not found)",source().toString().c_str(),Address(payload(),ZT_ADDRESS_LENGTH).toString().c_str());
+		}
+	} else {
+		TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doRENDEZVOUS(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		/*
+		 * At the moment, we only obey RENDEZVOUS if it comes from a designated
+		 * supernode. If relay offloading is implemented to scale the net, this
+		 * will need reconsideration.
+		 *
+		 * The reason is that RENDEZVOUS could technically be used to cause a
+		 * peer to send a weird encrypted UDP packet to an arbitrary IP:port.
+		 * The sender of RENDEZVOUS has no control over the content of this
+		 * packet, but it's still maybe something we want to not allow just
+		 * anyone to order due to possible DDOS or network forensic implications.
+		 * So if we diversify relays, we'll need some way of deciding whether the
+		 * sender is someone we should trust with a RENDEZVOUS hint. Or maybe
+		 * we just need rate limiting to prevent DDOS and amplification attacks.
+		 */
+		if (_r->topology->isSupernode(source())) {
+			Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+			SharedPtr<Peer> withPeer(_r->topology->getPeer(with));
+			if (withPeer) {
+				unsigned int port = at<uint16_t>(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT);
+				unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN];
+				if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
+					InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
+					TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",source().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
+					_r->sw->contact(withPeer,atAddr);
+				} else {
+					TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",source().toString().c_str(),_remoteAddress.toString().c_str());
+				}
+			} else {
+				TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",source().toString().c_str(),_remoteAddress.toString().c_str(),with.toString().c_str());
+			}
+		} else {
+			TRACE("ignored RENDEZVOUS from %s(%s): source not supernode",source().toString().c_str(),_remoteAddress.toString().c_str());
+		}
+	} catch (std::exception &ex) {
+		TRACE("dropped RENDEZVOUS from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		SharedPtr<Network> network(_r->nc->network(at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)));
+		if (network) {
+			if (network->isAllowed(source())) {
+				unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
+				if ((etherType != ZT_ETHERTYPE_ARP)&&(etherType != ZT_ETHERTYPE_IPV4)&&(etherType != ZT_ETHERTYPE_IPV6)) {
+					TRACE("dropped FRAME from %s: unsupported ethertype",source().toString().c_str());
+				} else if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
+					network->tap().put(source().toMAC(),network->tap().mac(),etherType,data() + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD);
+				}
+			} else {
+				TRACE("dropped FRAME from %s(%s): not a member of closed network %llu",source().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
+			}
+		} else {
+			TRACE("dropped FRAME from %s(%s): network %llu unknown",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
+		}
+	} catch (std::exception &ex) {
+		TRACE("dropped FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped FRAME from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
+		unsigned int numAccepted = 0;
+		uint64_t now = Utils::now();
+
+		// Iterate through 18-byte network,MAC,ADI tuples:
+		while ((ptr + 18) <= size()) {
+			uint64_t nwid = at<uint64_t>(ptr); ptr += 8;
+			SharedPtr<Network> network(_r->nc->network(nwid));
+			if ((network)&&(network->isAllowed(source()))) {
+				MAC mac(field(ptr,6)); ptr += 6;
+				uint32_t adi = at<uint32_t>(ptr); ptr += 4;
+				//TRACE("peer %s likes multicast group %s:%.8lx on network %llu",source().toString().c_str(),mac.toString().c_str(),(unsigned long)adi,nwid);
+				_r->multicaster->likesMulticastGroup(nwid,MulticastGroup(mac,adi),source(),now);
+				++numAccepted;
+			} else {
+				ptr += 10;
+				TRACE("ignored MULTICAST_LIKE from %s(%s): network %.16llx unknown, or sender is not a member of network",source().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)nwid);
+			}
+		}
+
+		Packet outp(source(),_r->identity.address(),Packet::VERB_OK);
+		outp.append((unsigned char)Packet::VERB_MULTICAST_LIKE);
+		outp.append(packetId());
+		outp.append((uint16_t)numAccepted);
+		outp.encrypt(peer->cryptKey());
+		outp.hmacSet(peer->macKey());
+		_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+	} catch (std::exception &ex) {
+		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		SharedPtr<Network> network(_r->nc->network(at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID)));
+		if ((network)&&(network->isAllowed(source()))) {
+			Address originalSubmitterAddress(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SUBMITTER_ADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+			MAC fromMac(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6));
+			MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DESTINATION_MAC,6)),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ADI));
+			unsigned int hops = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_HOP_COUNT];
+			unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE);
+			unsigned int datalen = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD_LENGTH);
+			unsigned int signaturelen = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SIGNATURE_LENGTH);
+			unsigned char *dataAndSignature = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PAYLOAD,datalen + signaturelen);
+
+			uint64_t mccrc = Multicaster::computeMulticastDedupCrc(network->id(),fromMac,mg,etherType,dataAndSignature,datalen);
+			uint64_t now = Utils::now();
+			bool isDuplicate = _r->multicaster->checkDuplicate(mccrc,now);
+
+			if (originalSubmitterAddress == _r->identity.address()) {
+				// Technically should not happen, since the original submitter is
+				// excluded from consideration as a propagation recipient.
+				TRACE("dropped boomerang MULTICAST_FRAME received from %s(%s)",source().toString().c_str(),_remoteAddress.toString().c_str());
+			} else if ((!isDuplicate)||(_r->topology->amSupernode())) {
+				//
+				// If I am a supernode, I will repeatedly propagate duplicates. That's
+				// because supernodes are used to bridge sparse multicast groups. Non-
+				// supernodes will ignore duplicates completely.
+				//
+				// TODO: supernodes should keep a local bloom filter too and OR it with
+				// the bloom from the packet in order to pick different recipients each
+				// time a multicast returns to them for repropagation.
+				//
+
+				SharedPtr<Peer> originalSubmitter(_r->topology->getPeer(originalSubmitterAddress));
+				if (!originalSubmitter) {
+					TRACE("requesting WHOIS on original multicast frame submitter %s",originalSubmitterAddress.toString().c_str());
+					_r->sw->requestWhois(originalSubmitterAddress);
+					_step = DECODE_WAITING_FOR_MULTICAST_FRAME_ORIGINAL_SENDER_LOOKUP;
+					return false; // try again if/when we get OK(WHOIS)
+				} else if (Multicaster::verifyMulticastPacket(originalSubmitter->identity(),network->id(),fromMac,mg,etherType,dataAndSignature,datalen,dataAndSignature + datalen,signaturelen)) {
+					// In checking the multicast rate, we don't re-check if this is
+					// a duplicate. That's because if isDuplicate is true it means
+					// we're a supernode and it's a second pass relay.
+					if ((isDuplicate)||(network->multicastRateGate(originalSubmitter->address(),datalen))) {
+						_r->multicaster->addToDedupHistory(mccrc,now);
+
+						// Even if we are a supernode, we still don't repeatedly inject
+						// duplicates into our own tap.
+						if (!isDuplicate)
+							network->tap().put(fromMac,mg.mac(),etherType,dataAndSignature,datalen);
+
+						if (++hops < ZT_MULTICAST_PROPAGATION_DEPTH) {
+							Address upstream(source()); // save this since we mangle it
+
+							Multicaster::MulticastBloomFilter bloom(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_BLOOM_FILTER,ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BYTES));
+							SharedPtr<Peer> propPeers[ZT_MULTICAST_PROPAGATION_BREADTH];
+							unsigned int np = _r->multicaster->pickNextPropagationPeers(
+								*(_r->prng),
+								*(_r->topology),
+								network->id(),
+								mg,
+								originalSubmitterAddress,
+								upstream,
+								bloom,
+								ZT_MULTICAST_PROPAGATION_BREADTH,
+								propPeers,
+								now);
+
+							// In a bit of a hack, we re-use this packet to repeat it
+							// to our multicast propagation recipients. Afterwords we
+							// return true just to be sure this is the end of this
+							// packet's life cycle, since it is now mangled.
+
+							setSource(_r->identity.address());
+							(*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_HOP_COUNT] = hops;
+							memcpy(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_BLOOM_FILTER,ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BYTES),bloom.data(),ZT_PROTO_VERB_MULTICAST_FRAME_BLOOM_FILTER_SIZE_BYTES);
+							compress();
+
+							for(unsigned int i=0;i<np;++i) {
+								//TRACE("propagating multicast from original node %s: %s -> %s",originalSubmitterAddress.toString().c_str(),upstream.toString().c_str(),propPeers[i]->address().toString().c_str());
+								// Re-use this packet to re-send multicast frame to everyone
+								// downstream from us.
+								newInitializationVector();
+								setDestination(propPeers[i]->address());
+								_r->sw->send(*this,true);
+							}
+
+							// Return here just to be safe, since this packet's state is no
+							// longer valid.
+							return true;
+						} else {
+							//TRACE("terminating MULTICAST_FRAME propagation from %s(%s): max depth reached",source().toString().c_str(),_remoteAddress.toString().c_str());
+						}
+					} else {
+						LOG("dropped MULTICAST_FRAME from original sender %s: rate limit overrun",originalSubmitter->address().toString().c_str());
+					}
+				} else {
+					TRACE("rejected MULTICAST_FRAME forwarded by %s(%s): failed signature check (falsely claims origin %s)",source().toString().c_str(),_remoteAddress.toString().c_str(),originalSubmitterAddress.toString().c_str());
+				}
+			} else {
+				TRACE("dropped duplicate MULTICAST_FRAME from %s(%s)",source().toString().c_str(),_remoteAddress.toString().c_str());
+			}
+		} else {
+			TRACE("dropped MULTICAST_FRAME from %s(%s): network %.16llx unknown or sender not allowed",source().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
+		}
+	} catch (std::exception &ex) {
+		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	// TODO: not implemented yet, will be needed for private networks.
+
+	return true;
+}
+
+bool PacketDecoder::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	char tmp[128];
+	try {
+		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID);
+#ifndef __WINDOWS__
+		if (_r->netconfService) {
+			unsigned int dictLen = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN);
+
+			Dictionary request;
+			if (dictLen)
+				request["meta"] = std::string((const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,dictLen),dictLen);
+			request["type"] = "netconf-request";
+			request["peerId"] = peer->identity().toString(false);
+			sprintf(tmp,"%llx",(unsigned long long)nwid);
+			request["nwid"] = tmp;
+			sprintf(tmp,"%llx",(unsigned long long)packetId());
+			request["requestId"] = tmp;
+			//TRACE("to netconf:\n%s",request.toString().c_str());
+			_r->netconfService->send(request);
+		} else {
+#endif // !__WINDOWS__
+			Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
+			outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
+			outp.append(packetId());
+			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
+			outp.append(nwid);
+			outp.encrypt(peer->cryptKey());
+			outp.hmacSet(peer->macKey());
+			_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
+#ifndef __WINDOWS__
+		}
+#endif // !__WINDOWS__
+	} catch (std::exception &exc) {
+		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool PacketDecoder::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
+{
+	try {
+		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REFRESH_IDX_NETWORK_ID);
+		SharedPtr<Network> nw(_r->nc->network(nwid));
+		if ((nw)&&(source() == nw->controller())) // only respond to requests from controller
+			nw->requestConfiguration();
+	} catch (std::exception &exc) {
+		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+} // namespace ZeroTier

node/PacketDecoder.hpp

@@ -0,0 +1,143 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_PACKETDECODER_HPP
+#define _ZT_PACKETDECODER_HPP
+
+#include <stdexcept>
+
+#include "Packet.hpp"
+#include "Demarc.hpp"
+#include "InetAddress.hpp"
+#include "Utils.hpp"
+#include "SharedPtr.hpp"
+#include "AtomicCounter.hpp"
+#include "Peer.hpp"
+
+namespace ZeroTier {
+
+class RuntimeEnvironment;
+
+/**
+ * Subclass of packet that handles the decoding of it
+ */
+class PacketDecoder : public Packet
+{
+	friend class SharedPtr<PacketDecoder>;
+
+public:
+	/**
+	 * Create a new packet-in-decode
+	 *
+	 * @param b Source buffer with raw packet data
+	 * @param localPort Local port on which packet was received
+	 * @param remoteAddress Address from which packet came
+	 * @throws std::out_of_range Range error processing packet
+	 */
+	template<unsigned int C2>
+	PacketDecoder(const Buffer<C2> &b,Demarc::Port localPort,const InetAddress &remoteAddress)
+ 		throw(std::out_of_range) :
+ 		Packet(b),
+ 		_receiveTime(Utils::now()),
+ 		_localPort(localPort),
+ 		_remoteAddress(remoteAddress),
+ 		_step(DECODE_WAITING_FOR_SENDER_LOOKUP),
+ 		__refCount()
+	{
+	}
+
+	/**
+	 * Attempt to decode this packet
+	 *
+	 * Note that this returns 'true' if processing is complete. This says nothing
+	 * about whether the packet was valid. A rejection is 'complete.'
+	 *
+	 * Once true is returned, this should not be called again.
+	 *
+	 * @param _r Runtime environment
+	 * @return True if decoding and processing is complete, false if caller should try again
+	 * @throws std::out_of_range Range error processing packet (should be discarded)
+	 * @throws std::runtime_error Other error processing packet (should be discarded)
+	 */
+	bool tryDecode(const RuntimeEnvironment *_r)
+		throw(std::out_of_range,std::runtime_error);
+
+	/**
+	 * @return Time of packet receipt
+	 */
+	inline uint64_t receiveTime() const throw() { return _receiveTime; }
+
+private:
+	struct _CBaddPeerFromHello_Data
+	{
+		const RuntimeEnvironment *renv;
+		Address source;
+		InetAddress remoteAddress;
+		int localPort;
+		unsigned int vMajor,vMinor,vRevision;
+		uint64_t helloPacketId;
+		uint64_t helloTimestamp;
+	};
+	static void _CBaddPeerFromHello(
+		void *arg, // _CBaddPeerFromHello_Data
+		const SharedPtr<Peer> &p,
+		Topology::PeerVerifyResult result);
+
+	static void _CBaddPeerFromWhois(
+		void *arg, // RuntimeEnvironment
+		const SharedPtr<Peer> &p,
+		Topology::PeerVerifyResult result);
+
+	// These are called internally to handle packet contents once it has
+	// been authenticated, decrypted, decompressed, and classified.
+	bool _doERROR(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doHELLO(const RuntimeEnvironment *_r);
+	bool _doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doWHOIS(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doRENDEZVOUS(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doMULTICAST_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+	bool _doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer);
+
+	uint64_t _receiveTime;
+	Demarc::Port _localPort;
+	InetAddress _remoteAddress;
+
+	enum {
+		DECODE_WAITING_FOR_SENDER_LOOKUP, // on initial receipt, we need peer's identity
+		DECODE_WAITING_FOR_MULTICAST_FRAME_ORIGINAL_SENDER_LOOKUP,
+	} _step;
+
+	AtomicCounter __refCount;
+};
+
+} // namespace ZeroTier
+
+#endif

node/Peer.cpp

@@ -30,6 +30,14 @@
 namespace ZeroTier {
 
 Peer::Peer() :
+	_id(),
+	_ipv4p(),
+	_ipv6p(),
+	_lastUnicastFrame(0),
+	_lastMulticastFrame(0),
+	_vMajor(0),
+	_vMinor(0),
+	_vRevision(0),
 	_dirty(false)
 {
 }
@@ -37,37 +45,44 @@ Peer::Peer() :
 Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	throw(std::runtime_error) :
 	_id(peerIdentity),
+	_ipv4p(),
+	_ipv6p(),
+	_lastUnicastFrame(0),
+	_lastMulticastFrame(0),
+	_vMajor(0),
+	_vMinor(0),
+	_vRevision(0),
 	_dirty(true)
 {
 	if (!myIdentity.agree(peerIdentity,_keys,sizeof(_keys)))
 		throw std::runtime_error("new peer identity key agreement failed");
 }
 
-void Peer::onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const InetAddress &fromAddr,unsigned int latency,unsigned int hops,Packet::Verb verb,uint64_t now)
+void Peer::onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const InetAddress &remoteAddr,unsigned int hops,Packet::Verb verb,uint64_t now)
 {
 	if (!hops) { // direct packet
-		WanPath *wp = (fromAddr.isV4() ? &_ipv4p : &_ipv6p);
-
+		WanPath *wp = (remoteAddr.isV4() ? &_ipv4p : &_ipv6p);
 		wp->lastReceive = now;
-		if (verb == Packet::VERB_FRAME)
-			wp->lastUnicastFrame = now;
-		if (latency)
-			wp->latency = latency;
 		wp->localPort = localPort;
 		if (!wp->fixed)
-			wp->addr = fromAddr;
+			wp->addr = remoteAddr;
+		_dirty = true;
+	}
 
+	if (verb == Packet::VERB_FRAME) {
+		_lastUnicastFrame = now;
+		_dirty = true;
+	} else if (verb == Packet::VERB_MULTICAST_FRAME) {
+		_lastMulticastFrame = now;
 		_dirty = true;
 	}
 }
 
-bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,bool relay,Packet::Verb verb,uint64_t now)
+bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,uint64_t now)
 {
 	if ((_ipv6p.isActive(now))||((!(_ipv4p.addr))&&(_ipv6p.addr))) {
 		if (_r->demarc->send(_ipv6p.localPort,_ipv6p.addr,data,len,-1)) {
 			_ipv6p.lastSend = now;
-			if (verb == Packet::VERB_FRAME)
-				_ipv6p.lastUnicastFrame = now;
 			_dirty = true;
 			return true;
 		}
@@ -76,8 +91,6 @@ bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,b
 	if (_ipv4p.addr) {
 		if (_r->demarc->send(_ipv4p.localPort,_ipv4p.addr,data,len,-1)) {
 			_ipv4p.lastSend = now;
-			if (verb == Packet::VERB_FRAME)
-				_ipv4p.lastUnicastFrame = now;
 			_dirty = true;
 			return true;
 		}
@@ -86,6 +99,17 @@ bool Peer::send(const RuntimeEnvironment *_r,const void *data,unsigned int len,b
 	return false;
 }
 
+void Peer::onSent(const RuntimeEnvironment *_r,bool relay,Packet::Verb verb,uint64_t now)
+{
+	if (verb == Packet::VERB_FRAME) {
+		_lastUnicastFrame = now;
+		_dirty = true;
+	} else if (verb == Packet::VERB_MULTICAST_FRAME) {
+		_lastMulticastFrame = now;
+		_dirty = true;
+	}
+}
+
 bool Peer::sendFirewallOpener(const RuntimeEnvironment *_r,uint64_t now)
 {
 	bool sent = false;

node/Peer.hpp

@@ -53,15 +53,16 @@
 #define ZT_PEER_MAX_SERIALIZED_LENGTH ( \
 	64 + \
 	IDENTITY_MAX_BINARY_SERIALIZED_LENGTH + \
-	(( \
-		(sizeof(uint64_t) * 5) + \
+	( ( \
+		(sizeof(uint64_t) * 4) + \
 		sizeof(uint16_t) + \
 		1 + \
 		sizeof(uint16_t) + \
 		16 + \
 		1 \
 	) * 2) + \
-	64 \
+	sizeof(uint64_t) + \
+	sizeof(uint64_t) \
 )
 
 namespace ZeroTier {
@@ -110,33 +111,42 @@ public:
 	/**
 	 * Must be called on authenticated packet receive from this peer
 	 * 
+	 * This must be called only after a packet has passed authentication
+	 * checking. Packets that fail are silently discarded.
+	 *
 	 * @param _r Runtime environment
 	 * @param localPort Local port on which packet was received
-	 * @param fromAddr Internet address of sender
-	 * @param latency Latency or 0 if unknown
+	 * @param remoteAddr Internet address of sender
 	 * @param hops ZeroTier (not IP) hops
 	 * @param verb Packet verb
 	 * @param now Current time
 	 */
-	void onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const InetAddress &fromAddr,unsigned int latency,unsigned int hops,Packet::Verb verb,uint64_t now);
+	void onReceive(const RuntimeEnvironment *_r,Demarc::Port localPort,const InetAddress &remoteAddr,unsigned int hops,Packet::Verb verb,uint64_t now);
 
 	/**
-	 * Send a UDP packet to this peer
-	 * 
-	 * If the active link is timed out (no receives for ping timeout ms), then
-	 * the active link number is incremented after send. This causes sends to
-	 * cycle through links if there is no clear active link. This also happens
-	 * if the send fails for some reason.
+	 * Send a packet to this peer
 	 * 
 	 * @param _r Runtime environment
 	 * @param data Data to send
 	 * @param len Length of packet
-	 * @param relay This is a relay on behalf of another peer (verb is ignored)
-	 * @param verb Packet verb (if not relay)
 	 * @param now Current time
 	 * @return True if packet appears to have been sent, false on local failure
 	 */
-	bool send(const RuntimeEnvironment *_r,const void *data,unsigned int len,bool relay,Packet::Verb verb,uint64_t now);
+	bool send(const RuntimeEnvironment *_r,const void *data,unsigned int len,uint64_t now);
+
+	/**
+	 * Must be called after a packet is successfully sent to this peer
+	 *
+	 * Note that 'relay' means we've sent a packet *from* this node to this
+	 * peer by relaying it, not that we have relayed a packet from somewhere
+	 * else to this peer. In the latter case this is not called.
+	 *
+	 * @param _r Runtime environment
+	 * @param relay If true, packet was sent indirectly via a relay
+	 * @param verb Packet verb
+	 * @param now Current time
+	 */
+	void onSent(const RuntimeEnvironment *_r,bool relay,Packet::Verb verb,uint64_t now);
 
 	/**
 	 * Send firewall opener to active link
@@ -195,7 +205,25 @@ public:
 	uint64_t lastUnicastFrame() const
 		throw()
 	{
-		return std::max(_ipv4p.lastUnicastFrame,_ipv6p.lastUnicastFrame);
+		return _lastUnicastFrame;
+	}
+
+	/**
+	 * @return Time of most recent multicast frame
+	 */
+	uint64_t lastMulticastFrame() const
+		throw()
+	{
+		return _lastMulticastFrame;
+	}
+
+	/**
+	 * @return Time of most recent frame of any kind (unicast or multicast)
+	 */
+	uint64_t lastFrame() const
+		throw()
+	{
+		return std::max(_lastUnicastFrame,_lastMulticastFrame);
 	}
 
 	/**
@@ -213,6 +241,21 @@ public:
 		return 0;
 	}
 
+	/**
+	 * @param addr Remote address
+	 * @param latency Latency measurment
+	 */
+	void setLatency(const InetAddress &addr,unsigned int latency)
+	{
+		if (addr == _ipv4p.addr) {
+			_ipv4p.latency = latency;
+			_dirty = true;
+		} else if (addr == _ipv6p.addr) {
+			_ipv6p.latency = latency;
+			_dirty = true;
+		}
+	}
+
 	/**
 	 * @return True if this peer has at least one direct IP address path
 	 */
@@ -290,6 +333,33 @@ public:
 		return (_keys + 32); // mac key is second 32-byte key
 	}
 
+	/**
+	 * Set the remote version of the peer (not persisted)
+	 *
+	 * @param vmaj Major version
+	 * @param vmin Minor version
+	 * @param vrev Revision
+	 */
+	inline void setRemoteVersion(unsigned int vmaj,unsigned int vmin,unsigned int vrev)
+	{
+		_vMajor = vmaj;
+		_vMinor = vmin;
+		_vRevision = vrev;
+	}
+
+	/**
+	 * @return Remote version in string form or '?' if unknown
+	 */
+	inline std::string remoteVersion() const
+	{
+		if ((_vMajor)||(_vMinor)||(_vRevision)) {
+			char tmp[32];
+			sprintf(tmp,"%u.%u.%u",_vMajor,_vMinor,_vRevision);
+			return std::string(tmp);
+		}
+		return std::string("?");
+	}
+
 	/**
 	 * Get and reset dirty flag
 	 * 
@@ -312,11 +382,13 @@ public:
 	inline void serialize(Buffer<C> &b)
 		throw(std::out_of_range)
 	{
-		b.append((unsigned char)1); // version
+		b.append((unsigned char)2); // version
 		b.append(_keys,sizeof(_keys));
 		_id.serialize(b,false);
 		_ipv4p.serialize(b);
 		_ipv6p.serialize(b);
+		b.append(_lastUnicastFrame);
+		b.append(_lastMulticastFrame);
 	}
 
 	template<unsigned int C>
@@ -325,14 +397,19 @@ public:
 	{
 		unsigned int p = startAt;
 
-		if (b[p++] != 1)
+		if (b[p++] != 2)
 			throw std::invalid_argument("Peer: deserialize(): version mismatch");
 
 		memcpy(_keys,b.field(p,sizeof(_keys)),sizeof(_keys)); p += sizeof(_keys);
 		p += _id.deserialize(b,p);
 		p += _ipv4p.deserialize(b,p);
 		p += _ipv6p.deserialize(b,p);
+		_lastUnicastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
+		_lastMulticastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 
+		_vMajor = 0;
+		_vMinor = 0;
+		_vRevision = 0;
 		_dirty = false;
 
 		return (p - startAt);
@@ -372,7 +449,6 @@ private:
 		WanPath() :
 			lastSend(0),
 			lastReceive(0),
-			lastUnicastFrame(0),
 			lastFirewallOpener(0),
 			localPort(Demarc::ANY_PORT),
 			latency(0),
@@ -393,7 +469,6 @@ private:
 		{
 			b.append(lastSend);
 			b.append(lastReceive);
-			b.append(lastUnicastFrame);
 			b.append(lastFirewallOpener);
 			b.append(Demarc::portToInt(localPort));
 			b.append((uint16_t)latency);
@@ -423,7 +498,6 @@ private:
 
 			lastSend = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 			lastReceive = b.template at<uint64_t>(p); p += sizeof(uint64_t);
-			lastUnicastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 			lastFirewallOpener = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 			localPort = Demarc::intToPort(b.template at<uint64_t>(p)); p += sizeof(uint64_t);
 			latency = b.template at<uint16_t>(p); p += sizeof(uint16_t);
@@ -449,9 +523,8 @@ private:
 
 		uint64_t lastSend;
 		uint64_t lastReceive;
-		uint64_t lastUnicastFrame;
 		uint64_t lastFirewallOpener;
-		Demarc::Port localPort; // ANY_PORT if not defined
+		Demarc::Port localPort; // ANY_PORT if not defined (size: uint64_t)
 		unsigned int latency; // 0 if never determined
 		InetAddress addr; // null InetAddress if path is undefined
 		bool fixed; // do not learn address from received packets
@@ -463,8 +536,12 @@ private:
 	WanPath _ipv4p;
 	WanPath _ipv6p;
 
+	uint64_t _lastUnicastFrame;
+	uint64_t _lastMulticastFrame;
+
 	// Fields below this line are not persisted with serialize()
 
+	unsigned int _vMajor,_vMinor,_vRevision;
 	bool _dirty;
 
 	AtomicCounter __refCount;
@@ -472,4 +549,13 @@ private:
 
 } // namespace ZeroTier
 
+// Add a swap() for shared ptr's to peers to speed up peer sorts
+namespace std {
+	template<>
+	inline void swap(ZeroTier::SharedPtr<ZeroTier::Peer> &a,ZeroTier::SharedPtr<ZeroTier::Peer> &b)
+	{
+		a.swap(b);
+	}
+}
+
 #endif

node/Range.hpp

@@ -0,0 +1,123 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_RANGE_HPP
+#define _ZT_RANGE_HPP
+
+namespace ZeroTier {
+
+/**
+ * A range of numeric values
+ *
+ * @tparam T Type, can be any numeric value (int, float, double, etc.)
+ */
+template<typename T>
+class Range
+{
+public:
+	/**
+	 * Construct an empty range
+	 */
+	Range()
+		throw() :
+		start(0),
+		end(0)
+	{
+	}
+
+	/**
+	 * @param s Starting value (inclusive)
+	 * @param e Ending value (exclusive)
+	 */
+	Range(T s,T e)
+		throw() :
+		start(s),
+		end(e)
+	{
+	}
+
+	/**
+	 * Construct a range containing from n to n+1 (thus only n for integers)
+	 *
+	 * @param n Number to contain
+	 */
+	Range(T n)
+		throw() :
+		start(n),
+		end(n+1)
+	{
+	}
+
+	/**
+	 * @return end - start
+	 */
+	inline T magnitude() const
+		throw()
+	{
+		return (end - start);
+	}
+
+	/**
+	 * @return True if range contains something (magnitude is nonzero)
+	 */
+	inline operator bool() const
+		throw()
+	{
+		return (end > start);
+	}
+
+	/**
+	 * @param v Value to test
+	 * @return True if value is between start (inclusive) and end (exclusive)
+	 */
+	inline bool operator()(const T &v) const
+		throw()
+	{
+		return ((v >= start)&&(v < end));
+	}
+
+	inline bool operator==(const Range &r) const throw() { return ((start == r.start)&&(end == r.end)); }
+	inline bool operator!=(const Range &r) const throw() { return (!(*this == r)); }
+	inline bool operator<(const Range &r) const throw() { return ((start < r.start) ? true : ((start == r.start) ? (end < r.end) : false)); }
+	inline bool operator>(const Range &r) const throw() { return (r < *this); }
+	inline bool operator<=(const Range &r) const throw() { return !(r < *this); }
+	inline bool operator>=(const Range &r) const throw() { return !(*this < r); }
+
+	/**
+	 * Start of range (may be modified directly)
+	 */
+	T start;
+
+	/**
+	 * End of range (may be modified directly)
+	 */
+	T end;
+};
+
+} // namespace ZeroTier
+
+#endif

node/RateLimiter.hpp

@@ -0,0 +1,138 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_RATELIMITER_HPP
+#define _ZT_RATELIMITER_HPP
+
+#include <math.h>
+#include "Utils.hpp"
+
+namespace ZeroTier {
+
+/**
+ * Burstable rate limiter
+ *
+ * This limits a transfer rate to a maximum bytes per second using an
+ * accounting method based on a balance rather than accumulating an
+ * average rate. The result is a burstable rate limit rather than a
+ * continuous rate limit; the link being limited may use all its balance
+ * at once or slowly over time. Balance constantly replenishes over time
+ * up to a configurable maximum balance.
+ */
+class RateLimiter
+{
+public:
+	/**
+	 * Limits to apply to a rate limiter
+	 *
+	 * Since many rate limiters may share the same fixed limit values,
+	 * save memory by breaking this out into a struct parameter that
+	 * can be passed into RateLimiter's methods.
+	 */
+	struct Limit
+	{
+		/**
+		 * Speed in bytes per second, or rate of balance accrual
+		 */
+		double bytesPerSecond;
+
+		/**
+		 * Maximum balance that can ever be accrued (should be > 0.0)
+		 */
+		double maxBalance;
+
+		/**
+		 * Minimum balance, or maximum allowable "debt" (should be <= 0.0)
+		 */
+		double minBalance;
+	};
+
+	/**
+	 * Create an uninitialized rate limiter
+	 *
+	 * init() must be called before this is used.
+	 */
+	RateLimiter() throw() {}
+
+	/**
+	 * @param preload Initial balance to place in account
+	 */
+	RateLimiter(double preload)
+		throw()
+	{
+		init(preload);
+	}
+
+	/**
+	 * Initialize or re-initialize rate limiter
+	 *
+	 * @param preload Initial balance to place in account
+	 */
+	inline void init(double preload)
+		throw()
+	{
+		_lastTime = Utils::nowf();
+		_balance = preload;
+	}
+
+	/**
+	 * Update balance based on current clock and supplied Limits bytesPerSecond and maxBalance
+	 *
+	 * @param lim Current limits in effect
+	 * @return New balance
+	 */
+	inline double updateBalance(const Limit &lim)
+		throw()
+	{
+		double lt = _lastTime;
+		double now = _lastTime = Utils::nowf();
+		return (_balance = fmin(lim.maxBalance,_balance + (lim.bytesPerSecond * (now - lt))));
+	}
+
+	/**
+	 * Update balance and test if a block of 'bytes' should be permitted to be transferred
+	 *
+	 * @param lim Current limits in effect
+	 * @param bytes Number of bytes that we wish to transfer
+	 * @return True if balance was sufficient
+	 */
+	inline bool gate(const Limit &lim,double bytes)
+		throw()
+	{
+		bool allow = (updateBalance(lim) >= bytes);
+		_balance = fmax(lim.minBalance,_balance - bytes);
+		return allow;
+	}
+
+private:
+	double _lastTime;
+	double _balance;
+};
+
+} // namespace ZeroTier
+
+#endif

node/RuntimeEnvironment.hpp

@@ -29,7 +29,9 @@
 #define _ZT_RUNTIMEENVIRONMENT_HPP
 
 #include <string>
+#include "Constants.hpp"
 #include "Identity.hpp"
+#include "Condition.hpp"
 
 namespace ZeroTier {
 
@@ -39,6 +41,9 @@ class Demarc;
 class Switch;
 class Topology;
 class SysEnv;
+class Multicaster;
+class CMWC4096;
+class Service;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -56,30 +61,42 @@ class RuntimeEnvironment
 {
 public:
 	RuntimeEnvironment() :
-		identity(),
 		log((Logger *)0),
-		nc((NodeConfig *)0),
+		prng((CMWC4096 *)0),
 		demarc((Demarc *)0),
+		multicaster((Multicaster *)0),
 		sw((Switch *)0),
-		topology((Topology *)0)
+		topology((Topology *)0),
+		sysEnv((SysEnv *)0),
+		nc((NodeConfig *)0)
+#ifndef __WINDOWS__
+		,netconfService((Service *)0)
+#endif
 	{
 	}
 
 	std::string homePath;
-	std::string autoconfUrlPrefix;
-	std::string configAuthorityIdentityStr;
-	std::string ownershipVerificationSecret;
-	std::string ownershipVerificationSecretHash; // base64 of SHA-256 X16 rounds
 
-	Identity configAuthority;
+	// signal() to prematurely interrupt main loop wait
+	Condition mainLoopWaitCondition;
+
 	Identity identity;
 
+	// Order matters a bit here. These are constructed in this order
+	// and then deleted in the opposite order on Node exit.
+
 	Logger *log; // may be null
-	NodeConfig *nc;
+	CMWC4096 *prng;
 	Demarc *demarc;
+	Multicaster *multicaster;
 	Switch *sw;
 	Topology *topology;
 	SysEnv *sysEnv;
+	NodeConfig *nc;
+
+#ifndef __WINDOWS__
+	Service *netconfService; // may be null
+#endif
 };
 
 } // namespace ZeroTier

node/Salsa20.cpp

@@ -5,6 +5,7 @@
  */
 
 #include "Salsa20.hpp"
+#include "Constants.hpp"
 
 #define ROTATE(v,c) (((v) << (c)) | ((v) >> (32 - (c))))
 #define XOR(v,w) ((v) ^ (w))

node/Service.cpp

@@ -0,0 +1,241 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include "Constants.hpp"
+
+#ifndef __WINDOWS__
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <signal.h>
+#include <time.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/select.h>
+#include <sys/wait.h>
+
+#include "Service.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Utils.hpp"
+#include "Logger.hpp"
+
+namespace ZeroTier {
+
+Service::Service(const RuntimeEnvironment *renv,const char *name,const char *path,void (*handler)(void *,Service &,const Dictionary &),void *arg) :
+	_r(renv),
+	_path(path),
+	_name(name),
+	_arg(arg),
+	_handler(handler),
+	_pid(-1),
+	_childStdin(0),
+	_childStdout(0),
+	_childStderr(0),
+	_run(true)
+{
+	_thread = Thread::start(this);
+}
+
+Service::~Service()
+{
+	_run = false;
+	long pid = _pid;
+	if (pid > 0) {
+		int st = 0;
+		::kill(pid,SIGTERM);
+		for(int i=0;i<20;++i) {
+			if (waitpid(pid,&st,WNOHANG) == pid) {
+				pid = 0;
+				break;
+			}
+			Thread::sleep(100);
+		}
+		if (pid > 0) {
+			::kill(pid,SIGKILL);
+			waitpid(pid,&st,0);
+		}
+	}
+	Thread::join(_thread);
+}
+
+bool Service::send(const Dictionary &msg)
+{
+	if (_childStdin <= 0)
+		return false;
+
+	std::string mser = msg.toString();
+	if (mser.length() > ZT_SERVICE_MAX_MESSAGE_SIZE)
+		return false;
+
+	// This can technically block. We'll fix this if it ends up being a
+	// problem.
+	uint32_t len = Utils::hton((uint32_t)mser.length());
+	if (write(_childStdin,&len,4) != 4)
+		return false;
+	if ((int)write(_childStdin,mser.data(),mser.length()) != (int)mser.length())
+		return false;
+
+	return true;
+}
+
+void Service::threadMain()
+	throw()
+{
+	char buf[131072];
+	fd_set readfds,writefds,exceptfds;
+	struct timeval tv;
+
+	std::string stderrBuf;
+	std::string stdoutBuf;
+	unsigned int stdoutExpecting = 0;
+
+	while (_run) {
+		if (_pid <= 0) {
+			LOG("launching service %s...",_name.c_str());
+
+			int in[2],out[2],err[2];
+			pipe(in);
+			pipe(out);
+			pipe(err);
+
+			long pid = vfork();
+			if (pid < 0) {
+				LOG("service %s terminating: could not fork!",_name.c_str());
+				return;
+			} else if (pid) {
+				// Parent
+				close(in[0]);
+				close(out[1]);
+				close(err[1]);
+				Thread::sleep(500); // give child time to start
+				_childStdin = in[1];
+				_childStdout = out[0];
+				_childStderr = err[0];
+				fcntl(_childStdout,F_SETFL,O_NONBLOCK);
+				fcntl(_childStderr,F_SETFL,O_NONBLOCK);
+				_pid = pid;
+			} else {
+				// Child
+				close(in[1]);
+				close(out[0]);
+				close(err[0]);
+				dup2(in[0],STDIN_FILENO);
+				dup2(out[1],STDOUT_FILENO);
+				dup2(err[1],STDERR_FILENO);
+				execl(_path.c_str(),_path.c_str(),_r->homePath.c_str(),(const char *)0);
+				exit(-1);
+			}
+		} else {
+			int st = 0;
+			if (waitpid(_pid,&st,WNOHANG) == _pid) {
+				if (_childStdin > 0) close(_childStdin);
+				_childStdin = 0;
+				if (_childStdout > 0) close(_childStdout);
+				if (_childStderr > 0) close(_childStderr);
+				_pid = 0;
+
+				if (!_run)
+					return;
+
+				LOG("service %s exited with exit code: %d, delaying 1s to attempt relaunch",_name.c_str(),st);
+
+				Thread::sleep(1000); // wait to relaunch
+				continue;
+			}
+		}
+
+		// If we've made it here, _pid is running last we checked.
+
+		FD_ZERO(&readfds);
+		FD_ZERO(&writefds);
+		FD_ZERO(&exceptfds);
+
+		FD_SET(_childStdout,&readfds);
+		FD_SET(_childStderr,&readfds);
+
+		tv.tv_sec = 1;
+		tv.tv_usec = 0;
+		select(std::max(_childStdout,_childStderr)+1,&readfds,&writefds,&exceptfds,&tv);
+
+		if (!_run) {
+			if (_childStdin > 0) close(_childStdin);
+			_childStdin = 0;
+			if (_childStdout > 0) close(_childStdout);
+			if (_childStderr > 0) close(_childStderr);
+			return;
+		}
+
+		if ((_childStderr > 0)&&(FD_ISSET(_childStderr,&readfds))) {
+			int n = (int)read(_childStderr,buf,sizeof(buf));
+			for(int i=0;i<n;++i) {
+				if ((buf[i] == '\r')||(buf[i] == '\n')) {
+					stderrBuf = Utils::trim(stderrBuf);
+					if (stderrBuf.length())
+						LOG("service %s: %s",_name.c_str(),stderrBuf.c_str());
+					stderrBuf = "";
+				} else stderrBuf.push_back(buf[i]);
+			}
+		}
+
+		if ((_childStdout > 0)&&(FD_ISSET(_childStdout,&readfds))) {
+			int n = (int)read(_childStdout,buf,sizeof(buf));
+			for(int i=0;i<n;++i) {
+				stdoutBuf.push_back(buf[i]);
+				if (stdoutExpecting) {
+					if (stdoutBuf.length() == stdoutExpecting) {
+						try {
+							_handler(_arg,*this,Dictionary(stdoutBuf));
+						} catch ( ... ) {
+							LOG("unexpected exception handling message from service %s",_name.c_str());
+						}
+						stdoutBuf = "";
+						stdoutExpecting = 0;
+					}
+				} else if (stdoutBuf.length() == 4) {
+					stdoutExpecting = Utils::ntoh(*((const uint32_t *)stdoutBuf.data()));
+					stdoutBuf = "";
+					if (stdoutExpecting > ZT_SERVICE_MAX_MESSAGE_SIZE) {
+						LOG("message size overrun from service %s: %u bytes -- restarting service",_name.c_str(),stdoutExpecting);
+						stdoutExpecting = 0;
+						kill(_pid,SIGKILL);
+						break;
+					}
+				}
+			}
+		}
+	}
+}
+
+} // namespace ZeroTier
+
+#endif // __WINDOWS__
+

node/Service.hpp

@@ -0,0 +1,132 @@
+/*
+ * ZeroTier One - Global Peer to Peer Ethernet
+ * Copyright (C) 2012-2013  ZeroTier Networks LLC
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef _ZT_SERVICE_HPP
+#define _ZT_SERVICE_HPP
+
+#include <string>
+#include <stdexcept>
+
+#include "Constants.hpp"
+#include "Dictionary.hpp"
+#include "Thread.hpp"
+#include "Mutex.hpp"
+
+/**
+ * Maximum size of a service message in bytes (sanity limit)
+ */
+#define ZT_SERVICE_MAX_MESSAGE_SIZE 131072
+
+namespace ZeroTier {
+
+class RuntimeEnvironment;
+
+#ifndef __WINDOWS__
+/**
+ * A subprocess that communicates with the host via a simple protocol
+ *
+ * This is currently only supported on *nix systems, and is used to implement
+ * special plugins that are used by supernodes and network configuration
+ * master nodes. Users will probably have no use for it.
+ *
+ * The simple binary protocol consists of a bidirectional stream of string-
+ * serialized Dictionaries prefixed by a 32-bit message length. Input
+ * messages are sent to the subprocess via its stdin, and output is read
+ * from its stdout. Messages printed by the subprocess on its stderr are
+ * logged via the standard Logger instance. If the subprocess dies, an
+ * attempt is made to restart it every second.
+ */
+class Service
+{
+public:
+	/**
+	 * Create and launch a new service
+	 *
+	 * @param renv Runtime environment
+	 * @param name Name of service
+	 * @param path Path to service binary
+	 * @param handler Handler function to call when service generates output
+	 * @param arg First argument to service
+	 */
+	Service(const RuntimeEnvironment *renv,
+	        const char *name,
+	        const char *path,
+	        void (*handler)(void *,Service &,const Dictionary &),
+	        void *arg);
+
+	~Service();
+
+	/**
+	 * Send a message to service subprocess
+	 *
+	 * @param msg Message in key/value dictionary form
+	 * @return True if message was sent
+	 */
+	bool send(const Dictionary &msg);
+
+	/**
+	 * @return Name of service
+	 */
+	inline const char *name() const
+		throw()
+	{
+		return _name.c_str();
+	}
+
+	/**
+	 * @return True if subprocess is running
+	 */
+	inline bool running() const
+		throw()
+	{
+		return (_pid > 0);
+	}
+
+	/**
+	 * Thread main method; do not call elsewhere
+	 */
+	void threadMain()
+		throw();
+
+private:
+	const RuntimeEnvironment *_r;
+	Thread _thread;
+	std::string _path;
+	std::string _name;
+	void *_arg;
+	void (*_handler)(void *,Service &,const Dictionary &);
+	long _pid;
+	int _childStdin;
+	int _childStdout;
+	int _childStderr;
+	volatile bool _run;
+};
+#endif // __WINDOWS__
+
+} // namespace ZeroTier
+
+#endif

node/SharedPtr.hpp

@@ -43,6 +43,9 @@ namespace ZeroTier {
  *
  * Because this is introspective, it is safe to apply to a naked pointer
  * multiple times provided there is always at least one holding SharedPtr.
+ *
+ * Once C++11 is ubiquitous, this and a few other things like Thread might get
+ * torn out for their standard equivalents.
  */
 template<typename T>
 class SharedPtr
@@ -88,6 +91,14 @@ public:
 		return *this;
 	}
 
+	inline void swap(SharedPtr &with)
+		throw()
+	{
+		T *tmp = _ptr;
+		_ptr = with._ptr;
+		with._ptr = tmp;
+	}
+
 	inline operator bool() const throw() { return (_ptr); }
 	inline T &operator*() const throw() { return *_ptr; }
 	inline T *operator->() const throw() { return _ptr; }

node/Switch.hpp

@@ -31,6 +31,7 @@
 #include <map>
 #include <set>
 #include <vector>
+#include <list>
 
 #include "Mutex.hpp"
 #include "MAC.hpp"
@@ -44,6 +45,8 @@
 #include "Network.hpp"
 #include "SharedPtr.hpp"
 #include "Demarc.hpp"
+#include "Multicaster.hpp"
+#include "PacketDecoder.hpp"
 
 namespace ZeroTier {
 
@@ -106,6 +109,16 @@ public:
 	 */
 	void sendHELLO(const Address &dest);
 
+	/**
+	 * Send a HELLO announcement immediately to the indicated address
+	 *
+	 * @param localPort Originating local port or ANY_PORT to pick
+	 * @param remoteAddr IP address to send to
+	 * @param dest Destination peer
+	 * @return True if send appears successful
+	 */
+	bool sendHELLO(const SharedPtr<Peer> &dest,Demarc::Port localPort,const InetAddress &remoteAddr);
+
 	/**
 	 * Send RENDEZVOUS to two peers to permit them to directly connect
 	 *
@@ -122,6 +135,14 @@ public:
 	 */
 	bool unite(const Address &p1,const Address &p2,bool force);
 
+	/**
+	 * Send NAT traversal messages to peer at the given candidate address
+	 *
+	 * @param peer Peer to contact
+	 * @param atAddr Address of peer
+	 */
+	void contact(const SharedPtr<Peer> &peer,const InetAddress &atAddr);
+
 	/**
 	 * Perform retries and other periodic timer tasks
 	 * 
@@ -139,70 +160,44 @@ public:
 	 */
 	void announceMulticastGroups(const std::map< SharedPtr<Network>,std::set<MulticastGroup> > &allMemberships);
 
+	/**
+	 * Request WHOIS on a given address
+	 *
+	 * @param addr Address to look up
+	 */
+	void requestWhois(const Address &addr);
+
+	/**
+	 * Run any processes that are waiting for this peer
+	 *
+	 * Called when we learn of a peer's identity from HELLO, OK(WHOIS), etc.
+	 *
+	 * @param peer New peer
+	 */
+	void doAnythingWaitingForPeer(const SharedPtr<Peer> &peer);
+
 private:
-	// Returned by _send() and _processRemotePacket() to indicate what happened
-	enum PacketServiceAttemptResult
-	{
-		PACKET_SERVICE_ATTEMPT_OK,
-		PACKET_SERVICE_ATTEMPT_PEER_UNKNOWN,
-		PACKET_SERVICE_ATTEMPT_SEND_FAILED
-	};
+	void _handleRemotePacketFragment(
+		Demarc::Port localPort,
+		const InetAddress &fromAddr,
+		const Buffer<4096> &data);
 
-	struct _CBaddPeerFromHello_Data
-	{
-		Switch *parent;
-		Address source;
-		InetAddress fromAddr;
-		int localPort;
-		unsigned int vMajor,vMinor,vRevision;
-		uint64_t helloPacketId;
-		uint64_t helloTimestamp;
-	};
-	static void _CBaddPeerFromHello(void *arg,const SharedPtr<Peer> &p,Topology::PeerVerifyResult result);
-	static void _CBaddPeerFromWhois(void *arg,const SharedPtr<Peer> &p,Topology::PeerVerifyResult result); // arg == this
+	void _handleRemotePacketHead(
+		Demarc::Port localPort,
+		const InetAddress &fromAddr,
+		const Buffer<4096> &data);
 
-	void _propagateMulticast(const SharedPtr<Network> &network,unsigned char *bloom,const MulticastGroup &mg,unsigned int mcHops,unsigned int mcLoadFactor,const MAC &from,unsigned int etherType,const void *data,unsigned int len);
-	PacketServiceAttemptResult _tryHandleRemotePacket(Demarc::Port localPort,const InetAddress &fromAddr,Packet &packet);
-	void _doHELLO(Demarc::Port localPort,const InetAddress &fromAddr,Packet &packet);
-	void _requestWhois(const Address &addr);
-	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
-	PacketServiceAttemptResult _trySend(const Packet &packet,bool encrypt);
-	void _retryPendingFor(const Address &addr);
+	Address _sendWhoisRequest(
+		const Address &addr,
+		const Address *peersAlreadyConsulted,
+		unsigned int numPeersAlreadyConsulted);
 
-	// Updates entry for crc in multicast history, returns true if already
-	// present in history and not expired.
-	inline bool _checkAndUpdateMulticastHistory(const MAC &fromMac,const MAC &toMulticastMac,const void *payload,unsigned int len,const uint64_t nwid,const uint64_t now)
-	{
-		uint64_t crc = Utils::crc64(0,fromMac.data,6);
-		crc = Utils::crc64(crc,toMulticastMac.data,6);
-		crc = Utils::crc64(crc,payload,len);
-		crc += nwid; // also include network ID
-
-		uint64_t earliest = 0xffffffffffffffffULL;
-		unsigned long earliestIdx = 0;
-		for(unsigned int i=0;i<ZT_MULTICAST_DEDUP_HISTORY_LENGTH;++i) {
-			if (_multicastHistory[i][0] == crc) {
-				uint64_t then = _multicastHistory[i][1];
-				_multicastHistory[i][1] = now;
-				return ((now - then) < ZT_MULTICAST_DEDUP_HISTORY_EXPIRE);
-			} else if (_multicastHistory[i][1] < earliest) {
-				earliest = _multicastHistory[i][1];
-				earliestIdx = i;
-			}
-		}
-
-		_multicastHistory[earliestIdx][0] = crc; // replace oldest entry
-		_multicastHistory[earliestIdx][1] = now;
-
-		return false;
-	}
+	bool _trySend(
+		const Packet &packet,
+		bool encrypt);
 
 	const RuntimeEnvironment *const _r;
 
-	// Multicast packet CRC64's for packets we've received recently, to reject
-	// duplicates during propagation. [0] is CRC64, [1] is time.
-	uint64_t _multicastHistory[ZT_MULTICAST_DEDUP_HISTORY_LENGTH][2];
-
 	struct WhoisRequest
 	{
 		uint64_t lastSent;
@@ -212,29 +207,28 @@ private:
 	std::map< Address,WhoisRequest > _outstandingWhoisRequests;
 	Mutex _outstandingWhoisRequests_m;
 
+	std::list< SharedPtr<PacketDecoder> > _rxQueue;
+	Mutex _rxQueue_m;
+
 	struct TXQueueEntry
 	{
+		TXQueueEntry() {}
+		TXQueueEntry(uint64_t ct,const Packet &p,bool enc) :
+			creationTime(ct),
+			packet(p),
+			encrypt(enc) {}
+
 		uint64_t creationTime;
 		Packet packet; // unencrypted/untagged for TX queue
 		bool encrypt;
 	};
-	std::multimap< Address,TXQueueEntry > _txQueue; // by destination address
+	std::multimap< Address,TXQueueEntry > _txQueue;
 	Mutex _txQueue_m;
 
-	struct RXQueueEntry
-	{
-		uint64_t creationTime;
-		Demarc::Port localPort;
-		Packet packet; // encrypted/tagged
-		InetAddress fromAddr;
-	};
-	std::multimap< Address,RXQueueEntry > _rxQueue; // by source address
-	Mutex _rxQueue_m;
-
 	struct DefragQueueEntry
 	{
 		uint64_t creationTime;
-		Packet frag0;
+		SharedPtr<PacketDecoder> frag0;
 		Packet::Fragment frags[ZT_MAX_PACKET_FRAGMENTS - 1];
 		unsigned int totalFragments; // 0 if only frag0 received, waiting for frags
 		uint32_t haveFragments; // bit mask, LSB to MSB
@@ -245,14 +239,22 @@ private:
 	std::map< Array< Address,2 >,uint64_t > _lastUniteAttempt; // key is always sorted in ascending order, for set-like behavior
 	Mutex _lastUniteAttempt_m;
 
-	struct RendezvousQueueEntry
+	struct ContactQueueEntry
 	{
-		InetAddress inaddr;
+		ContactQueueEntry() {}
+		ContactQueueEntry(const SharedPtr<Peer> &p,uint64_t ft,Demarc::Port lp,const InetAddress &a) :
+			peer(p),
+			fireAtTime(ft),
+			localPort(lp),
+			inaddr(a) {}
+
+		SharedPtr<Peer> peer;
 		uint64_t fireAtTime;
 		Demarc::Port localPort;
+		InetAddress inaddr;
 	};
-	std::map< Address,RendezvousQueueEntry > _rendezvousQueue;
-	Mutex _rendezvousQueue_m;
+	std::list<ContactQueueEntry> _contactQueue;
+	Mutex _contactQueue_m;
 };
 
 } // namespace ZeroTier

node/SysEnv.hpp

@@ -30,6 +30,8 @@
 
 #include <stdint.h>
 
+#include "NonCopyable.hpp"
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -37,7 +39,7 @@ class RuntimeEnvironment;
 /**
  * Local system environment monitoring utilities
  */
-class SysEnv
+class SysEnv : NonCopyable
 {
 public:
 	SysEnv(const RuntimeEnvironment *renv);

node/Thread.cpp

@@ -1,192 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include "Thread.hpp"
-
-#if defined(__APPLE__) || defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <stdexcept>
-
-extern "C" {
-static void *__m_thread_main(void *ptr)
-{
-	((ZeroTier::Thread *)ptr)->__intl_run();
-	return (void *)0;
-}
-}
-
-namespace ZeroTier {
-
-Thread::Thread() :
-	suicidalThread(false),
-	_impl(malloc(sizeof(pthread_t))),
-	_running()
-{
-	memset(_impl,0,sizeof(pthread_t));
-}
-
-Thread::~Thread()
-{
-	free(_impl);
-}
-
-void Thread::start()
-{
-	if (!*_running) {
-		++_running;
-		pthread_create((pthread_t *)_impl,(const pthread_attr_t *)0,&__m_thread_main,(void *)this);
-	}
-}
-
-void Thread::join()
-{
-	void *tmp;
-	if (*_running)
-		pthread_join(*((pthread_t *)_impl),&tmp);
-}
-
-void Thread::sleep(unsigned long ms)
-{
-	usleep(ms);
-}
-
-void Thread::__intl_run()
-{
-	for(;;) {
-		_notInit = false;
-		this->main();
-		if (suicidalThread) {
-			delete this;
-			return;
-		}
-		if (_notInit) // UGLY ASS HACK: see main()
-			usleep(50);
-		else break;
-	}
-	--_running;
-}
-
-void Thread::main()
-	throw()
-{
-	_notInit = true; // UGLY ASS HACK: retry if subclass has not defined virtual function pointer yet
-}
-
-} // namespace ZeroTier
-
-#endif
-
-#ifdef _WIN32
-
-#include <Windows.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-DWORD WINAPI __m_thread_main(LPVOID lpParam)
-{
-	((ZeroTier::Thread *)lpParam)->__intl_run();
-	return 0;
-}
-
-struct __m_thread_info
-{
-	HANDLE threadHandle;
-	DWORD threadId;
-	bool started;
-};
-
-namespace ZeroTier {
-
-Thread::Thread() :
-	suicidalThread(false),
-	_impl(malloc(sizeof(__m_thread_info))),
-	_running()
-{
-	memset(_impl,0,sizeof(__m_thread_info));
-}
-
-Thread::~Thread()
-{
-	if (((__m_thread_info *)_impl)->started)
-		CloseHandle(((__m_thread_info *)_impl)->threadHandle);
-	free(_impl);
-}
-
-void Thread::start()
-{
-	if (!*_running) {
-		++_running;
-		if ((((__m_thread_info *)_impl)->threadHandle = CreateThread(NULL,0,__m_thread_main,this,0,&(((__m_thread_info *)_impl)->threadId))) != NULL) {
-			((__m_thread_info *)_impl)->started = true;
-		}
-	}
-}
-
-void Thread::join()
-{
-	if (*_running)
-		WaitForSingleObject(((__m_thread_info *)_impl)->threadHandle,INFINITE);
-}
-
-void Thread::__intl_run()
-{
-	for(;;) {
-		_notInit = false;
-		this->main();
-		if (suicidalThread) {
-			delete this;
-			return;
-		}
-		if (_notInit)
-			Thread::sleep(50);
-		else break;
-	}
-	--_running;
-}
-
-void Thread::main()
-	throw()
-{
-	_notInit = true; // HACK: retry if subclass has not defined virtual function pointer yet
-}
-
-struct _Thread_RunInBackgroundData
-{
-	void (*func)(void *);
-	void *ptr;
-	HANDLE threadHandle;
-	DWORD threadId;
-};
-
-} // namespace ZeroTier
-
-#endif

node/Thread.hpp

@@ -28,67 +28,103 @@
 #ifndef _ZT_THREAD_HPP
 #define _ZT_THREAD_HPP
 
-#include "NonCopyable.hpp"
+#include <stdexcept>
+
+#include "Constants.hpp"
 #include "AtomicCounter.hpp"
 
+#ifdef __WINDOWS__
+
+todo need windows;
+
+#else
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <unistd.h>
+
 namespace ZeroTier {
 
+template<typename C>
+static void *___zt_threadMain(void *instance)
+{
+	try {
+		((C *)instance)->threadMain();
+	} catch ( ... ) {}
+	return (void *)0;
+}
+
 /**
- * Wrapper for OS-dependent thread functions like pthread_create, etc.
+ * A thread identifier, and static methods to start and join threads
  */
-class Thread : NonCopyable
+class Thread
 {
 public:
-	Thread();
-	virtual ~Thread();
+	Thread()
+		throw()
+	{
+		memset(&_tid,0,sizeof(_tid));
+	}
+
+	Thread(const Thread &t)
+		throw()
+	{
+		memcpy(&_tid,&(t._tid),sizeof(_tid));
+	}
+
+	inline Thread &operator=(const Thread &t)
+		throw()
+	{
+		memcpy(&_tid,&(t._tid),sizeof(_tid));
+		return *this;
+	}
 
 	/**
-	 * Start thread -- can only be called once
-	 */
-	void start();
-
-	/**
-	 * Wait for thread to terminate
+	 * Start a new thread
 	 *
-	 * More than one thread should not simultaneously use join().
+	 * @param instance Instance whose threadMain() method gets called by new thread
+	 * @return Thread identifier
+	 * @throws std::runtime_error Unable to create thread
+	 * @tparam C Class containing threadMain()
 	 */
-	void join();
+	template<typename C>
+	static inline Thread start(C *instance)
+		throw(std::runtime_error)
+	{
+		Thread t;
+		if (pthread_create(&t._tid,(const pthread_attr_t *)0,&___zt_threadMain<C>,instance))
+			throw std::runtime_error("pthread_create() failed, unable to create thread");
+		return t;
+	}
 
 	/**
-	 * @return True if thread is running
+	 * Join to a thread, waiting for it to terminate
+	 *
+	 * @param t Thread to join
 	 */
-	inline bool running() const { return (*_running > 0); }
-
-	/**
-	 * Internal bounce method; do not call or override
-	 */
-	void __intl_run();
+	static inline void join(const Thread &t)
+	{
+		pthread_join(t._tid,(void **)0);
+	}
 
 	/**
 	 * Sleep the current thread
 	 *
-	 * @param ms Milliseconds to sleep
+	 * @param ms Number of milliseconds to sleep
 	 */
-	static void sleep(unsigned long ms);
-
-protected:
-	/**
-	 * Override to set a thread main function
-	 */
-	virtual void main()
-		throw();
-
-	/**
-	 * Subclasses can set to true to cause Thread to delete itself on exit
-	 */
-	volatile bool suicidalThread;
+	static inline void sleep(unsigned long ms)
+	{
+		usleep(ms * 1000);
+	}
 
 private:
-	void *_impl;
-	AtomicCounter _running;
-	volatile bool _notInit;
+	pthread_t _tid;
 };
 
 } // namespace ZeroTier
 
+#endif // __WINDOWS__ / !__WINDOWS__
+
 #endif

node/Topology.cpp

@@ -25,8 +25,10 @@
  * LLC. Start here: http://www.zerotier.com/
  */
 
+#include <algorithm>
 #include "Topology.hpp"
 #include "NodeConfig.hpp"
+#include "CMWC4096.hpp"
 
 namespace ZeroTier {
 
@@ -36,8 +38,8 @@ namespace ZeroTier {
 
 Topology::Topology(const RuntimeEnvironment *renv,const char *dbpath)
 	throw(std::runtime_error) :
-	Thread(),
-	_r(renv)
+	_r(renv),
+	_amSupernode(false)
 {
 	if (KISSDB_open(&_dbm,dbpath,KISSDB_OPEN_MODE_RWCREAT,ZT_KISSDB_HASH_TABLE_SIZE,ZT_KISSDB_KEY_SIZE,ZT_KISSDB_VALUE_SIZE)) {
 		if (KISSDB_open(&_dbm,dbpath,KISSDB_OPEN_MODE_RWREPLACE,ZT_KISSDB_HASH_TABLE_SIZE,ZT_KISSDB_KEY_SIZE,ZT_KISSDB_VALUE_SIZE))
@@ -52,7 +54,7 @@ Topology::Topology(const RuntimeEnvironment *renv,const char *dbpath)
 
 	Utils::lockDownFile(dbpath,false); // node.db caches secrets
 
-	start();
+	_thread = Thread::start(this);
 }
 
 Topology::~Topology()
@@ -65,19 +67,18 @@ Topology::~Topology()
 		_peerDeepVerifyJobs.back().type = _PeerDeepVerifyJob::EXIT_THREAD;
 	}
 	_peerDeepVerifyJobs_c.signal();
-
-	while (running())
-		Thread::sleep(10); // wait for thread to terminate without join()
-
+	Thread::join(_thread);
 	KISSDB_close(&_dbm);
 }
 
 void Topology::setSupernodes(const std::map< Identity,std::vector<InetAddress> > &sn)
 {
 	Mutex::Lock _l(_supernodes_m);
+
 	_supernodes = sn;
 	_supernodeAddresses.clear();
 	_supernodePeers.clear();
+
 	for(std::map< Identity,std::vector<InetAddress> >::const_iterator i(sn.begin());i!=sn.end();++i) {
 		if (i->first != _r->identity) {
 			SharedPtr<Peer> p(getPeer(i->first.address()));
@@ -91,6 +92,8 @@ void Topology::setSupernodes(const std::map< Identity,std::vector<InetAddress> >
 		}
 		_supernodeAddresses.insert(i->first.address());
 	}
+
+	_amSupernode = (_supernodes.find(_r->identity) != _supernodes.end());
 }
 
 void Topology::addPeer(const SharedPtr<Peer> &candidate,void (*callback)(void *,const SharedPtr<Peer> &,Topology::PeerVerifyResult),void *arg)
@@ -125,9 +128,12 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 			return ap->second;
 	}
 
+	unsigned char ztatmp[ZT_ADDRESS_LENGTH];
+	zta.copyTo(ztatmp,ZT_ADDRESS_LENGTH);
+
 	Buffer<ZT_KISSDB_VALUE_SIZE> b(ZT_KISSDB_VALUE_SIZE);
 	_dbm_m.lock();
-	if (!KISSDB_get(&_dbm,zta.data(),b.data())) {
+	if (!KISSDB_get(&_dbm,ztatmp,b.data())) {
 		_dbm_m.unlock();
 
 		SharedPtr<Peer> p(new Peer());
@@ -145,23 +151,32 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 	return SharedPtr<Peer>();
 }
 
-SharedPtr<Peer> Topology::getBestSupernode(const Address *avoid,unsigned int avoidCount) const
+SharedPtr<Peer> Topology::getBestSupernode(const Address *avoid,unsigned int avoidCount,bool strictAvoid) const
 {
 	SharedPtr<Peer> bestSupernode;
-	unsigned long bestSupernodeLatency = 0xffff;
+	unsigned int bestSupernodeLatency = 0xffff;
 	uint64_t now = Utils::now();
 
 	Mutex::Lock _l(_supernodes_m);
 
+	if (_supernodePeers.empty())
+		return bestSupernode;
+
 	for(std::vector< SharedPtr<Peer> >::const_iterator sn=_supernodePeers.begin();sn!=_supernodePeers.end();) {
 		for(unsigned int i=0;i<avoidCount;++i) {
 			if (avoid[i] == (*sn)->address())
 				goto skip_and_try_next_supernode;
 		}
-		if ((*sn)->hasActiveDirectPath(now)) { // only consider those that responded to pings
+		if ((*sn)->hasActiveDirectPath(now)) {
 			unsigned int l = (*sn)->latency();
-			if ((l)&&(l <= bestSupernodeLatency)) {
-				bestSupernodeLatency = l;
+			if (bestSupernode) {
+				if ((l)&&(l < bestSupernodeLatency)) {
+					bestSupernodeLatency = l;
+					bestSupernode = *sn;
+				}
+			} else {
+				if (l)
+					bestSupernodeLatency = l;
 				bestSupernode = *sn;
 			}
 		}
@@ -169,14 +184,20 @@ skip_and_try_next_supernode:
 		++sn;
 	}
 
-	if (bestSupernode)
+	if ((bestSupernode)||(strictAvoid))
 		return bestSupernode;
 
 	for(std::vector< SharedPtr<Peer> >::const_iterator sn=_supernodePeers.begin();sn!=_supernodePeers.end();++sn) {
-		if ((*sn)->hasActiveDirectPath(now)) { // only consider those that responded to pings
+		if ((*sn)->hasActiveDirectPath(now)) {
 			unsigned int l = (*sn)->latency();
-			if ((l)&&(l <= bestSupernodeLatency)) {
-				bestSupernodeLatency = l;
+			if (bestSupernode) {
+				if ((l)&&(l < bestSupernodeLatency)) {
+					bestSupernodeLatency = l;
+					bestSupernode = *sn;
+				}
+			} else {
+				if (l)
+					bestSupernodeLatency = l;
 				bestSupernode = *sn;
 			}
 		}
@@ -185,16 +206,7 @@ skip_and_try_next_supernode:
 	if (bestSupernode)
 		return bestSupernode;
 
-	uint64_t bestSupernodeLastDirectReceive = 0;
-	for(std::vector< SharedPtr<Peer> >::const_iterator sn=_supernodePeers.begin();sn!=_supernodePeers.end();++sn) {
-		uint64_t l = (*sn)->lastDirectReceive();
-		if (l > bestSupernodeLastDirectReceive) {
-			bestSupernodeLastDirectReceive = l;
-			bestSupernode = *sn;
-		}
-	}
-
-	return bestSupernode;
+	return _supernodePeers[_r->prng->next32() % _supernodePeers.size()];
 }
 
 void Topology::clean()
@@ -207,105 +219,7 @@ void Topology::clean()
 	_peerDeepVerifyJobs_c.signal();
 }
 
-void Topology::likesMulticastGroup(uint64_t nwid,const MulticastGroup &mg,const Address &addr,uint64_t now)
-{
-	Mutex::Lock _l(_multicastGroupMembers_m);
-	_multicastGroupMembers[nwid][mg][addr] = now;
-}
-
-struct _PickMulticastPropagationPeersPeerPrioritySortOrder
-{
-	inline bool operator()(const SharedPtr<Peer> &p1,const SharedPtr<Peer> &p2) const
-	{
-		return (p1->lastUnicastFrame() >= p2->lastUnicastFrame());
-	}
-};
-#define _MAX_PEERS_TO_CONSIDER 256
-unsigned int Topology::pickMulticastPropagationPeers(uint64_t nwid,const Address &exclude,const void *propagationBloom,unsigned int propagationBloomSize,unsigned int count,const MulticastGroup &mg,SharedPtr<Peer> *peers)
-{
-	SharedPtr<Peer> possiblePeers[_MAX_PEERS_TO_CONSIDER];
-	unsigned int numPossiblePeers = 0;
-
-	if (count > _MAX_PEERS_TO_CONSIDER)
-		count = _MAX_PEERS_TO_CONSIDER;
-
-	Mutex::Lock _l1(_activePeers_m);
-	Mutex::Lock _l2(_supernodes_m);
-
-	// Grab known non-supernode peers in multicast group, excluding 'exclude'
-	// Also lazily clean up the _multicastGroupMembers structure
-	{
-		Mutex::Lock _l3(_multicastGroupMembers_m);
-		std::map< uint64_t,std::map< MulticastGroup,std::map< Address,uint64_t > > >::iterator mgm(_multicastGroupMembers.find(nwid));
-		if (mgm != _multicastGroupMembers.end()) {
-			std::map< MulticastGroup,std::map< Address,uint64_t > >::iterator g(mgm->second.find(mg));
-			if (g != mgm->second.end()) {
-				uint64_t now = Utils::now();
-				for(std::map< Address,uint64_t >::iterator m(g->second.begin());m!=g->second.end();) {
-					if ((now - m->second) < ZT_MULTICAST_LIKE_EXPIRE) {
-						std::map< Address,SharedPtr<Peer> >::const_iterator p(_activePeers.find(m->first));
-						if (p != _activePeers.end()) {
-							possiblePeers[numPossiblePeers++] = p->second;
-							if (numPossiblePeers > _MAX_PEERS_TO_CONSIDER)
-								break;
-						}
-						++m;
-					} else g->second.erase(m++);
-				}
-				if (!g->second.size())
-					mgm->second.erase(g);
-			}
-		}
-	}
-
-	// Sort non-supernode peers in descending order of most recent data
-	// exchange timestamp. This sorts by implicit social relationships -- who
-	// you are talking to are the people who get multicasts first.
-	std::sort(&(possiblePeers[0]),&(possiblePeers[numPossiblePeers]),_PickMulticastPropagationPeersPeerPrioritySortOrder());
-
-	// Tack on a supernode peer to the end if we don't have enough regular
-	// peers, using supernodes to bridge gaps in sparse multicast groups.
-	if (numPossiblePeers < count) {
-		SharedPtr<Peer> bestSupernode;
-		unsigned int bestSupernodeLatency = 0xffff;
-		for(std::vector< SharedPtr<Peer> >::const_iterator sn(_supernodePeers.begin());sn!=_supernodePeers.end();++sn) {
-			if (((*sn)->latency())&&((*sn)->latency() < bestSupernodeLatency)) {
-				bestSupernodeLatency = (*sn)->latency();
-				bestSupernode = *sn;
-			}
-		}
-		if (bestSupernode)
-			possiblePeers[numPossiblePeers++] = bestSupernode;
-	}
-
-	unsigned int num = 0;
-
-	// First, try to pick peers not in the propgation bloom filter
-	for(unsigned int i=0;i<numPossiblePeers;++i) {
-		if (!Utils::bloomContains(propagationBloom,propagationBloomSize,possiblePeers[i]->address().sum())) {
-			peers[num++] = possiblePeers[i];
-			if (num >= count)
-				return num;
-		}
-	}
-
-	// Next, pick other peers until full (without duplicates)
-	for(unsigned int i=0;i<numPossiblePeers;++i) {
-		for(unsigned int j=0;j<num;++j) {
-			if (peers[j] == possiblePeers[i])
-				goto check_next_peer;
-		}
-		peers[num++] = possiblePeers[i];
-		if (num >= count)
-			return num;
-check_next_peer:
-		continue;
-	}
-
-	return num;
-}
-
-void Topology::main()
+void Topology::threadMain()
 	throw()
 {
 	for(;;) {
@@ -390,11 +304,13 @@ void Topology::main()
 					for(std::map< Address,SharedPtr<Peer> >::iterator p(_activePeers.begin());p!=_activePeers.end();++p) {
 						if (p->second->getAndResetDirty()) {
 							try {
+								uint64_t atmp[ZT_ADDRESS_LENGTH];
+								p->second->identity().address().copyTo(atmp,ZT_ADDRESS_LENGTH);
 								Buffer<ZT_PEER_MAX_SERIALIZED_LENGTH> b;
 								p->second->serialize(b);
 								b.zeroUnused();
 								_dbm_m.lock();
-								if (KISSDB_put(&_dbm,p->second->identity().address().data(),b.data())) {
+								if (KISSDB_put(&_dbm,atmp,b.data())) {
 									TRACE("error writing %s to peer.db",p->second->identity().address().toString().c_str());
 								}
 								_dbm_m.unlock();
@@ -404,14 +320,6 @@ void Topology::main()
 						}
 					}
 				}
-				{
-					Mutex::Lock _l(_multicastGroupMembers_m);
-					for(std::map< uint64_t,std::map< MulticastGroup,std::map< Address,uint64_t > > >::iterator mgm(_multicastGroupMembers.begin());mgm!=_multicastGroupMembers.end();) {
-						if (_r->nc->hasNetwork(mgm->first))
-							++mgm;
-						else _multicastGroupMembers.erase(mgm++);
-					}
-				}
 				break;
 			case _PeerDeepVerifyJob::EXIT_THREAD:
 				TRACE("thread terminating...");
@@ -427,11 +335,13 @@ void Topology::_reallyAddPeer(const SharedPtr<Peer> &p)
 		_activePeers[p->identity().address()] = p;
 	}
 	try {
+		uint64_t atmp[ZT_ADDRESS_LENGTH];
+		p->address().copyTo(atmp,ZT_ADDRESS_LENGTH);
 		Buffer<ZT_PEER_MAX_SERIALIZED_LENGTH> b;
 		p->serialize(b);
 		b.zeroUnused();
 		_dbm_m.lock();
-		if (KISSDB_put(&_dbm,p->identity().address().data(),b.data())) {
+		if (KISSDB_put(&_dbm,atmp,b.data())) {
 			TRACE("error writing %s to peerdb",p->address().toString().c_str());
 		} else p->getAndResetDirty();
 		_dbm_m.unlock();

node/Topology.hpp

@@ -55,7 +55,7 @@ class RuntimeEnvironment;
 /**
  * Database of network topology
  */
-class Topology : protected Thread
+class Topology
 {
 public:
 	/**
@@ -74,7 +74,7 @@ public:
 	Topology(const RuntimeEnvironment *renv,const char *dbpath)
 		throw(std::runtime_error);
 
-	virtual ~Topology();
+	~Topology();
 
 	/**
 	 * Set up supernodes for this network
@@ -134,7 +134,7 @@ public:
 	 */
 	inline SharedPtr<Peer> getBestSupernode() const
 	{
-		return getBestSupernode((const Address *)0,0);
+		return getBestSupernode((const Address *)0,0,false);
 	}
 
 	/**
@@ -146,9 +146,10 @@ public:
 	 * 
 	 * @param avoid Nodes to avoid
 	 * @param avoidCount Number of nodes to avoid
+	 * @param strictAvoid If false, consider avoided supernodes anyway if no non-avoid supernodes are available
 	 * @return Supernode or NULL if none
 	 */
-	SharedPtr<Peer> getBestSupernode(const Address *avoid,unsigned int avoidCount) const;
+	SharedPtr<Peer> getBestSupernode(const Address *avoid,unsigned int avoidCount,bool strictAvoid) const;
 
 	/**
 	 * @param zta ZeroTier address
@@ -161,35 +162,16 @@ public:
 		return (_supernodeAddresses.count(zta) > 0);
 	}
 
+	/**
+	 * @return True if this node's identity is in the supernode set
+	 */
+	inline bool amSupernode() const { return _amSupernode; }
+
 	/**
 	 * Clean and flush database now (runs in the background)
 	 */
 	void clean();
 
-	/**
-	 * Pick peers for multicast propagation
-	 *
-	 * @param nwid Network ID
-	 * @param exclude Peer to exclude or zero address for none
-	 * @param propagationBloom Propgation bloom filter
-	 * @param propagationBloomSize Size of propagation bloom filter in BITS
-	 * @param count Number of peers desired (propagation breadth)
-	 * @param mg Multicast group
-	 * @param peers Array to receive peers (must be at least [count])
-	 * @return Number of peers actually picked
-	 */
-	unsigned int pickMulticastPropagationPeers(uint64_t nwid,const Address &exclude,const void *propagationBloom,unsigned int propagationBloomSize,unsigned int count,const MulticastGroup &mg,SharedPtr<Peer> *peers);
-
-	/**
-	 * Add or update last 'like' time for an address's membership in a multicast group
-	 *
-	 * @param nwid Network ID
-	 * @param mg Multicast group
-	 * @param addr ZeroTier address
-	 * @param now Current time
-	 */
-	void likesMulticastGroup(uint64_t nwid,const MulticastGroup &mg,const Address &addr,uint64_t now);
-
 	/**
 	 * Apply a function or function object to all peers
 	 *
@@ -218,7 +200,7 @@ public:
 
 		inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 		{
-			if ((p->hasDirectPath())&&((_now - p->lastFirewallOpener()) >= ZT_FIREWALL_OPENER_DELAY))
+			if ((p->hasDirectPath())&&((_now - std::max(p->lastFirewallOpener(),p->lastDirectSend())) >= ZT_FIREWALL_OPENER_DELAY))
 				_v.push_back(p);
 		}
 
@@ -241,7 +223,7 @@ public:
 
 		inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 		{
-			if (((p->hasActiveDirectPath(_now))||(t.isSupernode(p->address())))&&((_now - p->lastDirectSend()) >= ZT_PEER_DIRECT_PING_DELAY))
+			if ( ((t.isSupernode(p->address()))&&((_now - p->lastDirectReceive()) >= ZT_PEER_DIRECT_PING_DELAY)) || ((p->hasActiveDirectPath(_now))&&((_now - p->lastDirectSend()) >= ZT_PEER_DIRECT_PING_DELAY)) )
 				_v.push_back(p);
 		}
 
@@ -295,36 +277,9 @@ public:
 	};
 
 	/**
-	 * Dump peer I/O statistics to an open FILE (for status reporting and debug)
+	 * Thread main method; do not call elsewhere
 	 */
-	class DumpPeerStatistics
-	{
-	public:
-		DumpPeerStatistics(FILE *out) :
-			_out(out),
-			_now(Utils::now())
-		{
-			fprintf(_out,"Peer       Direct IPv4           Direct IPv6                                         Latency(ms)"ZT_EOL_S);
-		}
-
-		inline void operator()(Topology &t,const SharedPtr<Peer> &p)
-		{
-			InetAddress v4(p->ipv4ActivePath(_now));
-			InetAddress v6(p->ipv6ActivePath(_now));
-			fprintf(_out,"%-10s %-21s %-51s %u"ZT_EOL_S,
-				p->address().toString().c_str(),
-				((v4) ? v4.toString().c_str() : "(none)"),
-				((v6) ? v6.toString().c_str() : "(none)"),
-				p->latency());
-		}
-
-	private:
-		FILE *_out;
-		uint64_t _now;
-	};
-
-protected:
-	virtual void main()
+	void threadMain()
 		throw();
 
 private:
@@ -344,6 +299,7 @@ private:
 	};
 
 	const RuntimeEnvironment *const _r;
+	Thread _thread;
 
 	std::map< Address,SharedPtr<Peer> > _activePeers;
 	Mutex _activePeers_m;
@@ -357,12 +313,11 @@ private:
 	std::vector< SharedPtr<Peer> > _supernodePeers;
 	Mutex _supernodes_m;
 
+	// Set to true if my identity is in _supernodes
+	volatile bool _amSupernode;
+
 	KISSDB _dbm;
 	Mutex _dbm_m;
-
-	// Multicast group members by network ID, then multicast group
-	std::map< uint64_t,std::map< MulticastGroup,std::map< Address,uint64_t > > > _multicastGroupMembers;
-	Mutex _multicastGroupMembers_m;
 };
 
 } // namespace ZeroTier

node/UdpSocket.cpp

@@ -49,12 +49,12 @@
 namespace ZeroTier {
 
 UdpSocket::UdpSocket(
+	bool localOnly,
 	int localPort,
 	bool ipv6,
 	void (*packetHandler)(UdpSocket *,void *,const InetAddress &,const void *,unsigned int),
 	void *arg)
 	throw(std::runtime_error) :
-	Thread(),
 	_packetHandler(packetHandler),
 	_arg(arg),
 	_localPort(localPort),
@@ -87,7 +87,9 @@ UdpSocket::UdpSocket(
 		memset(&sin6,0,sizeof(sin6));
 		sin6.sin6_family = AF_INET6;
 		sin6.sin6_port = htons(localPort);
-		memcpy(&(sin6.sin6_addr),&in6addr_any,sizeof(struct in6_addr));
+		if (localOnly)
+			memcpy(&(sin6.sin6_addr.s6_addr),InetAddress::LO6.rawIpData(),16);
+		else memcpy(&(sin6.sin6_addr),&in6addr_any,sizeof(struct in6_addr));
 		if (::bind(_sock,(const struct sockaddr *)&sin6,sizeof(sin6))) {
 			::close(_sock);
 			throw std::runtime_error("unable to bind to port");
@@ -109,19 +111,27 @@ UdpSocket::UdpSocket(
 		memset(&sin,0,sizeof(sin));
 		sin.sin_family = AF_INET;
 		sin.sin_port = htons(localPort);
-		sin.sin_addr.s_addr = INADDR_ANY;
+		if (localOnly)
+			memcpy(&(sin.sin_addr.s_addr),InetAddress::LO4.rawIpData(),4);
+		else sin.sin_addr.s_addr = INADDR_ANY;
 		if (::bind(_sock,(const struct sockaddr *)&sin,sizeof(sin))) {
 			::close(_sock);
 			throw std::runtime_error("unable to bind to port");
 		}
 	}
 
-	start();
+	_thread = Thread::start(this);
 }
 
 UdpSocket::~UdpSocket()
 {
-	close(_sock);
+	int s = _sock;
+	_sock = 0;
+	if (s > 0) {
+		::shutdown(s,SHUT_RDWR);
+		::close(s);
+	}
+	Thread::join(_thread);
 }
 
 bool UdpSocket::send(const InetAddress &to,const void *data,unsigned int len,int hopLimit)
@@ -141,22 +151,25 @@ bool UdpSocket::send(const InetAddress &to,const void *data,unsigned int len,int
 	}
 }
 
-void UdpSocket::main()
+void UdpSocket::threadMain()
 	throw()
 {
-	char buf[32768];
+	char buf[65536];
 	InetAddress from;
 	socklen_t salen;
 	int n;
 
-	for(;;) {
+	while (_sock > 0) {
 		salen = from.saddrSpaceLen();
 		n = (int)recvfrom(_sock,buf,sizeof(buf),0,from.saddr(),&salen);
 		if (n < 0) {
 			if ((errno != EINTR)&&(errno != ETIMEDOUT))
 				break;
-		} else if (n > 0)
-			_packetHandler(this,_arg,from,buf,(unsigned int)n);
+		} else if (n > 0) {
+			try {
+				_packetHandler(this,_arg,from,buf,(unsigned int)n);
+			} catch ( ... ) {} // should never be thrown from here anyway...
+		}
 	}
 }
 

node/UdpSocket.hpp

@@ -40,12 +40,13 @@ namespace ZeroTier {
  *
  * The socket listens in a background thread and sends packets to Switch.
  */
-class UdpSocket : protected Thread
+class UdpSocket
 {
 public:
 	/**
 	 * Create and bind a local UDP socket
 	 *
+	 * @param localOnly If true, bind to loopback address only
 	 * @param localPort Local port to listen to
 	 * @param ipv6 If true, bind this as an IPv6 socket, otherwise IPv4
 	 * @param packetHandler Function to call when packets are read
@@ -53,13 +54,14 @@ public:
 	 * @throws std::runtime_error Unable to bind
 	 */
 	UdpSocket(
+		bool localOnly,
 		int localPort,
 		bool ipv6,
 		void (*packetHandler)(UdpSocket *,void *,const InetAddress &,const void *,unsigned int),
 		void *arg)
 		throw(std::runtime_error);
 
-	virtual ~UdpSocket();
+	~UdpSocket();
 
 	/**
 	 * @return Locally bound port
@@ -85,15 +87,18 @@ public:
 	bool send(const InetAddress &to,const void *data,unsigned int len,int hopLimit)
 		throw();
 
-protected:
-	virtual void main()
+	/**
+	 * Thread main method; do not call elsewhere
+	 */
+	void threadMain()
 		throw();
 
 private:
+	Thread _thread;
 	void (*_packetHandler)(UdpSocket *,void *,const InetAddress &,const void *,unsigned int);
 	void *_arg;
 	int _localPort;
-	int _sock;
+	volatile int _sock;
 	bool _v6;
 	Mutex _sendLock;
 };

node/Utils.cpp

@@ -28,22 +28,28 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
-#include "Utils.hpp"
-#include "Mutex.hpp"
+#include <stdarg.h>
 
-#if defined(__APPLE__) || defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
+#include "Constants.hpp"
+
+#ifdef __UNIX_LIKE__
 #include <unistd.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/uio.h>
+#include <dirent.h>
 #endif
 
-#ifdef _WIN32
+#ifdef __WINDOWS__
 #include <Windows.h>
 #endif
 
 #include <sys/stat.h>
-#include <openssl/rand.h>
+
+#include "Utils.hpp"
+#include "Mutex.hpp"
 
 namespace ZeroTier {
 
@@ -213,6 +219,29 @@ const char Utils::base64DecMap[128] = {
 static const char *DAY_NAMES[7] = { "Sun","Mon","Tue","Wed","Thu","Fri","Sat" };
 static const char *MONTH_NAMES[12] = { "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec" };
 
+std::map<std::string,bool> Utils::listDirectory(const char *path)
+{
+	struct dirent de;
+	struct dirent *dptr;
+	std::map<std::string,bool> r;
+
+	DIR *d = opendir(path);
+	if (!d)
+		return r;
+
+	dptr = (struct dirent *)0;
+	for(;;) {
+		if (readdir_r(d,&de,&dptr))
+			break;
+		if (dptr) {
+			if ((strcmp(dptr->d_name,"."))&&(strcmp(dptr->d_name,"..")))
+				r[std::string(dptr->d_name)] = (dptr->d_type == DT_DIR);
+		} else break;
+	}
+
+	return r;
+}
+
 std::string Utils::base64Encode(const void *data,unsigned int len)
 {
 	if (!len)
@@ -283,31 +312,6 @@ std::string Utils::base64Decode(const char *data,unsigned int len)
 	return out.substr(0,outLen);
 }
 
-const char *Utils::etherTypeName(const unsigned int etherType)
-{
-	static char tmp[6];
-	switch(etherType) {
-		case ZT_ETHERTYPE_IPV4:
-			return "IPV4";
-		case ZT_ETHERTYPE_ARP:
-			return "ARP";
-		case ZT_ETHERTYPE_RARP:
-			return "RARP";
-		case ZT_ETHERTYPE_ATALK:
-			return "ATALK";
-		case ZT_ETHERTYPE_AARP:
-			return "AARP";
-		case ZT_ETHERTYPE_IPX_A:
-			return "IPX_A";
-		case ZT_ETHERTYPE_IPX_B:
-			return "IPX_B";
-		case ZT_ETHERTYPE_IPV6:
-			return "IPV6";
-	}
-	sprintf(tmp,"%.4x",etherType);
-	return tmp; // technically not thread safe, but we're only going to see this in debugging if ever
-}
-
 std::string Utils::hex(const void *data,unsigned int len)
 {
 	std::string r;
@@ -374,26 +378,36 @@ unsigned int Utils::unhex(const char *hex,void *buf,unsigned int len)
 
 void Utils::getSecureRandom(void *buf,unsigned int bytes)
 {
-	unsigned char tmp[16384];
-	while (!RAND_bytes((unsigned char *)buf,bytes)) {
-#if defined(__APPLE__) || defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-		FILE *rf = fopen("/dev/urandom","r");
-		if (rf) {
-			fread(tmp,sizeof(tmp),1,rf);
-			fclose(rf);
-			RAND_seed(tmp,sizeof(tmp));
-		} else {
-			fprintf(stderr,"FATAL: could not open /dev/urandom\n");
-			exit(-1);
+#ifdef __UNIX_LIKE__
+	static Mutex randomLock;
+	static char randbuf[32768];
+	static unsigned int randptr = sizeof(randbuf);
+
+	Mutex::Lock _l(randomLock);
+	for(unsigned int i=0;i<bytes;++i) {
+		if (randptr >= sizeof(randbuf)) {
+			int fd = ::open("/dev/urandom",O_RDONLY);
+			if (fd < 0) {
+				fprintf(stderr,"FATAL ERROR: unable to open /dev/urandom: %s"ZT_EOL_S,strerror(errno));
+				exit(-1);
+			}
+			if ((int)::read(fd,randbuf,sizeof(randbuf)) != (int)sizeof(randbuf)) {
+				fprintf(stderr,"FATAL ERROR: unable to read from /dev/urandom"ZT_EOL_S);
+				exit(-1);
+			}
+			::close(fd);
+			randptr = 0;
 		}
-#else
-#ifdef _WIN32
-		error need win32;
-#else
-		error;
-#endif
-#endif
+		((char *)buf)[i] = randbuf[randptr++];
 	}
+
+#else // !__UNIX_LIKE__
+#ifdef __WINDOWS__
+	probably use windows capi...;
+#else // !__WINDOWS__
+	no getSecureRandom() implementation!
+#endif // __WINDOWS__
+#endif // __UNIX_LIKE__
 }
 
 void Utils::lockDownFile(const char *path,bool isDir)
@@ -555,4 +569,20 @@ std::string Utils::trim(const std::string &s)
 	return s.substr(start,end - start);
 }
 
+void Utils::stdsprintf(std::string &s,const char *fmt,...)
+	throw(std::bad_alloc,std::length_error)
+{
+	char buf[65536];
+	va_list ap;
+
+	va_start(ap,fmt);
+	int n = vsnprintf(buf,sizeof(buf),fmt,ap);
+	va_end(ap);
+
+	if ((n >= (int)sizeof(buf))||(n < 0))
+		throw std::length_error("printf result too large");
+
+	s.append(buf);
+}
+
 } // namespace ZeroTier

node/Utils.hpp

@@ -34,26 +34,23 @@
 #include <time.h>
 #include <sys/time.h>
 #include <arpa/inet.h>
+
 #include <string>
 #include <stdexcept>
 #include <vector>
+#include <map>
 
 #include "../ext/lz4/lz4.h"
 #include "../ext/lz4/lz4hc.h"
-#include "../ext/huffandpuff/huffman.h"
+
+#ifdef __WINDOWS__
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif
 
 #include "Constants.hpp"
 
-/* Ethernet frame types that might be relevant to us */
-#define ZT_ETHERTYPE_IPV4 0x0800
-#define ZT_ETHERTYPE_ARP 0x0806
-#define ZT_ETHERTYPE_RARP 0x8035
-#define ZT_ETHERTYPE_ATALK 0x809b
-#define ZT_ETHERTYPE_AARP 0x80f3
-#define ZT_ETHERTYPE_IPX_A 0x8137
-#define ZT_ETHERTYPE_IPX_B 0x8138
-#define ZT_ETHERTYPE_IPV6 0x86dd
-
 /**
  * Maximum compression/decompression block size (do not change)
  */
@@ -68,10 +65,35 @@ class Utils
 {
 public:
 	/**
-	 * @param etherType Ethernet type ID
-	 * @return Name of Ethernet protocol (e.g. ARP, IPV4)
+	 * Delete a file
+	 *
+	 * @param path Path to delete
+	 * @return True if delete was successful
 	 */
-	static const char *etherTypeName(const unsigned int etherType);
+	static inline bool rm(const char *path)
+		throw()
+	{
+#ifdef __WINDOWS__
+		foo;
+#else
+		return (unlink(path) == 0);
+#endif
+	}
+	static inline bool rm(const std::string &path)
+		throw()
+	{
+		return rm(path.c_str());
+	}
+
+	/**
+	 * List a directory's contents
+	 *
+	 * @param path Path to list
+	 * @param files Set to fill with files
+	 * @param directories Set to fill with directories
+	 * @return Map of entries and whether or not they are also directories (empty on failure)
+	 */
+	static std::map<std::string,bool> listDirectory(const char *path);
 
 	/**
 	 * @param data Data to convert to hex
@@ -102,18 +124,6 @@ public:
 	 */
 	static void getSecureRandom(void *buf,unsigned int bytes);
 
-	/**
-	 * @tparam T Integer type to fill and return
-	 * @return Random int using secure random source
-	 */
-	template<typename T>
-	static inline T randomInt()
-	{
-		T foo = 0; // prevents valgrind warnings
-		getSecureRandom(&foo,sizeof(foo));
-		return foo;
-	}
-
 	/**
 	 * Set modes on a file to something secure
 	 * 
@@ -136,6 +146,15 @@ public:
 	 */
 	static uint64_t getLastModified(const char *path);
 
+	/**
+	 * @param path Path to check
+	 * @return True if file or directory exists at path location
+	 */
+	static inline bool fileExists(const char *path)
+	{
+		return (getLastModified(path) != 0);
+	}
+
 	/**
 	 * @param t64 Time in ms since epoch
 	 * @return RFC1123 date string
@@ -192,7 +211,6 @@ public:
 	template<typename I,typename O>
 	static inline void compress(I begin,I end,O out)
 	{
-		char huffheap[HUFFHEAP_SIZE];
 		unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
 		char *buf = new char[bufLen * 2];
 		char *buf2 = buf + bufLen;
@@ -226,16 +244,9 @@ public:
 					continue;
 				}
 
-				unsigned long huffCompressedLen = huffman_compress((const unsigned char *)buf2,lz4CompressedLen,(unsigned char *)buf,bufLen,huffheap);
-				if ((!huffCompressedLen)||((int)huffCompressedLen >= lz4CompressedLen)) {
-					l = hton((uint32_t)lz4CompressedLen); // lz4 only
-					out((const void *)&l,4);
-					out((const void *)buf2,(unsigned int)lz4CompressedLen);
-				} else {
-					l = hton((uint32_t)0x80000000 | (uint32_t)huffCompressedLen); // lz4 with huffman
-					out((const void *)&l,4);
-					out((const void *)buf,(unsigned int)huffCompressedLen);
-				}
+				l = hton((uint32_t)lz4CompressedLen); // lz4 only
+				out((const void *)&l,4);
+				out((const void *)buf2,(unsigned int)lz4CompressedLen);
 			}
 
 			delete [] buf;
@@ -258,7 +269,6 @@ public:
 	template<typename I,typename O>
 	static inline bool decompress(I begin,I end,O out)
 	{
-		char huffheap[HUFFHEAP_SIZE];
 		volatile char i32c[4];
 		void *const i32cp = (void *)i32c;
 		unsigned int bufLen = LZ4_compressBound(ZT_COMPRESSION_BLOCK_SIZE);
@@ -295,23 +305,10 @@ public:
 						return false;
 					}
 
-					if ((_compressedSize & 0x80000000)) { // lz4 and huffman
-						unsigned long lz4CompressedSize = huffman_decompress((const unsigned char *)buf,compressedSize,(unsigned char *)buf2,bufLen,huffheap);
-						if (lz4CompressedSize) {
-							if (LZ4_uncompress_unknownOutputSize(buf2,buf,lz4CompressedSize,bufLen) != (int)originalSize) {
-								delete [] buf;
-								return false;
-							} else out((const void *)buf,(unsigned int)originalSize);
-						} else {
-							delete [] buf;
-							return false;
-						}
-					} else { // lz4 only
-						if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) {
-							delete [] buf;
-							return false;
-						} else out((const void *)buf2,(unsigned int)originalSize);
-					}
+					if (LZ4_uncompress_unknownOutputSize(buf,buf2,compressedSize,bufLen) != (int)originalSize) {
+						delete [] buf;
+						return false;
+					} else out((const void *)buf2,(unsigned int)originalSize);
 				} else { // stored
 					if (originalSize > bufLen) {
 						delete [] buf;
@@ -350,6 +347,17 @@ public:
 		return ( (1000ULL * (uint64_t)tv.tv_sec) + (uint64_t)(tv.tv_usec / 1000) );
 	};
 
+	/**
+	 * @return Current time in seconds since epoch, to the highest available resolution
+	 */
+	static inline double nowf()
+		throw()
+	{
+		struct timeval tv;
+		gettimeofday(&tv,(struct timezone *)0);
+		return ( ((double)tv.tv_sec) + (((double)tv.tv_usec) / 1000000.0) );
+	}
+
 	/**
 	 * Read the full contents of a file into a string buffer
 	 *
@@ -419,6 +427,18 @@ public:
 	 */
 	static std::string trim(const std::string &s);
 
+	/**
+	 * Like sprintf, but appends to std::string
+	 *
+	 * @param s String to append to
+	 * @param fmt Printf format string
+	 * @param ... Format arguments
+	 * @throws std::bad_alloc Memory allocation failure
+	 * @throws std::length_error Format + args exceeds internal buffer maximum
+	 */
+	static void stdsprintf(std::string &s,const char *fmt,...)
+		throw(std::bad_alloc,std::length_error);
+
 	/**
 	 * Count the number of bits set in an integer
 	 *
@@ -476,38 +496,6 @@ public:
 		return ((*aptr & mask) == (*aptr & mask));
 	}
 
-	/**
-	 * Add a value to a bloom filter
-	 *
-	 * Note that bloom filter methods depend on n being evenly distributed, so
-	 * it's the job of the caller to implement any hashing.
-	 *
-	 * @param bits Bloom filter data (must be filterSize / 8 bytes in length)
-	 * @param filterSize Size of bloom filter in BITS
-	 * @param n Number to add
-	 */
-	static inline void bloomAdd(void *bits,unsigned int filterSize,unsigned int n)
-		throw()
-	{
-		n %= filterSize;
-		((unsigned char *)bits)[n / 8] |= (0x80 >> (n % 8));
-	}
-
-	/**
-	 * Test for a value in a bloom filter
-	 *
-	 * @param bits Bloom filter data (must be filterSize / 8 bytes in length)
-	 * @param filterSize Size of bloom filter in BITS
-	 * @param n Number to test
-	 * @return True if number might be in filter
-	 */
-	static inline bool bloomContains(const void *bits,unsigned int filterSize,unsigned int n)
-		throw()
-	{
-		n %= filterSize;
-		return ((((const unsigned char *)bits)[n / 8] & (0x80 >> (n % 8))));
-	}
-
 	/**
 	 * Compute CRC64
 	 *

objects.mk

@@ -1,9 +1,4 @@
 OBJS=\
-	ext/http-parser/http_parser.o \
-	ext/huffandpuff/huffman.o \
-	ext/jsoncpp/src/json_reader.o \
-	ext/jsoncpp/src/json_value.o \
-	ext/jsoncpp/src/json_writer.o \
 	ext/kissdb/kissdb.o \
 	ext/lz4/lz4hc.o \
 	ext/lz4/lz4.o \
@@ -11,8 +6,8 @@ OBJS=\
 	node/Demarc.o \
 	node/EllipticCurveKeyPair.o \
 	node/EthernetTap.o \
+	node/Filter.o \
 	node/HMAC.o \
-	node/Http.o \
 	node/Identity.o \
 	node/InetAddress.o \
 	node/Logger.o \
@@ -20,12 +15,12 @@ OBJS=\
 	node/Node.o \
 	node/NodeConfig.o \
 	node/Packet.o \
-	node/Pack.o \
+	node/PacketDecoder.o \
 	node/Peer.o \
 	node/Salsa20.o \
+	node/Service.o \
 	node/Switch.o \
 	node/SysEnv.o \
-	node/Thread.o \
 	node/Topology.o \
 	node/UdpSocket.o \
 	node/Utils.o

packtool.cpp

@@ -1,182 +0,0 @@
-/*
- * ZeroTier One - Global Peer to Peer Ethernet
- * Copyright (C) 2012-2013  ZeroTier Networks LLC
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <iostream>
-#include <string>
-
-#include "node/Identity.hpp"
-#include "node/Pack.hpp"
-#include "node/Utils.hpp"
-
-#include <unistd.h>
-
-using namespace ZeroTier;
-
-static void printHelp(const char *pn)
-{
-	std::cout << "Usage: " << pn << " <command> [<args>]" << std::endl << std::endl;
-	std::cout << "Commands:" << std::endl;
-	std::cout << "	list <packfile> [<identity.secret/public>]" << std::endl;
-	std::cout << "	create <packfile> <identity.secret> <file> [<file> ...]" << std::endl;
-	std::cout << "	extract <packfile> <destination directory>" << std::endl;
-	std::cout << std::endl << "To check signatures, use 'list' with an identity argument." << std::endl;
-}
-
-static Pack *readPack(const char *path)
-{
-	std::string tmp;
-	if (!Utils::readFile(path,tmp))
-		return (Pack *)0;
-	Pack *p = new Pack();
-	if (!p->deserialize(tmp)) {
-		delete p;
-		return (Pack *)0;
-	}
-	return p;
-}
-
-int main(int argc,char **argv)
-{
-	if (argc < 2) {
-		printHelp(argv[0]);
-		return -1;
-	}
-
-	if (!strcmp(argv[1],"list")) {
-		if (argc < 3) {
-			printHelp(argv[0]);
-			return -1;
-		}
-
-		Pack *pack = readPack(argv[2]);
-		if (!pack) {
-			std::cout << "Could not read " << argv[2] << std::endl;
-			return -1;
-		}
-
-		std::vector<const Pack::Entry *> entries = pack->getAll();
-		for(std::vector<const Pack::Entry *>::iterator e=entries.begin();e!=entries.end();++e) {
-			std::cout << (*e)->name << '\t' << (*e)->content.length() << '\t' << Utils::hex((*e)->sha256,32) << "\tsigned by: " << (*e)->signedBy.toString() << std::endl;
-		}
-
-		if (argc >= 4) {
-			std::string idser;
-			if (!Utils::readFile(argv[3],idser)) {
-				std::cout << "Unable to read identity from " << argv[3] << std::endl;
-				return -1;
-			}
-			Identity id;
-			if (!id.fromString(idser)) {
-				std::cout << "Invalid identity" << std::endl;
-				return -1;
-			}
-
-			entries = pack->verifyAll(id,true);
-			for(std::vector<const Pack::Entry *>::iterator e=entries.begin();e!=entries.end();++e) {
-				std::cout << "!!! Signature verification FAILED for: " << (*e)->name << std::endl;
-			}
-			if (!entries.size())
-				std::cout << "Signature for all entries verified OK" << std::endl;
-		}
-
-		delete pack;
-	} else if (!strcmp(argv[1],"create")) {
-		if (argc < 5) {
-			printHelp(argv[0]);
-			return -1;
-		}
-
-		std::string idser;
-		if (!Utils::readFile(argv[3],idser)) {
-			std::cout << "Unable to read identity from " << argv[3] << std::endl;
-			return -1;
-		}
-		Identity id;
-		if (!id.fromString(idser)) {
-			std::cout << "Invalid identity" << std::endl;
-			return -1;
-		}
-
-		if (!id.hasPrivate()) {
-			std::cout << "Identity must include private key to sign" << std::endl;
-			return -1;
-		}
-
-		Pack pack;
-		for(int i=4;i<argc;++i) {
-			std::string fdata;
-			if (!Utils::readFile(argv[i],fdata)) {
-				std::cout << "Unable to read " << argv[i] << std::endl;
-				return -1;
-			}
-			pack.put(std::string(argv[i]),fdata);
-			std::cout << "Added " << argv[i] << std::endl;
-		}
-		if (!pack.signAll(id)) {
-			std::cout << "Unable to sign with identity" << std::endl;
-			return -1;
-		} else std::cout << "Signed all entries with identity " << id.address().toString() << std::endl;
-		std::string packser = pack.serialize();
-
-		if (!Utils::writeFile(argv[2],packser)) {
-			std::cout << "Unable to write " << argv[2] << std::endl;
-			return -1;
-		}
-		std::cout << "Wrote " << packser.length() << " bytes to " << argv[2] << std::endl;
-	} else if (!strcmp(argv[1],"extract")) {
-		if (argc < 4) {
-			printHelp(argv[0]);
-			return -1;
-		}
-
-		Pack *pack = readPack(argv[2]);
-		if (!pack) {
-			std::cout << "Could not read " << argv[2] << std::endl;
-			return -1;
-		}
-
-		if (chdir(argv[3])) {
-			std::cout << "Unable to change to " << argv[3] << " for output." << std::endl;
-			return -1;
-		}
-
-		std::vector<const Pack::Entry *> entries = pack->getAll();
-		for(std::vector<const Pack::Entry *>::iterator e=entries.begin();e!=entries.end();++e) {
-			if (!Utils::writeFile((*e)->name.c_str(),(*e)->content))
-				std::cout << "Error writing " << (*e)->name << std::endl;
-			else std::cout << "Wrote " << (*e)->name << " (" << (*e)->content.length() << ")" << std::endl;
-		}
-	} else {
-		printHelp(argv[0]);
-		return -1;
-	}
-
-	return 0;
-}

selftest.cpp

@@ -43,11 +43,44 @@
 #include "node/HMAC.hpp"
 #include "node/MAC.hpp"
 #include "node/Peer.hpp"
-#include "node/Http.hpp"
 #include "node/Condition.hpp"
+#include "node/NodeConfig.hpp"
+#include "node/Dictionary.hpp"
+
+#include <openssl/rand.h>
 
 using namespace ZeroTier;
 
+// ---------------------------------------------------------------------------
+// Override libcrypto default RAND_ with Utils::getSecureRandom(), which uses
+// a system strong random source. This is because OpenSSL libcrypto's default
+// RAND_ implementation uses uninitialized memory as one of its entropy
+// sources, which plays havoc with all kinds of debuggers and auditing tools.
+
+static void _zeroTier_rand_cleanup() {}
+static void _zeroTier_rand_add(const void *buf, int num, double add_entropy) {}
+static int _zeroTier_rand_status() { return 1; }
+static void _zeroTier_rand_seed(const void *buf, int num) {}
+static int _zeroTier_rand_bytes(unsigned char *buf, int num)
+{
+	Utils::getSecureRandom(buf,num);
+	return 1;
+}
+static RAND_METHOD _zeroTierRandMethod = {
+	_zeroTier_rand_seed,
+	_zeroTier_rand_bytes,
+	_zeroTier_rand_cleanup,
+	_zeroTier_rand_add,
+	_zeroTier_rand_bytes,
+	_zeroTier_rand_status
+};
+static void _initLibCrypto()
+{
+	RAND_set_rand_method(&_zeroTierRandMethod);
+}
+
+// ---------------------------------------------------------------------------
+
 static unsigned char fuzzbuf[1048576];
 
 static const char *hmacShaTV0Key = "key";
@@ -63,6 +96,9 @@ static int testCrypto()
 	unsigned char buf1[16384];
 	unsigned char buf2[sizeof(buf1)],buf3[sizeof(buf1)];
 
+	//Utils::getSecureRandom(buf1,1024);
+	//std::cout << "[crypto] getSecureRandom() -> " << Utils::hex(buf1,1024) << std::endl;
+
 	std::cout << "[crypto] Testing ECDSA... "; std::cout.flush();
 	for(unsigned int k=0;k<64;++k) {
 		EllipticCurveKeyPair kp;
@@ -193,6 +229,9 @@ static int testPacket()
 	unsigned char salsaKey[32],hmacKey[32];
 	Packet a,b;
 
+	a.zeroAll();
+	b.zeroAll();
+
 	for(unsigned int i=0;i<32;++i) {
 		salsaKey[i] = (unsigned char)rand();
 		hmacKey[i] = (unsigned char)rand();
@@ -200,12 +239,15 @@ static int testPacket()
 
 	std::cout << "[packet] Testing Packet encoder/decoder... ";
 
-	a = Packet();
-	a.setVerb(Packet::VERB_HELLO);
+	a.reset(Address(),Address(),Packet::VERB_HELLO);
 	for(int i=0;i<32;++i)
 		a.append("supercalifragilisticexpealidocious",strlen("supercalifragilisticexpealidocious"));
 
 	b = a;
+	if (a != b) {
+		std::cout << "FAIL (assign)" << std::endl;
+		return -1;
+	}
 
 	a.compress();
 	unsigned int complen = a.size();
@@ -266,40 +308,71 @@ static int testOther()
 	}
 	std::cout << "PASS" << std::endl;
 
+	std::cout << "[other] Testing command bus encode/decode... "; std::cout.flush();
+	try {
+		static char key[32] = { 0 };
+		for(unsigned int k=0;k<1000;++k) {
+			std::vector<std::string> original;
+			for(unsigned int i=0,j=rand() % 256,l=(rand() % 1024)+1;i<j;++i)
+				original.push_back(std::string(l,'x'));
+			std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> > packets(NodeConfig::encodeControlMessage(key,1,original));
+			//std::cout << packets.size() << ' '; std::cout.flush();
+			std::vector<std::string> after;
+			for(std::vector< Buffer<ZT_NODECONFIG_MAX_PACKET_SIZE> >::iterator i(packets.begin());i!=packets.end();++i) {
+				unsigned long convId = 9999;
+				if (!NodeConfig::decodeControlMessagePacket(key,i->data(),i->size(),convId,after)) {
+					std::cout << "FAIL (decode)" << std::endl;
+					return -1;
+				}
+				if (convId != 1) {
+					std::cout << "FAIL (conversation ID)" << std::endl;
+					return -1;
+				}
+			}
+			if (after != original) {
+				std::cout << "FAIL (compare)" << std::endl;
+				return -1;
+			}
+		}
+	} catch (std::exception &exc) {
+		std::cout << "FAIL (" << exc.what() << ")" << std::endl;
+		return -1;
+	}
+	std::cout << "PASS" << std::endl;
+
+	std::cout << "[other] Testing Dictionary... "; std::cout.flush();
+	for(int k=0;k<10000;++k) {
+		Dictionary a,b;
+		int nk = rand() % 32;
+		for(int q=0;q<nk;++q) {
+			std::string k,v;
+			int kl = (rand() % 512);
+			int vl = (rand() % 512);
+			for(int i=0;i<kl;++i)
+				k.push_back((char)rand());
+			for(int i=0;i<vl;++i)
+				v.push_back((char)rand());
+			a[k] = v;
+		}
+		std::string aser = a.toString();
+		b.fromString(aser);
+		if (a != b) {
+			std::cout << "FAIL!" << std::endl;
+			return -1;
+		}
+	}
+	std::cout << "PASS" << std::endl;
+
 	return 0;
 }
 
-static Condition testHttpDoneCondition;
-
-static bool testHttpHandler(Http::Request *req,void *arg,const std::string &url,int code,const std::map<std::string,std::string> &headers,const std::string &body)
-{
-	if (code)
-		std::cout << "[net] " << url << " " << code << " bytes: " << body.length() << std::endl;
-	else std::cout << "[net] " << url << " FAILED: " << body << std::endl;
-	testHttpDoneCondition.signal();
-	return false;
-}
-
-static int testNet()
-{
-	std::cout << "[net] GET http://www.uc.edu/" << std::endl;
-	new Http::Request(Http::HTTP_METHOD_GET,"http://www.uc.edu/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
-	testHttpDoneCondition.wait();
-	std::cout << "[net] GET http://zerotier.com/" << std::endl;
-	new Http::Request(Http::HTTP_METHOD_GET,"http://zerotier.com/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
-	testHttpDoneCondition.wait();
-	std::cout << "[net] GET http://www.google.com/" << std::endl;
-	new Http::Request(Http::HTTP_METHOD_GET,"http://www.google.com/",Http::EMPTY_HEADERS,std::string(),&testHttpHandler,(void *)0);
-	testHttpDoneCondition.wait();
-}
-
 int main(int argc,char **argv)
 {
 	int r = 0;
 
+	_initLibCrypto();
 	srand(time(0));
 
-	r |= testNet();
 	r |= testCrypto();
 	r |= testPacket();
 	r |= testOther();