Lots of cleanup, more work on certificates, some security fixes.

netconf-service/netconf.cpp

@@ -161,19 +161,23 @@ int main(int argc,char **argv)
 		try {
 			const std::string &reqType = request.get("type");
 			if (reqType == "netconf-request") { // NETWORK_CONFIG_REQUEST packet
+				// Deserialize querying peer identity and network ID
 				Identity peerIdentity(request.get("peerId"));
 				uint64_t nwid = strtoull(request.get("nwid").c_str(),(char **)0,16);
+				std::string fromAddr(request.get("from"));
+
+				// Meta-information from node, such as (future) geo-location stuff
 				Dictionary meta;
 				if (request.contains("meta"))
 					meta.fromString(request.get("meta"));
 
-				// Do quick signature check / sanity check
+				// Check validity of node's identity, ignore request on failure
 				if (!peerIdentity.locallyValidate()) {
 					fprintf(stderr,"identity failed validity check: %s\n",peerIdentity.toString(false).c_str());
 					continue;
 				}
 
-				// Save identity if unknown
+				// Save node's identity if unknown
 				{
 					Query q = dbCon->query();
 					q << "SELECT identity FROM Node WHERE id = " << peerIdentity.address().toInt();
@@ -196,17 +200,21 @@ int main(int argc,char **argv)
 					}
 				}
 
-				// Update lastSeen
+				// Update lastSeen for Node, which is always updated on a netconf request
 				{
 					Query q = dbCon->query();
 					q << "UPDATE Node SET lastSeen = " << Utils::now() << " WHERE id = " << peerIdentity.address().toInt();
 					q.exec();
 				}
 
+				// Look up core network information
 				bool isOpen = false;
-				unsigned int mpb = 0;
+				unsigned int multicastPrefixBits = 0;
-				unsigned int md = 0;
+				unsigned int multicastDepth = 0;
-				std::string name,desc;
+				bool emulateArp = false;
+				bool emulateNdp = false;
+				std::string name;
+				std::string desc;
 				{
 					Query q = dbCon->query();
 					q << "SELECT name,`desc`,isOpen,multicastPrefixBits,multicastDepth FROM Network WHERE id = " << nwid;
@@ -215,8 +223,10 @@ int main(int argc,char **argv)
 						name = rs[0]["name"].c_str();
 						desc = rs[0]["desc"].c_str();
 						isOpen = ((int)rs[0]["isOpen"] > 0);
-						mpb = (unsigned int)rs[0]["multicastPrefixBits"];
+						emulateArp = ((int)rs[0]["emulateArp"] > 0);
-						md = (unsigned int)rs[0]["multicastDepth"];
+						emulateNdp = ((int)rs[0]["emulateNdp"] > 0);
+						multicastPrefixBits = (unsigned int)rs[0]["multicastPrefixBits"];
+						multicastDepth = (unsigned int)rs[0]["multicastDepth"];
 					} else {
 						Dictionary response;
 						response["peer"] = peerIdentity.address().toString();
@@ -231,10 +241,34 @@ int main(int argc,char **argv)
 						write(STDOUT_FILENO,&respml,4);
 						write(STDOUT_FILENO,respm.data(),respm.length());
 						stdoutWriteLock.unlock();
-						continue;
+						continue; // ABORT, wait for next request
 					}
 				}
 
+				// Check membership if this is a closed network
+				if (!isOpen) {
+					Query q = dbCon->query();
+					q << "SELECT Node_id FROM NetworkNodes WHERE Network_id = " << nwid << " AND Node_id = " << peerIdentity.address().toInt();
+					StoreQueryResult rs = q.store();
+					if (!rs.num_rows()) {
+						Dictionary response;
+						response["peer"] = peerIdentity.address().toString();
+						response["nwid"] = request.get("nwid");
+						response["type"] = "netconf-response";
+						response["requestId"] = request.get("requestId");
+						response["error"] = "ACCESS_DENIED";
+						std::string respm = response.toString();
+						uint32_t respml = (uint32_t)htonl((uint32_t)respm.length());
+
+						stdoutWriteLock.lock();
+						write(STDOUT_FILENO,&respml,4);
+						write(STDOUT_FILENO,respm.data(),respm.length());
+						stdoutWriteLock.unlock();
+						continue; // ABORT, wait for next request
+					}
+				}
+
+				// Get list of etherTypes in comma-delimited hex format
 				std::string etherTypeWhitelist;
 				{
 					Query q = dbCon->query();
@@ -247,6 +281,7 @@ int main(int argc,char **argv)
 					}
 				}
 
+				// Get multicast group rates in dictionary format
 				Dictionary multicastRates;
 				{
 					Query q = dbCon->query();
@@ -267,40 +302,16 @@ int main(int argc,char **argv)
 						if (mac) {
 							sprintf(buf,"%.12llx/%lx",(mac & 0xffffffffffffULL),(unsigned long)rs[i]["multicastGroupAdi"]);
 							multicastRates[buf] = buf2;
-						} else multicastRates["*"] = buf2;
+						} else { // zero MAC indicates default for unmatching multicast groups
+							multicastRates["*"] = buf2;
+						}
 					}
 				}
 
-				Dictionary netconf;
+				// Check for (or assign?) static IP address assignments
-
+				std::string ipv4Static;
-				sprintf(buf,"%.16llx",(unsigned long long)nwid);
+				std::string ipv6Static;
-				netconf["nwid"] = buf;
-				netconf["o"] = (isOpen ? "1" : "0");
-				netconf["name"] = name;
-				netconf["desc"] = desc;
-				netconf["et"] = etherTypeWhitelist;
-				netconf["mr"] = multicastRates.toString();
-				sprintf(buf,"%llx",(unsigned long long)Utils::now());
-				netconf["ts"] = buf;
-				netconf["peer"] = peerIdentity.address().toString();
-				if (mpb) {
-					sprintf(buf,"%x",mpb);
-					netconf["mpb"] = buf;
-				}
-				if (md) {
-					sprintf(buf,"%x",md);
-					netconf["md"] = buf;
-				}
-
-				if (!isOpen) {
-					// TODO: handle closed networks, look up private membership,
-					// generate signed cert.
-				}
-
-				std::string ipv4Static,ipv6Static;
-
 				{
-					// Check for IPv4 static assignments
 					Query q = dbCon->query();
 					q << "SELECT INET_NTOA(ip) AS ip,netmaskBits FROM IPv4Static WHERE Node_id = " << peerIdentity.address().toInt() << " AND Network_id = " << nwid;
 					StoreQueryResult rs = q.store();
@@ -363,16 +374,42 @@ int main(int argc,char **argv)
 					}
 				}
 
-				// Add static assignments to netconf, if any
+				// Update activity table for this network to indicate peer's participation
-				if (ipv4Static.length()) {
+				{
-					netconf["ipv4Static"] = ipv4Static; // TODO: remove, old name
+					Query q = dbCon->query();
-					netconf["v4s"] = ipv4Static;
+					q << "INSERT INTO NetworkActivity (Network_id,Node_id,lastActivityTime,lastActivityFrom) VALUES (" << nwid << "," << peerIdentity.address().toInt() << "," << Utils::now() << "," << fromAddr << ") ON DUPLICATE KEY UPDATE lastActivityTime = VALUES(lastActivityTime),lastActivityFrom = VALUES(lastActivityFrom)";
-				}
+					q.exec();
-				if (ipv6Static.length()) {
-					netconf["v6s"] = ipv6Static;
 				}
 
-				{ // Create and send service bus response with payload attached as 'netconf'
+				// Assemble response dictionary to send to peer
+				Dictionary netconf;
+				sprintf(buf,"%.16llx",(unsigned long long)nwid);
+				netconf["nwid"] = buf;
+				netconf["peer"] = peerIdentity.address().toString();
+				netconf["name"] = name;
+				netconf["desc"] = desc;
+				netconf["o"] = (isOpen ? "1" : "0");
+				netconf["et"] = etherTypeWhitelist;
+				netconf["mr"] = multicastRates.toString();
+				sprintf(buf,"%llx",(unsigned long long)Utils::now());
+				netconf["ts"] = buf;
+				netconf["eARP"] = (emulateArp ? "1" : "0");
+				netconf["eNDP"] = (emulateNdp ? "1" : "0");
+				if (multicastPrefixBits) {
+					sprintf(buf,"%x",multicastPrefixBits);
+					netconf["mpb"] = buf;
+				}
+				if (multicastDepth) {
+					sprintf(buf,"%x",multicastDepth);
+					netconf["md"] = buf;
+				}
+				if (ipv4Static.length())
+					netconf["v4s"] = ipv4Static;
+				if (ipv6Static.length())
+					netconf["v6s"] = ipv6Static;
+
+				// Send netconf as service bus response
+				{
 					Dictionary response;
 					response["peer"] = peerIdentity.address().toString();
 					response["nwid"] = request.get("nwid");
@@ -386,6 +423,8 @@ int main(int argc,char **argv)
 					write(STDOUT_FILENO,&respml,4);
 					write(STDOUT_FILENO,respm.data(),respm.length());
 					stdoutWriteLock.unlock();
+
+					// LOOP, wait for next request
 				}
 			}
 		} catch (std::exception &exc) {

node/CertificateOfMembership.cpp

@@ -163,15 +163,10 @@ bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other) c
 
 		// Compare to determine if the absolute value of the difference
 		// between these two parameters is within our maxDelta.
-		uint64_t a = _qualifiers[myidx].value;
+		const uint64_t a = _qualifiers[myidx].value;
-		uint64_t b = other._qualifiers[myidx].value;
+		const uint64_t b = other._qualifiers[myidx].value;
-		if (a >= b) {
+		if (((a >= b) ? (a - b) : (b - a)) > _qualifiers[myidx].maxDelta)
-			if ((a - b) > _qualifiers[myidx].maxDelta)
 			return false;
-		} else {
-			if ((b - a) > _qualifiers[myidx].maxDelta)
-				return false;
-		}
 
 		++myidx;
 	}

node/CertificateOfMembership.hpp

@@ -50,10 +50,8 @@ namespace ZeroTier {
  * These contain an id, a value, and a maximum delta.
  *
  * The ID is arbitrary and should be assigned using a scheme that makes
- * every ID globally unique. ID 0 is reserved for the always-present
+ * every ID globally unique. IDs beneath 65536 are reserved for global
- * validity timestamp and range, and ID 1 is reserved for the always-present
+ * assignment by ZeroTier Networks.
- * network ID. IDs less than 65536 are reserved for future global
- * assignment.
  *
  * The value's meaning is ID-specific and isn't important here. What's
  * important is the value and the third member of the tuple: the maximum
@@ -83,21 +81,107 @@ public:
 	};
 
 	/**
-	 * Reserved COM IDs
+	 * Reserved qualifier IDs
 	 *
 	 * IDs below 65536 should be considered reserved for future global
 	 * assignment here.
+	 *
+	 * Addition of new required fields requires that code in hasRequiredFields
+	 * be updated as well.
 	 */
 	enum ReservedId
 	{
-		COM_RESERVED_ID_TIMESTAMP = 0, // timestamp, max delta defines cert life
+		/**
-		COM_RESERVED_ID_NETWORK_ID = 1 // network ID, max delta always 0
+		 * Timestamp of certificate issue in milliseconds since epoch
+		 *
+		 * maxDelta here defines certificate lifetime, and certs are lazily
+		 * pushed to other peers on a net with a frequency of 1/2 this time.
+		 */
+		COM_RESERVED_ID_TIMESTAMP = 0,
+
+		/**
+		 * Network ID for which certificate was issued
+		 *
+		 * maxDelta here is zero, since this must match.
+		 */
+		COM_RESERVED_ID_NETWORK_ID = 1,
+
+		/**
+		 * ZeroTier address to whom certificate was issued
+		 *
+		 * maxDelta will be 0xffffffffffffffff here since it's permitted to differ
+		 * from peers obviously.
+		 */
+		COM_RESERVED_ID_ISSUED_TO = 2
 	};
 
+	/**
+	 * Create an empty certificate
+	 */
 	CertificateOfMembership() { memset(_signature.data,0,_signature.size()); }
+
+	/**
+	 * Create from required fields common to all networks
+	 *
+	 * @param timestamp Timestamp of cert
+	 * @param timestampMaxDelta Maximum variation between timestamps on this net
+	 * @param nwid Network ID
+	 * @param issuedTo Certificate recipient
+	 */
+	CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo)
+	{
+		_qualifiers.push_back(_Qualifier(COM_RESERVED_ID_TIMESTAMP,timestamp,timestampMaxDelta));
+		_qualifiers.push_back(_Qualifier(COM_RESERVED_ID_NETWORK_ID,nwid,0));
+		_qualifiers.push_back(_Qualifier(COM_RESERVED_ID_ISSUED_TO,issuedTo.toInt(),0xffffffffffffffffULL));
+		memset(_signature.data,0,_signature.size());
+	}
+
+	/**
+	 * Create from string-serialized data
+	 *
+	 * @param s String-serialized COM
+	 */
 	CertificateOfMembership(const char *s) { fromString(s); }
+
+	/**
+	 * Create from string-serialized data
+	 *
+	 * @param s String-serialized COM
+	 */
 	CertificateOfMembership(const std::string &s) { fromString(s.c_str()); }
 
+	/**
+	 * Create from binary-serialized COM in buffer
+	 *
+	 * @param b Buffer to deserialize from
+	 * @param startAt Position to start in buffer
+	 */
+	template<unsigned int C>
+	CertificateOfMembership(const Buffer<C> &b,unsigned int startAt = 0)
+		throw(std::out_of_range,std::invalid_argument)
+	{
+		deserialize(b,startAt);
+	}
+
+	/**
+	 * Check for presence of all required fields common to all networks
+	 *
+	 * @return True if all required fields are present
+	 */
+	inline bool hasRequiredFields() const
+		throw()
+	{
+		if (_qualifiers.size() < 3)
+			return false;
+		if (_qualifiers[0].id != COM_RESERVED_ID_TIMESTAMP)
+			return false;
+		if (_qualifiers[1].id != COM_RESERVED_ID_NETWORK_ID)
+			return false;
+		if (_qualifiers[2].id != COM_RESERVED_ID_ISSUED_TO)
+			return false;
+		return true;
+	}
+
 	/**
 	 * @return Maximum delta for mandatory timestamp field or 0 if field missing
 	 */
@@ -111,6 +195,45 @@ public:
 		return 0ULL;
 	}
 
+	/**
+	 * @return Timestamp for this cert in ms since epoch (according to netconf's clock)
+	 */
+	inline Address timestamp() const
+		throw()
+	{
+		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
+			if (q->id == COM_RESERVED_ID_TIMESTAMP)
+				return Address(q->value);
+		}
+		return Address();
+	}
+
+	/**
+	 * @return Address to which this cert was issued
+	 */
+	inline Address issuedTo() const
+		throw()
+	{
+		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
+			if (q->id == COM_RESERVED_ID_ISSUED_TO)
+				return Address(q->value);
+		}
+		return Address();
+	}
+
+	/**
+	 * @return Network ID for which this cert was issued
+	 */
+	inline uint64_t networkId() const
+		throw()
+	{
+		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
+			if (q->id == COM_RESERVED_ID_NETWORK_ID)
+				return q->value;
+		}
+		return 0ULL;
+	}
+
 	/**
 	 * Add or update a qualifier in this certificate
 	 *
@@ -186,7 +309,7 @@ public:
 		throw(std::out_of_range)
 	{
 		b.append((unsigned char)COM_UINT64_ED25519);
-		b.append((uint32_t)_qualifiers.size());
+		b.append((uint16_t)_qualifiers.size());
 		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
 			b.append(q->id);
 			b.append(q->value);
@@ -209,10 +332,15 @@ public:
 		if (b[p++] != COM_UINT64_ED25519)
 			throw std::invalid_argument("unknown certificate of membership type");
 
-		unsigned int numq = b.template at<uint32_t>(p); p += sizeof(uint32_t);
+		unsigned int numq = b.template at<uint16_t>(p); p += sizeof(uint16_t);
+		uint64_t lastId = 0;
 		for(unsigned int i=0;i<numq;++i) {
+			uint64_t tmp = b.template at<uint64_t>(p);
+			if (tmp < lastId)
+				throw std::invalid_argument("certificate qualifiers are not sorted");
+			else lastId = tmp;
 			_qualifiers.push_back(_Qualifier(
-					b.template at<uint64_t>(p),
+					tmp,
 					b.template at<uint64_t>(p + 8),
 					b.template at<uint64_t>(p + 16)
 				));
@@ -247,8 +375,8 @@ private:
 		inline bool operator<(const _Qualifier &q) const throw() { return (id < q.id); } // for sort
 	};
 
-	std::vector<_Qualifier> _qualifiers; // sorted by id and unique
 	Address _signedBy;
+	std::vector<_Qualifier> _qualifiers; // sorted by id and unique
 	C25519::Signature _signature;
 };
 

node/Network.cpp

@@ -39,6 +39,9 @@
 #include "Switch.hpp"
 #include "Packet.hpp"
 #include "Utils.hpp"
+#include "Buffer.hpp"
+
+#define ZT_NETWORK_CERT_WRITE_BUF_SIZE 524288
 
 namespace ZeroTier {
 
@@ -51,6 +54,7 @@ const char *Network::statusString(const Status s)
 		case NETWORK_WAITING_FOR_FIRST_AUTOCONF: return "WAITING_FOR_FIRST_AUTOCONF";
 		case NETWORK_OK: return "OK";
 		case NETWORK_ACCESS_DENIED: return "ACCESS_DENIED";
+		case NETWORK_NOT_FOUND: return "NOT_FOUND";
 	}
 	return "(invalid)";
 }
@@ -58,15 +62,13 @@ const char *Network::statusString(const Status s)
 Network::~Network()
 {
 	delete _tap;
-
 	if (_destroyOnDelete) {
-		std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf");
+		Utils::rm(std::string(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf"));
-		std::string mcdbPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts");
+		Utils::rm(std::string(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts"));
-		Utils::rm(confPath);
-		Utils::rm(mcdbPath);
 	} else {
 		// Causes flush of membership certs to disk
 		clean();
+		_dumpMulticastCerts();
 	}
 }
 
@@ -87,21 +89,24 @@ SharedPtr<Network> Network::newInstance(const RuntimeEnvironment *renv,uint64_t
 	nw->_r = renv;
 	nw->_tap = new EthernetTap(renv,tag,renv->identity.address().toMAC(),ZT_IF_MTU,&_CBhandleTapData,nw.ptr());
 	nw->_isOpen = false;
+	nw->_emulateArp = false;
+	nw->_emulateNdp = false;
 	nw->_multicastPrefixBits = ZT_DEFAULT_MULTICAST_PREFIX_BITS;
 	nw->_multicastDepth = ZT_DEFAULT_MULTICAST_DEPTH;
+	nw->_status = NETWORK_WAITING_FOR_FIRST_AUTOCONF;
 	memset(nw->_etWhitelist,0,sizeof(nw->_etWhitelist));
 	nw->_id = id;
 	nw->_lastConfigUpdate = 0;
 	nw->_destroyOnDelete = false;
-	if (nw->controller() == renv->identity.address()) // sanity check, this isn't supported for now
+	if (nw->controller() == renv->identity.address()) // netconf masters can't really join networks
-		throw std::runtime_error("cannot add a network for which I am the netconf master");
+		throw std::runtime_error("cannot join a network for which I am the netconf master");
 	nw->_restoreState();
 	nw->_ready = true; // enable handling of Ethernet frames
 	nw->requestConfiguration();
 	return nw;
 }
 
-void Network::setConfiguration(const Network::Config &conf)
+void Network::setConfiguration(const Network::Config &conf,bool saveToDisk)
 {
 	Mutex::Lock _l(_lock);
 	try {
@@ -113,6 +118,8 @@ void Network::setConfiguration(const Network::Config &conf)
 			_mcRates = conf.multicastRates();
 			_staticAddresses = conf.staticAddresses();
 			_isOpen = conf.isOpen();
+			_emulateArp = conf.emulateArp();
+			_emulateNdp = conf.emulateNdp();
 			_multicastPrefixBits = conf.multicastPrefixBits();
 			_multicastDepth = conf.multicastDepth();
 
@@ -121,18 +128,21 @@ void Network::setConfiguration(const Network::Config &conf)
 			_tap->setIps(_staticAddresses);
 			_tap->setDisplayName((std::string("ZeroTier One [") + conf.name() + "]").c_str());
 
-			// Expand ethertype whitelist into fast-lookup bit field
+			// Expand ethertype whitelist into fast-lookup bit field (more memoization)
 			memset(_etWhitelist,0,sizeof(_etWhitelist));
 			std::set<unsigned int> wl(conf.etherTypes());
 			for(std::set<unsigned int>::const_iterator t(wl.begin());t!=wl.end();++t)
 				_etWhitelist[*t / 8] |= (unsigned char)(1 << (*t % 8));
 
-			// Save most recent configuration to disk in networks.d
+			_status = NETWORK_OK;
+
+			if (saveToDisk) {
 				std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf");
 				if (!Utils::writeFile(confPath.c_str(),conf.toString())) {
 					LOG("error: unable to write network configuration file at: %s",confPath.c_str());
 				}
 			}
+		}
 	} catch ( ... ) {
 		// If conf is invalid, reset everything
 		_configuration = Config();
@@ -141,6 +151,9 @@ void Network::setConfiguration(const Network::Config &conf)
 		_mcRates = MulticastRates();
 		_staticAddresses.clear();
 		_isOpen = false;
+		_emulateArp = false;
+		_emulateNdp = false;
+		_status = NETWORK_WAITING_FOR_FIRST_AUTOCONF;
 
 		_lastConfigUpdate = 0;
 		LOG("unexpected exception handling config for network %.16llx, retrying fetch...",(unsigned long long)_id);
@@ -150,7 +163,7 @@ void Network::setConfiguration(const Network::Config &conf)
 void Network::requestConfiguration()
 {
 	if (controller() == _r->identity.address()) {
-		// FIXME: Right now the netconf master cannot be a member of its own nets
+		// netconf master cannot be a member of its own nets
 		LOG("unable to request network configuration for network %.16llx: I am the network master, cannot query self",(unsigned long long)_id);
 		return;
 	}
@@ -162,11 +175,13 @@ void Network::requestConfiguration()
 	_r->sw->send(outp,true);
 }
 
-void Network::addMembershipCertificate(const Address &peer,const CertificateOfMembership &cert)
+void Network::addMembershipCertificate(const CertificateOfMembership &cert)
 {
 	Mutex::Lock _l(_lock);
-	if (!_isOpen)
+	// We go ahead and accept certs provisionally even if _isOpen is true, since
-		_membershipCertificates[peer] = cert;
+	// that might be changed in short order if the user is fiddling in the UI.
+	// These will be purged on clean() for open networks eventually.
+	_membershipCertificates[cert.issuedTo()] = cert;
 }
 
 bool Network::isAllowed(const Address &peer) const
@@ -175,80 +190,55 @@ bool Network::isAllowed(const Address &peer) const
 	try {
 		Mutex::Lock _l(_lock);
 		if (_isOpen)
-			return true;
+			return true; // network is public
 		std::map<Address,CertificateOfMembership>::const_iterator pc(_membershipCertificates.find(peer));
 		if (pc == _membershipCertificates.end())
-			return false;
+			return false; // no certificate on file
-		return _myCertificate.agreesWith(pc->second);
+		return _myCertificate.agreesWith(pc->second); // is other cert valid against ours?
 	} catch (std::exception &exc) {
 		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
 	} catch ( ... ) {
 		TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer.toString().c_str());
 	}
-	return false;
+	return false; // default position on any failure
 }
 
 void Network::clean()
 {
-	std::string mcdbPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts");
-
 	Mutex::Lock _l(_lock);
-
+	uint64_t timestampMaxDelta = _myCertificate.timestampMaxDelta();
-	if ((!_id)||(_isOpen)) {
+	if (_isOpen) {
+		// Open (public) networks do not track certs or cert pushes at all.
 		_membershipCertificates.clear();
-		Utils::rm(mcdbPath);
+		_lastPushedMembershipCertificate.clear();
-	} else {
+	} else if (timestampMaxDelta) {
-		FILE *mcdb = fopen(mcdbPath.c_str(),"wb");
+		// Clean certificates that are no longer valid from the cache.
-		bool writeError = false;
+		for(std::map<Address,CertificateOfMembership>::iterator c=(_membershipCertificates.begin());c!=_membershipCertificates.end();) {
-		if (!mcdb) {
+			if (_myCertificate.agreesWith(c->second))
-			LOG("error: unable to open membership cert database at: %s",mcdbPath.c_str());
+				++c;
-		} else {
+			else _membershipCertificates.erase(c++);
-			if ((writeError)||(fwrite("MCDB0",5,1,mcdb) != 1)) // version
-				writeError = true;
 		}
 
-		for(std::map<Address,CertificateOfMembership>::iterator i=(_membershipCertificates.begin());i!=_membershipCertificates.end();) {
+		// Clean entries from the last pushed tracking map if they're so old as
-			if (_myCertificate.agreesWith(i->second)) {
+		// to be no longer relevant.
-				if ((!writeError)&&(mcdb)) {
+		uint64_t forgetIfBefore = Utils::now() - (timestampMaxDelta * 3);
-					char tmp[ZT_ADDRESS_LENGTH];
+		for(std::map<Address,uint64_t>::iterator lp(_lastPushedMembershipCertificate.begin());lp!=_lastPushedMembershipCertificate.end();) {
-					i->first.copyTo(tmp,ZT_ADDRESS_LENGTH);
+			if (lp->second < forgetIfBefore)
-					if ((writeError)||(fwrite(tmp,ZT_ADDRESS_LENGTH,1,mcdb) != 1))
+				_lastPushedMembershipCertificate.erase(lp++);
-						writeError = true;
+			else ++lp;
-					std::string c(i->second.toString());
-					uint32_t cl = Utils::hton((uint32_t)c.length());
-					if ((writeError)||(fwrite(&cl,sizeof(cl),1,mcdb) != 1))
-						writeError = true;
-					if ((writeError)||(fwrite(c.data(),c.length(),1,mcdb) != 1))
-						writeError = true;
-				}
-				++i;
-			} else _membershipCertificates.erase(i++);
-		}
-
-		if (mcdb)
-			fclose(mcdb);
-		if (writeError) {
-			Utils::rm(mcdbPath);
-			LOG("error: unable to write to membership cert database at: %s",mcdbPath.c_str());
 		}
 	}
 }
 
-Network::Status Network::status() const
-{
-	Mutex::Lock _l(_lock);
-	if (_configuration)
-		return NETWORK_OK;
-	return NETWORK_WAITING_FOR_FIRST_AUTOCONF;
-}
-
 void Network::_CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data)
 {
-	if (!((Network *)arg)->_ready)
+	if (!((Network *)arg)->isUp())
 		return;
+
 	const RuntimeEnvironment *_r = ((Network *)arg)->_r;
 	if (_r->shutdownInProgress)
 		return;
+
 	try {
 		_r->sw->onLocalEthernet(SharedPtr<Network>((Network *)arg),from,to,etherType,data);
 	} catch (std::exception &exc) {
@@ -261,17 +251,14 @@ void Network::_CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned
 void Network::_pushMembershipCertificate(const Address &peer,bool force,uint64_t now)
 {
 	uint64_t timestampMaxDelta = _myCertificate.timestampMaxDelta();
-	if (!timestampMaxDelta) {
+	if (!timestampMaxDelta)
-		LOG("unable to push my certificate to %s for network %.16llx: certificate invalid, missing required timestamp field",peer.toString().c_str(),_id);
+		return; // still waiting on my own cert
-		return; // required field missing!
-	}
 
 	uint64_t &lastPushed = _lastPushedMembershipCertificate[peer];
 	if ((force)||((now - lastPushed) > (timestampMaxDelta / 2))) {
 		lastPushed = now;
 
 		Packet outp(peer,_r->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
-		outp.append((uint64_t)_id);
 		_myCertificate.serialize(outp);
 		_r->sw->send(outp,true);
 	}
@@ -279,21 +266,114 @@ void Network::_pushMembershipCertificate(const Address &peer,bool force,uint64_t
 
 void Network::_restoreState()
 {
-	std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf");
+	if (!_id)
+		return; // sanity check
+
+	Buffer<ZT_NETWORK_CERT_WRITE_BUF_SIZE> buf;
+
+	std::string idstr(idString());
+	std::string confPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idstr + ".conf");
+	std::string mcdbPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idstr + ".mcerts");
+
+	// Read configuration file containing last config from netconf master
+	{
 		std::string confs;
 		if (Utils::readFile(confPath.c_str(),confs)) {
 			try {
 				if (confs.length())
-				setConfiguration(Config(confs));
+					setConfiguration(Config(confs),false);
-		} catch ( ... ) {} // ignore invalid config on disk, we will re-request
+			} catch ( ... ) {} // ignore invalid config on disk, we will re-request from netconf master
 		} else {
 			// If the conf file isn't present, "touch" it so we'll remember
 			// the existence of this network.
-		FILE *tmp = fopen(confPath.c_str(),"w");
+			FILE *tmp = fopen(confPath.c_str(),"wb");
 			if (tmp)
 				fclose(tmp);
 		}
-	// TODO: restore membership certs
+	}
+
+	// Read most recent multicast cert dump
+	if ((!_isOpen)&&(Utils::fileExists(mcdbPath.c_str()))) {
+		CertificateOfMembership com;
+		Mutex::Lock _l(_lock);
+
+		_membershipCertificates.clear();
+
+		try {
+			FILE *mcdb = fopen(mcdbPath.c_str(),"rb");
+			if (mcdb) {
+				for(;;) {
+					long rlen = (long)fread(buf.data() + buf.size(),1,ZT_NETWORK_CERT_WRITE_BUF_SIZE - buf.size(),mcdb);
+					if (rlen <= 0)
+						break;
+					buf.setSize(buf.size() + (unsigned int)rlen);
+					unsigned int ptr = 0;
+					while ((ptr < (ZT_NETWORK_CERT_WRITE_BUF_SIZE / 2))&&(ptr < buf.size())) {
+						ptr += com.deserialize(buf,ptr);
+						if (com.issuedTo())
+							_membershipCertificates[com.issuedTo()] = com;
+					}
+					if (ptr) {
+						memmove(buf.data(),buf.data() + ptr,buf.size() - ptr);
+						buf.setSize(buf.size() - ptr);
+					}
+				}
+			}
+		} catch ( ... ) {
+			// Membership cert dump file invalid. We'll re-learn them off the net.
+			_membershipCertificates.clear();
+			Utils::rm(mcdbPath);
+		}
+	}
+}
+
+void Network::_dumpMulticastCerts()
+{
+	Buffer<ZT_NETWORK_CERT_WRITE_BUF_SIZE> buf;
+	std::string mcdbPath(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts");
+	Mutex::Lock _l(_lock);
+
+	if ((!_id)||(_isOpen)) {
+		Utils::rm(mcdbPath);
+		return;
+	}
+
+	FILE *mcdb = fopen(mcdbPath.c_str(),"wb");
+	if (!mcdb)
+		return;
+	if (fwrite("ZTMCD0",6,1,mcdb) != 1) {
+		Utils::rm(mcdbPath);
+		return;
+	}
+
+	for(std::map<Address,CertificateOfMembership>::iterator c=(_membershipCertificates.begin());c!=_membershipCertificates.end();++c) {
+		try {
+			c->second.serialize(buf);
+			if (buf.size() >= (ZT_NETWORK_CERT_WRITE_BUF_SIZE / 2)) {
+				if (fwrite(buf.data(),buf.size(),1,mcdb) != 1) {
+					fclose(mcdb);
+					Utils::rm(mcdbPath);
+					return;
+				}
+				buf.clear();
+			}
+		} catch ( ... ) {
+			// Sanity check... no cert will ever be big enough to overflow buf
+			fclose(mcdb);
+			Utils::rm(mcdbPath);
+			return;
+		}
+	}
+
+	if (buf.size()) {
+		if (fwrite(buf.data(),buf.size(),1,mcdb) != 1) {
+			fclose(mcdb);
+			Utils::rm(mcdbPath);
+			return;
+		}
+	}
+
+	fclose(mcdb);
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -250,6 +250,28 @@ public:
 			else return CertificateOfMembership(cm->second);
 		}
 
+		/**
+		 * @return True if this network emulates IPv4 ARP for assigned addresses
+		 */
+		inline bool emulateArp() const
+		{
+			const_iterator e(find("eARP"));
+			if (e == end())
+				return false;
+			else return (e->second == "1");
+		}
+
+		/**
+		 * @return True if this network emulates IPv6 NDP for assigned addresses
+		 */
+		inline bool emulateNdp() const
+		{
+			const_iterator e(find("eNDP"));
+			if (e == end())
+				return false;
+			else return (e->second == "1");
+		}
+
 		/**
 		 * @return Multicast rates for this network
 		 */
@@ -343,7 +365,8 @@ public:
 	{
 		NETWORK_WAITING_FOR_FIRST_AUTOCONF,
 		NETWORK_OK,
-		NETWORK_ACCESS_DENIED
+		NETWORK_ACCESS_DENIED,
+		NETWORK_NOT_FOUND
 	};
 
 	/**
@@ -424,6 +447,26 @@ public:
 		return _isOpen;
 	}
 
+	/**
+	 * @return True if this network emulates IPv4 ARP for assigned addresses
+	 */
+	inline bool emulateArp() const
+		throw()
+	{
+		Mutex::Lock _l(_lock);
+		return _emulateArp;
+	}
+
+	/**
+	 * @return True if this network emulates IPv6 NDP for assigned addresses
+	 */
+	inline bool emulateNdp() const
+		throw()
+	{
+		Mutex::Lock _l(_lock);
+		return _emulateNdp;
+	}
+
 	/**
 	 * Update multicast groups for this network's tap
 	 *
@@ -451,8 +494,9 @@ public:
 	 * internally when an old config is reloaded from disk.
 	 *
 	 * @param conf Configuration in key/value dictionary form
+	 * @param saveToDisk IF true (default), write config to disk
 	 */
-	void setConfiguration(const Config &conf);
+	void setConfiguration(const Config &conf,bool saveToDisk = true);
 
 	/**
 	 * Causes this network to request an updated configuration from its master node now
@@ -460,14 +504,13 @@ public:
 	void requestConfiguration();
 
 	/**
-	 * Add or update a peer's membership certificate
+	 * Add or update a membership certificate
 	 *
 	 * The certificate must already have been validated via signature checking.
 	 *
-	 * @param peer Peer that owns certificate
+	 * @param cert Certificate of membership
-	 * @param cert Certificate itself
 	 */
-	void addMembershipCertificate(const Address &peer,const CertificateOfMembership &cert);
+	void addMembershipCertificate(const CertificateOfMembership &cert);
 
 	/**
 	 * Push our membership certificate to a peer
@@ -523,10 +566,35 @@ public:
 	 */
 	inline uint64_t lastConfigUpdate() const throw() { return _lastConfigUpdate; }
 
+	/** 
+	 * Force this network's status to a particular state based on config reply
+	 */
+	inline void forceStatusTo(const Status s)
+		throw()
+	{
+		Mutex::Lock _l(_lock);
+		_status = s;
+	}
+
 	/**
 	 * @return Status of this network
 	 */
-	Status status() const;
+	inline Status status() const
+		throw()
+	{
+		Mutex::Lock _l(_lock);
+		return _status;
+	}
+
+	/**
+	 * @return True if this network is in "OK" status and can accept traffic from us
+	 */
+	inline bool isUp() const
+		throw()
+	{
+		Mutex::Lock _l(_lock);
+		return ((_status == NETWORK_OK)&&(_ready));
+	}
 
 	/**
 	 * Determine whether frames of a given ethernet type are allowed on this network
@@ -567,9 +635,10 @@ public:
 	}
 
 	/**
+	 * @param fromPeer Peer attempting to bridge other Ethernet peers onto network
 	 * @return True if this network allows bridging
 	 */
-	inline bool permitsBridging() const
+	inline bool permitsBridging(const Address &fromPeer) const
 		throw()
 	{
 		return false; // TODO: bridging not implemented yet
@@ -589,6 +658,7 @@ private:
 	static void _CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data);
 	void _pushMembershipCertificate(const Address &peer,bool force,uint64_t now);
 	void _restoreState();
+	void _dumpMulticastCerts();
 
 	const RuntimeEnvironment *_r;
 
@@ -612,9 +682,14 @@ private:
 	MulticastRates _mcRates;
 	std::set<InetAddress> _staticAddresses;
 	bool _isOpen;
+	bool _emulateArp;
+	bool _emulateNdp;
 	unsigned int _multicastPrefixBits;
 	unsigned int _multicastDepth;
 
+	// Network status
+	Status _status;
+
 	// Ethertype whitelist bit field, set from config, for really fast lookup
 	unsigned char _etWhitelist[65536 / 8];
 

node/Node.cpp

@@ -246,6 +246,8 @@ static void _netconfServiceMessageHandler(void *renv,Service &svc,const Dictiona
 					const std::string &err = msg.get("error");
 					if (err == "OBJ_NOT_FOUND")
 						errCode = Packet::ERROR_OBJ_NOT_FOUND;
+					else if (err == "ACCESS_DENIED")
+						errCode = Packet::ERROR_NETWORK_ACCESS_DENIED;
 
 					Packet outp(peerAddress,_r->identity.address(),Packet::VERB_ERROR);
 					outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);

node/NodeConfig.cpp

@@ -125,7 +125,7 @@ void NodeConfig::clean()
 		n->second->clean();
 }
 
-// Macro used in execute()
+// Macro used in execute() to push lines onto the return packet
 #undef _P
 #define _P(f,...) { r.push_back(std::string()); Utils::stdsprintf(r.back(),(f),##__VA_ARGS__); }
 
@@ -161,18 +161,30 @@ std::vector<std::string> NodeConfig::execute(const char *command)
 	std::vector<std::string> r;
 	std::vector<std::string> cmd(Utils::split(command,"\r\n \t","\\","'"));
 
-	//
+	/* Not coincidentally, response type codes correspond with HTTP
-	// Not coincidentally, response type codes correspond with HTTP
+	 * status codes. Technically a little arbitrary, but would maybe
-	// status codes.
+	 * make things easier if we wanted to slap some kind of web API
-	//
+	 * in front of this thing. */
 
 	if ((cmd.empty())||(cmd[0] == "help")) {
 		_P("200 help help");
+		_P("200 help info");
 		_P("200 help listpeers");
 		_P("200 help listnetworks");
 		_P("200 help join <network ID>");
 		_P("200 help leave <network ID>");
 		_P("200 help terminate [<reason>]");
+	} else if (cmd[0] == "info") {
+		bool isOnline = false;
+		uint64_t now = Utils::now();
+		std::vector< SharedPtr<Peer> > snp(_r->topology->supernodePeers());
+		for(std::vector< SharedPtr<Peer> >::const_iterator sn(snp.begin());sn!=snp.end();++sn) {
+			if ((*sn)->hasActiveDirectPath(now)) {
+				isOnline = true;
+				break;
+			}
+		}
+		_P("200 info %s %s %s",_r->identity.address().toString().c_str(),(isOnline ? "ONLINE" : "OFFLINE"),Node::versionString());
 	} else if (cmd[0] == "listpeers") {
 		_P("200 listpeers <ztaddr> <ipv4> <ipv6> <latency> <version>");
 		_r->topology->eachPeer(_DumpPeerStatistics(r));
@@ -187,8 +199,7 @@ std::vector<std::string> NodeConfig::execute(const char *command)
 					tmp.push_back(',');
 				tmp.append(i->toString());
 			}
-			// TODO: display network status, such as "permission denied to closed
+
-			// network" or "waiting".
 			_P("200 listnetworks %.16llx %s %s %s %s",
 				(unsigned long long)nw->first,
 				Network::statusString(nw->second->status()),
@@ -202,7 +213,7 @@ std::vector<std::string> NodeConfig::execute(const char *command)
 			if (nwid > 0) {
 				Mutex::Lock _l(_networks_m);
 				if (_networks.count(nwid)) {
-					_P("400 already a member of %.16llx",(unsigned long long)nwid);
+					_P("409 already a member of %.16llx",(unsigned long long)nwid);
 				} else {
 					try {
 						SharedPtr<Network> nw(Network::newInstance(_r,nwid));

node/Packet.cpp

@@ -63,6 +63,7 @@ const char *Packet::errorString(ErrorCode e)
 		case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION";
 		case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION";
 		case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE";
+		case ERROR_NETWORK_ACCESS_DENIED: return "NETWORK_ACCESS_DENIED";
 	}
 	return "(unknown)";
 }

node/Packet.hpp

@@ -56,6 +56,9 @@
  *   * New crypto completely changes key agreement cipher
  * 4 - 0.6.0 ...
  *   * New identity format based on hashcash design
+ *
+ * This isn't going to change again for a long time unless your
+ * author wakes up again at 4am with another great idea. :P
  */
 #define ZT_PROTO_VERSION 4
 
@@ -196,6 +199,8 @@
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN 2
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME_LEN + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN)
 
+#define ZT_PROTO_VERB_NETWORK_MEMBERSHIP_CERTIFICATE_IDX_CERTIFICATE (ZT_PACKET_IDX_PAYLOAD)
+
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID + 8)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN + 2)
@@ -551,12 +556,12 @@ public:
 		 */
 		VERB_MULTICAST_LIKE = 9,
 
-		/* Network member certificate for sending peer:
+		/* Network member certificate:
-		 *   <[8] 64-bit network ID>
 		 *   <[...] serialized certificate of membership>
 		 *
-		 * OK is generated on acceptance. ERROR is returned on failure. In both
+		 * Certificate contains network ID, peer it was issued for, etc.
-		 * cases the payload is the network ID.
+		 *
+		 * OK/ERROR are not generated.
 		 */
 		VERB_NETWORK_MEMBERSHIP_CERTIFICATE = 10,
 
@@ -623,7 +628,10 @@ public:
 		ERROR_UNSUPPORTED_OPERATION = 5,
 
 		/* Message to private network rejected -- no unexpired certificate on file */
-		ERROR_NEED_MEMBERSHIP_CERTIFICATE = 6
+		ERROR_NEED_MEMBERSHIP_CERTIFICATE = 6,
+
+		/* Tried to join network, but you're not a member */
+		ERROR_NETWORK_ACCESS_DENIED = 7
 	};
 
 	/**

node/PacketDecoder.cpp

@@ -64,6 +64,10 @@ bool PacketDecoder::tryDecode(const RuntimeEnvironment *_r)
 			// packet and are waiting for the lookup of the original sender
 			// for a multicast frame. So check to see if we've got it.
 			return _doMULTICAST_FRAME(_r,peer);
+		} else if (_step == DECODE_WAITING_FOR_NETWORK_MEMBERSHIP_CERTIFICATE_SIGNER_LOOKUP) {
+			// In this state we have already authenticated and decoded the
+			// packet and we're waiting for the identity of the cert's signer.
+			return _doNETWORK_MEMBERSHIP_CERTIFICATE(_r,peer);
 		}
 
 		if (!dearmor(peer->key())) {
@@ -134,15 +138,22 @@ bool PacketDecoder::_doERROR(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 		switch(errorCode) {
 			case Packet::ERROR_OBJ_NOT_FOUND:
 				if (inReVerb == Packet::VERB_WHOIS) {
-					// TODO: abort WHOIS if sender is a supernode
+					if (_r->topology->isSupernode(source()))
+						_r->sw->cancelWhoisRequest(Address(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH));
 				}
 				break;
 			case Packet::ERROR_IDENTITY_COLLISION:
 				// TODO: if it comes from a supernode, regenerate a new identity
+				// if (_r->topology->isSupernode(source())) {}
 				break;
-			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE:
+			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
-				// TODO: send member certificate
+				// TODO: this allows anyone to request a membership cert, which is
-				break;
+				// harmless until these contain possibly privacy-sensitive info.
+				// Then we'll need to be more careful.
+				SharedPtr<Network> network(_r->nc->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				if (network)
+					network->pushMembershipCertificate(source(),true,Utils::now());
+			}	break;
 			default:
 				break;
 		}
@@ -177,6 +188,9 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 		SharedPtr<Peer> peer(_r->topology->getPeer(id.address()));
 		if (peer) {
 			if (peer->identity() != id) {
+				// Sorry, someone beat you to that address. What are the odds?
+				// Well actually they're around two in 2^40. You should play
+				// the lottery.
 				unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 				if (_r->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
 					TRACE("rejected HELLO from %s(%s): address already claimed",source().toString().c_str(),_remoteAddress.toString().c_str());
@@ -189,8 +203,11 @@ bool PacketDecoder::_doHELLO(const RuntimeEnvironment *_r)
 					_r->demarc->send(_localPort,_remoteAddress,outp.data(),outp.size(),-1);
 				}
 				return true;
+			} // else continue and send OK since we already know thee...
+		} else {
+			// Learn a new peer
+			peer = _r->topology->addPeer(SharedPtr<Peer>(new Peer(_r->identity,id)));
 		}
-		} else peer = _r->topology->addPeer(SharedPtr<Peer>(new Peer(_r->identity,id)));
 
 		peer->onReceive(_r,_localPort,_remoteAddress,hops(),Packet::VERB_HELLO,Utils::now());
 		peer->setRemoteVersion(vMajor,vMinor,vRevision);
@@ -217,6 +234,7 @@ bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &pe
 {
 	try {
 		Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
+		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
 		switch(inReVerb) {
 			case Packet::VERB_HELLO: {
 				// OK from HELLO permits computation of latency.
@@ -252,9 +270,7 @@ bool PacketDecoder::_doOK(const RuntimeEnvironment *_r,const SharedPtr<Peer> &pe
 					}
 				}
 			}	break;
-			default:
+			default: break;
-				//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
-				break;
 		}
 	} catch (std::exception &ex) {
 		TRACE("dropped OK from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
@@ -412,12 +428,19 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		const unsigned int signatureLen = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen);
 		const unsigned char *const signature = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen + 2,signatureLen);
 
+		// Check multicast signature to verify original sender
 		const unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + frameLen;
 		if (!originPeer->identity().verify(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION,signedPartLen),signedPartLen,signature,signatureLen)) {
 			TRACE("dropped MULTICAST_FRAME from %s(%s): failed signature verification, claims to be from %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
 			return true;
 		}
 
+		// Security check to prohibit multicasts that are really Ethernet unicasts
+		if (!dest.mac().isMulticast()) {
+			TRACE("dropped MULTICAST_FRAME from %s(%s): %s is not a multicast/broadcast address",source().toString().c_str(),_remoteAddress.toString().c_str(),dest.mac().toString().c_str());
+			return true;
+		}
+
 #ifdef ZT_TRACE_MULTICAST
 		char mct[256];
 		unsigned int startingFifoItems = 0;
@@ -430,18 +453,11 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		_r->demarc->send(Demarc::ANY_PORT,ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct),-1);
 #endif
 
-		// Security check to prohibit multicasts that are really Ethernet unicasts
-		if (!dest.mac().isMulticast()) {
-			TRACE("dropped MULTICAST_FRAME from %s(%s): %s is not a multicast/broadcast address",source().toString().c_str(),_remoteAddress.toString().c_str(),dest.mac().toString().c_str());
-			return true;
-		}
-
-		bool rateLimitsExceeded = false;
 		unsigned int maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
 		SharedPtr<Network> network(_r->nc->network(nwid));
 
 		if ((origin == _r->identity.address())||(_r->mc->deduplicate(nwid,guid))) {
-			// Ordinary frames will drop duplicates. Supernodes keep propagating
+			// Ordinary nodes will drop duplicates. Supernodes keep propagating
 			// them since they're used as hubs to link disparate clusters of
 			// members of the same multicast group.
 			if (!_r->topology->amSupernode()) {
@@ -453,16 +469,19 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 				return true;
 			}
 		} else {
-			// Supernodes however won't do this more than once. If the supernode
+			// If we are actually a member of this network (will just about always
-			// does happen to be a member of the network -- which is usually not
+			// be the case unless we're a supernode), check to see if we should
-			// true -- we don't want to see a ton of copies of the same frame on
+			// inject the packet. This also gives us an opportunity to check things
-			// its tap device. Also double or triple counting bandwidth metrics
+			// like multicast bandwidth constraints.
-			// for the same frame would not be fair.
 			if (network) {
 				maxDepth = std::min((unsigned int)ZT_MULTICAST_GLOBAL_MAX_DEPTH,network->multicastDepth());
+				if (!maxDepth)
+					maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
+
 				if (!network->isAllowed(origin)) {
 					TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: sender %s not allowed or we don't have a certificate",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),origin.toString().c_str());
 
+					// Tell them we need a certificate
 					Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
 					outp.append((unsigned char)Packet::VERB_FRAME);
 					outp.append(packetId());
@@ -473,30 +492,33 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 
 					// We do not terminate here, since if the member just has an out of
 					// date cert or hasn't sent us a cert yet we still want to propagate
-					// the message so multicast works.
+					// the message so multicast keeps working downstream.
-				} else if ((!network->permitsBridging())&&(!origin.wouldHaveMac(sourceMac))) {
+				} else if ((!network->permitsBridging(origin))&&(!origin.wouldHaveMac(sourceMac))) {
-					TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),sourceMac.toString().c_str(),origin.toString().c_str());
+					// This *does* terminate propagation, since it's technically a
+					// security violation of the network's bridging policy. But if we
+					// were to keep propagating it wouldn't hurt anything, just waste
+					// bandwidth as everyone else would reject it too.
+					TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),sourceMac.toString().c_str(),origin.toString().c_str());
+					return true;
 				} else if (!network->permitsEtherType(etherType)) {
-					TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: ethertype %u is not allowed",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),etherType);
+					// Ditto for this-- halt propagation if this is for an ethertype
+					// this network doesn't allow. Same principle as bridging test.
+					TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: ethertype %u is not allowed",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),etherType);
+					return true;
 				} else if (!network->updateAndCheckMulticastBalance(origin,dest,frameLen)) {
-					rateLimitsExceeded = true;
+					// Rate limits can only be checked by members of this network, but
-				} else {
+					// there should be enough of them that over-limit multicasts get
-					network->tap().put(sourceMac,dest.mac(),etherType,frame,frameLen);
+					// their propagation aborted.
-				}
-			}
-		}
-
-		// We can only really know if rate limit was exceeded if we're a member of
-		// this network. This will nearly always be true for anyone getting a
-		// multicast except supernodes, so the net effect will be to truncate
-		// multicast propagation if the rate limit is exceeded.
-		if (rateLimitsExceeded) {
 #ifdef ZT_TRACE_MULTICAST
 					Utils::snprintf(mct,sizeof(mct),"%c %s dropped %.16llx: rate limits exceeded",(_r->topology->amSupernode() ? 'S' : '-'),_r->identity.address().toString().c_str(),guid);
 					_r->demarc->send(Demarc::ANY_PORT,ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct),-1);
 #endif
 					TRACE("dropped MULTICAST_FRAME from %s(%s): rate limits exceeded for sender %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
 					return true;
+				} else {
+					network->tap().put(sourceMac,dest.mac(),etherType,frame,frameLen);
+				}
+			}
 		}
 
 		if (depth == 0xffff) {
@@ -550,7 +572,8 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 			*(newFifoPtr++) = (unsigned char)0;
 
 		// If we're forwarding a packet within a private network that we are
-		// a member of, also propagate our cert forward if needed.
+		// a member of, also propagate our cert if needed. This propagates
+		// it to everyone including people who will receive this multicast.
 		if (network)
 			network->pushMembershipCertificate(newFifo,sizeof(newFifo),false,Utils::now());
 
@@ -616,13 +639,52 @@ bool PacketDecoder::_doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedP
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
-
 	return true;
 }
 
 bool PacketDecoder::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *_r,const SharedPtr<Peer> &peer)
 {
-	// TODO: not implemented yet, will be needed for private networks.
+	try {
+		CertificateOfMembership com(*this,ZT_PROTO_VERB_NETWORK_MEMBERSHIP_CERTIFICATE_IDX_CERTIFICATE);
+		if (!com.hasRequiredFields()) {
+			TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): invalid cert: at least one required field is missing",source().toString().c_str(),_remoteAddress.toString().c_str());
+			return true;
+		} else if (com.signedBy()) {
+			SharedPtr<Peer> signer(_r->topology->getPeer(com.signedBy()));
+			if (signer) {
+				if (com.verify(signer->identity())) {
+					uint64_t nwid = com.networkId();
+					SharedPtr<Network> network(_r->nc->network(nwid));
+					if (network) {
+						if (network->controller() == signer) {
+							network->addMembershipCertificate(com);
+							return true;
+						} else {
+							TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): signer %s is not the controller for network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),signer->address().toString().c_str(),(unsigned long long)nwid);
+							return true;
+						}
+					} else {
+						TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): not a member of network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)nwid);
+						return true;
+					}
+				} else {
+					TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): failed signature verification for signer %s",source().toString().c_str(),_remoteAddress.toString().c_str(),signer->address().toString().c_str());
+					return true;
+				}
+			} else {
+				_r->sw->requestWhois(com.signedBy());
+				_step = DECODE_WAITING_FOR_NETWORK_MEMBERSHIP_CERTIFICATE_SIGNER_LOOKUP;
+				return false;
+			}
+		} else {
+			TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): invalid cert: no signature",source().toString().c_str(),_remoteAddress.toString().c_str());
+			return true;
+		}
+	} catch (std::exception &ex) {
+		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
+	} catch ( ... ) {
+		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
 	return true;
 }
 
@@ -644,6 +706,7 @@ bool PacketDecoder::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *_r,const
 			request["nwid"] = tmp;
 			Utils::snprintf(tmp,sizeof(tmp),"%llx",(unsigned long long)packetId());
 			request["requestId"] = tmp;
+			request["from"] = _remoteAddress.toString();
 			//TRACE("to netconf:\n%s",request.toString().c_str());
 			_r->netconfService->send(request);
 		} else {

node/PacketDecoder.hpp

@@ -131,6 +131,7 @@ private:
 	enum {
 		DECODE_WAITING_FOR_SENDER_LOOKUP, // on initial receipt, we need peer's identity
 		DECODE_WAITING_FOR_MULTICAST_FRAME_ORIGINAL_SENDER_LOOKUP,
+		DECODE_WAITING_FOR_NETWORK_MEMBERSHIP_CERTIFICATE_SIGNER_LOOKUP
 	} _step;
 
 	AtomicCounter __refCount;

node/Switch.cpp

@@ -466,6 +466,12 @@ void Switch::requestWhois(const Address &addr)
 		_sendWhoisRequest(addr,(const Address *)0,0);
 }
 
+void Switch::cancelWhoisRequest(const Address &addr)
+{
+	Mutex::Lock _l(_outstandingWhoisRequests_m);
+	_outstandingWhoisRequests.erase(addr);
+}
+
 void Switch::doAnythingWaitingForPeer(const SharedPtr<Peer> &peer)
 {
 	{

node/Switch.hpp

@@ -179,6 +179,13 @@ public:
 	 */
 	void requestWhois(const Address &addr);
 
+	/**
+	 * Cancel WHOIS for an address
+	 *
+	 * @param addr Address to cancel
+	 */
+	void cancelWhoisRequest(const Address &addr);
+
 	/**
 	 * Run any processes that are waiting for this peer
 	 *