Tons more refactoring: simplify Network, move explicit management of Tap out, redo COM serialization, etc.

include/ZeroTierOne.h

@@ -526,14 +526,19 @@ typedef long (*ZT1_DataStoreGetFunction)(ZT1_Node *,const char *,void *,unsigned
 /**
  * Function to store an object in the data store
  *
- * Parameters: (1) node, (2) object name, (3) object data, (4) object size.
+ * Parameters: (1) node, (2) object name, (3) object data, (4) object size,
+ * and (5) secure? (bool). If secure is true, the file should be set readable
+ * and writable only to the user running ZeroTier One. What this means is
+ * platform-specific.
+ *
  * Name semantics are the same as the get function. This must return zero on
  * success. You can return any OS-specific error code on failure, as these
  * may be visible in logs or error messages and might aid in debugging.
  *
- * A call to write 0 bytes can safely be interpreted as a delete operation.
+ * A call to write 0 bytes with a null data pointer should be interpreted
+ * as a delete operation. The secure flag is not meaningful in this case.
  */
-typedef int (*ZT1_DataStorePutFunction)(ZT1_Node *,const char *,const void *,unsigned long);
+typedef int (*ZT1_DataStorePutFunction)(ZT1_Node *,const char *,const void *,unsigned long,int);
 
 /**
  * Function to send a ZeroTier packet out over the wire
@@ -570,6 +575,7 @@ typedef void (*ZT1_VirtualNetworkFrameFunction)(ZT1_Node *,uint64_t,uint64_t,uin
  * will generate an identity.
  *
  * @param node Result: pointer is set to new node instance on success
+ * @param now Current clock in milliseconds
  * @param dataStoreGetFunction Function called to get objects from persistent storage
  * @param dataStorePutFunction Function called to put objects in persistent storage
  * @param virtualNetworkConfigCallback Function to be called when virtual LANs are created, deleted, or their config parameters change
@@ -578,6 +584,7 @@ typedef void (*ZT1_VirtualNetworkFrameFunction)(ZT1_Node *,uint64_t,uint64_t,uin
  */
 enum ZT1_ResultCode ZT1_Node_new(
 	ZT1_Node **node,
+	uint64_t now,
 	ZT1_DataStoreGetFunction *dataStoreGetFunction,
 	ZT1_DataStorePutFunction *dataStorePutFunction,
 	ZT1_WirePacketSendFunction *wirePacketSendFunction,
@@ -644,7 +651,7 @@ enum ZT1_ResultCode ZT1_Node_processVirtualNetworkFrame(
  * @param nextCallDeadline Result: set to deadline for next call to one of the three processXXX() methods
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-enum ZT1_Resultcode ZT1_Node_processNothing(ZT1_Node *node,uint64_t now,uint64_t *nextCallDeadline);
+enum ZT1_ResultCode ZT1_Node_processNothing(ZT1_Node *node,uint64_t now,uint64_t *nextCallDeadline);
 
 /**
  * Join a network

netconf/SqliteNetworkConfigMaster.cpp

@@ -137,7 +137,7 @@ SqliteNetworkConfigMaster::~SqliteNetworkConfigMaster()
 	}
 }
 
-NetworkConfigMaster::ResultCode SqliteNetworkConfigMaster::doNetworkConfigRequest(const InetAddress &fromAddr,uint64_t packetId,const Identity &identity,uint64_t nwid,const Dictionary &metaData,uint64_t haveRevision,Dictionary &netconf)
+NetworkConfigMaster::ResultCode SqliteNetworkConfigMaster::doNetworkConfigRequest(const InetAddress &fromAddr,const Identity &identity,uint64_t nwid,const Dictionary &metaData,uint64_t haveRevision,Dictionary &netconf)
 {
 	Mutex::Lock _l(_lock);
 

netconf/SqliteNetworkConfigMaster.hpp

@@ -54,7 +54,6 @@ public:
 
 	virtual NetworkConfigMaster::ResultCode doNetworkConfigRequest(
 		const InetAddress &fromAddr,
-		uint64_t packetId,
 		const Identity &identity,
 		uint64_t nwid,
 		const Dictionary &metaData,

node/CertificateOfMembership.hpp

@@ -40,6 +40,7 @@
 #include "Address.hpp"
 #include "C25519.hpp"
 #include "Identity.hpp"
+#include "Utils.hpp"
 
 namespace ZeroTier {
 
@@ -314,9 +315,80 @@ public:
 	 */
 	inline const Address &signedBy() const throw() { return _signedBy; }
 
+	/**
+	 * Serialize to std::string or compatible class
+	 *
+	 * @param b String or other class supporting push_back() and append() like std::string
+	 */
+	template<typename T>
+	inline void serialize2(T &b) const
+	{
+		uint64_t tmp[3];
+		char tmp2[ZT_ADDRESS_LENGTH];
+		b.push_back((char)COM_UINT64_ED25519);
+		b.push_back((char)((_qualifiers.size() >> 8) & 0xff));
+		b.push_back((char)(_qualifiers.size() & 0xff));
+		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
+			tmp[0] = Utils::hton(q->id);
+			tmp[1] = Utils::hton(q->value);
+			tmp[2] = Utils::hton(q->maxDelta);
+			b.append(reinterpret_cast<const char *>(reinterpret_cast<void *>(tmp)),sizeof(tmp));
+		}
+		_signedBy.copyTo(tmp2,ZT_ADDRESS_LENGTH);
+		b.append(tmp2,ZT_ADDRESS_LENGTH);
+		if (_signedBy)
+			b.append((const char *)_signature.data,_signature.size());
+	}
+
+	/**
+	 * Deserialize from std::string::iterator or compatible iterator or char* pointer
+	 *
+	 * @param p Iterator
+	 * @param end End of buffer
+	 */
+	template<typename T>
+	inline void deserialize2(T &p,const T &end)
+	{
+		uint64_t tmp[3];
+		char tmp2[ZT_ADDRESS_LENGTH];
+		unsigned int qcount;
+
+		_qualifiers.clear();
+		_signedBy.zero();
+
+		if (p == end) throw std::out_of_range("incomplete certificate of membership");
+		if (*(p++) != (char)COM_UINT64_ED25519) throw std::invalid_argument("unknown certificate of membership type");
+
+		if (p == end) throw std::out_of_range("incomplete certificate of membership");
+		qcount = (unsigned int)*(p++) << 8;
+		if (p == end) throw std::out_of_range("incomplete certificate of membership");
+		qcount |= (unsigned int)*(p++);
+
+		for(unsigned int i=0;i<qcount;++i) {
+			char *p2 = reinterpret_cast<char *>(reinterpret_cast<void *>(tmp));
+			for(unsigned int j=0;j<sizeof(tmp);++j) {
+				if (p == end) throw std::out_of_range("incomplete certificate of membership");
+				*(p2++) = *(p++);
+			}
+			_qualifiers.push_back(_Qualifier(Utils::ntoh(tmp[0]),Utils::ntoh(tmp[1]),Utils::ntoh(tmp[2])));
+		}
+
+		for(unsigned int j=0;j<ZT_ADDRESS_LENGTH;++j) {
+			if (p == end) throw std::out_of_range("incomplete certificate of membership");
+			tmp2[j] = *(p++);
+		}
+		_signedBy.setTo(tmp2,ZT_ADDRESS_LENGTH);
+
+		if (_signedBy) {
+			for(unsigned int j=0;j<_signature.size();++j) {
+				if (p == end) throw std::out_of_range("incomplete certificate of membership");
+				_signature.data[j] = (unsigned char)*(p++);
+			}
+		}
+	}
+
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
-		throw(std::out_of_range)
 	{
 		b.append((unsigned char)COM_UINT64_ED25519);
 		b.append((uint16_t)_qualifiers.size());
@@ -332,7 +404,6 @@ public:
 
 	template<unsigned int C>
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-		throw(std::out_of_range,std::invalid_argument)
 	{
 		unsigned int p = startAt;
 

node/IncomingPacket.cpp

@@ -308,14 +308,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 			} break;
 
 			case Packet::VERB_NETWORK_CONFIG_REQUEST: {
-				SharedPtr<Network> nw(RR->nc->network(at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID)));
+				SharedPtr<Network> nw(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID)));
 				if ((nw)&&(nw->controller() == source())) {
 					unsigned int dictlen = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN);
 					std::string dict((const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,dictlen),dictlen);
 					if (dict.length()) {
 						if (nw->setConfiguration(Dictionary(dict)) == 2) { // 2 == accepted and actually new
 							/* If this configuration was indeed new, we do another
-							 * netconf request with its timestamp. We do this in
+							 * netconf request with its revision. We do this in
 							 * order to (a) tell the netconf server we got it (it
 							 * won't send a duplicate if ts == current), and (b)
 							 * get another one if the netconf is changing rapidly
@@ -323,15 +323,15 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 							 *
 							 * Note that we don't do this for netconf masters with
 							 * versions <= 1.0.3, since those regenerate a new netconf
-							 * with a new timestamp every time. In that case this double
+							 * with a new revision every time. In that case this double
 							 * confirmation would create a race condition. */
-							if (peer->atLeastVersion(1,0,3)) {
+							if ((peer->atLeastVersion(1,0,3))&&(nc->revision() > 0)) {
 								SharedPtr<NetworkConfig> nc(nw->config2());
 								if ((nc)&&(nc->timestamp() > 0)) { // sanity check
 									Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CONFIG_REQUEST);
 									outp.append((uint64_t)nw->id());
 									outp.append((uint16_t)0); // no meta-data
-							                outp.append((uint64_t)nc->timestamp());
+									outp.append((uint64_t)nc->revision());
 									outp.armor(peer->key(),true);
 									_fromSock->send(_remoteAddress,outp.data(),outp.size());
 								}
@@ -624,15 +624,17 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID);
 		unsigned int metaDataLength = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN);
 		Dictionary metaData((const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength),metaDataLength);
-		uint64_t haveTimestamp = 0;
+		uint64_t haveRevision = 0;
 		if ((ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength + 8) <= size())
-			haveTimestamp = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength);
+			haveRevision = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT + metaDataLength);
 
-		peer->received(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,Utils::now());
+		const unsigned int h = hops();
+		uint64_t pid = packetId();
+		peer->received(RR,_fromSock,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,RR->node->now());
 
 		if (RR->netconfMaster) {
 			Dictionary netconf;
-			switch(RR->netconfMaster->doNetworkConfigRequest(_remoteAddress,packetId(),peer->identity(),nwid,metaData,haveTimestamp,netconf)) {
+			switch(RR->netconfMaster->doNetworkConfigRequest((h > 0) ? InetAddress() : _remoteAddress,peer->identity(),nwid,metaData,haveRevision,netconf)) {
 				case NetworkConfigMaster::NETCONF_QUERY_OK: {
 					std::string netconfStr(netconf.toString());
 					if (netconfStr.length() > 0xffff) { // sanity check since field ix 16-bit
@@ -640,7 +642,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					} else {
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 						outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
-						outp.append(packetId());
+						outp.append(pid);
 						outp.append(nwid);
 						outp.append((uint16_t)netconfStr.length());
 						outp.append(netconfStr.data(),netconfStr.length());
@@ -657,7 +659,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 				case NetworkConfigMaster::NETCONF_QUERY_OBJECT_NOT_FOUND: {
 					Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
 					outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
-					outp.append(packetId());
+					outp.append(pid);
 					outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
@@ -666,7 +668,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 				case NetworkConfigMaster::NETCONF_QUERY_ACCESS_DENIED: {
 					Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
 					outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
-					outp.append(packetId());
+					outp.append(pid);
 					outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
@@ -682,7 +684,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 		} else {
 			Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
 			outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
-			outp.append(packetId());
+			outp.append(pid);
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
 			outp.armor(peer->key(),true);

node/Network.cpp

@@ -33,83 +33,85 @@
 #include "Constants.hpp"
 #include "Network.hpp"
 #include "RuntimeEnvironment.hpp"
-#include "NodeConfig.hpp"
 #include "Switch.hpp"
 #include "Packet.hpp"
 #include "Buffer.hpp"
-#include "EthernetTap.hpp"
+#include "NetworkConfigMaster.hpp"
-#include "EthernetTapFactory.hpp"
-
-#define ZT_NETWORK_CERT_WRITE_BUF_SIZE 131072
 
 namespace ZeroTier {
 
 const ZeroTier::MulticastGroup Network::BROADCAST(ZeroTier::MAC(0xff),0);
 
-const char *Network::statusString(const Status s)
+Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
-	throw()
+	RR(renv),
+	_id(nwid),
+	_mac(renv->identity.address(),nwid),
+	_enabled(true),
+	_lastConfigUpdate(0),
+	_destroyed(false),
+	_netconfFailure(NETCONF_FAILURE_NONE)
 {
-	switch(s) {
+	char confn[128],mcdbn[128];
-		case NETWORK_INITIALIZING: return "INITIALIZING";
+	Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id);
-		case NETWORK_WAITING_FOR_FIRST_AUTOCONF: return "WAITING_FOR_FIRST_AUTOCONF";
+	Utils::snprintf(mcdbn,sizeof(mcdbn),"networks.d/%.16llx.mcerts",_id);
-		case NETWORK_OK: return "OK";
+
-		case NETWORK_ACCESS_DENIED: return "ACCESS_DENIED";
+	if (_id == ZT_TEST_NETWORK_ID) {
-		case NETWORK_NOT_FOUND: return "NOT_FOUND";
+		applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address()));
-		case NETWORK_INITIALIZATION_FAILED: return "INITIALIZATION_FAILED";
+
-		case NETWORK_NO_MORE_DEVICES: return "NO_MORE_DEVICES";
+		// Save a one-byte CR to persist membership in the test network
+		RR->node->dataStorePut(confn,"\n",1,false);
+	} else {
+		bool gotConf = false;
+		try {
+			std::string conf(RR->node->dataStoreGet(confn));
+			if (conf.length()) {
+				setConfiguration(Dictionary(conf),false);
+				gotConf = true;
 			}
-	return "(invalid)";
+		} catch ( ... ) {} // ignore invalids, we'll re-request
+
+		if (!gotConf) {
+			// Save a one-byte CR to persist membership while we request a real netconf
+			RR->node->dataStorePut(confn,"\n",1,false);
+		}
+
+		try {
+			std::string mcdb(RR->node->dataStoreGet(mcdbn));
+			if (mcdb.length() > 6) {
+				const char *p = mcdb.data();
+				const char *e = p + mcdb.length();
+				if (!memcmp("ZTMCD0",p,6)) {
+					p += 6;
+					Mutex::Lock _l(_lock);
+					while (p != e) {
+						CertificateOfMembership com;
+						com.deserialize2(p,e);
+						if (!com)
+							break;
+						_membershipCertificates.insert(std::pair< Address,CertificateOfMembership >(com.issuedTo(),com));
+					}
+				}
+			}
+		} catch ( ... ) {} // ignore invalid MCDB, we'll re-learn from peers
+	}
+
+	requestConfiguration();
 }
 
 Network::~Network()
 {
-	_lock.lock();
-	if ((_setupThread)&&(!_destroyed)) {
-		_lock.unlock();
-		Thread::join(_setupThread);
-	} else _lock.unlock();
-
-	{
-		Mutex::Lock _l(_lock);
-		if (_tap)
-			RR->tapFactory->close(_tap,_destroyed);
-	}
-
 	if (_destroyed) {
-		Utils::rm(std::string(RR->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf"));
+		char n[128];
-		Utils::rm(std::string(RR->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts"));
+		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
+		RR->node->dataStoreDelete(n);
+		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
+		RR->node->dataStoreDelete(n);
 	} else {
 		clean();
 		_dumpMembershipCerts();
 	}
 }
 
-SharedPtr<Network> Network::newInstance(const RuntimeEnvironment *renv,NodeConfig *nc,uint64_t id)
-{
-	SharedPtr<Network> nw(new Network());
-	nw->_id = id;
-	nw->_nc = nc;
-	nw->_mac.fromAddress(renv->identity.address(),id);
-	nw->RR = renv;
-	nw->_tap = (EthernetTap *)0;
-	nw->_enabled = true;
-	nw->_lastConfigUpdate = 0;
-	nw->_destroyed = false;
-	nw->_netconfFailure = NETCONF_FAILURE_NONE;
-
-	if (nw->controller() == renv->identity.address()) // TODO: fix Switch to allow packets to self
-		throw std::runtime_error("cannot join a network for which I am the netconf master");
-
-	try {
-		nw->_restoreState();
-		nw->requestConfiguration();
-	} catch ( ... ) {
-		nw->_lastConfigUpdate = 0; // call requestConfiguration() again
-	}
-
-	return nw;
-}
-
 // Function object used by rescanMulticastGroups()
 class AnnounceMulticastGroupsToPeersWithActiveDirectPaths
 {
@@ -154,105 +156,16 @@ private:
 	std::vector<Address> _supernodeAddresses;
 };
 
-bool Network::rescanMulticastGroups()
-{
-	bool updated = false;
-
-	{
-		Mutex::Lock _l(_lock);
-		EthernetTap *t = _tap;
-		if (t) {
-			// Grab current groups from the local tap
-			updated = t->updateMulticastGroups(_myMulticastGroups);
-
-			// Merge in learned groups from any hosts bridged in behind us
-			for(std::map<MulticastGroup,uint64_t>::const_iterator mg(_multicastGroupsBehindMe.begin());mg!=_multicastGroupsBehindMe.end();++mg)
-				_myMulticastGroups.insert(mg->first);
-
-			// Add or remove BROADCAST group based on broadcast enabled netconf flag
-			if ((_config)&&(_config->enableBroadcast())) {
-				if (!_myMulticastGroups.count(BROADCAST)) {
-					_myMulticastGroups.insert(BROADCAST);
-					updated = true;
-				}
-			} else {
-				if (_myMulticastGroups.count(BROADCAST)) {
-					_myMulticastGroups.erase(BROADCAST);
-					updated = true;
-				}
-			}
-		}
-	}
-
-	if (updated) {
-		AnnounceMulticastGroupsToPeersWithActiveDirectPaths afunc(RR,this);
-		RR->topology->eachPeer<AnnounceMulticastGroupsToPeersWithActiveDirectPaths &>(afunc);
-	}
-
-	return updated;
-}
-
 bool Network::applyConfiguration(const SharedPtr<NetworkConfig> &conf)
 {
 	Mutex::Lock _l(_lock);
-
 	if (_destroyed)
 		return false;
-
 	try {
 		if ((conf->networkId() == _id)&&(conf->issuedTo() == RR->identity.address())) {
-			std::vector<InetAddress> oldStaticIps;
-			if (_config)
-				oldStaticIps = _config->staticIps();
-
 			_config = conf;
-
+			_lastConfigUpdate = RR->node->now();
-			_lastConfigUpdate = Utils::now();
 			_netconfFailure = NETCONF_FAILURE_NONE;
-
-			EthernetTap *t = _tap;
-			if (t) {
-				char fname[1024];
-				_mkNetworkFriendlyName(fname,sizeof(fname));
-				t->setFriendlyName(fname);
-
-				// Remove previously configured static IPs that are gone
-				for(std::vector<InetAddress>::const_iterator oldip(oldStaticIps.begin());oldip!=oldStaticIps.end();++oldip) {
-					if (std::find(_config->staticIps().begin(),_config->staticIps().end(),*oldip) == _config->staticIps().end())
-						t->removeIP(*oldip);
-				}
-
-				// Add new static IPs that were not in previous config
-				for(std::vector<InetAddress>::const_iterator newip(_config->staticIps().begin());newip!=_config->staticIps().end();++newip) {
-					if (std::find(oldStaticIps.begin(),oldStaticIps.end(),*newip) == oldStaticIps.end())
-						t->addIP(*newip);
-				}
-
-#ifdef __APPLE__
-				// Make sure there's an IPv6 link-local address on Macs if IPv6 is enabled
-				// Other OSes don't need this -- Mac seems not to want to auto-assign
-				// This might go away once we integrate properly w/Mac network setup stuff.
-				if (_config->permitsEtherType(ZT_ETHERTYPE_IPV6)) {
-					bool haveV6LinkLocal = false;
-					std::set<InetAddress> ips(t->ips());
-					for(std::set<InetAddress>::const_iterator i(ips.begin());i!=ips.end();++i) {
-						if ((i->isV6())&&(i->isLinkLocal())) {
-							haveV6LinkLocal = true;
-							break;
-						}
-					}
-					if (!haveV6LinkLocal)
-						t->addIP(InetAddress::makeIpv6LinkLocal(_mac));
-				}
-#endif // __APPLE__
-
-				// ... IPs that were never controlled by static assignment are left
-				// alone, as these may be DHCP or user-configured.
-			} else {
-				if (!_setupThread)
-					_setupThread = Thread::start<Network>(this);
-			}
-
 			return true;
 		} else {
 			LOG("ignored invalid configuration for network %.16llx (configuration contains mismatched network ID or issued-to address)",(unsigned long long)_id);
@@ -262,7 +175,6 @@ bool Network::applyConfiguration(const SharedPtr<NetworkConfig> &conf)
 	} catch ( ... ) {
 		LOG("ignored invalid configuration for network %.16llx (unknown exception)",(unsigned long long)_id);
 	}
-
 	return false;
 }
 
@@ -277,12 +189,9 @@ int Network::setConfiguration(const Dictionary &conf,bool saveToDisk)
 		}
 		if (applyConfiguration(newConfig)) {
 			if (saveToDisk) {
-				std::string confPath(RR->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf");
+				char n[128];
-				if (!Utils::writeFile(confPath.c_str(),conf.toString())) {
+				Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
-					LOG("error: unable to write network configuration file at: %s",confPath.c_str());
+				RR->node->dataStorePut(n,conf.toString(),true);
-				} else {
-					Utils::lockDownFile(confPath.c_str(),false);
-				}
 			}
 			return 2; // OK and configuration has changed
 		}
@@ -298,9 +207,26 @@ void Network::requestConfiguration()
 		return;
 
 	if (controller() == RR->identity.address()) {
-		// netconf master cannot be a member of its own nets
+		if (RR->netconfMaster) {
-		LOG("unable to request network configuration for network %.16llx: I am the network master, cannot query self",(unsigned long long)_id);
+			SharedPtr<NetworkConfig> nconf(config2());
+			Dictionary newconf;
+			switch(RR->netconfMaster->doNetworkConfigRequest(InetAddress(),RR->identity,_id,Dictionary(),(nconf) ? nconf->revision() : (uint64_t)0,newconf)) {
+				case NetworkConfigMaster::NETCONF_QUERY_OK:
+					this->setConfiguration(newconf,true);
 					return;
+				case NetworkConfigMaster::NETCONF_QUERY_OBJECT_NOT_FOUND:
+					this->setNotFound();
+					return;
+				case NetworkConfigMaster::NETCONF_QUERY_ACCESS_DENIED:
+					this->setAccessDenied();
+					return;
+				default:
+					return;
+			}
+		} else {
+			this->setNotFound();
+			return;
+		}
 	}
 
 	TRACE("requesting netconf for network %.16llx from netconf master %s",(unsigned long long)_id,controller().toString().c_str());
@@ -310,7 +236,7 @@ void Network::requestConfiguration()
 	{
 		Mutex::Lock _l(_lock);
 		if (_config)
-			outp.append((uint64_t)_config->timestamp());
+			outp.append((uint64_t)_config->revision());
 		else outp.append((uint64_t)0);
 	}
 	RR->sw->send(outp,true);
@@ -494,178 +420,24 @@ void Network::destroy()
 	_tap = (EthernetTap *)0;
 }
 
-// Ethernet tap creation thread -- required on some platforms where tap
-// creation may be time consuming (e.g. Windows). Thread exits after tap
-// device setup.
-void Network::threadMain()
-	throw()
-{
-	char fname[1024],lcentry[128];
-	Utils::snprintf(lcentry,sizeof(lcentry),"_dev_for_%.16llx",(unsigned long long)_id);
-
-	EthernetTap *t = (EthernetTap *)0;
-	try {
-		std::string desiredDevice(_nc->getLocalConfig(lcentry));
-		_mkNetworkFriendlyName(fname,sizeof(fname));
-
-		t = RR->tapFactory->open(_mac,ZT_IF_MTU,ZT_DEFAULT_IF_METRIC,_id,(desiredDevice.length() > 0) ? desiredDevice.c_str() : (const char *)0,fname,_CBhandleTapData,this);
-
-		std::string dn(t->deviceName());
-		if ((dn.length())&&(dn != desiredDevice))
-			_nc->putLocalConfig(lcentry,dn);
-	} catch (std::exception &exc) {
-		delete t;
-		t = (EthernetTap *)0;
-		LOG("network %.16llx failed to initialize: %s",_id,exc.what());
-		_netconfFailure = NETCONF_FAILURE_INIT_FAILED;
-	} catch ( ... ) {
-		delete t;
-		t = (EthernetTap *)0;
-		LOG("network %.16llx failed to initialize: unknown error",_id);
-		_netconfFailure = NETCONF_FAILURE_INIT_FAILED;
-	}
-
-	{
-		Mutex::Lock _l(_lock);
-		if (_tap) // the tap creation thread can technically be re-launched, though this isn't done right now
-			RR->tapFactory->close(_tap,false);
-		_tap = t;
-		if (t) {
-			if (_config) {
-				for(std::vector<InetAddress>::const_iterator newip(_config->staticIps().begin());newip!=_config->staticIps().end();++newip)
-					t->addIP(*newip);
-			}
-			t->setEnabled(_enabled);
-		}
-	}
-
-	rescanMulticastGroups();
-}
-
-void Network::_CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data)
-{
-	SharedPtr<Network> network((Network *)arg,true);
-	if ((!network)||(!network->_enabled)||(network->status() != NETWORK_OK))
-		return;
-	try {
-		network->RR->sw->onLocalEthernet(network,from,to,etherType,data);
-	} catch (std::exception &exc) {
-		TRACE("unexpected exception handling local packet: %s",exc.what());
-	} catch ( ... ) {
-		TRACE("unexpected exception handling local packet");
-	}
-}
-
-void Network::_restoreState()
-{
-	Buffer<ZT_NETWORK_CERT_WRITE_BUF_SIZE> buf;
-
-	std::string idstr(idString());
-	std::string confPath(RR->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idstr + ".conf");
-	std::string mcdbPath(RR->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idstr + ".mcerts");
-
-	if (_id == ZT_TEST_NETWORK_ID) {
-		applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address()));
-
-		// "Touch" path to this ID to remember test network membership
-		FILE *tmp = fopen(confPath.c_str(),"w");
-		if (tmp) fclose(tmp);
-	} else {
-		// Read configuration file containing last config from netconf master
-		{
-			std::string confs;
-			if (Utils::readFile(confPath.c_str(),confs)) {
-				try {
-					if (confs.length())
-						setConfiguration(Dictionary(confs),false);
-				} catch ( ... ) {} // ignore invalid config on disk, we will re-request from netconf master
-			} else {
-				// "Touch" path to remember membership in lieu of real config from netconf master
-				FILE *tmp = fopen(confPath.c_str(),"w");
-				if (tmp) fclose(tmp);
-			}
-		}
-	}
-
-	{ // Read most recent membership cert dump if there is one
-		Mutex::Lock _l(_lock);
-		if ((_config)&&(!_config->isPublic())&&(Utils::fileExists(mcdbPath.c_str()))) {
-			CertificateOfMembership com;
-
-			_membershipCertificates.clear();
-
-			FILE *mcdb = fopen(mcdbPath.c_str(),"rb");
-			if (mcdb) {
-				try {
-					char magic[6];
-					if ((fread(magic,6,1,mcdb) == 1)&&(!memcmp("ZTMCD0",magic,6))) {
-						long rlen = 0;
-						do {
-							long rlen = (long)fread(const_cast<char *>(static_cast<const char *>(buf.data())) + buf.size(),1,ZT_NETWORK_CERT_WRITE_BUF_SIZE - buf.size(),mcdb);
-							if (rlen < 0) rlen = 0;
-							buf.setSize(buf.size() + (unsigned int)rlen);
-							unsigned int ptr = 0;
-							while ((ptr < (ZT_NETWORK_CERT_WRITE_BUF_SIZE / 2))&&(ptr < buf.size())) {
-								ptr += com.deserialize(buf,ptr);
-								if (com.issuedTo())
-									_membershipCertificates[com.issuedTo()] = com;
-							}
-							buf.behead(ptr);
-						} while (rlen > 0);
-						fclose(mcdb);
-					} else {
-						fclose(mcdb);
-						Utils::rm(mcdbPath);
-					}
-				} catch ( ... ) {
-					// Membership cert dump file invalid. We'll re-learn them off the net.
-					_membershipCertificates.clear();
-					fclose(mcdb);
-					Utils::rm(mcdbPath);
-				}
-			}
-		}
-	}
-}
-
 void Network::_dumpMembershipCerts()
 {
-	Buffer<ZT_NETWORK_CERT_WRITE_BUF_SIZE> buf;
+	char n[128];
-	std::string mcdbPath(RR->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts");
+	std::string buf("ZTMCD0");
+
+	Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
+
 	Mutex::Lock _l(_lock);
 
-	if (!_config)
+	if ((!_config)||(_config.isPublic())||(_membershipCertificates.size() == 0)) {
-		return;
+		RR->node->dataStoreDelete(n);
-
-	if ((!_id)||(_config->isPublic())) {
-		Utils::rm(mcdbPath);
 		return;
 	}
 
-	FILE *mcdb = fopen(mcdbPath.c_str(),"wb");
+	for(std::map<Address,CertificateOfMembership>::iterator c(_membershipCertificates.begin());c!=_membershipCertificates.end();++c)
-	if (!mcdb)
+		c->second.serialize2(buf);
-		return;
 
-	if (fwrite("ZTMCD0",6,1,mcdb) != 1) {
+	RR->node->dataStorePut(n,buf,true);
-		fclose(mcdb);
-		Utils::rm(mcdbPath);
-		return;
-	}
-
-	for(std::map<Address,CertificateOfMembership>::iterator c=(_membershipCertificates.begin());c!=_membershipCertificates.end();++c) {
-		buf.clear();
-		c->second.serialize(buf);
-		if (buf.size() > 0) {
-			if (fwrite(buf.data(),buf.size(),1,mcdb) != 1) {
-				fclose(mcdb);
-				Utils::rm(mcdbPath);
-				return;
-			}
-		}
-	}
-
-	fclose(mcdb);
-	Utils::lockDownFile(mcdbPath.c_str(),false);
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -31,7 +31,6 @@
 #include <stdint.h>
 
 #include <string>
-#include <set>
 #include <map>
 #include <vector>
 #include <algorithm>
@@ -57,48 +56,23 @@
 namespace ZeroTier {
 
 class RuntimeEnvironment;
-class NodeConfig;
 
 /**
  * A virtual LAN
- *
- * Networks can be open or closed. Each network has an ID whose most
- * significant 40 bits are the ZeroTier address of the node that should
- * be contacted for network configuration. The least significant 24
- * bits are arbitrary, allowing up to 2^24 networks per managing
- * node.
- *
- * Open networks do not track membership. Anyone is allowed to communicate
- * over them. For closed networks, each peer must distribute a certificate
- * regularly that proves that they are allowed to communicate.
  */
 class Network : NonCopyable
 {
 	friend class SharedPtr<Network>;
-	friend class NodeConfig;
-
-private:
-	// Only NodeConfig can create, only SharedPtr can delete
-
-	// Actual construction happens in newInstance()
-	Network() throw() {}
-	~Network();
-
-	/**
-	 * Create a new Network instance and restore any saved state
-	 *
-	 * If there is no saved state, a dummy .conf is created on disk to remember
-	 * this network across restarts.
-	 *
-	 * @param renv Runtime environment
-	 * @param nc Parent NodeConfig
-	 * @param id Network ID
-	 * @return Reference counted pointer to new network
-	 * @throws std::runtime_error Unable to create tap device or other fatal error
-	 */
-	static SharedPtr<Network> newInstance(const RuntimeEnvironment *renv,NodeConfig *nc,uint64_t id);
 
 public:
+	/**
+	 * @param renv Runtime environment
+	 * @param nwid Network ID
+	 */
+	Network(const RuntimeEnvironment *renv,uint64_t nwid);
+
+	~Network();
+
 	/**
 	 * Broadcast multicast group: ff:ff:ff:ff:ff:ff / 0
 	 */
@@ -118,13 +92,6 @@ public:
 		NETWORK_NO_MORE_DEVICES = 6             // OS cannot create any more tap devices (some OSes have a limit)
 	};
 
-	/**
-	 * @param s Status
-	 * @return String description
-	 */
-	static const char *statusString(const Status s)
-		throw();
-
 	/**
 	 * @return Network ID
 	 */
@@ -136,26 +103,9 @@ public:
 	inline Address controller() throw() { return Address(_id >> 24); }
 
 	/**
-	 * @return Network ID in hexadecimal form
+	 * @return Latest list of multicast groups for this network's tap
 	 */
-	inline std::string idString()
+	inline std::vector<MulticastGroup> multicastGroups() const
-	{
-		char buf[64];
-		Utils::snprintf(buf,sizeof(buf),"%.16llx",(unsigned long long)_id);
-		return std::string(buf);
-	}
-
-	/**
-	 * Rescan multicast groups for this network's tap and update peers on change
-	 *
-	 * @return True if internal multicast group set has changed since last update
-	 */
-	bool rescanMulticastGroups();
-
-	/**
-	 * @return Latest set of multicast groups for this network's tap
-	 */
-	inline std::set<MulticastGroup> multicastGroups() const
 	{
 		Mutex::Lock _l(_lock);
 		return _myMulticastGroups;
@@ -168,7 +118,7 @@ public:
 	bool subscribedToMulticastGroup(const MulticastGroup &mg) const
 	{
 		Mutex::Lock _l(_lock);
-		return (_myMulticastGroups.find(mg) != _myMulticastGroups.end());
+		return (std::find(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg) != _myMulticastGroups.end());
 	}
 
 	/**
@@ -311,72 +261,11 @@ public:
 		return _config;
 	}
 
-	/**
-	 * Inject a frame into tap (if it's created and network is enabled)
-	 *
-	 * @param from Origin MAC
-	 * @param to Destination MC
-	 * @param etherType Ethernet frame type
-	 * @param data Frame data
-	 * @param len Frame length
-	 */
-	inline void tapPut(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
-	{
-		Mutex::Lock _l(_lock);
-		if (!_enabled)
-			return;
-		EthernetTap *t = _tap;
-		if (t)
-			t->put(from,to,etherType,data,len);
-	}
-
-	/**
-	 * Call injectPacketFromHost() on tap if it exists
-	 *
-	 * @param from Source MAC
-	 * @param to Destination MAC
-	 * @param etherType Ethernet frame type
-	 * @param data Packet data
-	 * @param len Packet length
-	 */
-	inline bool tapInjectPacketFromHost(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
-	{
-		Mutex::Lock _l(_lock);
-		EthernetTap *t = _tap;
-		if (t)
-			return t->injectPacketFromHost(from,to,etherType,data,len);
-		return false;
-	}
-
-	/**
-	 * @return Tap device name or empty string if still initializing
-	 */
-	inline std::string tapDeviceName() const
-	{
-		Mutex::Lock _l(_lock);
-		EthernetTap *t = _tap;
-		if (t)
-			return t->deviceName();
-		else return std::string();
-	}
-
 	/**
 	 * @return Ethernet MAC address for this network's local interface
 	 */
 	inline const MAC &mac() const throw() { return _mac; }
 
-	/**
-	 * @return Set of IPs currently assigned to interface
-	 */
-	inline std::set<InetAddress> ips() const
-	{
-		Mutex::Lock _l(_lock);
-		EthernetTap *t = _tap;
-		if (t)
-			return t->ips();
-		return std::set<InetAddress>();
-	}
-
 	/**
 	 * Shortcut for config()->permitsBridging(), returns false if no config
 	 *
@@ -446,9 +335,6 @@ public:
 	void destroy();
 
 private:
-	static void _CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data);
-
-	void _restoreState();
 	void _dumpMembershipCerts();
 
 	inline void _mkNetworkFriendlyName(char *buf,unsigned int len)
@@ -459,14 +345,12 @@ private:
 		else Utils::snprintf(buf,len,"ZeroTier One [%.16llx]",(unsigned long long)_id);
 	}
 
-	uint64_t _id;
-	NodeConfig *_nc; // parent NodeConfig object
-	MAC _mac; // local MAC address
 	const RuntimeEnvironment *RR;
-	EthernetTap *volatile _tap; // tap device or NULL if not initialized yet
+	uint64_t _id;
+	MAC _mac; // local MAC address
 	volatile bool _enabled;
 
-	std::set< MulticastGroup > _myMulticastGroups; // multicast groups that we belong to including those behind us (updated periodically)
+	std::vector< MulticastGroup > _myMulticastGroups; // multicast groups that we belong to including those behind us (updated periodically)
 	std::map< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups bridged to us and when we last saw activity on each
 	std::map< MulticastGroup,BandwidthAccount > _multicastRateAccounts;
 

node/NetworkConfigMaster.hpp

@@ -70,9 +70,8 @@ public:
 	 * On internal server errors, the 'error' field in result can be filled in
 	 * to indicate the error.
 	 *
-	 * @param fromAddr Originating IP address
+	 * @param fromAddr Originating wire address or null address if packet is not direct (or from self)
-	 * @param packetId 64-bit packet ID
+	 * @param identity Originating peer ZeroTier identity
-	 * @param member Originating peer ZeroTier identity
 	 * @param nwid 64-bit network ID
 	 * @param metaData Meta-data bundled with request (empty if none)
 	 * @param haveRevision Network revision ID sent by requesting peer or 0 if none
@@ -81,7 +80,6 @@ public:
 	 */
 	virtual NetworkConfigMaster::ResultCode doNetworkConfigRequest(
 		const InetAddress &fromAddr,
-		uint64_t packetId,
 		const Identity &identity,
 		uint64_t nwid,
 		const Dictionary &metaData,

node/Node.cpp

@@ -45,6 +45,7 @@
 namespace ZeroTier {
 
 Node::Node(
+	uint64_t now,
 	ZT1_DataStoreGetFunction *dataStoreGetFunction,
 	ZT1_DataStorePutFunction *dataStorePutFunction,
 	ZT1_WirePacketSendFunction *wirePacketSendFunction,
@@ -60,7 +61,7 @@ Node::Node(
 	_statusCallback(statusCallback),
 	_networks(),
 	_networks_m(),
-	_now(0),
+	_now(now),
 	_timeOfLastPacketReceived(0),
 	_timeOfLastPrivilegedPacket(0),
 	_spamCounter(0)
@@ -129,14 +130,18 @@ ZT1_ResultCode Node::join(uint64_t nwid)
 	Mutex::Lock _l(_networks_m);
 	SharedPtr<Network> &nw = _networks[nwid];
 	if (!nw)
-		nw = new Network();
+		nw = SharedPtr<Network>(new Network(RR,nwid));
 	return ZT1_RESULT_OK;
 }
 
 ZT1_ResultCode Node::leave(uint64_t nwid)
 {
 	Mutex::Lock _l(_networks_m);
-	_networks.erase(nwid);
+	std::map< uint64_t,SharedPtr<Network> >::iterator nw(_networks.find(nwid));
+	if (nw != _networks.end()) {
+		nw->second->destroy();
+		_networks.erase(nw);
+	}
 }
 
 ZT1_ResultCode Node::multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
@@ -180,6 +185,7 @@ extern "C" {
 
 enum ZT1_ResultCode ZT1_Node_new(
 	ZT1_Node **node,
+	uint64_t now,
 	ZT1_DataStoreGetFunction *dataStoreGetFunction,
 	ZT1_DataStorePutFunction *dataStorePutFunction,
 	ZT1_WirePacketSendFunction *wirePacketSendFunction,
@@ -189,7 +195,7 @@ enum ZT1_ResultCode ZT1_Node_new(
 {
 	*node = (ZT1_Node *)0;
 	try {
-		*node = reinterpret_cast<ZT1_Node *>(new ZeroTier::Node(dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigCallback,statusCallback));
+		*node = reinterpret_cast<ZT1_Node *>(new ZeroTier::Node(now,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigCallback,statusCallback));
 		return ZT1_RESULT_OK;
 	} catch (std::bad_alloc &exc) {
 		return ZT1_RESULT_ERROR_OUT_OF_MEMORY;

node/Node.hpp

@@ -56,6 +56,7 @@ class Node
 {
 public:
 	Node(
+		uint64_t now,
 		ZT1_DataStoreGetFunction *dataStoreGetFunction,
 		ZT1_DataStorePutFunction *dataStorePutFunction,
 		ZT1_WirePacketSendFunction *wirePacketSendFunction,
@@ -185,6 +186,19 @@ public:
 		return ((nw == _networks.end()) ? SharedPtr<Network>() : nw->second);
 	}
 
+	inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure)
+	{
+	}
+	inline bool dataStorePut(const char *name,const std::string &data,bool secure) { return dataStorePut(name,(const void *)data.data(),(unsigned int)data.length(),secure); }
+
+	inline std::string dataStoreGet(const char *name)
+	{
+	}
+
+	inline void dataStoreDelete(const char *name)
+	{
+	}
+
 private:
 	RuntimeEnvironment *RR;
 

node/Packet.hpp

@@ -596,10 +596,10 @@ public:
 		 *   <[8] 64-bit network ID>
 		 *   <[2] 16-bit length of request meta-data dictionary>
 		 *   <[...] string-serialized request meta-data>
-		 *  [<[8] 64-bit timestamp of netconf we currently have>]
+		 *  [<[8] 64-bit revision of netconf we currently have>]
 		 *
 		 * This message requests network configuration from a node capable of
-		 * providing it. If the optional timestamp is included, a response is
+		 * providing it. If the optional revision is included, a response is
 		 * only generated if there is a newer network configuration available.
 		 *
 		 * OK response payload:
@@ -614,7 +614,7 @@ public:
 		 * a given network.
 		 *
 		 * When a new network configuration is received, another config request
-		 * should be sent with the new netconf's timestamp. This confirms receipt
+		 * should be sent with the new netconf's revision. This confirms receipt
 		 * and also causes any subsequent changes to rapidly propagate as this
 		 * cycle will repeat until there are no changes. This is optional but
 		 * recommended behavior.