Clean up handling of COMs, network access control, and fix a backward compatiblity issue.

node/IncomingPacket.cpp

@@ -552,7 +552,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		bool approved = false;
 		if (network) {
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
-				if (!network->isAllowed(peer)) {
+				if (!network->gate(peer,verb(),packetId())) {
 					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
 				} else {
 					const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
@@ -591,7 +591,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 						network->addCredential(com);
 				}
 
-				if (!network->isAllowed(peer)) {
+				if (!network->gate(peer,verb(),packetId())) {
 					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id());
 					peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false);
 					return true;
@@ -619,7 +619,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 								peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 								return true;
 							}
-						} else if (to != network->mac()) {
+						} else if ( (to != network->mac()) && (!to.isMulticast()) ) {
 							if (!network->config().permitsBridging(RR->identity.address())) {
 								TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id());
 								peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
@@ -934,7 +934,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 
 			// Check membership after we've read any included COM, since
 			// that cert might be what we needed.
-			if (!network->isAllowed(peer)) {
+			if (!network->gate(peer,verb(),packetId())) {
 				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
 				peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
 				return true;

node/Membership.cpp

@@ -24,13 +24,13 @@
 #include "Packet.hpp"
 #include "Node.hpp"
 
-#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 4)
+#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 3)
 
 namespace ZeroTier {
 
 void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap)
 {
-	if ((now - _lastPushAttempt) < 1000ULL)
+	if ((now - _lastPushAttempt) < 2000ULL)
 		return;
 	_lastPushAttempt = now;
 
@@ -99,9 +99,11 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe
 	const int vr = com.verify(RR);
 
 	if (vr == 0) {
-		TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId());
+		if (com.timestamp().first >= _com.timestamp().first) {
-		if (com.timestamp().first > _com.timestamp().first) {
+			TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId());
 			_com = com;
+		} else {
+			TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED but not used (OK but older than current)",com.issuedTo().toString().c_str(),com.networkId());
 		}
 	} else {
 		TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr);

node/Membership.hpp

@@ -154,6 +154,21 @@ public:
 		return nconf.com.agreesWith(_com);
 	}
 
+	/**
+	 * @return True if this member has been on this network recently (or network is public)
+	 */
+	inline bool recentlyAllowedOnNetwork(const NetworkConfig &nconf) const
+	{
+		if (nconf.isPublic())
+			return true;
+		if (_com) {
+			const uint64_t a = _com.timestamp().first;
+			const std::pair<uint64_t,uint64_t> b(nconf.com.timestamp());
+			return ((a <= b.first) ? ((b.first - a) <= ZT_PEER_ACTIVITY_TIMEOUT) : true);
+		}
+		return false;
+	}
+
 	/**
 	 * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time
 	 *

node/Network.cpp

@@ -877,7 +877,7 @@ void Network::multicastSubscribe(const MulticastGroup &mg)
 			return;
 		_myMulticastGroups.push_back(mg);
 		std::sort(_myMulticastGroups.begin(),_myMulticastGroups.end());
-		_announceMulticastGroups(&mg);
+		_pushStateToMembers(&mg);
 	}
 }
 
@@ -1062,6 +1062,36 @@ void Network::requestConfiguration()
 	_inboundConfigChunks.clear();
 }
 
+bool Network::gate(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId)
+{
+	Mutex::Lock _l(_lock);
+	try {
+		if (_config) {
+			Membership &m = _membership(peer->address());
+			const bool allow = m.isAllowedOnNetwork(_config);
+			if (allow) {
+				const uint64_t now = RR->node->now();
+				m.sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0);
+				if (m.shouldLikeMulticasts(now)) {
+					_announceMulticastGroupsTo(peer->address(),_allMulticastGroups());
+					m.likingMulticasts(now);
+				}
+			} else if (m.recentlyAllowedOnNetwork(_config)) {
+				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
+				outp.append((uint8_t)verb);
+				outp.append(packetId);
+				outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
+				outp.append(_id);
+				RR->sw->send(outp,true);
+			}
+			return allow;
+		}
+	} catch ( ... ) {
+		TRACE("gate() check failed for peer %s: unexpected exception",peer->address().toString().c_str());
+	}
+	return false;
+}
+
 void Network::clean()
 {
 	const uint64_t now = RR->node->now();
@@ -1135,7 +1165,7 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now)
 	const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size();
 	_multicastGroupsBehindMe.set(mg,now);
 	if (tmp != _multicastGroupsBehindMe.size())
-		_announceMulticastGroups(&mg);
+		_pushStateToMembers(&mg);
 }
 
 void Network::destroy()
@@ -1200,33 +1230,18 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	}
 }
 
-bool Network::_isAllowed(const SharedPtr<Peer> &peer) const
+void Network::_pushStateToMembers(const MulticastGroup *const newMulticastGroup)
-{
-	// Assumes _lock is locked
-	try {
-		if (_config) {
-			const Membership *const m = _memberships.get(peer->address());
-			if (m)
-				return m->isAllowedOnNetwork(_config);
-		}
-	} catch ( ... ) {
-		TRACE("isAllowed() check failed for peer %s: unexpected exception",peer->address().toString().c_str());
-	}
-	return false;
-}
-
-void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis)
 {
 	// Assumes _lock is locked
 	const uint64_t now = RR->node->now();
 
 	std::vector<MulticastGroup> groups;
-	if (onlyThis)
+	if (newMulticastGroup)
-		groups.push_back(*onlyThis);
+		groups.push_back(*newMulticastGroup);
 	else groups = _allMulticastGroups();
 
-	if ((onlyThis)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) {
+	if ((newMulticastGroup)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) {
-		if (!onlyThis)
+		if (!newMulticastGroup)
 			_lastAnnouncedMulticastGroupsUpstream = now;
 
 		// Announce multicast groups to upstream peers (roots, etc.) and also send
@@ -1255,7 +1270,7 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis)
 	// piecemeal on-demand fashion.
 	const std::vector<Address> anchors(_config.anchors());
 	for(std::vector<Address>::const_iterator a(anchors.begin());a!=anchors.end();++a)
-		_memberships[*a];
+		_membership(*a);
 
 	// Send MULTICAST_LIKE(s) to all members of this network
 	{
@@ -1263,11 +1278,13 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis)
 		Membership *m = (Membership *)0;
 		Hashtable<Address,Membership>::Iterator i(_memberships);
 		while (i.next(a,m)) {
-			if ((onlyThis)||(m->shouldLikeMulticasts(now))) {
+			if ( (m->recentlyAllowedOnNetwork(_config)) || (std::find(anchors.begin(),anchors.end(),*a) != anchors.end()) ) {
-				if (!onlyThis)
-					m->likingMulticasts(now);
 				m->sendCredentialsIfNeeded(RR,RR->node->now(),*a,_config,(const Capability *)0);
-				_announceMulticastGroupsTo(*a,groups);
+				if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) {
+					if (!newMulticastGroup)
+						m->likingMulticasts(now);
+					_announceMulticastGroupsTo(*a,groups);
+				}
 			}
 		}
 	}
@@ -1314,15 +1331,7 @@ std::vector<MulticastGroup> Network::_allMulticastGroups() const
 Membership &Network::_membership(const Address &a)
 {
 	// assumes _lock is locked
-	const unsigned long ms = _memberships.size();
+	return _memberships[a];
-	Membership &m = _memberships[a];
-	if (ms != _memberships.size()) {
-		const uint64_t now = RR->node->now();
-		m.sendCredentialsIfNeeded(RR,now,a,_config,(const Capability *)0);
-		_announceMulticastGroupsTo(a,_allMulticastGroups());
-		m.likingMulticasts(now);
-	}
-	return m;
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -48,7 +48,6 @@ namespace ZeroTier {
 
 class RuntimeEnvironment;
 class Peer;
-class _MulticastAnnounceAll;
 
 /**
  * A virtual LAN
@@ -56,7 +55,6 @@ class _MulticastAnnounceAll;
 class Network : NonCopyable
 {
 	friend class SharedPtr<Network>;
-	friend class _MulticastAnnounceAll; // internal function object
 
 public:
 	/**
@@ -250,14 +248,14 @@ public:
 	void requestConfiguration();
 
 	/**
+	 * Membership check gate for incoming packets related to this network
+	 *
 	 * @param peer Peer to check
+	 * @param verb Packet verb
+	 * @param packetId Packet ID
 	 * @return True if peer is allowed to communicate on this network
 	 */
-	inline bool isAllowed(const SharedPtr<Peer> &peer) const
+	bool gate(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId);
-	{
-		Mutex::Lock _l(_lock);
-		return _isAllowed(peer);
-	}
 
 	/**
 	 * Perform cleanup and possibly save state
@@ -265,12 +263,12 @@ public:
 	void clean();
 
 	/**
-	 * Announce multicast groups to all members, anchors, etc.
+	 * Push state to members such as multicast group memberships and latest COM (if needed)
 	 */
-	inline void announceMulticastGroups()
+	inline void pushStateToMembers()
 	{
 		Mutex::Lock _l(_lock);
-		_announceMulticastGroups((const MulticastGroup *)0);
+		_pushStateToMembers((const MulticastGroup *)0);
 	}
 
 	/**
@@ -408,11 +406,11 @@ public:
 private:
 	ZT_VirtualNetworkStatus _status() const;
 	void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked
-	bool _isAllowed(const SharedPtr<Peer> &peer) const;
+	bool _gate(const SharedPtr<Peer> &peer);
-	void _announceMulticastGroups(const MulticastGroup *const onlyThis);
+	void _pushStateToMembers(const MulticastGroup *const newMulticastGroup);
 	void _announceMulticastGroupsTo(const Address &peer,const std::vector<MulticastGroup> &allMulticastGroups);
 	std::vector<MulticastGroup> _allMulticastGroups() const;
-	Membership &_membership(const Address &a); // also lazily sends COM and MULTICAST_LIKE(s) if this is a new member
+	Membership &_membership(const Address &a);
 
 	const RuntimeEnvironment *RR;
 	void *_uPtr;

node/Node.cpp

@@ -263,7 +263,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 				for(std::vector< std::pair< uint64_t,SharedPtr<Network> > >::const_iterator n(_networks.begin());n!=_networks.end();++n) {
 					if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig()))
 						needConfig.push_back(n->second);
-					n->second->announceMulticastGroups();
+					n->second->pushStateToMembers();
 				}
 			}
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)

node/Packet.cpp

@@ -62,6 +62,7 @@ const char *Packet::errorString(ErrorCode e)
 		case ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND";
 		case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION";
 		case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION";
+		case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE";
 		case ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED";
 		case ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST";
 	}

node/Packet.hpp

@@ -1067,6 +1067,9 @@ public:
 		/* Verb or use case not supported/enabled by this node */
 		ERROR_UNSUPPORTED_OPERATION = 0x05,
 
+		/* Network membership certificate update needed */
+		ERROR_NEED_MEMBERSHIP_CERTIFICATE = 0x06,
+
 		/* Tried to join network, but you're not a member */
 		ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */