Multicaster needs to be global, not per-network, and a bunch of other stuff.

node/IncomingPacket.cpp

@@ -109,6 +109,10 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 				return _doNETWORK_CONFIG_REQUEST(RR,peer);
 			case Packet::VERB_NETWORK_CONFIG_REFRESH:
 				return _doNETWORK_CONFIG_REFRESH(RR,peer);
+			case Packet::VERB_MULTICAST_GATHER:
+				return _doMULTICAST_GATHER(RR,peer);
+			case Packet::VERB_MULTICAST_FRAME:
+				return _doMULTICAST_FRAME(RR,peer);
 		}
 	} else {
 		_step = DECODE_WAITING_FOR_SENDER_LOOKUP; // should already be this...
@@ -124,9 +128,10 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID);
 		Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE];
 
-		TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+		//TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
 
 		switch(errorCode) {
+
 			case Packet::ERROR_OBJ_NOT_FOUND:
 				if (inReVerb == Packet::VERB_WHOIS) {
 					if (RR->topology->isSupernode(source()))
@@ -137,22 +142,29 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 						network->setNotFound();
 				}
 				break;
+
 			case Packet::ERROR_IDENTITY_COLLISION:
 				// TODO: if it comes from a supernode, regenerate a new identity
 				// if (RR->topology->isSupernode(source())) {}
 				break;
+
 			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
 				SharedPtr<Network> network(RR->nc->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 				if (network)
 					network->pushMembershipCertificate(source(),true,Utils::now());
 			}	break;
+
 			case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
 				SharedPtr<Network> network(RR->nc->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 				if ((network)&&(network->controller() == source()))
 					network->setAccessDenied();
 			}	break;
-			default:
+
-				break;
+			// TODO
+			//case Packet::ERROR_UNWANTED_MULTICAST: {
+			//}	break;
+
+			default: break;
 		}
 
 		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,Utils::now());
@@ -174,10 +186,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		uint64_t timestamp = at<uint64_t>(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP);
 		Identity id(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
 
-		if (protoVersion != ZT_PROTO_VERSION) {
+		if (protoVersion < ZT_PROTO_VERSION_MIN) {
-			TRACE("dropped HELLO from %s(%s): protocol version mismatch (%u, expected %u)",source().toString().c_str(),_remoteAddress.toString().c_str(),protoVersion,(unsigned int)ZT_PROTO_VERSION);
+			TRACE("dropped HELLO from %s(%s): protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
 		}
+
 		if (!id.locallyValidate()) {
 			TRACE("dropped HELLO from %s(%s): identity invalid",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
@@ -246,7 +259,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		}
 
 		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP,Utils::now());
-		peer->setRemoteVersion(vMajor,vMinor,vRevision);
+		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
 
 		// If a supernode has a version higher than ours, this causes a software
 		// update check to run now.
@@ -280,14 +293,23 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
 
 		switch(inReVerb) {
+
 			case Packet::VERB_HELLO: {
 				unsigned int latency = std::min((unsigned int)(Utils::now() - at<uint64_t>(ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP)),(unsigned int)0xffff);
+				unsigned int vProto = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION];
 				unsigned int vMajor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION];
 				unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION];
 				unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION);
+
+				if (vProto < ZT_PROTO_VERSION_MIN) {
+					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
+					return true;
+				}
+
 				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency);
+
 				peer->addDirectLatencyMeasurment(latency);
-				peer->setRemoteVersion(vMajor,vMinor,vRevision);
+				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
 				// If a supernode has a version higher than ours, this causes a software
 				// update check to run now. This might bum-rush download.zerotier.com, but
@@ -296,6 +318,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				if ((RR->updater)&&(RR->topology->isSupernode(peer->address())))
 					RR->updater->sawRemoteVersion(vMajor,vMinor,vRevision);
 			}	break;
+
 			case Packet::VERB_WHOIS: {
 				// Right now only supernodes are allowed to send OK(WHOIS) to prevent
 				// poisoning attacks. Further decentralization will require some other
@@ -306,6 +329,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 						RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr<Peer>(new Peer(RR->identity,id))));
 				}
 			} break;
+
 			case Packet::VERB_NETWORK_CONFIG_REQUEST: {
 				SharedPtr<Network> nw(RR->nc->network(at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID)));
 				if ((nw)&&(nw->controller() == source())) {
@@ -319,8 +343,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					}
 				}
 			}	break;
-			default:
+
-				break;
+			case Packet::VERB_MULTICAST_GATHER: {
+			}	break;
+
+			case Packet::VERB_MULTICAST_FRAME: {
+			}	break;
+
+			default: break;
 		}
 
 		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,Utils::now());
@@ -430,11 +460,6 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 				network->tapPut(MAC(peer->address(),network->id()),network->mac(),etherType,data() + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD);
 
-				/* Source moves "closer" to us in multicast propagation priority when
-				 * we receive unicast frames from it. This is called "implicit social
-				 * ordering" in other docs. */
-				RR->mc->bringCloser(network->id(),peer->address());
-
 				peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,Utils::now());
 				return true;
 			}
@@ -455,10 +480,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 		SharedPtr<Network> network(RR->nc->network(at<uint64_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID)));
 		if (network) {
 			if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
-				if ((*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS] != 0) {
+				unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS];
-					TRACE("dropped EXT_FRAME due to unknown flags");
-					return true;
-				}
 
 				if (!network->isAllowed(peer->address())) {
 					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
@@ -466,14 +488,30 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 					return true;
 				}
 
-				unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE);
+				unsigned int comLen = 0;
+				if ((flags & 0x01) != 0) {
+					CertificateOfMembership com;
+					comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
+					if (com.hasRequiredFields())
+						network->addMembershipCertificate(com);
+				}
+
+				// Everything after flags must be adjusted based on the length
+				// of the certificate, if there was one...
+
+				unsigned int etherType = at<uint16_t>(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE);
 				if (!network->config()->permitsEtherType(etherType)) {
 					TRACE("dropped EXT_FRAME from %s(%s): ethertype %.4x not allowed on network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned int)etherType,(unsigned long long)network->id());
 					return true;
 				}
 
-				const MAC to(field(ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO);
+				const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO);
-				const MAC from(field(ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM);
+				const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM);
+
+				if (to.isMulticast()) {
+					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: destination is multicast, must use MULTICAST_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
+					return true;
+				}
 
 				if ((!from)||(from.isMulticast())||(from == network->mac())||(!to)) {
 					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source or destination MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
@@ -488,7 +526,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 						TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
 						return true;
 					}
-				} // else: it is valid to send a non-bridged packet this way instead of as FRAME, but this isn't done by current code
+				}
 
 				// If it's not to us, we must be allowed to bridge into this network
 				if (to != network->mac()) {
@@ -498,12 +536,9 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 					}
 				}
 
-				network->tapPut(from,to,etherType,data() + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,size() - ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD);
+				unsigned int payloadLen = size() - (comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD);
-
+				if (payloadLen)
-				/* Source moves "closer" to us in multicast propagation priority when
+					network->tapPut(from,to,etherType,field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
-				 * we receive unicast frames from it. This is called "implicit social
-				 * ordering" in other docs. */
-				RR->mc->bringCloser(network->id(),peer->address());
 
 				peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,Utils::now());
 			}
@@ -518,8 +553,17 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 	return true;
 }
 
-bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
+bool IncomingPacket::_doP5_MULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
+	// This handles the old deprecated "P5" multicast frame, and will
+	// go away once there are no longer nodes using this on the network.
+	// We handle these old nodes by accepting these as simple multicasts
+	// and if we are a supernode performing individual relaying of them
+	// to all older nodes that expect them. This won't be too expensive
+	// though since there aren't likely to be many older nodes left after
+	// we do a software update.
+
+#if 0 // old code preserved below
 	try {
 		Address origin(Address(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ORIGIN,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_ORIGIN),ZT_ADDRESS_LENGTH));
 		SharedPtr<Peer> originPeer(RR->topology->getPeer(origin));
@@ -763,6 +807,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
+#endif
 
 	return true;
 }
@@ -774,15 +819,8 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 		uint64_t now = Utils::now();
 
 		// Iterate through 18-byte network,MAC,ADI tuples
-		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18) {
+		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18)
-			uint64_t nwid = at<uint64_t>(ptr);
+			RR->mc->subscribe(now,at<uint64_t>(ptr),MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),Address(),src);
-			SharedPtr<Network> network(RR->nc->network(nwid));
-			if ((RR->topology->amSupernode())||((network)&&(network->isAllowed(peer->address())))) {
-				RR->mc->likesGroup(nwid,src,MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),now);
-				if (network)
-					network->pushMembershipCertificate(peer->address(),false,now);
-			}
-		}
 
 		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,now);
 	} catch (std::exception &ex) {
@@ -833,6 +871,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 {
 	try {
 		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID);
+
 #ifndef __WINDOWS__
 		if (RR->netconfService) {
 			char tmp[128];
@@ -853,6 +892,10 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			RR->netconfService->send(request);
 		} else {
 #endif // !__WINDOWS__
+
+			// Send unsupported operation if there is no netconf service
+			// configured on this node (or if this is a Windows machine,
+			// which doesn't support that at all).
 			Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR);
 			outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
 			outp.append(packetId());
@@ -860,9 +903,11 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			outp.append(nwid);
 			outp.armor(peer->key(),true);
 			_fromSock->send(_remoteAddress,outp.data(),outp.size());
+
 #ifndef __WINDOWS__
 		}
 #endif // !__WINDOWS__
+
 		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,Utils::now());
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
@@ -895,14 +940,111 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 
 bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
+	try {
+		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID);
+		MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI));
+		unsigned int gatherLimit = at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT);
+
+		if (gatherLimit) {
+			Packet outp(source(),RR->identity.address(),Packet::VERB_OK);
+			outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER);
+			outp.append(packetId());
+			outp.append(nwid);
+			mg.mac().appendTo(outp);
+			outp.append((uint32_t)mg.adi());
+			if (RR->mc->gather(RR,nwid,mg,outp,gatherLimit)) {
+				outp.armor(peer->key(),true);
+				_fromSock->send(_remoteAddress,outp.data(),outp.size());
+			}
+		}
+
+		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,Utils::now());
+	} catch (std::exception &exc) {
+		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
 }
 
 bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
-}
+	try {
+		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
+		unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
+		unsigned int gatherLimit = at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT);
 
-void IncomingPacket::_handleMulticastGatherResponse(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,unsigned int startIdx)
+		SharedPtr<Network> network(RR->nc->network(nwid)); // will be NULL if not a member
-{
+		if (network) {
+			if (!network->isAllowed(peer->address())) {
+				TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
+				_sendErrorNeedCertificate(RR,peer,network->id());
+				return true;
+			}
+
+			unsigned int comLen = 0;
+			if ((flags & 0x01) != 0) {
+				CertificateOfMembership com;
+				comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
+				if (com.hasRequiredFields())
+					network->addMembershipCertificate(com);
+			}
+
+			// Everything after gatherLimit is relative to the size of the
+			// attached certificate, if any.
+
+			MulticastGroup to(MAC(field(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at<uint32_t>(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
+			MAC from(field(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6),6);
+			unsigned int etherType = at<uint16_t>(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE);
+			unsigned int payloadLen = size() - (comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME);
+
+			if (!payloadLen)
+				return true;
+
+			if (!to.mac().isMulticast()) {
+				TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
+				return true;
+			}
+
+			if ((!from)||(from.isMulticast())||(from == network->mac())) {
+				TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
+				return true;
+			}
+
+			// If it's not from the sending peer, they must be allowed to bridge into this network
+			if (from != MAC(peer->address(),network->id())) {
+				if (network->permitsBridging(peer->address())) {
+					network->learnBridgeRoute(from,peer->address());
+				} else {
+					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
+					return true;
+				}
+			}
+
+			network->tapPut(from,to,etherType,field(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,payloadLen),payloadLen);
+		}
+
+		if (gatherLimit) {
+			Packet outp(source(),RR->identity.address(),Packet::VERB_OK);
+			outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
+			outp.append(packetId());
+			outp.append(nwid);
+			to.mac().appendTo(outp);
+			outp.append((uint32_t)to.adi());
+			outp.append((unsigned char)0x01); // flag 0x01 = contains gather results
+			if (RR->mc->gather(RR,nwid,to,outp,gatherLimit)) {
+				outp.armor(peer->key(),true);
+				_fromSock->send(_remoteAddress,outp.data(),outp.size());
+			}
+		}
+
+		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,Utils::now());
+	} catch (std::exception &exc) {
+		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
 }
 
 void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid)

node/IncomingPacket.hpp

@@ -127,9 +127,6 @@ private:
 	bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 
-	// Both OK(MULTICAST_GATHER) and OK(MULTICAST_FRAME) can carry this payload
-	void _handleMulticastGatherResponse(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,unsigned int startIdx);
-
 	// Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to join
 	void _sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid);
 

node/Multicaster.cpp

@@ -34,6 +34,7 @@
 #include "Switch.hpp"
 #include "Packet.hpp"
 #include "Peer.hpp"
+#include "CMWC4096.hpp"
 #include "CertificateOfMembership.hpp"
 #include "RuntimeEnvironment.hpp"
 
@@ -47,10 +48,75 @@ Multicaster::~Multicaster()
 {
 }
 
-void Multicaster::send(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership *com,unsigned int limit,uint64_t now,const MulticastGroup &mg,const MAC &src,unsigned int etherType,const void *data,unsigned int len)
+unsigned int Multicaster::gather(const RuntimeEnvironment *RR,uint64_t nwid,MulticastGroup &mg,Packet &appendTo,unsigned int limit) const
+{
+	unsigned char *p;
+	unsigned int n = 0,i,rptr;
+	uint64_t a,done[(ZT_PROTO_MAX_PACKET_LENGTH / 5) + 1];
+
+	Mutex::Lock _l(_groups_m);
+
+	std::map< std::pair<uint64_t,MulticastGroup>,MulticastGroupStatus >::const_iterator gs(_groups.find(std::pair<uint64_t,MulticastGroup>(nwid,mg)));
+	if ((gs == _groups.end())||(gs->second.members.empty())) {
+		appendTo.append((uint32_t)0);
+		appendTo.append((uint16_t)0);
+		return 0;
+	}
+
+	if (limit > gs->second.members.size())
+		limit = (unsigned int)gs->second.members.size();
+	if (limit > 0xffff) // sanity check -- this won't fit in a packet anyway
+		limit = 0xffff;
+
+	appendTo.append((uint32_t)gs->second.members.size());
+	unsigned int nAt = appendTo.size();
+	appendTo.append((uint16_t)0); // set to n later
+
+	while ((n < limit)&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_PROTO_MAX_PACKET_LENGTH)) {
+		// Pick a member at random -- if we've already picked it,
+		// keep circling the buffer until we find one we haven't.
+		// This won't loop forever since limit <= members.size().
+		rptr = (unsigned int)RR->prng->next32();
+restart_member_scan:
+		a = gs->second.members[rptr % (unsigned int)gs->second.members.size()].address.toInt();
+		for(i=0;i<n;++i) {
+			if (done[i] == a) {
+				++rptr;
+				goto restart_member_scan;
+			}
+		}
+
+		// Log that we've picked this one
+		done[n++] = a;
+
+		// Append to packet
+		p = (unsigned char *)appendTo.appendField(ZT_ADDRESS_LENGTH);
+		*(p++) = (unsigned char)((a >> 32) & 0xff);
+		*(p++) = (unsigned char)((a >> 24) & 0xff);
+		*(p++) = (unsigned char)((a >> 16) & 0xff);
+		*(p++) = (unsigned char)((a >> 8) & 0xff);
+		*p = (unsigned char)(a & 0xff);
+	}
+
+	appendTo.setAt(nAt,(uint16_t)n);
+
+	return n;
+}
+
+void Multicaster::send(
+	const RuntimeEnvironment *RR,
+	const CertificateOfMembership *com,
+	unsigned int limit,
+	uint64_t now,
+	uint64_t nwid,
+	const MulticastGroup &mg,
+	const MAC &src,
+	unsigned int etherType,
+	const void *data,
+	unsigned int len)
 {
 	Mutex::Lock _l(_groups_m);
-	MulticastGroupStatus &gs = _groups[mg];
+	MulticastGroupStatus &gs = _groups[std::pair<uint64_t,MulticastGroup>(nwid,mg)];
 
 	if (gs.members.size() >= limit) {
 		// If we already have enough members, just send and we're done -- no need for TX queue
@@ -95,13 +161,13 @@ void Multicaster::send(const RuntimeEnvironment *RR,uint64_t nwid,const Certific
 	}
 }
 
-void Multicaster::clean(const RuntimeEnvironment *RR,uint64_t now,unsigned int limit)
+void Multicaster::clean(const RuntimeEnvironment *RR,uint64_t now)
 {
 	Mutex::Lock _l(_groups_m);
-	for(std::map< MulticastGroup,MulticastGroupStatus >::iterator mm(_groups.begin());mm!=_groups.end();) {
+	for(std::map< std::pair<uint64_t,MulticastGroup>,MulticastGroupStatus >::iterator mm(_groups.begin());mm!=_groups.end();) {
 		// Remove expired outgoing multicasts from multicast TX queue
 		for(std::list<OutboundMulticast>::iterator tx(mm->second.txQueue.begin());tx!=mm->second.txQueue.end();) {
-			if ((tx->expired(now))||(tx->sentToCount() >= limit))
+			if (tx->expired(now))
 				mm->second.txQueue.erase(tx++);
 			else ++tx;
 		}
@@ -152,7 +218,7 @@ void Multicaster::clean(const RuntimeEnvironment *RR,uint64_t now,unsigned int l
 	}
 }
 
-void Multicaster::_add(const RuntimeEnvironment *RR,uint64_t now,MulticastGroupStatus &gs,const Address &learnedFrom,const Address &member)
+void Multicaster::_add(uint64_t now,MulticastGroupStatus &gs,const Address &learnedFrom,const Address &member)
 {
 	// assumes _groups_m is locked
 

node/Multicaster.hpp

@@ -47,6 +47,7 @@ namespace ZeroTier {
 
 class RuntimeEnvironment;
 class CertificateOfMembership;
+class Packet;
 
 /**
  * Database of known multicast peers within a network
@@ -82,20 +83,40 @@ public:
 	~Multicaster();
 
 	/**
-	 * Add or update a member in a multicast group and send any pending multicasts
+	 * Add or update a member in a multicast group
 	 *
-	 * @param RR Runtime environment
 	 * @param now Current time
+	 * @param nwid Network ID
 	 * @param mg Multicast group
 	 * @param learnedFrom Address from which we learned this member or NULL/0 Address if direct
 	 * @param member New member address
 	 */
-	inline void add(const RuntimeEnvironment *RR,uint64_t now,const MulticastGroup &mg,const Address &learnedFrom,const Address &member)
+	inline void subscribe(uint64_t now,uint64_t nwid,const MulticastGroup &mg,const Address &learnedFrom,const Address &member)
 	{
 		Mutex::Lock _l(_groups_m);
-		_add(RR,now,_groups[mg],learnedFrom,member);
+		_add(now,_groups[std::pair<uint64_t,MulticastGroup>(nwid,mg)],learnedFrom,member);
 	}
 
+	/**
+	 * Append gather results to a packet by choosing registered multicast recipients at random
+	 *
+	 * This appends the following fields to the packet:
+	 *   <[4] 32-bit total number of known members in this multicast group>
+	 *   <[2] 16-bit number of members enumerated in this packet>
+	 *   <[...] series of 5-byte ZeroTier addresses of enumerated members>
+	 *
+	 * If zero is returned, the first two fields will still have been appended.
+	 *
+	 * @param RR Runtime environment
+	 * @param nwid Network ID
+	 * @param mg Multicast group
+	 * @param appendTo Packet to append to
+	 * @param limit Maximum number of 5-byte addresses to append
+	 * @return Number of addresses appended
+	 * @throws std::out_of_range Buffer overflow writing to packet
+	 */
+	unsigned int gather(const RuntimeEnvironment *RR,uint64_t nwid,MulticastGroup &mg,Packet &appendTo,unsigned int limit) const;
+
 	/**
 	 * Send a multicast
 	 *
@@ -112,10 +133,10 @@ public:
 	 */
 	void send(
 		const RuntimeEnvironment *RR,
-		uint64_t nwid,
 		const CertificateOfMembership *com,
 		unsigned int limit,
 		uint64_t now,
+		uint64_t nwid,
 		const MulticastGroup &mg,
 		const MAC &src,
 		unsigned int etherType,
@@ -127,14 +148,13 @@ public:
 	 *
 	 * @param RR Runtime environment
 	 * @param now Current time
-	 * @param limit Multicast limit
 	 */
-	void clean(const RuntimeEnvironment *RR,uint64_t now,unsigned int limit);
+	void clean(const RuntimeEnvironment *RR,uint64_t now);
 
 private:
-	void _add(const RuntimeEnvironment *RR,uint64_t now,MulticastGroupStatus &gs,const Address &learnedFrom,const Address &member);
+	void _add(uint64_t now,MulticastGroupStatus &gs,const Address &learnedFrom,const Address &member);
 
-	std::map< MulticastGroup,MulticastGroupStatus > _groups;
+	std::map< std::pair<uint64_t,MulticastGroup>,MulticastGroupStatus > _groups;
 	Mutex _groups_m;
 };
 

node/Network.cpp

@@ -282,43 +282,38 @@ bool Network::isAllowed(const Address &peer) const
 void Network::clean()
 {
 	uint64_t now = Utils::now();
-	{
+	Mutex::Lock _l(_lock);
-		Mutex::Lock _l(_lock);
 
-		if (_destroyed)
+	if (_destroyed)
-			return;
+		return;
 
-		if ((_config)&&(_config->isPublic())) {
+	if ((_config)&&(_config->isPublic())) {
-			// Open (public) networks do not track certs or cert pushes at all.
+		// Open (public) networks do not track certs or cert pushes at all.
-			_membershipCertificates.clear();
+		_membershipCertificates.clear();
-			_lastPushedMembershipCertificate.clear();
+		_lastPushedMembershipCertificate.clear();
-		} else if (_config) {
+	} else if (_config) {
-			// Clean certificates that are no longer valid from the cache.
+		// Clean certificates that are no longer valid from the cache.
-			for(std::map<Address,CertificateOfMembership>::iterator c=(_membershipCertificates.begin());c!=_membershipCertificates.end();) {
+		for(std::map<Address,CertificateOfMembership>::iterator c=(_membershipCertificates.begin());c!=_membershipCertificates.end();) {
-				if (_config->com().agreesWith(c->second))
+			if (_config->com().agreesWith(c->second))
-					++c;
+				++c;
-				else _membershipCertificates.erase(c++);
+			else _membershipCertificates.erase(c++);
-			}
-
-			// Clean entries from the last pushed tracking map if they're so old as
-			// to be no longer relevant.
-			uint64_t forgetIfBefore = now - (_config->com().timestampMaxDelta() * 3ULL);
-			for(std::map<Address,uint64_t>::iterator lp(_lastPushedMembershipCertificate.begin());lp!=_lastPushedMembershipCertificate.end();) {
-				if (lp->second < forgetIfBefore)
-					_lastPushedMembershipCertificate.erase(lp++);
-				else ++lp;
-			}
 		}
 
-		// Clean learned multicast groups if we haven't heard from them in a while
+		// Clean entries from the last pushed tracking map if they're so old as
-		for(std::map<MulticastGroup,uint64_t>::iterator mg(_multicastGroupsBehindMe.begin());mg!=_multicastGroupsBehindMe.end();) {
+		// to be no longer relevant.
-			if ((now - mg->second) > (ZT_MULTICAST_LIKE_EXPIRE * 2))
+		uint64_t forgetIfBefore = now - (_config->com().timestampMaxDelta() * 3ULL);
-				_multicastGroupsBehindMe.erase(mg++);
+		for(std::map<Address,uint64_t>::iterator lp(_lastPushedMembershipCertificate.begin());lp!=_lastPushedMembershipCertificate.end();) {
-			else ++mg;
+			if (lp->second < forgetIfBefore)
+				_lastPushedMembershipCertificate.erase(lp++);
+			else ++lp;
 		}
 	}
-	{
+
-		_multicaster.clean(RR,now,(_config) ? _config->multicastLimit() : (unsigned int)ZT_MULTICAST_DEFAULT_LIMIT);
+	// Clean learned multicast groups if we haven't heard from them in a while
+	for(std::map<MulticastGroup,uint64_t>::iterator mg(_multicastGroupsBehindMe.begin());mg!=_multicastGroupsBehindMe.end();) {
+		if ((now - mg->second) > (ZT_MULTICAST_LIKE_EXPIRE * 2))
+			_multicastGroupsBehindMe.erase(mg++);
+		else ++mg;
 	}
 }
 

node/Network.hpp

@@ -227,35 +227,6 @@ public:
 			_pushMembershipCertificate(peer,force,now);
 	}
 
-	/**
-	 * Send a multicast packet to the members of a group
-	 *
-	 * This performs no rate checking or other permission checking.
-	 *
-	 * @param mg Multicast group destination
-	 * @param src Source Ethernet MAC address
-	 * @param etherType Ethernet frame type
-	 * @param data Payload data
-	 * @param len Length of payload
-	 */
-	inline void sendMulticast(const MulticastGroup &mg,const MAC &src,unsigned int etherType,const void *data,unsigned int len)
-	{
-		Mutex::Lock _l(_lock);
-		if (!_config)
-			return;
-		_multicaster.send(
-			RR,
-			_id,
-			(((!_config->isPublic())&&(_config->com())) ? &(_config->com()) : (const CertificateOfMembership *)0),
-			_config->multicastLimit(),
-			Utils::now(),
-			mg,
-			src,
-			etherType,
-			data,
-			len);
-	}
-
 	/**
 	 * @param peer Peer address to check
 	 * @return True if peer is allowed to communicate on this network
@@ -477,7 +448,6 @@ private:
 	std::set< MulticastGroup > _myMulticastGroups; // multicast groups that we belong to including those behind us (updated periodically)
 	std::map< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups bridged to us and when we last saw activity on each
 	std::map< MulticastGroup,BandwidthAccount > _multicastRateAccounts;
-	Multicaster _multicaster;
 
 	std::map<MAC,Address> _remoteBridgeRoutes; // remote addresses where given MACs are reachable
 

node/Node.cpp

@@ -69,6 +69,7 @@
 #include "NodeConfig.hpp"
 #include "Network.hpp"
 #include "MulticastGroup.hpp"
+#include "Multicaster.hpp"
 #include "Mutex.hpp"
 #include "Service.hpp"
 #include "SoftwareUpdater.hpp"
@@ -112,6 +113,7 @@ struct _NodeImpl
 		delete renv.topology; renv.topology = (Topology *)0;        // now we no longer need routing info
 		delete renv.sm;       renv.sm = (SocketManager *)0;         // close all sockets
 		delete renv.sw;       renv.sw = (Switch *)0;                // order matters less from here down
+		delete renv.mc;       renv.mc = (Multicaster *)0;
 		delete renv.antiRec;  renv.antiRec = (AntiRecursion *)0;
 		delete renv.http;     renv.http = (HttpClient *)0;
 		delete renv.prng;     renv.prng = (CMWC4096 *)0;
@@ -380,6 +382,7 @@ Node::ReasonForTermination Node::run()
 
 		RR->http = new HttpClient();
 		RR->antiRec = new AntiRecursion();
+		RR->mc = new Multicaster();
 		RR->sw = new Switch(_r);
 		RR->sm = new SocketManager(impl->udpPort,impl->tcpPort,&_CBztTraffic,_r);
 		RR->topology = new Topology(RR,Utils::fileExists((RR->homePath + ZT_PATH_SEPARATOR_S + "iddb.d").c_str()));
@@ -602,8 +605,8 @@ Node::ReasonForTermination Node::run()
 			// Do periodic tasks in submodules.
 			if ((now - lastClean) >= ZT_DB_CLEAN_PERIOD) {
 				lastClean = now;
-				RR->mc->clean();
 				RR->topology->clean();
+				RR->mc->clean(RR,now);
 				RR->nc->clean();
 				if (RR->updater)
 					RR->updater->checkIfMaxIntervalExceeded(now);

node/Packet.hpp

@@ -54,7 +54,7 @@
  * 3 - 0.5.0 ... 0.6.0
  *   * Yet another multicast redesign
  *   * New crypto completely changes key agreement cipher
- * 4 - 0.6.0 ...
+ * 4 - 0.6.0 ... 0.9.2
  *   * New identity format based on hashcash design
  *
  * This isn't going to change again for a long time unless your
@@ -62,6 +62,11 @@
  */
 #define ZT_PROTO_VERSION 4
 
+/**
+ * Minimum supported protocol version
+ */
+#define ZT_PROTO_VERSION_MIN 4
+
 /**
  * Maximum hop count allowed by packet structure (3 bits, 0-7)
  * 
@@ -184,6 +189,7 @@
 #define ZT_PROTO_VERB_EXT_FRAME_LEN_NETWORK_ID 8
 #define ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS (ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID + ZT_PROTO_VERB_EXT_FRAME_LEN_NETWORK_ID)
 #define ZT_PROTO_VERB_EXT_FRAME_LEN_FLAGS 1
+#define ZT_PROTO_VERB_EXT_FRAME_IDX_COM (ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS + ZT_PROTO_VERB_EXT_FRAME_LEN_FLAGS)
 #define ZT_PROTO_VERB_EXT_FRAME_IDX_TO (ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS + ZT_PROTO_VERB_EXT_FRAME_LEN_FLAGS)
 #define ZT_PROTO_VERB_EXT_FRAME_LEN_TO 6
 #define ZT_PROTO_VERB_EXT_FRAME_IDX_FROM (ZT_PROTO_VERB_EXT_FRAME_IDX_TO + ZT_PROTO_VERB_EXT_FRAME_LEN_TO)
@@ -227,8 +233,7 @@
 #define ZT_PROTO_VERB_P5_MULTICAST_FRAME_IDX_FRAME_LEN (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_ETHERTYPE)
 #define ZT_PROTO_VERB_P5_MULTICAST_FRAME_LEN_FRAME_LEN 2
 #define ZT_PROTO_VERB_P5_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME_LEN + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN)
-
+#define ZT_PROTO_VERB_P5_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE 0x01
-#define ZT_PROTO_VERB_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE 0x01
 
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID + 8)
@@ -239,12 +244,15 @@
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS + 1)
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC + 6)
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI + 4)
-#define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT + 4)
 
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID + 8)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
-// remainder are relative to offset after COM -- might do this with macros at some point
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT + 4)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI + 4)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC + 6)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC + 6)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + 2)
 
 #define ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP (ZT_PROTO_VERB_OK_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION (ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP + 8)
@@ -528,8 +536,7 @@ public:
 		 */
 		VERB_FRAME = 6,
 
-		/*
+		/* Full Ethernet frame with MAC addressing and optional fields:
-		 * Full Ethernet frame with MAC addressing and optional fields:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
 		 *  [<[...] certificate of network membership>]
@@ -545,6 +552,8 @@ public:
 		 * superset of VERB_FRAME. They're used for bridging or when we
 		 * want to attach a certificate since FRAME does not support that.
 		 *
+		 * Multicast frames may not be sent as EXT_FRAME.
+		 *
 		 * ERROR may be generated if a membership certificate is needed for a
 		 * closed network. Payload will be network ID.
 		 */
@@ -698,6 +707,7 @@ public:
 		 *   <[8] 64-bit network ID>
 		 *   <[6] MAC address of multicast group being queried>
 		 *   <[4] 32-bit ADI for multicast group being queried>
+		 *   [begin gather results -- these same fields can be in OK(MULTICAST_FRAME)]
 		 *   <[4] 32-bit total number of known members in this multicast group>
 		 *   <[2] 16-bit number of members enumerated in this packet>
 		 *   <[...] series of 5-byte ZeroTier addresses of enumerated members>
@@ -731,18 +741,27 @@ public:
 		 *
 		 * This is similar to EXT_FRAME but carries a multicast, and is sent
 		 * out to recipients on a multicast list. It may also specify a desired
-		 * number of multicast peers to gather, which is called "implicit
+		 * number of multicast peers to gather if additional multicast peers
-		 * gathering." It is only allowed if certificate authentication permits
+		 * for this group are desired.
-		 * these peers to communicate on this network.
 		 *
 		 * (ADI precedes MAC here so that everything from destination MAC forward
 		 * could be treated as a raw Ethernet frame.)
 		 *
+		 * OK responses are optional and are currently only returned if gathering
+		 * of additional multicast peers is requested.
+		 *
 		 * OK response payload:
-		 *   <[1] flags, 0x01 if implicit gathering results are included>
+		 *   <[8] 64-bit network ID>
-		 *   [... same as OK(MULTICAST_GATHER) payload if flag 0x01 is set ...]
+		 *   <[6] MAC address of multicast group>
+		 *   <[4] 32-bit ADI for multicast group>
+		 *   <[1] flags>
+		 *  [<[...] implicit gather results if flag 0x01 is set>]
+		 *
+		 * Flags:
+		 *   0x01 - OK include implicit gather results
 		 *
 		 * ERROR response payload:
+		 *   <[8] 64-bit network ID>
 		 *   <[6] multicast group MAC>
 		 *   <[4] 32-bit multicast group ADI>
 		 *

node/Peer.hpp

@@ -50,7 +50,7 @@
 #include "NonCopyable.hpp"
 #include "Mutex.hpp"
 
-#define ZT_PEER_SERIALIZATION_VERSION 11
+#define ZT_PEER_SERIALIZATION_VERSION 12
 
 namespace ZeroTier {
 
@@ -351,21 +351,25 @@ public:
 	/**
 	 * Set the currently known remote version of this peer's client
 	 *
+	 * @param vproto Protocol version
 	 * @param vmaj Major version
 	 * @param vmin Minor version
 	 * @param vrev Revision
 	 */
-	inline void setRemoteVersion(unsigned int vmaj,unsigned int vmin,unsigned int vrev)
+	inline void setRemoteVersion(unsigned int vproto,unsigned int vmaj,unsigned int vmin,unsigned int vrev)
-		throw()
 	{
-		_vMajor = vmaj;
+		_vProto = (uint16_t)vproto;
-		_vMinor = vmin;
+		_vMajor = (uint16_t)vmaj;
-		_vRevision = vrev;
+		_vMinor = (uint16_t)vmin;
+		_vRevision = (uint16_t)vrev;
 	}
 
+	inline unsigned int remoteVersionProtocol() const throw() { return _vProto; }
+
 	inline unsigned int remoteVersionMajor() const throw() { return _vMajor; }
 	inline unsigned int remoteVersionMinor() const throw() { return _vMinor; }
 	inline unsigned int remoteVersionRevision() const throw() { return _vRevision; }
+
 	inline bool remoteVersionKnown() const throw() { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); }
 
 	/**
@@ -413,6 +417,7 @@ public:
 		b.append(_lastUnicastFrame);
 		b.append(_lastMulticastFrame);
 		b.append(_lastAnnouncedTo);
+		b.append((uint16_t)_vProto);
 		b.append((uint16_t)_vMajor);
 		b.append((uint16_t)_vMinor);
 		b.append((uint16_t)_vRevision);
@@ -438,6 +443,7 @@ public:
 		_lastUnicastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 		_lastMulticastFrame = b.template at<uint64_t>(p); p += sizeof(uint64_t);
 		_lastAnnouncedTo = b.template at<uint64_t>(p); p += sizeof(uint64_t);
+		_vProto = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 		_vMajor = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 		_vMinor = b.template at<uint16_t>(p); p += sizeof(uint16_t);
 		_vRevision = b.template at<uint16_t>(p); p += sizeof(uint16_t);
@@ -463,9 +469,10 @@ private:
 	volatile uint64_t _lastUnicastFrame;
 	volatile uint64_t _lastMulticastFrame;
 	volatile uint64_t _lastAnnouncedTo;
-	volatile unsigned int _vMajor;
+	volatile uint16_t _vProto;
-	volatile unsigned int _vMinor;
+	volatile uint16_t _vMajor;
-	volatile unsigned int _vRevision;
+	volatile uint16_t _vMinor;
+	volatile uint16_t _vRevision;
 	volatile unsigned int _latency;
 
 	Mutex _lock;

node/RuntimeEnvironment.hpp

@@ -44,6 +44,7 @@ class Service;
 class Node;
 class SoftwareUpdater;
 class SocketManager;
+class Multicaster;
 class AntiRecursion;
 class EthernetTapFactory;
 class RoutingTable;
@@ -78,10 +79,12 @@ public:
 		prng((CMWC4096 *)0),
 		http((HttpClient *)0),
 		antiRec((AntiRecursion *)0),
+		mc((Multicaster *)0),
 		sw((Switch *)0),
 		sm((SocketManager *)0),
 		topology((Topology *)0),
 		nc((NodeConfig *)0),
+		node((Node *)0),
 		updater((SoftwareUpdater *)0)
 #ifndef __WINDOWS__
 		,netconfService((Service *)0)
@@ -127,6 +130,7 @@ public:
 	CMWC4096 *prng;
 	HttpClient *http;
 	AntiRecursion *antiRec;
+	Multicaster *mc;
 	Switch *sw;
 	SocketManager *sm;
 	Topology *topology;