Certificate of membership works now... had to fix multicast propagation so COM is pushed with multicast, which makes tremendous sense in retrospect.

node/Network.hpp

@@ -207,32 +207,6 @@ public:
 			_pushMembershipCertificate(peer,force,now);
 	}
 
-	/**
-	 * Push membership certificate to a packed zero-terminated array of addresses
-	 *
-	 * This pushes to all peers in peers[] (length must be a multiple of 5) until
-	 * len is reached or a null address is encountered.
-	 *
-	 * @param peers Packed array of 5-byte big-endian addresses
-	 * @param len Length of peers[] in total (bytes, not addresses)
-	 * @param force If true, push even if we've already done so within required time frame
-	 * @param now Current time
-	 */
-	inline void pushMembershipCertificate(const void *peers,unsigned int len,bool force,uint64_t now)
-	{
-		Mutex::Lock _l(_lock);
-		if ((_config)&&(!_config->isOpen())&&(_config->com())) {
-			for(unsigned int i=0;i<len;i+=ZT_ADDRESS_LENGTH) {
-				if ((i + ZT_ADDRESS_LENGTH) > len)
-					break;
-				Address a((char *)peers + i,ZT_ADDRESS_LENGTH);
-				if (a)
-					_pushMembershipCertificate(a,force,now);
-				else break;
-			}
-		}
-	}
-
 	/**
 	 * @param peer Peer address to check
 	 * @return True if peer is allowed to communicate on this network

node/Packet.hpp

@@ -205,6 +205,8 @@
 #define ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN 2
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME_LEN + ZT_PROTO_VERB_MULTICAST_FRAME_LEN_FRAME_LEN)
 
+#define ZT_PROTO_VERB_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE 0x01
+
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID + 8)
 #define ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT (ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN + 2)
@@ -507,6 +509,11 @@ public:
 		 *   [... end of signed portion ...]
 		 *   <[2] 16-bit length of signature>
 		 *   <[...] signature (currently Ed25519/SHA-512, 96 bytes in length)>
+		 *  [<[...] network membership certificate (optional)>]
+		 *
+		 * Flags:
+		 *   0x01 - Multicast frame includes network membership certificate
+		 *          for original sender for this network.
 		 *
 		 * When a multicast frame is received:
 		 *

node/PacketDecoder.cpp

@@ -436,7 +436,7 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		unsigned char *const bloom = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM);
 
 		// These fields don't -- they're signed by the original sender
-		// const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
+		const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
 		const uint16_t bloomNonce = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_BLOOM_NONCE);
 		const unsigned int prefixBits = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_PREFIX_BITS];
@@ -450,6 +450,27 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		const unsigned int signatureLen = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen);
 		const unsigned char *const signature = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen + 2,signatureLen);
 
+		SharedPtr<Network> network(_r->nc->network(nwid));
+
+		// Grab, verify, and learn certificate if any -- provided we are a member of this network
+		// Note: we can do this before verification of the actual packet, since the certificate
+		// has its own separate signature.
+		if (((flags & ZT_PROTO_VERB_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE))&&(network)) {
+			CertificateOfMembership originCom(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME + frameLen + 2 + signatureLen);
+			Address signedBy(originCom.signedBy());
+			if ((originCom.networkId() == nwid)&&(signedBy == network->controller())) {
+				SharedPtr<Peer> signingPeer(_r->topology->getPeer(signedBy));
+				if (!signingPeer) {
+					// Technically this shouldn't happen, but handle it anyway...
+					_r->sw->requestWhois(signedBy);
+					_step = DECODE_WAITING_FOR_MULTICAST_FRAME_ORIGINAL_SENDER_LOOKUP; // causes processing to come back here
+					return false;
+				} else if (originCom.verify(signingPeer->identity())) {
+					network->addMembershipCertificate(originCom);
+				}
+			}
+		}
+
 		// Check multicast signature to verify original sender
 		const unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + frameLen;
 		if (!originPeer->identity().verify(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION,signedPartLen),signedPartLen,signature,signatureLen)) {
@@ -490,7 +511,6 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 #endif
 
 		unsigned int maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
-		SharedPtr<Network> network(_r->nc->network(nwid));
 
 		if ((origin == _r->identity.address())||(_r->mc->deduplicate(nwid,guid))) {
 			// Ordinary nodes will drop duplicates. Supernodes keep propagating
@@ -660,12 +680,6 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		while (newFifoPtr != newFifoEnd)
 			*(newFifoPtr++) = (unsigned char)0;
 
-		// If we're forwarding a packet within a private network that we are
-		// a member of, also propagate our cert if needed. This propagates
-		// it to everyone including people who will receive this multicast.
-		if (network)
-			network->pushMembershipCertificate(newFifo,sizeof(newFifo),false,Utils::now());
-
 		// First element in newFifo[] is next hop
 		Address nextHop(newFifo,ZT_ADDRESS_LENGTH);
 		if ((!nextHop)&&(!_r->topology->amSupernode())) {

node/Switch.cpp

@@ -117,7 +117,6 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		unsigned char *const fifoEnd = fifo + sizeof(fifo);
 		const unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + data.size();
 		const SharedPtr<Peer> supernode(_r->topology->getBestSupernode());
-		uint64_t now = Utils::now();
 
 		for(unsigned int prefix=0,np=((unsigned int)2 << (nconf->multicastPrefixBits() - 1));prefix<np;++prefix) {
 			memset(bloom,0,sizeof(bloom));
@@ -134,17 +133,11 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 				else continue;
 			}
 
-			// If network is not open, make sure all recipients have our membership
-			// certificate if we haven't sent it recently. As the multicast goes
-			// further down the line, peers beyond the first batch will ask us for
-			// our membership certificate if they need it.
-			network->pushMembershipCertificate(fifo,sizeof(fifo),false,now);
-
 			Packet outp(firstHop,_r->identity.address(),Packet::VERB_MULTICAST_FRAME);
 			outp.append((uint16_t)0);
 			outp.append(fifo + ZT_ADDRESS_LENGTH,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO); // remainder of fifo is loaded into packet
 			outp.append(bloom,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM);
-			outp.append((unsigned char)0);
+			outp.append((nconf->com()) ? (unsigned char)ZT_PROTO_VERB_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE : (unsigned char)0);
 			outp.append(network->id());
 			outp.append(bloomNonce);
 			outp.append((unsigned char)nconf->multicastPrefixBits());
@@ -164,6 +157,9 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			outp.append((uint16_t)sig.size());
 			outp.append(sig.data,sig.size());
 
+			if (nconf->com())
+				nconf->com().serialize(outp);
+
 			outp.compress();
 			send(outp,true);
 		}
@@ -194,8 +190,6 @@ void Switch::send(const Packet &packet,bool encrypt)
 		return;
 	}
 
-	//TRACE(">> %.16llx %s -> %s (size: %u) (enc: %s)",(unsigned long long)packet.packetId(),packet.source().toString().c_str(),packet.destination().toString().c_str(),packet.size(),(encrypt ? "yes" : "no"));
-
 	if (!_trySend(packet,encrypt)) {
 		Mutex::Lock _l(_txQueue_m);
 		_txQueue.insert(std::pair< Address,TXQueueEntry >(packet.destination(),TXQueueEntry(Utils::now(),packet,encrypt)));