Add UDP MTU configurability.

include/ZeroTierOne.h

@@ -93,13 +93,17 @@ extern "C" {
 #define ZT_MAX_MTU 10000
 
 /**
- * Default payload MTU for UDP packets
+ * Minimum UDP payload size allowed
+ */
+#define ZT_MIN_PHYSMTU 1400
+
+/**
+ * Default UDP payload size (physical path MTU) not including UDP and IP overhead
  *
  * This is 1500 - IPv6 UDP overhead - PPPoE overhead and is safe for 99.9% of
  * all Internet links.
  */
 #define ZT_DEFAULT_PHYSMTU 1444
-#define ZT_UDP_DEFAULT_PAYLOAD_MTU 1444
 
 /**
  * Maximum physical UDP payload
@@ -172,9 +176,9 @@ extern "C" {
 #define ZT_MAX_PEER_NETWORK_PATHS 4
 
 /**
- * Maximum number of trusted physical network paths
+ * Maximum number of path configurations that can be set
  */
-#define ZT_MAX_TRUSTED_PATHS 16
+#define ZT_MAX_CONFIGURABLE_PATHS 32
 
 /**
  * Maximum number of rules per capability
@@ -1058,11 +1062,6 @@ typedef struct
 	 */
 	unsigned int mtu;
 
-	/**
-	 * Recommended MTU to avoid fragmentation at the physical layer (hint)
-	 */
-	unsigned int physicalMtu;
-
 	/**
 	 * If nonzero, the network this port belongs to indicates DHCP availability
 	 *
@@ -1132,6 +1131,21 @@ typedef struct
 	unsigned long networkCount;
 } ZT_VirtualNetworkList;
 
+/**
+ * Physical path configuration
+ */
+typedef struct {
+	/**
+	 * If non-zero set this physical network path to be trusted to disable encryption and authentication
+	 */
+	uint64_t trustedPathId;
+
+	/**
+	 * Physical path MTU from ZT_MIN_PHYSMTU and ZT_MAX_PHYSMTU or <= 0 to use default
+	 */
+	int mtu;
+} ZT_PhysicalPathConfiguration;
+
 /**
  * Physical network path to a peer
  */
@@ -1856,27 +1870,14 @@ ZT_SDK_API int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,ui
 ZT_SDK_API void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance);
 
 /**
- * Set trusted paths
+ * Set configuration for a given physical path
- *
- * A trusted path is a physical network (network/bits) over which both
- * encryption and authentication can be skipped to improve performance.
- * Each trusted path must have a non-zero unique ID that is the same across
- * all participating nodes.
- *
- * We don't recommend using trusted paths at all unless you really *need*
- * near-bare-metal performance. Even on a LAN authentication and encryption
- * are never a bad thing, and anything that introduces an "escape hatch"
- * for encryption should be treated with the utmost care.
- *
- * Calling with NULL pointers for networks and ids and a count of zero clears
- * all trusted paths.
  *
  * @param node Node instance
- * @param networks Array of [count] networks
+ * @param pathNetwork Network/CIDR of path or NULL to clear the cache and reset all paths to default
- * @param ids Array of [count] corresponding non-zero path IDs (zero path IDs are ignored)
+ * @param pathConfig Path configuration or NULL to erase this entry and therefore reset it to NULL
- * @param count Number of trusted paths-- values greater than ZT_MAX_TRUSTED_PATHS are clipped
+ * @return OK or error code
  */
-ZT_SDK_API void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);
+ZT_SDK_API enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig);
 
 /**
  * Get ZeroTier One version

node/Multicaster.cpp

@@ -111,7 +111,7 @@ unsigned int Multicaster::gather(const Address &queryingPeer,uint64_t nwid,const
 		// Members are returned in random order so that repeated gather queries
 		// will return different subsets of a large multicast group.
 		k = 0;
-		while ((added < limit)&&(k < s->members.size())&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_UDP_DEFAULT_PAYLOAD_MTU)) {
+		while ((added < limit)&&(k < s->members.size())&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_PROTO_MAX_PACKET_LENGTH)) {
 			rptr = (unsigned int)RR->node->prng();
 
 restart_member_scan:

node/Network.cpp

@@ -1346,7 +1346,6 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	ec->status = _status();
 	ec->type = (_config) ? (_config.isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE;
 	ec->mtu = (_config) ? _config.mtu : ZT_DEFAULT_MTU;
-	ec->physicalMtu = ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 16);
 	ec->dhcp = 0;
 	std::vector<Address> ab(_config.activeBridges());
 	ec->bridge = ((_config.allowPassiveBridging())||(std::find(ab.begin(),ab.end(),RR->identity.address()) != ab.end())) ? 1 : 0;

node/Node.cpp

@@ -561,9 +561,9 @@ uint64_t Node::prng()
 	return z + y;
 }
 
-void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count)
+ZT_ResultCode Node::setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork, const ZT_PhysicalPathConfiguration *pathConfig)
 {
-	RR->topology->setTrustedPaths(reinterpret_cast<const InetAddress *>(networks),ids,count);
+	return ZT_RESULT_OK;
 }
 
 World Node::planet() const
@@ -815,7 +815,7 @@ enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,u
 	}
 }
 
-ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId)
+enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->deorbit(tptr,moonWorldId);
@@ -902,11 +902,13 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 	} catch ( ... ) {}
 }
 
-void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count)
+enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig)
 {
 	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->setTrustedPaths(networks,ids,count);
+		return reinterpret_cast<ZeroTier::Node *>(node)->setPhysicalPathConfiguration(pathNetwork,pathConfig);
-	} catch ( ... ) {}
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
 }
 
 void ZT_version(int *major,int *minor,int *revision)

node/Node.hpp

@@ -192,7 +192,7 @@ public:
 	inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast<struct sockaddr_storage *>(&addr)) != 0) : false ); }
 
 	uint64_t prng();
-	void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);
+	ZT_ResultCode setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig);
 
 	World planet() const;
 	std::vector<World> moons() const;

node/Packet.hpp

@@ -225,12 +225,8 @@
 
 /**
  * Packet buffer size (can be changed)
- *
- * The current value is big enough for ZT_MAX_PACKET_FRAGMENTS, the pragmatic
- * packet fragment limit, times the default UDP MTU. Most packets won't be
- * this big.
  */
-#define ZT_PROTO_MAX_PACKET_LENGTH (ZT_MAX_PACKET_FRAGMENTS * ZT_UDP_DEFAULT_PAYLOAD_MTU)
+#define ZT_PROTO_MAX_PACKET_LENGTH (ZT_MAX_PACKET_FRAGMENTS * ZT_DEFAULT_PHYSMTU)
 
 /**
  * Minimum viable packet length (a.k.a. header length)

node/Switch.cpp

@@ -722,10 +722,13 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 		return false;
 	}
 
-	unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU);
+	unsigned int mtu = ZT_DEFAULT_PHYSMTU;
+	uint64_t trustedPathId = 0;
+	RR->topology->getOutboundPathInfo(viaPath->address(),mtu,trustedPathId);
+
+	unsigned int chunkSize = std::min(packet.size(),mtu);
 	packet.setFragmented(chunkSize < packet.size());
 
-	const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address());
 	if (trustedPathId) {
 		packet.setTrusted(trustedPathId);
 	} else {
@@ -737,13 +740,13 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 			// Too big for one packet, fragment the rest
 			unsigned int fragStart = chunkSize;
 			unsigned int remaining = packet.size() - chunkSize;
-			unsigned int fragsRemaining = (remaining / (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH));
+			unsigned int fragsRemaining = (remaining / (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH));
-			if ((fragsRemaining * (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining)
+			if ((fragsRemaining * (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining)
 				++fragsRemaining;
 			const unsigned int totalFragments = fragsRemaining + 1;
 
 			for(unsigned int fno=1;fno<totalFragments;++fno) {
-				chunkSize = std::min(remaining,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH));
+				chunkSize = std::min(remaining,(unsigned int)(mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH));
 				Packet::Fragment frag(packet,fragStart,chunkSize,fno,totalFragments);
 				viaPath->send(RR,tPtr,frag.data(),frag.size(),now);
 				fragStart += chunkSize;

node/Topology.cpp

@@ -65,7 +65,7 @@ static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x0
 
 Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) :
 	RR(renv),
-	_trustedPathCount(0),
+	_numConfiguredPhysicalPaths(0),
 	_amRoot(false)
 {
 	uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];

node/Topology.hpp

@@ -339,6 +339,41 @@ public:
 	 */
 	inline bool amRoot() const { return _amRoot; }
 
+	/**
+	 * Get info about a path
+	 *
+	 * The supplied result variables are not modified if no special config info is found.
+	 *
+	 * @param physicalAddress Physical endpoint address
+	 * @param mtu Variable set to MTU
+	 * @param trustedPathId Variable set to trusted path ID
+	 */
+	inline void getOutboundPathInfo(const InetAddress &physicalAddress,unsigned int &mtu,uint64_t &trustedPathId)
+	{
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
+			if (_physicalPathConfig[i].first.containsAddress(physicalAddress)) {
+				trustedPathId = _physicalPathConfig[i].second.trustedPathId;
+				mtu = _physicalPathConfig[i].second.mtu;
+				return;
+			}
+		}
+	}
+
+	/**
+	 * Get the payload MTU for an outbound physical path (returns default if not configured)
+	 *
+	 * @param physicalAddress Physical endpoint address
+	 * @return MTU
+	 */
+	inline unsigned int getOutboundPathMtu(const InetAddress &physicalAddress)
+	{
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
+			if (_physicalPathConfig[i].first.containsAddress(physicalAddress))
+				return _physicalPathConfig[i].second.mtu;
+		}
+		return ZT_DEFAULT_PHYSMTU;
+	}
+
 	/**
 	 * Get the outbound trusted path ID for a physical address, or 0 if none
 	 *
@@ -347,9 +382,9 @@ public:
 	 */
 	inline uint64_t getOutboundPathTrust(const InetAddress &physicalAddress)
 	{
-		for(unsigned int i=0;i<_trustedPathCount;++i) {
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
-			if (_trustedPathNetworks[i].containsAddress(physicalAddress))
+			if (_physicalPathConfig[i].first.containsAddress(physicalAddress))
-				return _trustedPathIds[i];
+				return _physicalPathConfig[i].second.trustedPathId;
 		}
 		return 0;
 	}
@@ -362,30 +397,48 @@ public:
 	 */
 	inline bool shouldInboundPathBeTrusted(const InetAddress &physicalAddress,const uint64_t trustedPathId)
 	{
-		for(unsigned int i=0;i<_trustedPathCount;++i) {
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
-			if ((_trustedPathIds[i] == trustedPathId)&&(_trustedPathNetworks[i].containsAddress(physicalAddress)))
+			if ((_physicalPathConfig[i].second.trustedPathId == trustedPathId)&&(_physicalPathConfig[i].first.containsAddress(physicalAddress)))
 				return true;
 		}
 		return false;
 	}
 
 	/**
-	 * Set trusted paths in this topology
+	 * Set or clear physical path configuration (called via Node::setPhysicalPathConfiguration)
-	 *
-	 * @param networks Array of networks (prefix/netmask bits)
-	 * @param ids Array of trusted path IDs
-	 * @param count Number of trusted paths (if larger than ZT_MAX_TRUSTED_PATHS overflow is ignored)
 	 */
-	inline void setTrustedPaths(const InetAddress *networks,const uint64_t *ids,unsigned int count)
+	inline void setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig)
 	{
-		if (count > ZT_MAX_TRUSTED_PATHS)
+		if (!pathNetwork) {
-			count = ZT_MAX_TRUSTED_PATHS;
+			_numConfiguredPhysicalPaths = 0;
-		Mutex::Lock _l(_trustedPaths_m);
+		} else {
-		for(unsigned int i=0;i<count;++i) {
+			std::map<InetAddress,ZT_PhysicalPathConfiguration> cpaths;
-			_trustedPathIds[i] = ids[i];
+			for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i)
-			_trustedPathNetworks[i] = networks[i];
+				cpaths[_physicalPathConfig[i].first] = _physicalPathConfig[i].second;
+
+			if (pathConfig) {
+				ZT_PhysicalPathConfiguration pc(*pathConfig);
+
+				if (pc.mtu <= 0)
+					pc.mtu = ZT_DEFAULT_PHYSMTU;
+				else if (pc.mtu < ZT_MIN_PHYSMTU)
+					pc.mtu = ZT_MIN_PHYSMTU;
+				else if (pc.mtu > ZT_MAX_PHYSMTU)
+					pc.mtu = ZT_MAX_PHYSMTU;
+
+				cpaths[*(reinterpret_cast<const InetAddress *>(pathNetwork))] = pc;
+			} else {
+				cpaths.erase(*(reinterpret_cast<const InetAddress *>(pathNetwork)));
+			}
+
+			unsigned int cnt = 0;
+			for(std::map<InetAddress,ZT_PhysicalPathConfiguration>::const_iterator i(cpaths.begin());((i!=cpaths.end())&&(cnt<ZT_MAX_CONFIGURABLE_PATHS));++i) {
+				_physicalPathConfig[cnt].first = i->first;
+				_physicalPathConfig[cnt].second = i->second;
+				++cnt;
+			}
+			_numConfiguredPhysicalPaths = cnt;
 		}
-		_trustedPathCount = count;
 	}
 
 	/**
@@ -414,10 +467,8 @@ private:
 
 	const RuntimeEnvironment *const RR;
 
-	uint64_t _trustedPathIds[ZT_MAX_TRUSTED_PATHS];
+	std::pair<InetAddress,ZT_PhysicalPathConfiguration> _physicalPathConfig[ZT_MAX_CONFIGURABLE_PATHS];
-	InetAddress _trustedPathNetworks[ZT_MAX_TRUSTED_PATHS];
+	volatile unsigned int _numConfiguredPhysicalPaths;
-	unsigned int _trustedPathCount;
-	Mutex _trustedPaths_m;
 
 	Hashtable< Address,SharedPtr<Peer> > _peers;
 	Mutex _peers_m;

service/OneService.cpp

@@ -564,16 +564,14 @@ public:
 
 			// Read local configuration
 			{
-				uint64_t trustedPathIds[ZT_MAX_TRUSTED_PATHS];
+				std::map<InetAddress,ZT_PhysicalPathConfiguration> ppc;
-				InetAddress trustedPathNetworks[ZT_MAX_TRUSTED_PATHS];
-				unsigned int trustedPathCount = 0;
 
 				// LEGACY: support old "trustedpaths" flat file
 				FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r");
 				if (trustpaths) {
 					fprintf(stderr,"WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S);
 					char buf[1024];
-					while ((fgets(buf,sizeof(buf),trustpaths))&&(trustedPathCount < ZT_MAX_TRUSTED_PATHS)) {
+					while (fgets(buf,sizeof(buf),trustpaths)) {
 						int fno = 0;
 						char *saveptr = (char *)0;
 						uint64_t trustedPathId = 0;
@@ -587,9 +585,8 @@ public:
 							++fno;
 						}
 						if ( (trustedPathId != 0) && ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathNetwork.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (trustedPathNetwork.netmaskBits() > 0) ) {
-							trustedPathIds[trustedPathCount] = trustedPathId;
+							ppc[trustedPathNetwork].trustedPathId = trustedPathId;
-							trustedPathNetworks[trustedPathCount] = trustedPathNetwork;
+							ppc[trustedPathNetwork].mtu = 0; // use default
-							++trustedPathCount;
 						}
 					}
 					fclose(trustpaths);
@@ -618,12 +615,10 @@ public:
 							if (phy.value().is_object()) {
 								uint64_t tpid;
 								if ((tpid = OSUtils::jsonInt(phy.value()["trustedPathId"],0ULL)) != 0ULL) {
-									if ( ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (net.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (net.netmaskBits() > 0) ) {
+									if ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6))
-										trustedPathIds[trustedPathCount] = tpid;
+										ppc[net].trustedPathId = tpid;
-										trustedPathNetworks[trustedPathCount] = net;
-										++trustedPathCount;
-									}
 								}
+								ppc[net].mtu = (int)OSUtils::jsonInt(phy.value()["mtu"],0ULL); // 0 means use default
 							}
 						}
 					}
@@ -638,8 +633,10 @@ public:
 				}
 
 				// Set trusted paths if there are any
-				if (trustedPathCount)
+				if (ppc.size() > 0) {
-					_node->setTrustedPaths(reinterpret_cast<const struct sockaddr_storage *>(trustedPathNetworks),trustedPathIds,trustedPathCount);
+					for(std::map<InetAddress,ZT_PhysicalPathConfiguration>::iterator i(ppc.begin());i!=ppc.end();++i)
+						_node->setPhysicalPathConfiguration(reinterpret_cast<const struct sockaddr_storage *>(&(i->first)),&(i->second));
+				}
 			}
 
 			// Apply other runtime configuration from local.conf

service/README.md

@@ -14,7 +14,8 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al
 	"physical": { /* Settings that apply to physical L2/L3 network paths. */
 		"NETWORK/bits": { /* Network e.g. 10.0.0.0/24 or fd00::/32 */
 			"blacklist": true|false, /* If true, blacklist this path for all ZeroTier traffic */
-			"trustedPathId": 0|!0 /* If present and nonzero, define this as a trusted path (see below) */
+			"trustedPathId": 0|!0, /* If present and nonzero, define this as a trusted path (see below) */
+			"mtu": 0|!0 /* if present and non-zero, set UDP maximum payload MTU for this path */
 		} /* ,... additional networks */
 	},
 	"virtual": { /* Settings applied to ZeroTier virtual network devices (VL1) */