User-configurable physical MTU for individual links

This patch allows users to specify the physical layer MTU for individual links
when in multipath mode. For example:

{
  "settings":
  {
    "defaultBondingPolicy": "custom-balance-xor",
    "policies":
    {
      "custom-balance-xor":
      {
        "basePolicy": "balance-xor",
        "failoverInterval": 5000,
        "links": {
          "weird_5g_link": { "mtu": 1300 },
          "enp5s0": { "mtu": 1400  }
        }
      }
    }
  }
}

node/Bond.cpp

@@ -184,7 +184,7 @@ SharedPtr<Link> Bond::getLinkBySocket(const std::string& policyAlias, uint64_t l
 	auto search = _interfaceToLinkMap[policyAlias].find(ifnameStr);
 	if (search == _interfaceToLinkMap[policyAlias].end()) {
 		if (createIfNeeded) {
-			SharedPtr<Link> s = new Link(ifnameStr, 0, 0, true, ZT_BOND_SLAVE_MODE_PRIMARY, "");
+			SharedPtr<Link> s = new Link(ifnameStr, 0, 0, 0, true, ZT_BOND_SLAVE_MODE_PRIMARY, "");
 			_interfaceToLinkMap[policyAlias].insert(std::pair<std::string, SharedPtr<Link> >(ifnameStr, s));
 			return s;
 		}
@@ -1253,6 +1253,7 @@ void Bond::estimatePathQuality(int64_t now)
 			if (link) {
 				int linkSpeed = link->capacity();
 				_paths[i].p->_givenLinkSpeed = linkSpeed;
+				_paths[i].p->_mtu = link->mtu();
 				maxObservedLinkCap = linkSpeed > maxObservedLinkCap ? linkSpeed : maxObservedLinkCap;
 			}
 		}

node/Bond.hpp

@@ -122,9 +122,10 @@ class Link {
 	 * @param mode
 	 * @param failoverToLinkStr
 	 */
-	Link(std::string ifnameStr, uint8_t ipvPref, uint32_t capacity, bool enabled, uint8_t mode, std::string failoverToLinkStr)
+	Link(std::string ifnameStr, uint8_t ipvPref, uint16_t mtu, uint32_t capacity, bool enabled, uint8_t mode, std::string failoverToLinkStr)
 		: _ifnameStr(ifnameStr)
 		, _ipvPref(ipvPref)
+		, _mtu(mtu)
 		, _capacity(capacity)
 		, _relativeCapacity(0.0)
 		, _enabled(enabled)
@@ -226,6 +227,14 @@ class Link {
 		return _ipvPref;
 	}
 
+	/**
+	 * @return The MTU for this link (as specified by the user.)
+	 */
+	inline uint16_t mtu()
+	{
+		return _mtu;
+	}
+
 	/**
 	 * @return The mode (e.g. primary/spare) for this link (as specified by the user.)
 	 */
@@ -260,6 +269,11 @@ class Link {
 	 */
 	uint8_t _ipvPref;
 
+	/**
+	 * The physical-layer MTU for this link
+	 */
+	uint16_t _mtu;
+
 	/**
 	 * User-specified capacity of this link
 	 */

node/Path.hpp

@@ -92,6 +92,7 @@ public:
 		_valid(true),
 		_eligible(false),
 		_bonded(false),
+		_mtu(0),
 		_givenLinkSpeed(0),
 		_relativeQuality(0),
 		_latency(0xffff),
@@ -112,6 +113,7 @@ public:
 		_valid(true),
 		_eligible(false),
 		_bonded(false),
+		_mtu(0),
 		_givenLinkSpeed(0),
 		_relativeQuality(0),
 		_latency(0xffff),
@@ -334,6 +336,11 @@ public:
 	 */
 	inline unsigned int bonded() const { return _bonded; }
 
+	/**
+	 * @return Whether the user-specified MTU for this path (determined by MTU for parent link)
+	 */
+	inline unsigned int mtu() const { return _mtu; }
+
 	/**
 	 * @return Given link capacity as reported by the bonding layer
 	 */
@@ -370,6 +377,7 @@ private:
 	volatile bool _valid;
 	volatile bool _eligible;
 	volatile bool _bonded;
+	volatile uint16_t _mtu;
 	volatile uint32_t _givenLinkSpeed;
 	volatile float _relativeQuality;
 

node/Switch.cpp

@@ -1009,7 +1009,8 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
 			Mutex::Lock _l(peer->_paths_m);
 			for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 				if (peer->_paths[i].p && peer->_paths[i].p->alive(now)) {
-					_sendViaSpecificPath(tPtr,peer,peer->_paths[i].p,now,packet,encrypt,flowId);
+					uint16_t userSpecifiedMtu = peer->_paths[i].p->mtu();
+					_sendViaSpecificPath(tPtr,peer,peer->_paths[i].p, userSpecifiedMtu,now,packet,encrypt,flowId);
 				}
 			}
 			return true;
@@ -1025,7 +1026,8 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
 				}
 			}
 			if (viaPath) {
-				_sendViaSpecificPath(tPtr,peer,viaPath,now,packet,encrypt,flowId);
+				uint16_t userSpecifiedMtu = viaPath->mtu();
+				_sendViaSpecificPath(tPtr,peer,viaPath,userSpecifiedMtu,now,packet,encrypt,flowId);
 				return true;
 			}
 		}
@@ -1033,12 +1035,15 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
 	return false;
 }
 
-void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,int64_t now,Packet &packet,bool encrypt,int32_t flowId)
+void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,uint16_t userSpecifiedMtu, int64_t now,Packet &packet,bool encrypt,int32_t flowId)
 {
 	unsigned int mtu = ZT_DEFAULT_PHYSMTU;
 	uint64_t trustedPathId = 0;
 	RR->topology->getOutboundPathInfo(viaPath->address(),mtu,trustedPathId);
 
+	if (userSpecifiedMtu > 0) {
+		mtu = userSpecifiedMtu;
+	}
 	unsigned int chunkSize = std::min(packet.size(),mtu);
 	packet.setFragmented(chunkSize < packet.size());
 

node/Switch.hpp

@@ -207,7 +207,7 @@ public:
 private:
 	bool _shouldUnite(const int64_t now,const Address &source,const Address &destination);
 	bool _trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId = ZT_QOS_NO_FLOW); // packet is modified if return is true
-	void _sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,int64_t now,Packet &packet,bool encrypt,int32_t flowId);
+	void _sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path> viaPath,uint16_t userSpecifiedMtu, int64_t now,Packet &packet,bool encrypt,int32_t flowId);
 
 	const RuntimeEnvironment *const RR;
 	int64_t _lastBeaconResponse;

service/OneService.cpp

@@ -2075,6 +2075,7 @@ public:
 					bool enabled = OSUtils::jsonInt(link["enabled"],true);
 					uint32_t capacity = OSUtils::jsonInt(link["capacity"],0);
 					uint8_t ipvPref = OSUtils::jsonInt(link["ipvPref"],0);
+					uint16_t mtu = OSUtils::jsonInt(link["mtu"],0);
 					std::string failoverToStr(OSUtils::jsonString(link["failoverTo"],""));
 					// Mode
 					std::string linkModeStr(OSUtils::jsonString(link["mode"],"spare"));
@@ -2091,7 +2092,7 @@ public:
 						failoverToStr = "";
 						enabled = false;
 					}
-					_node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,capacity,enabled,linkMode,failoverToStr));
+					_node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,mtu,capacity,enabled,linkMode,failoverToStr));
 				}
 				std::string linkSelectMethodStr(OSUtils::jsonString(customPolicy["activeReselect"],"optimize"));
 				if (linkSelectMethodStr == "always") {