Default route override and route management in general now works in Linux.

osdep/Binder.hpp

@@ -38,12 +38,17 @@
 #include <sys/wait.h>
 #include <unistd.h>
 #include <ifaddrs.h>
+#ifdef __LINUX__
+#include <sys/ioctl.h>
+#include <net/if.h>
+#endif
 #endif
 
 #include <string>
 #include <vector>
 #include <algorithm>
 #include <utility>
+#include <map>
 
 #include "../node/NonCopyable.hpp"
 #include "../node/InetAddress.hpp"
@@ -124,14 +129,9 @@ public:
 	template<typename PHY_HANDLER_TYPE,typename INTERFACE_CHECKER>
 	void refresh(Phy<PHY_HANDLER_TYPE> &phy,unsigned int port,INTERFACE_CHECKER &ifChecker)
 	{
-		std::vector<InetAddress> localIfAddrs;
+		std::map<InetAddress,std::string> localIfAddrs;
-		std::vector<_Binding> newBindings;
-		std::vector<std::string>::const_iterator si;
-		std::vector<InetAddress>::const_iterator ii;
-		typename std::vector<_Binding>::const_iterator bi;
 		PhySocket *udps;
 		//PhySocket *tcps;
-		InetAddress ip;
 		Mutex::Lock _l(_lock);
 
 #ifdef __WINDOWS__
@@ -149,11 +149,10 @@ public:
 							default: break;
 							case InetAddress::IP_SCOPE_PSEUDOPRIVATE:
 							case InetAddress::IP_SCOPE_GLOBAL:
-							//case InetAddress::IP_SCOPE_LINK_LOCAL:
 							case InetAddress::IP_SCOPE_SHARED:
 							case InetAddress::IP_SCOPE_PRIVATE:
 								ip.setPort(port);
-								localIfAddrs.push_back(ip);
+								localIfAddrs.insert(std::pair<InetAddress,std::string>(ip,std::string()));
 								break;
 						}
 					}
@@ -171,17 +170,16 @@ public:
 			ifa = ifatbl;
 			while (ifa) {
 				if ((ifa->ifa_name)&&(ifa->ifa_addr)) {
-					ip = *(ifa->ifa_addr);
+					InetAddress ip = *(ifa->ifa_addr);
 					if (ifChecker.shouldBindInterface(ifa->ifa_name,ip)) {
 						switch(ip.ipScope()) {
 							default: break;
 							case InetAddress::IP_SCOPE_PSEUDOPRIVATE:
 							case InetAddress::IP_SCOPE_GLOBAL:
-							//case InetAddress::IP_SCOPE_LINK_LOCAL:
 							case InetAddress::IP_SCOPE_SHARED:
 							case InetAddress::IP_SCOPE_PRIVATE:
 								ip.setPort(port);
-								localIfAddrs.push_back(ip);
+								localIfAddrs.insert(std::pair<InetAddress,std::string>(ip,std::string(ifa->ifa_name)));
 								break;
 						}
 					}
@@ -194,38 +192,52 @@ public:
 #endif
 
 		// Default to binding to wildcard if we can't enumerate addresses
-		if (localIfAddrs.size() == 0) {
+		if (localIfAddrs.empty()) {
-			localIfAddrs.push_back(InetAddress((uint32_t)0,port));
+			localIfAddrs.insert(std::pair<InetAddress,std::string>(InetAddress((uint32_t)0,port),std::string()));
-			localIfAddrs.push_back(InetAddress((const void *)"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0",16,port));
+			localIfAddrs.insert(std::pair<InetAddress,std::string>(InetAddress((const void *)"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0",16,port),std::string()));
 		}
 
 		// Close any old bindings to anything that doesn't exist anymore
-		for(bi=_bindings.begin();bi!=_bindings.end();++bi) {
+		for(typename std::vector<_Binding>::const_iterator bi(_bindings.begin());bi!=_bindings.end();++bi) {
-			if (std::find(localIfAddrs.begin(),localIfAddrs.end(),bi->address) == localIfAddrs.end()) {
+			if (localIfAddrs.find(bi->address) == localIfAddrs.end()) {
 				phy.close(bi->udpSock,false);
 				phy.close(bi->tcpListenSock,false);
 			}
 		}
 
-		for(ii=localIfAddrs.begin();ii!=localIfAddrs.end();++ii) {
+		std::vector<_Binding> newBindings;
-			// Copy over bindings that still are valid
+		for(std::map<InetAddress,std::string>::const_iterator ii(localIfAddrs.begin());ii!=localIfAddrs.end();++ii) {
-			for(bi=_bindings.begin();bi!=_bindings.end();++bi) {
+			typename std::vector<_Binding>::const_iterator bi(_bindings.begin());
-				if (bi->address == *ii) {
+			while (bi != _bindings.end()) {
+				if (bi->address == ii->first) {
 					newBindings.push_back(*bi);
 					break;
 				}
+				++bi;
 			}
 
-			// Add new bindings
 			if (bi == _bindings.end()) {
-				udps = phy.udpBind(reinterpret_cast<const struct sockaddr *>(&(*ii)),(void *)0,ZT_UDP_DESIRED_BUF_SIZE);
+				udps = phy.udpBind(reinterpret_cast<const struct sockaddr *>(&(ii->first)),(void *)0,ZT_UDP_DESIRED_BUF_SIZE);
 				if (udps) {
 					//tcps = phy.tcpListen(reinterpret_cast<const struct sockaddr *>(&ii),(void *)0);
 					//if (tcps) {
+#ifdef __LINUX__
+						// Bind Linux sockets to their device so routes tha we manage do not override physical routes (wish all platforms had this!)
+						if (ii->second.length() > 0) {
+							int fd = (int)Phy<PHY_HANDLER_TYPE>::getDescriptor(udps);
+							char tmp[256];
+							Utils::scopy(tmp,sizeof(tmp),ii->second.c_str());
+							if (fd >= 0) {
+								if (setsockopt(fd,SOL_SOCKET,SO_BINDTODEVICE,tmp,strlen(tmp)) != 0) {
+									fprintf(stderr,"WARNING: unable to set SO_BINDTODEVICE to bind %s to %s\n",ii->first.toIpString().c_str(),ii->second.c_str());
+								}
+							}
+						}
+#endif // __LINUX__
 						newBindings.push_back(_Binding());
 						newBindings.back().udpSock = udps;
 						//newBindings.back().tcpListenSock = tcps;
-						newBindings.back().address = *ii;
+						newBindings.back().address = ii->first;
 					//} else {
 					//	phy.close(udps,false);
 					//}
@@ -233,12 +245,6 @@ public:
 			}
 		}
 
-		/*
-		for(bi=newBindings.begin();bi!=newBindings.end();++bi) {
-			printf("Binder: bound to %s\n",bi->address.toString().c_str());
-		}
-		*/
-
 		// Swapping pointers and then letting the old one fall out of scope is faster than copying again
 		_bindings.swap(newBindings);
 	}

osdep/ManagedRoute.cpp

@@ -55,6 +55,7 @@
 
 #define ZT_BSD_ROUTE_CMD "/sbin/route"
 #define ZT_LINUX_IP_COMMAND "/sbin/ip"
+#define ZT_LINUX_IP_COMMAND_2 "/usr/sbin/ip"
 
 namespace ZeroTier {
 
@@ -258,6 +259,26 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress
 #ifdef __LINUX__ // ----------------------------------------------------------
 #define ZT_ROUTING_SUPPORT_FOUND 1
 
+static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *localInterface)
+{
+	long p = (long)fork();
+	if (p > 0) {
+		int exitcode = -1;
+		::waitpid(p,&exitcode,0);
+	} else if (p == 0) {
+		::close(STDOUT_FILENO);
+		::close(STDERR_FILENO);
+		if (via) {
+			::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"via",via.toIpString().c_str(),(const char *)0);
+			::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"via",via.toIpString().c_str(),(const char *)0);
+		} else if ((localInterface)&&(localInterface[0])) {
+			::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"dev",localInterface,(const char *)0);
+			::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString().c_str(),"dev",localInterface,(const char *)0);
+		}
+		::_exit(-1);
+	}
+}
+
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------
@@ -271,6 +292,17 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress
 
 } // anonymous namespace
 
+/* Linux NOTE: for default route override, some Linux distributions will
+ * require a change to the rp_filter parameter.
+ *
+ * sudo sysctl net.ipv4.conf.all.rp_filter=2
+ *
+ * Add to /etc/sysctl.conf or /etc/sysctl.d/... to make permanent.
+ *
+ * This is true of CentOS/RHEL 6+ and possibly others. This is because
+ * Linux default route override implies asymmetric routes, which then
+ * trigger Linux's "martian packet" filter. */
+
 bool ManagedRoute::sync()
 {
 	if ((_target.isDefaultRoute())||((_target.ss_family == AF_INET)&&(_target.netmaskBits() < 32))) {
@@ -357,6 +389,12 @@ bool ManagedRoute::sync()
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 
+		if (!_applied) {
+			_routeCmd("replace",leftt,_via,(_via) ? _device : (const char *)0);
+			_routeCmd("replace",rightt,_via,(_via) ? _device : (const char *)0);
+			_applied = true;
+		}
+
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------
@@ -364,8 +402,6 @@ bool ManagedRoute::sync()
 #endif // __WINDOWS__ --------------------------------------------------------
 
 	} else {
-		/* For non-default routes, IPv4 /32, and IPv6 non-default routes, we just
-		 * add the route itself. */
 
 #ifdef __BSD__ // ------------------------------------------------------------
 
@@ -398,14 +434,7 @@ bool ManagedRoute::sync()
 void ManagedRoute::remove()
 {
 	if (_applied) {
-		if (_target.isDefaultRoute()) {
+		if ((_target.isDefaultRoute())||((_target.ss_family == AF_INET)&&(_target.netmaskBits() < 32))) {
-			/* In ZeroTier we use a forked-route trick to override the default
-			* with a more specific one while leaving the original system route
-			* intact. We also create a shadow more specific route to the
-			* original gateway that is device-bound so that ZeroTier's device
-			* bound ports go via the physical Internet link. This has to be
-			* done *slightly* differently on different platforms. */
-
 			InetAddress leftt,rightt;
 			_forkTarget(_target,leftt,rightt);
 
@@ -415,7 +444,6 @@ void ManagedRoute::remove()
 				_routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0);
 				_routeCmd("delete",rightt,_systemVia,_systemDevice,(const char *)0);
 			}
-
 			if (_via) {
 				_routeCmd("delete",leftt,_via,(const char *)0,(const char *)0);
 				_routeCmd("delete",rightt,_via,(const char *)0,(const char *)0);
@@ -428,6 +456,9 @@ void ManagedRoute::remove()
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 
+			_routeCmd("del",leftt,_via,(_via) ? _device : (const char *)0);
+			_routeCmd("del",rightt,_via,(_via) ? _device : (const char *)0);
+
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------
@@ -448,6 +479,8 @@ void ManagedRoute::remove()
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 
+			_routeCmd("del",_target,_via,(_via) ? _device : (const char *)0);
+
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------