From fe5aad3cef526d8e968747d8653ef7c9d6221492 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Oct 2020 10:40:34 -0400 Subject: [PATCH 01/35] Some Mac feth tap changes to hopefully fix slow-leave problem. --- osdep/MacEthernetTap.cpp | 54 ++++++++++++++++++++++++++++++++----- osdep/MacEthernetTap.hpp | 1 + osdep/MacEthernetTapAgent.c | 8 +++--- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/osdep/MacEthernetTap.cpp b/osdep/MacEthernetTap.cpp index 1e538e883..a1d68c5ae 100644 --- a/osdep/MacEthernetTap.cpp +++ b/osdep/MacEthernetTap.cpp @@ -79,6 +79,7 @@ MacEthernetTap::MacEthernetTap( _homePath(homePath), _mtu(mtu), _metric(metric), + _devNo(0), _agentStdin(-1), _agentStdout(-1), _agentStderr(-1), @@ -97,7 +98,7 @@ MacEthernetTap::MacEthernetTap( agentPath.push_back(ZT_PATH_SEPARATOR); agentPath.append("MacEthernetTapAgent"); if (!OSUtils::fileExists(agentPath.c_str())) - throw std::runtime_error("MacEthernetTapAgent not installed in ZeroTier home"); + throw std::runtime_error("MacEthernetTapAgent not present in ZeroTier home"); Mutex::Lock _gl(globalTapCreateLock); // only make one at a time @@ -112,7 +113,7 @@ MacEthernetTap::MacEthernetTap( while (p) { int nameLen = (int)strlen(p->ifa_name); // Delete feth# from feth0 to feth9999, but don't touch >10000. - if ((!strncmp(p->ifa_name,"feth",4))&&(nameLen >= 5)&&(nameLen < 9)&&(deleted.count(std::string(p->ifa_name)) == 0)) { + if ((!strncmp(p->ifa_name,"feth",4))&&(nameLen >= 5)&&(nameLen <= 8)&&(deleted.count(std::string(p->ifa_name)) == 0)) { deleted.insert(std::string(p->ifa_name)); const char *args[4]; args[0] = "/sbin/ifconfig"; @@ -156,10 +157,11 @@ MacEthernetTap::MacEthernetTap( if (devNo < 100) devNo = 100; } else { + _dev = devstr; + _devNo = devNo; break; } } - _dev = devstr; if (::pipe(_shutdownSignalPipe)) throw std::runtime_error("pipe creation failed"); @@ -204,22 +206,60 @@ MacEthernetTap::MacEthernetTap( MacEthernetTap::~MacEthernetTap() { + char tmp[64]; + const char *args[4]; + pid_t pid0,pid1; + MacDNSHelper::removeDNS(_nwid); - + Mutex::Lock _gl(globalTapCreateLock); ::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit - Thread::join(_thread); ::close(_shutdownSignalPipe[0]); ::close(_shutdownSignalPipe[1]); + int ec = 0; - ::kill(_agentPid,SIGTERM); - ::waitpid(_agentPid,&ec,0); + ::kill(_agentPid,SIGKILL); + ::close(_agentStdin); ::close(_agentStdout); ::close(_agentStderr); ::close(_agentStdin2); ::close(_agentStdout2); ::close(_agentStderr2); + + ::waitpid(_agentPid,&ec,0); + + args[0] = "/sbin/ifconfig"; + args[1] = _dev.c_str(); + args[2] = "destroy"; + args[3] = (char *)0; + pid0 = vfork(); + if (pid0 == 0) { + execv(args[0],const_cast(args)); + _exit(-1); + } + + snprintf(tmp,sizeof(tmp),"feth%u",_devNo + 5000); + //args[0] = "/sbin/ifconfig"; + args[1] = tmp; + //args[2] = "destroy"; + //args[3] = (char *)0; + pid1 = vfork(); + if (pid1 == 0) { + execv(args[0],const_cast(args)); + _exit(-1); + } + + if (pid0 > 0) { + int rv = 0; + waitpid(pid0,&rv,0); + } + if (pid1 > 0) { + int rv = 0; + waitpid(pid1,&rv,0); + } + + Thread::join(_thread); } void MacEthernetTap::setEnabled(bool en) { _enabled = en; } diff --git a/osdep/MacEthernetTap.hpp b/osdep/MacEthernetTap.hpp index c9f9a3b22..4b02999b9 100644 --- a/osdep/MacEthernetTap.hpp +++ b/osdep/MacEthernetTap.hpp @@ -72,6 +72,7 @@ private: Mutex _putLock; unsigned int _mtu; unsigned int _metric; + unsigned int _devNo; int _shutdownSignalPipe[2]; int _agentStdin,_agentStdout,_agentStderr,_agentStdin2,_agentStdout2,_agentStderr2; long _agentPid; diff --git a/osdep/MacEthernetTapAgent.c b/osdep/MacEthernetTapAgent.c index 361018e28..3a44eadd2 100644 --- a/osdep/MacEthernetTapAgent.c +++ b/osdep/MacEthernetTapAgent.c @@ -29,13 +29,13 @@ * is limited to 2048. AF_NDRV packet injection is required to inject * ZeroTier's large MTU frames. * - * Benchmarks show that this performs similarly to the old tap.kext driver, - * and a kext is no longer required. Splitting it off into an agent will - * also make it easier to have zerotier-one itself drop permissions. - * * All this stuff is basically undocumented. A lot of tracing through * the Darwin/XNU kernel source was required to figure out how to make * this actually work. + * + * We hope to develop a DriverKit-based driver in the near-mid future to + * replace this weird hack, but it works for now through Big Sur in our + * testing. * * See also: * From f2c490345c579ce2bae10f141156929f71d1fca3 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Oct 2020 16:14:59 -0400 Subject: [PATCH 02/35] Remove unnecessary includes that can cause compile problems. --- node/Constants.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/Constants.hpp b/node/Constants.hpp index 3a329b0db..70085d0c0 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -107,8 +107,6 @@ #include #include #include -#include -#include #endif #if (defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(ZT_ARCH_ARM_HAS_NEON)) From c9fe04d8ea9cd68733b39a64e1bb2449b97e19fc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 30 Oct 2020 17:42:15 -0400 Subject: [PATCH 03/35] Version bump in prep for 1.6.0 beta 2 --- version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.h b/version.h index 320c8c323..a5167c3dd 100644 --- a/version.h +++ b/version.h @@ -27,7 +27,7 @@ /** * Revision */ -#define ZEROTIER_ONE_VERSION_REVISION 0 +#define ZEROTIER_ONE_VERSION_REVISION 1 /** * Build version From 763d1cc3dc476af9e367aeba5b373e6a63f61e32 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 2 Nov 2020 14:00:35 -0500 Subject: [PATCH 04/35] Some more changes for slow network leaving on Mac. --- osdep/MacEthernetTap.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/osdep/MacEthernetTap.cpp b/osdep/MacEthernetTap.cpp index a1d68c5ae..6c8c7b9f0 100644 --- a/osdep/MacEthernetTap.cpp +++ b/osdep/MacEthernetTap.cpp @@ -214,19 +214,9 @@ MacEthernetTap::~MacEthernetTap() Mutex::Lock _gl(globalTapCreateLock); ::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit - ::close(_shutdownSignalPipe[0]); - ::close(_shutdownSignalPipe[1]); int ec = 0; ::kill(_agentPid,SIGKILL); - - ::close(_agentStdin); - ::close(_agentStdout); - ::close(_agentStderr); - ::close(_agentStdin2); - ::close(_agentStdout2); - ::close(_agentStderr2); - ::waitpid(_agentPid,&ec,0); args[0] = "/sbin/ifconfig"; @@ -496,6 +486,15 @@ void MacEthernetTap::threadMain() */ } } + + ::close(_agentStdin); + ::close(_agentStdout); + ::close(_agentStderr); + ::close(_agentStdin2); + ::close(_agentStdout2); + ::close(_agentStderr2); + ::close(_shutdownSignalPipe[0]); + ::close(_shutdownSignalPipe[1]); } void MacEthernetTap::setDns(const char *domain, const std::vector &servers) From 52679d2784882fb85886f5b96d439a62b343dc86 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 2 Nov 2020 19:02:07 -0500 Subject: [PATCH 05/35] Simplify and fix managed route logic. --- osdep/ManagedRoute.cpp | 23 ++++++++++++++ osdep/ManagedRoute.hpp | 24 ++------------- service/OneService.cpp | 68 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 88 insertions(+), 27 deletions(-) diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 3094c32d1..cf727fd95 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -388,6 +388,29 @@ static bool _winHasRoute(const NET_LUID &interfaceLuid, const NET_IFINDEX &inter } // anonymous namespace +ManagedRoute::ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device) +{ + _target = target; + _via = via; + _src = src; + if (via.ss_family == AF_INET) + _via.setPort(32); + else if (via.ss_family == AF_INET6) + _via.setPort(128); + if (src.ss_family == AF_INET) { + _src.setPort(32); + } else if (src.ss_family == AF_INET6) { + _src.setPort(128); + } + Utils::scopy(_device,sizeof(_device),device); + _systemDevice[0] = (char)0; +} + +ManagedRoute::~ManagedRoute() +{ + this->remove(); +} + /* Linux NOTE: for default route override, some Linux distributions will * require a change to the rp_filter parameter. A value of '1' will prevent * default route override from working properly. diff --git a/osdep/ManagedRoute.hpp b/osdep/ManagedRoute.hpp index 9cdf3f264..004aea48f 100644 --- a/osdep/ManagedRoute.hpp +++ b/osdep/ManagedRoute.hpp @@ -36,28 +36,8 @@ class ManagedRoute friend class SharedPtr; public: - ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device) - { - _target = target; - _via = via; - _src = src; - if (via.ss_family == AF_INET) - _via.setPort(32); - else if (via.ss_family == AF_INET6) - _via.setPort(128); - if (src.ss_family == AF_INET) { - _src.setPort(32); - } else if (src.ss_family == AF_INET6) { - _src.setPort(128); - } - Utils::scopy(_device,sizeof(_device),device); - _systemDevice[0] = (char)0; - } - - ~ManagedRoute() - { - this->remove(); - } + ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device); + ~ManagedRoute(); /** * Set or update currently set route diff --git a/service/OneService.cpp b/service/OneService.cpp index c37f09001..5b3ec7f29 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -529,7 +529,7 @@ public: std::shared_ptr tap; ZT_VirtualNetworkConfig config; // memcpy() of raw config from core std::vector managedIps; - std::list< SharedPtr > managedRoutes; + std::map< InetAddress, SharedPtr > managedRoutes; NetworkSettings settings; }; std::map _nets; @@ -1932,15 +1932,72 @@ public: } if (syncRoutes) { - char tapdev[64]; + // Get tap device name (use LUID in hex on Windows) and IP addresses. + char tapdevbuf[64]; + std::string tapdev; #if defined(__WINDOWS__) && !defined(ZT_SDK) - OSUtils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value); + OSUtils::ztsnprintf(tapdevbuf,sizeof(tapdevbuf),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value); + tapdev = tapdevbuf; #else - Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str()); + tapdev = n.tap->deviceName(); #endif - std::vector myIps(n.tap->ips()); + // Add routes not already added. + std::set haveRouteTargets; + for(unsigned int i=0;i(&(n.config.routes[i].target)); + const InetAddress *const via = reinterpret_cast(&(n.config.routes[i].via)); + + const InetAddress *src = NULL; + for (unsigned int j=0; j(&(n.config.assignedAddresses[j])); + if (target->isV4() && tmp->isV4()) { + src = reinterpret_cast(&(n.config.assignedAddresses[j])); + break; + } else if (target->isV6() && tmp->isV6()) { + src = reinterpret_cast(&(n.config.assignedAddresses[j])); + break; + } + } + if (!src) + src = &NULL_INET_ADDR; + + if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) ) + continue; + + // Ignore routes implied by local managed IPs since adding the IP adds the route +#ifndef __APPLE__ + bool haveRoute = false; + for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { + if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) { + haveRoute = true; + break; + } + } + if (haveRoute) + continue; +#endif + + haveRouteTargets.insert(*target); + +#ifndef ZT_SDK + SharedPtr &mr = n.managedRoutes[*target]; + if (!mr) + mr.set(new ManagedRoute(*target, *via, *src, tapdev.c_str())); + mr->sync(); +#endif + } + + // Destroy managed routes no longer in n.managedRoutes. + for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();) { + if (haveRouteTargets.find(r->first) == haveRouteTargets.end()) + n.managedRoutes.erase(r++); + else ++r; + } + + // This is the old logic for applying managed routes, and is kept around for now for documentation purposes. +#if 0 // Nuke applied routes that are no longer in n.config.routes[] and/or are not allowed for(std::list< SharedPtr >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) { bool haveRoute = false; @@ -2014,6 +2071,7 @@ public: n.managedRoutes.pop_back(); #endif } +#endif // end old managed route apply logic } if (syncDns) { From 317263b31c93f8fdf34502fadb7be561de3439d1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 2 Nov 2020 19:09:11 -0500 Subject: [PATCH 06/35] Remove unused variable on non-windows --- service/OneService.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/OneService.cpp b/service/OneService.cpp index 5b3ec7f29..7cf28d3fd 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1933,9 +1933,9 @@ public: if (syncRoutes) { // Get tap device name (use LUID in hex on Windows) and IP addresses. - char tapdevbuf[64]; std::string tapdev; #if defined(__WINDOWS__) && !defined(ZT_SDK) + char tapdevbuf[64]; OSUtils::ztsnprintf(tapdevbuf,sizeof(tapdevbuf),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value); tapdev = tapdevbuf; #else From 90f18f7ee741aca66579165b7807f27ed6a6d685 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 6 Nov 2020 11:01:45 -0500 Subject: [PATCH 07/35] Fix for ZTO-33 (Jira), only assign routes if there is a viable source IP. --- node/InetAddress.hpp | 46 ++++++++++++++++++++++++++++++++++++++++++ node/Utils.hpp | 16 +++++++++++++++ service/OneService.cpp | 22 ++++++++++---------- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp index 67f70d2a9..a9a35dd20 100644 --- a/node/InetAddress.hpp +++ b/node/InetAddress.hpp @@ -453,6 +453,52 @@ struct InetAddress : public sockaddr_storage */ bool isNetwork() const; + /** + * Find the total number of prefix bits that match between this IP and another + * + * @param b Second IP to compare with + * @return Number of matching prefix bits or 0 if none match or IPs are of different families (e.g. v4 and v6) + */ + inline unsigned int matchingPrefixBits(const InetAddress &b) const + { + unsigned int c = 0; + if (ss_family == b.ss_family) { + switch(ss_family) { + case AF_INET: { + uint32_t ip0 = Utils::ntoh((uint32_t)reinterpret_cast(this)->sin_addr.s_addr); + uint32_t ip1 = Utils::ntoh((uint32_t)reinterpret_cast(&b)->sin_addr.s_addr); + while ((ip0 >> 31) == (ip1 >> 31)) { + ip0 <<= 1; + ip1 <<= 1; + if (++c == 32) + break; + } + } break; + case AF_INET6: { + const uint8_t *ip0 = reinterpret_cast(reinterpret_cast(this)->sin6_addr.s6_addr); + const uint8_t *ip1 = reinterpret_cast(reinterpret_cast(&b)->sin6_addr.s6_addr); + for(unsigned int i=0;i<16;++i) { + if (ip0[i] == ip1[i]) { + c += 8; + } else { + uint8_t ip0b = ip0[i]; + uint8_t ip1b = ip1[i]; + uint8_t bit = 0x80; + while (bit != 0) { + if ((ip0b & bit) != (ip1b & bit)) + break; + ++c; + bit >>= 1; + } + break; + } + } + } break; + } + } + return c; + } + /** * @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP */ diff --git a/node/Utils.hpp b/node/Utils.hpp index ec898fc0a..685fdf591 100644 --- a/node/Utils.hpp +++ b/node/Utils.hpp @@ -94,6 +94,22 @@ public: static const CPUIDRegisters CPUID; #endif + /** + * Compute the log2 (most significant bit set) of a 32-bit integer + * + * @param v Integer to compute + * @return log2 or 0 if v is 0 + */ + static inline unsigned int log2(uint32_t v) + { + uint32_t r = (v > 0xffff) << 4; v >>= r; + uint32_t shift = (v > 0xff) << 3; v >>= shift; r |= shift; + shift = (v > 0xf) << 2; v >>= shift; r |= shift; + shift = (v > 0x3) << 1; v >>= shift; r |= shift; + r |= (v >> 1); + return (unsigned int)r; + } + /** * Perform a time-invariant binary comparison * diff --git a/service/OneService.cpp b/service/OneService.cpp index 7cf28d3fd..30f241196 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1949,21 +1949,21 @@ public: const InetAddress *const target = reinterpret_cast(&(n.config.routes[i].target)); const InetAddress *const via = reinterpret_cast(&(n.config.routes[i].via)); + // Make sure we are allowed to set this managed route. + if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) ) + continue; + + // Find an IP on the interface that can be a source IP, abort if no IPs assigned. const InetAddress *src = NULL; - for (unsigned int j=0; j(&(n.config.assignedAddresses[j])); - if (target->isV4() && tmp->isV4()) { - src = reinterpret_cast(&(n.config.assignedAddresses[j])); - break; - } else if (target->isV6() && tmp->isV6()) { - src = reinterpret_cast(&(n.config.assignedAddresses[j])); - break; + unsigned int mostMatchingPrefixBits = 0; + for(std::vector::const_iterator i(myIps.begin());i!=myIps.end();++i) { + const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target); + if (matchingPrefixBits >= mostMatchingPrefixBits) { + mostMatchingPrefixBits = matchingPrefixBits; + src = &(*i); } } if (!src) - src = &NULL_INET_ADDR; - - if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) ) continue; // Ignore routes implied by local managed IPs since adding the IP adds the route From dd65680150e592eb704ab7dd64dc3ba7e5a77714 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 6 Nov 2020 11:18:41 -0800 Subject: [PATCH 08/35] Windows build fixes, version bump in AIP. --- ext/installfiles/windows/ZeroTier One.aip | 54 +++++++++---------- node/AES_aesni.cpp | 22 ++++++++ windows/ZeroTierOne/ZeroTierOne.vcxproj | 4 +- .../ZeroTierOne/ZeroTierOne.vcxproj.filters | 6 +++ 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/ext/installfiles/windows/ZeroTier One.aip b/ext/installfiles/windows/ZeroTier One.aip index a5b4f4158..955248c4b 100644 --- a/ext/installfiles/windows/ZeroTier One.aip +++ b/ext/installfiles/windows/ZeroTier One.aip @@ -1,5 +1,5 @@ - + @@ -25,10 +25,10 @@ - + - + @@ -58,7 +58,7 @@ - + @@ -131,7 +131,7 @@ - + @@ -465,28 +465,28 @@ - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/AES_aesni.cpp b/node/AES_aesni.cpp index 6fe705ecf..a185b1b36 100644 --- a/node/AES_aesni.cpp +++ b/node/AES_aesni.cpp @@ -26,7 +26,9 @@ namespace { const __m128i s_sseSwapBytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul"))) +#endif __m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept { y = _mm_shuffle_epi8(y, s_sseSwapBytes); @@ -57,7 +59,9 @@ __m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept #define ZT_AES_VAES512 1 +#ifdef __GNUC__ __attribute__((__target__("sse4,aes,avx,avx2,vaes,avx512f,avx512bw"))) +#endif void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept { const __m512i kk0 = _mm512_broadcast_i32x4(k[0]); @@ -107,7 +111,9 @@ void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, co #define ZT_AES_VAES256 1 +#ifdef __GNUC__ __attribute__((__target__("sse4,aes,avx,avx2,vaes"))) +#endif void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept { const __m256i kk0 = _mm256_broadcastsi128_si256(k[0]); @@ -175,7 +181,9 @@ void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, co #endif // does compiler support AVX2 and AVX512 AES intrinsics? +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul"))) +#endif __m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept { __m128i x, y; @@ -190,7 +198,9 @@ __m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept return x; } +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul"))) +#endif __m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept { __m128i x, y, z; @@ -208,7 +218,9 @@ __m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept } // anonymous namespace +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul"))) +#endif void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept { __m128i y = _mm_loadu_si128(reinterpret_cast(_y)); @@ -274,7 +286,9 @@ void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept _rp = len; // len is always less than 16 here } +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul,aes"))) +#endif void AES::GMAC::p_aesNIFinish(uint8_t tag[16]) noexcept { __m128i y = _mm_loadu_si128(reinterpret_cast(_y)); @@ -345,7 +359,9 @@ void AES::GMAC::p_aesNIFinish(uint8_t tag[16]) noexcept _mm_storeu_si128(reinterpret_cast<__m128i *>(tag), _mm_xor_si128(_mm_shuffle_epi8(t4, s_sseSwapBytes), encIV)); } +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes"))) +#endif void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) noexcept { const __m128i dd = _mm_set_epi64x(0, (long long)_ctr[0]); @@ -542,7 +558,9 @@ void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) n _ctr[1] = Utils::hton(c1); } +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul"))) +#endif void AES::p_init_aesni(const uint8_t *key) noexcept { __m128i t1, t2, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13; @@ -604,7 +622,9 @@ void AES::p_init_aesni(const uint8_t *key) noexcept p_k.ni.h2[3] = _mm_xor_si128(_mm_shuffle_epi32(hhhh, 78), hhhh); } +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul"))) +#endif void AES::p_encrypt_aesni(const void *const in, void *const out) const noexcept { __m128i tmp = _mm_loadu_si128((const __m128i *)in); @@ -625,7 +645,9 @@ void AES::p_encrypt_aesni(const void *const in, void *const out) const noexcept _mm_storeu_si128((__m128i *)out, _mm_aesenclast_si128(tmp, p_k.ni.k[14])); } +#ifdef __GNUC__ __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul"))) +#endif void AES::p_decrypt_aesni(const void *in, void *out) const noexcept { __m128i tmp = _mm_loadu_si128((const __m128i *)in); diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj b/windows/ZeroTierOne/ZeroTierOne.vcxproj index 75dcad6ce..9b2b996ab 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj @@ -51,6 +51,8 @@ + + @@ -443,7 +445,7 @@ ZT_EXPORT;FD_SETSIZE=1024;STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="apply";ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;ZT_BUILD_PLATFORM=2;ZT_BUILD_ARCHITECTURE=2;%(PreprocessorDefinitions) MultiThreaded - NotSet + StreamingSIMDExtensions2 true AnySuitable Speed diff --git a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters index f474c19b5..cca0c5ce3 100644 --- a/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters +++ b/windows/ZeroTierOne/ZeroTierOne.vcxproj.filters @@ -285,6 +285,12 @@ Source Files\node + + Source Files\node + + + Source Files\node + From 83a2dcb65abad787413e9ad3485b6bb11decdc87 Mon Sep 17 00:00:00 2001 From: Vince Date: Sat, 7 Nov 2020 14:16:03 -0800 Subject: [PATCH 09/35] FreeBSD ARMv6, ARMv7, and Aarch64 support --- make-bsd.mk | 7 ++++--- node/Utils.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/make-bsd.mk b/make-bsd.mk index 62a6d73e6..da7a60125 100644 --- a/make-bsd.mk +++ b/make-bsd.mk @@ -80,16 +80,16 @@ ifeq ($(CC_MACH),armv6kz) endif ifeq ($(CC_MACH),armv7) ZT_ARCHITECTURE=3 - override DEFS+=-DZT_NO_TYPE_PUNNING + override DEFS+=-DZT_NO_TYPE_PUNNING -DZT_AES_NO_ACCEL ZT_USE_ARM32_NEON_ASM_SALSA2012=1 endif ifeq ($(CC_MACH),arm64) ZT_ARCHITECTURE=4 - override DEFS+=-DZT_NO_TYPE_PUNNING + override DEFS+=-DZT_NO_TYPE_PUNNING -march=armv8-a+crypto endif ifeq ($(CC_MACH),aarch64) ZT_ARCHITECTURE=4 - override DEFS+=-DZT_NO_TYPE_PUNNING + override DEFS+=-DZT_NO_TYPE_PUNNING -march=armv8-a+crypto endif ifeq ($(CC_MACH),mipsel) ZT_ARCHITECTURE=5 @@ -124,6 +124,7 @@ ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1) override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012 override CORE_OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o override ASFLAGS+=-meabi=5 + override LDFLAGS+=-Wl,-z,notext endif override DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\"" diff --git a/node/Utils.cpp b/node/Utils.cpp index 1acd5e1bf..091a2d5c9 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -50,6 +50,38 @@ #include #endif +#if defined(__FreeBSD__) +#include +#include +static inline long getauxval(int caps) { + long hwcaps = 0; + elf_aux_info(caps, &hwcaps, sizeof(hwcaps)); + return hwcaps; +} +#endif + +// If these are not even defined, then they're not supported at all +#ifndef HWCAP_AES +#define HWCAP_AES 0 +#endif + +#ifndef HWCAP_CRC32 +#define HWCAP_CRC32 0 +#endif + +#ifndef HWCAP_PMULL +#define HWCAP_PMULL 0 +#endif + +#ifndef HWCAP_SHA1 +#define HWCAP_SHA1 0 +#endif + +#ifndef HWCAP_SHA2 +#define HWCAP_SHA2 0 +#endif + + namespace ZeroTier { const uint64_t Utils::ZERO256[4] = {0ULL,0ULL,0ULL,0ULL}; From 7280fcdec29a201504ce047305074ab92016a9eb Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 9 Nov 2020 19:54:32 -0500 Subject: [PATCH 10/35] Only define FreeBSD hwcaps stuff if ARM crypto is enabled for compiled. --- node/Utils.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/node/Utils.cpp b/node/Utils.cpp index 091a2d5c9..4d32c9b3a 100644 --- a/node/Utils.cpp +++ b/node/Utils.cpp @@ -50,37 +50,42 @@ #include #endif +#ifdef ZT_ARCH_ARM_HAS_NEON + +#ifdef __LINUX__ +#include +#include +#endif + #if defined(__FreeBSD__) #include #include -static inline long getauxval(int caps) { +static inline long getauxval(int caps) +{ long hwcaps = 0; elf_aux_info(caps, &hwcaps, sizeof(hwcaps)); return hwcaps; } #endif -// If these are not even defined, then they're not supported at all +// If these are not even defined, obviously they are not supported. #ifndef HWCAP_AES #define HWCAP_AES 0 #endif - #ifndef HWCAP_CRC32 #define HWCAP_CRC32 0 #endif - #ifndef HWCAP_PMULL #define HWCAP_PMULL 0 #endif - #ifndef HWCAP_SHA1 #define HWCAP_SHA1 0 #endif - #ifndef HWCAP_SHA2 #define HWCAP_SHA2 0 #endif +#endif // ZT_ARCH_ARM_HAS_NEON namespace ZeroTier { From 53ba413d329f6449a0f0870c2e15a554db5d5793 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 9 Nov 2020 20:52:49 -0500 Subject: [PATCH 11/35] Dont use AES in HELLO. --- node/Peer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/Peer.cpp b/node/Peer.cpp index 3aa070e88..afdc657f8 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -409,7 +409,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA outp.cryptField(_key,startCryptedPortionAt,outp.size() - startCryptedPortionAt); if (atAddress) { - outp.armor(_key,false,aesKeysIfSupported()); // false == don't encrypt full payload, but add MAC + outp.armor(_key,false,nullptr); // false == don't encrypt full payload, but add MAC RR->node->expectReplyTo(outp.packetId()); RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); } else { From e9e20fdad89c4ce79489e85a63d45faf6bc0a97f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 10 Nov 2020 16:16:52 -0500 Subject: [PATCH 12/35] Increase multicast announce frequency a little. --- node/Constants.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/Constants.hpp b/node/Constants.hpp index 70085d0c0..6cd4aa709 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -186,6 +186,9 @@ */ #define ZT_ADDRESS_LENGTH_HEX 10 +/** + * Size of symmetric key (only the first 32 bits are used for some ciphers) + */ #define ZT_SYMMETRIC_KEY_SIZE 48 /** @@ -253,7 +256,7 @@ /** * Period for multicast LIKE announcements */ -#define ZT_MULTICAST_ANNOUNCE_PERIOD 120000 +#define ZT_MULTICAST_ANNOUNCE_PERIOD 60000 /** * Delay between explicit MULTICAST_GATHER requests for a given multicast channel From d2f2abe5252c5059dee5207e1b4614b24a8d4e96 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Tue, 10 Nov 2020 16:30:55 -0500 Subject: [PATCH 13/35] Add force-exit timeout to get around some of the exit hang issues users have observed. Will punt full diagnosis to V2 which rewrites all these code paths anyway. --- one.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/one.cpp b/one.cpp index c8a9dddf4..aba3bac2b 100644 --- a/one.cpp +++ b/one.cpp @@ -1480,8 +1480,13 @@ static int idtool(int argc,char **argv) static void _sighandlerHup(int sig) { } +static void _sighandlerReallyQuit(int sig) +{ + exit(0); +} static void _sighandlerQuit(int sig) { + alarm(5); // force exit after 5s OneService *s = zt1Service; if (s) s->terminate(); @@ -1873,7 +1878,7 @@ int main(int argc,char **argv) signal(SIGIO,SIG_IGN); signal(SIGUSR1,SIG_IGN); signal(SIGUSR2,SIG_IGN); - signal(SIGALRM,SIG_IGN); + signal(SIGALRM,&_sighandlerReallyQuit); signal(SIGINT,&_sighandlerQuit); signal(SIGTERM,&_sighandlerQuit); signal(SIGQUIT,&_sighandlerQuit); From 9480ff1f3789463f5bb522280450dcdec46060f4 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Nov 2020 11:46:09 -0500 Subject: [PATCH 14/35] Fix some timestamp signedness mismatches. --- node/Multicaster.cpp | 56 +++++++++++++++++++++----------------------- node/Multicaster.hpp | 6 ++--- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 3b48b799e..8556acc67 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -371,39 +371,37 @@ void Multicaster::send( void Multicaster::clean(int64_t now) { - { - Mutex::Lock _l(_groups_m); - Multicaster::Key *k = (Multicaster::Key *)0; - MulticastGroupStatus *s = (MulticastGroupStatus *)0; - Hashtable::Iterator mm(_groups); - while (mm.next(k,s)) { - for(std::list::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) { - if ((tx->expired(now))||(tx->atLimit())) - s->txQueue.erase(tx++); - else ++tx; - } + Mutex::Lock _l(_groups_m); + Multicaster::Key *k = (Multicaster::Key *)0; + MulticastGroupStatus *s = (MulticastGroupStatus *)0; + Hashtable::Iterator mm(_groups); + while (mm.next(k,s)) { + for(std::list::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) { + if ((tx->expired(now))||(tx->atLimit())) + s->txQueue.erase(tx++); + else ++tx; + } - unsigned long count = 0; - { - std::vector::iterator reader(s->members.begin()); - std::vector::iterator writer(reader); - while (reader != s->members.end()) { - if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) { - *writer = *reader; - ++writer; - ++count; - } - ++reader; + unsigned long count = 0; + { + std::vector::iterator reader(s->members.begin()); + std::vector::iterator writer(reader); + while (reader != s->members.end()) { + if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) { + *writer = *reader; + ++writer; + ++count; } + ++reader; } + } - if (count) { - s->members.resize(count); - } else if (s->txQueue.empty()) { - _groups.erase(*k); - } else { - s->members.clear(); - } + if (count) { + s->members.resize(count); + } else if (s->txQueue.empty()) { + _groups.erase(*k); + } else { + s->members.clear(); } } } diff --git a/node/Multicaster.hpp b/node/Multicaster.hpp index 55c09eb3b..5809c67fc 100644 --- a/node/Multicaster.hpp +++ b/node/Multicaster.hpp @@ -138,7 +138,7 @@ public: unsigned int len); /** - * Clean up and resort database + * Clean database * * @param RR Runtime environment * @param now Current time @@ -172,14 +172,14 @@ private: inline bool operator!=(const Address &a) const { return (address != a); } Address address; - uint64_t timestamp; // time of last notification + int64_t timestamp; // time of last notification }; struct MulticastGroupStatus { MulticastGroupStatus() : lastExplicitGather(0) {} - uint64_t lastExplicitGather; + int64_t lastExplicitGather; std::list txQueue; // pending outbound multicasts std::vector members; // members of this group }; From d735a1d04cf1d8236d531210e7361607d2774a13 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Nov 2020 12:49:56 -0500 Subject: [PATCH 15/35] Reorder some stuff in LinuxEthernetTap as possible workaround for MAC-set weirdness. --- osdep/LinuxEthernetTap.cpp | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index 1d6bc430a..f83688fd3 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -170,37 +170,17 @@ LinuxEthernetTap::LinuxEthernetTap( ::ioctl(_fd,TUNSETPERSIST,0); // valgrind may generate a false alarm here - // Open an arbitrary socket to talk to netlink int sock = socket(AF_INET,SOCK_DGRAM,0); if (sock <= 0) { ::close(_fd); throw std::runtime_error("unable to open netlink socket"); } - // Set MAC address - ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER; - mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6); - if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to configure TAP hardware (MAC) address"); - return; - } - - // Set MTU - ifr.ifr_ifru.ifru_mtu = (int)mtu; - if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to configure TAP MTU"); - } - if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { ::close(_fd); throw std::runtime_error("unable to set flags on file descriptor for TAP device"); } - /* Bring interface up */ if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { ::close(_fd); ::close(sock); @@ -213,6 +193,21 @@ LinuxEthernetTap::LinuxEthernetTap( throw std::runtime_error("unable to set TAP interface flags"); } + ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER; + mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6); + if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) { + ::close(_fd); + ::close(sock); + throw std::runtime_error("unable to configure TAP hardware (MAC) address"); + return; + } + ifr.ifr_ifru.ifru_mtu = (int)mtu; + if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) { + ::close(_fd); + ::close(sock); + throw std::runtime_error("unable to configure TAP MTU"); + } + ::close(sock); // Set close-on-exec so that devices cannot persist if we fork/exec for update From 2d489a8679c1103f1256d0ec1f1056883186269d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Nov 2020 14:42:56 -0500 Subject: [PATCH 16/35] Another possible workaround for what seems to be a Linux bug in some newer kernels. --- osdep/LinuxEthernetTap.cpp | 61 +++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index f83688fd3..859d0673b 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -166,31 +166,39 @@ LinuxEthernetTap::LinuxEthernetTap( throw std::runtime_error("unable to configure TUN/TAP device for TAP operation"); } - _dev = ifr.ifr_name; - ::ioctl(_fd,TUNSETPERSIST,0); // valgrind may generate a false alarm here - int sock = socket(AF_INET,SOCK_DGRAM,0); + const int sock = socket(AF_INET,SOCK_DGRAM,0); if (sock <= 0) { ::close(_fd); throw std::runtime_error("unable to open netlink socket"); } - if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { - ::close(_fd); - throw std::runtime_error("unable to set flags on file descriptor for TAP device"); - } + _dev = ifr.ifr_name; - if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to get TAP interface flags"); - } - ifr.ifr_flags |= IFF_UP; - if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to set TAP interface flags"); + // Set/check loop is a workaround for a weird likely kernel bug in which + // the interface doesn't come up right away when set to up. This causes + // settings like the MAC address to not "take." + for(;;) { + if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { + ::close(_fd); + ::close(sock); + throw std::runtime_error("unable to get TAP interface flags"); + } + ifr.ifr_flags |= IFF_UP; + if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) { + ::close(_fd); + ::close(sock); + throw std::runtime_error("unable to bring up TAP interface"); + } + if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { + ::close(_fd); + ::close(sock); + throw std::runtime_error("unable to get TAP interface flags"); + } + usleep(1000); + if ((ifr.ifr_flags & IFF_UP) != 0) + break; } ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER; @@ -201,6 +209,7 @@ LinuxEthernetTap::LinuxEthernetTap( throw std::runtime_error("unable to configure TAP hardware (MAC) address"); return; } + ifr.ifr_ifru.ifru_mtu = (int)mtu; if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) { ::close(_fd); @@ -208,6 +217,11 @@ LinuxEthernetTap::LinuxEthernetTap( throw std::runtime_error("unable to configure TAP MTU"); } + if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { + ::close(_fd); + throw std::runtime_error("unable to set flags on file descriptor for TAP device"); + } + ::close(sock); // Set close-on-exec so that devices cannot persist if we fork/exec for update @@ -215,19 +229,6 @@ LinuxEthernetTap::LinuxEthernetTap( (void)::pipe(_shutdownSignalPipe); - /* - globalDeviceMap[nwids] = _dev; - devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w"); - if (devmapf) { - gdmEntry = globalDeviceMap.begin(); - while (gdmEntry != globalDeviceMap.end()) { - fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str()); - ++gdmEntry; - } - fclose(devmapf); - } - */ - _thread = Thread::start(this); } From 36e167013a8e8aca0223dddc3d177734c65c470e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Nov 2020 15:12:16 -0500 Subject: [PATCH 17/35] More Linux tap shotgun debugging. --- osdep/LinuxEthernetTap.cpp | 105 +++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 56 deletions(-) diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index 859d0673b..360147eb1 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -167,63 +167,8 @@ LinuxEthernetTap::LinuxEthernetTap( } ::ioctl(_fd,TUNSETPERSIST,0); // valgrind may generate a false alarm here - - const int sock = socket(AF_INET,SOCK_DGRAM,0); - if (sock <= 0) { - ::close(_fd); - throw std::runtime_error("unable to open netlink socket"); - } - _dev = ifr.ifr_name; - // Set/check loop is a workaround for a weird likely kernel bug in which - // the interface doesn't come up right away when set to up. This causes - // settings like the MAC address to not "take." - for(;;) { - if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to get TAP interface flags"); - } - ifr.ifr_flags |= IFF_UP; - if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to bring up TAP interface"); - } - if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to get TAP interface flags"); - } - usleep(1000); - if ((ifr.ifr_flags & IFF_UP) != 0) - break; - } - - ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER; - mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6); - if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to configure TAP hardware (MAC) address"); - return; - } - - ifr.ifr_ifru.ifru_mtu = (int)mtu; - if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) { - ::close(_fd); - ::close(sock); - throw std::runtime_error("unable to configure TAP MTU"); - } - - if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { - ::close(_fd); - throw std::runtime_error("unable to set flags on file descriptor for TAP device"); - } - - ::close(sock); - // Set close-on-exec so that devices cannot persist if we fork/exec for update ::fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC); @@ -460,7 +405,55 @@ void LinuxEthernetTap::threadMain() int n,nfds,r; char getBuf[ZT_MAX_MTU + 64]; - Thread::sleep(500); + Thread::sleep(100); + + { + struct ifreq ifr; + memset(&ifr,0,sizeof(ifr)); + + strcpy(ifr.ifr_name,_dev.c_str()); + + const int sock = socket(AF_INET,SOCK_DGRAM,0); + if (sock <= 0) + return; + + if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) { + ::close(sock); + printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n"); + return; + } + ifr.ifr_flags |= IFF_UP; + if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) { + ::close(sock); + printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n"); + return; + } + + Thread::sleep(500); + + ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER; + mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6); + if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) { + ::close(sock); + printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n"); + return; + } + + ifr.ifr_ifru.ifru_mtu = (int)mtu; + if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) { + ::close(sock); + printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n"); + return; + } + + if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) { + ::close(sock); + printf("WARNING: ioctl() failed setting up Linux tap device (set non-blocking)\n"); + return; + } + + ::close(sock); + } FD_ZERO(&readfds); FD_ZERO(&nullfds); From 59dd71187e7523dbfaf0d1ea33c15a828ed068f1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Nov 2020 15:14:08 -0500 Subject: [PATCH 18/35] Build fix. --- osdep/LinuxEthernetTap.cpp | 3 ++- osdep/LinuxEthernetTap.hpp | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index 360147eb1..5804745e3 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -80,6 +80,7 @@ LinuxEthernetTap::LinuxEthernetTap( _handler(handler), _arg(arg), _nwid(nwid), + _mac(mac), _homePath(homePath), _mtu(mtu), _fd(0), @@ -432,7 +433,7 @@ void LinuxEthernetTap::threadMain() Thread::sleep(500); ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER; - mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6); + _mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6); if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) { ::close(sock); printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n"); diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp index 0cef1cb9f..68fdf2461 100644 --- a/osdep/LinuxEthernetTap.hpp +++ b/osdep/LinuxEthernetTap.hpp @@ -63,6 +63,7 @@ private: void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int); void *_arg; uint64_t _nwid; + MAC _mac; Thread _thread; std::string _homePath; std::string _dev; From eadca9dd75dac9ddece04934222ed8cef74f7016 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 11 Nov 2020 15:29:01 -0500 Subject: [PATCH 19/35] Build fix. --- osdep/LinuxEthernetTap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp index 5804745e3..275443bd4 100644 --- a/osdep/LinuxEthernetTap.cpp +++ b/osdep/LinuxEthernetTap.cpp @@ -440,7 +440,7 @@ void LinuxEthernetTap::threadMain() return; } - ifr.ifr_ifru.ifru_mtu = (int)mtu; + ifr.ifr_ifru.ifru_mtu = (int)_mtu; if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) { ::close(sock); printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n"); From 1316ee3127ede4c81830ec3c15db7bbffdaf066e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Nov 2020 20:33:31 -0500 Subject: [PATCH 20/35] Push credentials in multicast. --- node/Switch.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/Switch.cpp b/node/Switch.cpp index b2040455b..7d81aca2c 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -481,6 +481,8 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const return; } + network->pushCredentialsIfNeeded(tPtr,toZT,RR->node->now()); + RR->mc->send( tPtr, RR->node->now(), From cc42d6f4e0b692eec5a8fd4d886958e59238e0f8 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Nov 2020 20:40:10 -0500 Subject: [PATCH 21/35] Build fix, and move multicast pushCredentialsIfNeeded. --- node/Multicaster.cpp | 9 +++++++++ node/Switch.cpp | 2 -- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 8556acc67..b6ae652de 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -247,6 +247,7 @@ void Multicaster::send( for(unsigned int i=0;iidentity.address())&&(activeBridges[i] != origin)) { + network->pushCredentialsIfNeeded(tPtr,activeBridges[i],RR->node->now()); out.sendOnly(RR,tPtr,activeBridges[i]); // optimization: don't use dedup log if it's a one-pass send if (++count >= limit) break; @@ -257,6 +258,7 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { const Address ma(gs.members[indexes[idx++]].address); if ((std::find(activeBridges,activeBridges + activeBridgeCount,ma) == (activeBridges + activeBridgeCount))&&(ma != origin)) { + network->pushCredentialsIfNeeded(tPtr,ma,RR->node->now()); out.sendOnly(RR,tPtr,ma); // optimization: don't use dedup log if it's a one-pass send ++count; } @@ -347,6 +349,7 @@ void Multicaster::send( for(unsigned int i=0;iidentity.address()) { + network->pushCredentialsIfNeeded(tPtr,activeBridges[i],RR->node->now()); out.sendAndLog(RR,tPtr,activeBridges[i]); if (++count >= limit) break; @@ -357,6 +360,7 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { Address ma(gs.members[indexes[idx++]].address); if (std::find(activeBridges,activeBridges + activeBridgeCount,ma) == (activeBridges + activeBridgeCount)) { + network->pushCredentialsIfNeeded(tPtr,ma,RR->node->now()); out.sendAndLog(RR,tPtr,ma); ++count; } @@ -414,6 +418,10 @@ void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup if (member == RR->identity.address()) return; + SharedPtr network(RR->node->network(nwid)); + if (!network) + return; + std::vector::iterator m(std::lower_bound(gs.members.begin(),gs.members.end(),member)); if (m != gs.members.end()) { if (m->address == member) { @@ -429,6 +437,7 @@ void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup if (tx->atLimit()) gs.txQueue.erase(tx++); else { + network->pushCredentialsIfNeeded(tPtr,member,RR->node->now()); tx->sendIfNew(RR,tPtr,member); if (tx->atLimit()) gs.txQueue.erase(tx++); diff --git a/node/Switch.cpp b/node/Switch.cpp index 7d81aca2c..b2040455b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -481,8 +481,6 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const return; } - network->pushCredentialsIfNeeded(tPtr,toZT,RR->node->now()); - RR->mc->send( tPtr, RR->node->now(), From 1a106bca3be9dd1cee8e890e4ec21916167168bc Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Nov 2020 20:53:14 -0500 Subject: [PATCH 22/35] Revert "Build fix, and move multicast pushCredentialsIfNeeded." This reverts commit cc42d6f4e0b692eec5a8fd4d886958e59238e0f8. --- node/Multicaster.cpp | 9 --------- node/Switch.cpp | 2 ++ 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index b6ae652de..8556acc67 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -247,7 +247,6 @@ void Multicaster::send( for(unsigned int i=0;iidentity.address())&&(activeBridges[i] != origin)) { - network->pushCredentialsIfNeeded(tPtr,activeBridges[i],RR->node->now()); out.sendOnly(RR,tPtr,activeBridges[i]); // optimization: don't use dedup log if it's a one-pass send if (++count >= limit) break; @@ -258,7 +257,6 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { const Address ma(gs.members[indexes[idx++]].address); if ((std::find(activeBridges,activeBridges + activeBridgeCount,ma) == (activeBridges + activeBridgeCount))&&(ma != origin)) { - network->pushCredentialsIfNeeded(tPtr,ma,RR->node->now()); out.sendOnly(RR,tPtr,ma); // optimization: don't use dedup log if it's a one-pass send ++count; } @@ -349,7 +347,6 @@ void Multicaster::send( for(unsigned int i=0;iidentity.address()) { - network->pushCredentialsIfNeeded(tPtr,activeBridges[i],RR->node->now()); out.sendAndLog(RR,tPtr,activeBridges[i]); if (++count >= limit) break; @@ -360,7 +357,6 @@ void Multicaster::send( while ((count < limit)&&(idx < gs.members.size())) { Address ma(gs.members[indexes[idx++]].address); if (std::find(activeBridges,activeBridges + activeBridgeCount,ma) == (activeBridges + activeBridgeCount)) { - network->pushCredentialsIfNeeded(tPtr,ma,RR->node->now()); out.sendAndLog(RR,tPtr,ma); ++count; } @@ -418,10 +414,6 @@ void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup if (member == RR->identity.address()) return; - SharedPtr network(RR->node->network(nwid)); - if (!network) - return; - std::vector::iterator m(std::lower_bound(gs.members.begin(),gs.members.end(),member)); if (m != gs.members.end()) { if (m->address == member) { @@ -437,7 +429,6 @@ void Multicaster::_add(void *tPtr,int64_t now,uint64_t nwid,const MulticastGroup if (tx->atLimit()) gs.txQueue.erase(tx++); else { - network->pushCredentialsIfNeeded(tPtr,member,RR->node->now()); tx->sendIfNew(RR,tPtr,member); if (tx->atLimit()) gs.txQueue.erase(tx++); diff --git a/node/Switch.cpp b/node/Switch.cpp index b2040455b..7d81aca2c 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -481,6 +481,8 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const return; } + network->pushCredentialsIfNeeded(tPtr,toZT,RR->node->now()); + RR->mc->send( tPtr, RR->node->now(), From 26a0cbcd73d01f72b7ba918ef2faaf27f29a9f60 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Nov 2020 20:53:23 -0500 Subject: [PATCH 23/35] Revert "Push credentials in multicast." This reverts commit 1316ee3127ede4c81830ec3c15db7bbffdaf066e. --- node/Switch.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/node/Switch.cpp b/node/Switch.cpp index 7d81aca2c..b2040455b 100644 --- a/node/Switch.cpp +++ b/node/Switch.cpp @@ -481,8 +481,6 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr &network,const return; } - network->pushCredentialsIfNeeded(tPtr,toZT,RR->node->now()); - RR->mc->send( tPtr, RR->node->now(), From 44af828aa42b129ccef5bec5d752ccb41d0828cf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 12 Nov 2020 22:21:43 -0500 Subject: [PATCH 24/35] Tweak multicast settings to prevent failures due to TX queue overflow. --- node/Constants.hpp | 2 +- node/Multicaster.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/node/Constants.hpp b/node/Constants.hpp index 6cd4aa709..3445e2613 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -224,7 +224,7 @@ /** * How often Topology::clean() and Network::clean() and similar are called, in ms */ -#define ZT_HOUSEKEEPING_PERIOD 60000 +#define ZT_HOUSEKEEPING_PERIOD 30000 /** * Delay between WHOIS retries in ms diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp index 8556acc67..4856b88ee 100644 --- a/node/Multicaster.cpp +++ b/node/Multicaster.cpp @@ -262,9 +262,8 @@ void Multicaster::send( } } } else { - if (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) { - RR->t->outgoingNetworkFrameDropped(tPtr,network,src,mg.mac(),etherType,0,len,"multicast TX queue is full"); - return; + while (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) { + gs.txQueue.pop_front(); } const unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1; From 020d882e82ff0cc3d6c4e0cf9e0cd4fc228dec3d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 16:02:59 -0500 Subject: [PATCH 25/35] Route assignment fix. --- service/OneService.cpp | 96 +++++------------------------------------- 1 file changed, 10 insertions(+), 86 deletions(-) diff --git a/service/OneService.cpp b/service/OneService.cpp index 30f241196..323dabad9 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1885,9 +1885,9 @@ public: } // Match only an IP from a vector of IPs -- used in syncManagedStuff() - bool matchIpOnly(const std::vector &ips,const InetAddress &ip) const + inline bool matchIpOnly(const std::set &ips,const InetAddress &ip) const { - for(std::vector::const_iterator i(ips.begin());i!=ips.end();++i) { + for(std::set::const_iterator i(ips.begin());i!=ips.end();++i) { if (i->ipsEqual(ip)) return true; } @@ -1933,17 +1933,19 @@ public: if (syncRoutes) { // Get tap device name (use LUID in hex on Windows) and IP addresses. - std::string tapdev; #if defined(__WINDOWS__) && !defined(ZT_SDK) char tapdevbuf[64]; OSUtils::ztsnprintf(tapdevbuf,sizeof(tapdevbuf),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value); - tapdev = tapdevbuf; + std::string tapdev(tapdevbuf); #else - tapdev = n.tap->deviceName(); + std::string tapdev(n.tap->deviceName()); #endif - std::vector myIps(n.tap->ips()); - // Add routes not already added. + std::vector tapIps(n.tap->ips()); + std::set myIps(tapIps.begin(), tapIps.end()); + for(unsigned int i=0;i haveRouteTargets; for(unsigned int i=0;i(&(n.config.routes[i].target)); @@ -1956,7 +1958,7 @@ public: // Find an IP on the interface that can be a source IP, abort if no IPs assigned. const InetAddress *src = NULL; unsigned int mostMatchingPrefixBits = 0; - for(std::vector::const_iterator i(myIps.begin());i!=myIps.end();++i) { + for(std::set::const_iterator i(myIps.begin());i!=myIps.end();++i) { const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target); if (matchingPrefixBits >= mostMatchingPrefixBits) { mostMatchingPrefixBits = matchingPrefixBits; @@ -1989,89 +1991,11 @@ public: #endif } - // Destroy managed routes no longer in n.managedRoutes. for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();) { if (haveRouteTargets.find(r->first) == haveRouteTargets.end()) n.managedRoutes.erase(r++); else ++r; } - - // This is the old logic for applying managed routes, and is kept around for now for documentation purposes. -#if 0 - // Nuke applied routes that are no longer in n.config.routes[] and/or are not allowed - for(std::list< SharedPtr >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) { - bool haveRoute = false; - if ( (checkIfManagedIsAllowed(n,(*mr)->target())) && (((*mr)->via().ss_family != (*mr)->target().ss_family)||(!matchIpOnly(myIps,(*mr)->via()))) ) { - for(unsigned int i=0;i(&(n.config.routes[i].target)); - const InetAddress *const via = reinterpret_cast(&(n.config.routes[i].via)); - if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (strcmp(tapdev,(*mr)->device())==0) ) ) { - haveRoute = true; - break; - } - } - } - if (haveRoute) { - ++mr; - } else { - n.managedRoutes.erase(mr++); - } - } - - // Apply routes in n.config.routes[] that we haven't applied yet, and sync those we have in case shadow routes need to change - for(unsigned int i=0;i(&(n.config.routes[i].target)); - const InetAddress *const via = reinterpret_cast(&(n.config.routes[i].via)); - - const InetAddress *src = NULL; - for (unsigned int j=0; j(&(n.config.assignedAddresses[j])); - if (target->isV4() && tmp->isV4()) { - src = reinterpret_cast(&(n.config.assignedAddresses[j])); - break; - } else if (target->isV6() && tmp->isV6()) { - src = reinterpret_cast(&(n.config.assignedAddresses[j])); - break; - } - } - if (!src) - src = &NULL_INET_ADDR; - - if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) ) - continue; - - bool haveRoute = false; - - // Ignore routes implied by local managed IPs since adding the IP adds the route -#ifndef __APPLE__ - for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { - if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) { - haveRoute = true; - break; - } - } -#endif - if (haveRoute) - continue; -#ifndef ZT_SDK - // If we've already applied this route, just sync it and continue - for(std::list< SharedPtr >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) { - if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (tapdev == (*mr)->device()) ) ) { - haveRoute = true; - (*mr)->sync(); - break; - } - } - if (haveRoute) - continue; - - // Add and apply new routes - n.managedRoutes.push_back(SharedPtr(new ManagedRoute(*target,*via,*src,tapdev))); - if (!n.managedRoutes.back()->sync()) - n.managedRoutes.pop_back(); -#endif - } -#endif // end old managed route apply logic } if (syncDns) { From 16a878adb1f2f7bd68918e501bb7e949d4a5ad4f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 16:18:00 -0500 Subject: [PATCH 26/35] ... route fix test --- service/OneService.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/service/OneService.cpp b/service/OneService.cpp index 323dabad9..e370fe374 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1968,19 +1968,6 @@ public: if (!src) continue; - // Ignore routes implied by local managed IPs since adding the IP adds the route -#ifndef __APPLE__ - bool haveRoute = false; - for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { - if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) { - haveRoute = true; - break; - } - } - if (haveRoute) - continue; -#endif - haveRouteTargets.insert(*target); #ifndef ZT_SDK From 6c5a0977115ab869a41d59c5928c24edc6e506f1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 18:16:14 -0500 Subject: [PATCH 27/35] Use LinuxNetLink for routes on Linux... still needs testing/debugging on an actual Linux machine. --- osdep/ManagedRoute.cpp | 29 ++++++++++++++++++++++------- service/OneService.cpp | 19 +++++++++++++++++-- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index cf727fd95..8bab2174c 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -49,6 +49,9 @@ #include #include "ManagedRoute.hpp" +#ifdef __LINUX__ +#include "LinuxNetLink.hpp" +#endif #define ZT_BSD_ROUTE_CMD "/sbin/route" #define ZT_LINUX_IP_COMMAND "/sbin/ip" @@ -269,6 +272,8 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress #ifdef __LINUX__ // ---------------------------------------------------------- #define ZT_ROUTING_SUPPORT_FOUND 1 +// This has been replaced by LinuxNetLink +/* static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *localInterface) { long p = (long)fork(); @@ -289,6 +294,7 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress ::_exit(-1); } } +*/ #endif // __LINUX__ ---------------------------------------------------------- @@ -393,15 +399,19 @@ ManagedRoute::ManagedRoute(const InetAddress &target,const InetAddress &via,cons _target = target; _via = via; _src = src; - if (via.ss_family == AF_INET) + + if (_via.ss_family == AF_INET) { _via.setPort(32); - else if (via.ss_family == AF_INET6) + } else if (_via.ss_family == AF_INET6) { _via.setPort(128); - if (src.ss_family == AF_INET) { + } + + if (_src.ss_family == AF_INET) { _src.setPort(32); - } else if (src.ss_family == AF_INET6) { + } else if (_src.ss_family == AF_INET6) { _src.setPort(128); } + Utils::scopy(_device,sizeof(_device),device); _systemDevice[0] = (char)0; } @@ -510,11 +520,15 @@ bool ManagedRoute::sync() if (!_applied.count(leftt)) { _applied[leftt] = false; // boolean unused - _routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().delRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().addRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); + //_routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device); } if ((rightt)&&(!_applied.count(rightt))) { _applied[rightt] = false; // boolean unused - _routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().delRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().addRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); + //_routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device); } #endif // __LINUX__ ---------------------------------------------------------- @@ -562,7 +576,8 @@ void ManagedRoute::remove() #endif // __BSD__ ------------------------------------------------------------ #ifdef __LINUX__ // ---------------------------------------------------------- - _routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device); + //_routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().delRoute(r->first,_via,_src,(_via) ? (const char *)0 : _device); #endif // __LINUX__ ---------------------------------------------------------- #ifdef __WINDOWS__ // -------------------------------------------------------- diff --git a/service/OneService.cpp b/service/OneService.cpp index e370fe374..3e899358c 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1951,12 +1951,13 @@ public: const InetAddress *const target = reinterpret_cast(&(n.config.routes[i].target)); const InetAddress *const via = reinterpret_cast(&(n.config.routes[i].via)); - // Make sure we are allowed to set this managed route. + // Make sure we are allowed to set this managed route, and that 'via' is not our IP. The latter + // avoids setting routes via the router on the router. if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) ) continue; // Find an IP on the interface that can be a source IP, abort if no IPs assigned. - const InetAddress *src = NULL; + const InetAddress *src = nullptr; unsigned int mostMatchingPrefixBits = 0; for(std::set::const_iterator i(myIps.begin());i!=myIps.end();++i) { const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target); @@ -1968,6 +1969,20 @@ public: if (!src) continue; + // Ignore routes implied by local managed IPs since adding the IP adds the route. + // Apple on the other hand seems to need this at least on some versions. +#ifndef __APPLE__ + bool haveRoute = false; + for(std::vector::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) { + if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) { + haveRoute = true; + break; + } + } + if (haveRoute) + continue; +#endif + haveRouteTargets.insert(*target); #ifndef ZT_SDK From e8be28734bccf10085fae9401adb1238b9c2c2bd Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 19:07:54 -0500 Subject: [PATCH 28/35] Linux netlink and route setting... work in progress. --- osdep/LinuxNetLink.cpp | 58 ++++++++++++++++++++++-------------------- osdep/ManagedRoute.cpp | 4 +-- service/OneService.cpp | 13 +++++++++- 3 files changed, 44 insertions(+), 31 deletions(-) diff --git a/osdep/LinuxNetLink.cpp b/osdep/LinuxNetLink.cpp index 4c6d21a87..8eff09229 100644 --- a/osdep/LinuxNetLink.cpp +++ b/osdep/LinuxNetLink.cpp @@ -13,6 +13,8 @@ #include "../node/Constants.hpp" +#define ZT_NETLINK_TRACE + #ifdef __LINUX__ #include "LinuxNetLink.hpp" @@ -85,7 +87,7 @@ void LinuxNetLink::_setSocketTimeout(int fd, int seconds) tv.tv_sec = seconds; tv.tv_usec = 0; if(setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)) != 0) { -#ifdef ZT_TRACE +#ifdef ZT_NETLINK_TRACE fprintf(stderr, "setsockopt failed: %s\n", strerror(errno)); #endif } @@ -119,8 +121,8 @@ int LinuxNetLink::_doRecv(int fd) if(nlp->nlmsg_type == NLMSG_ERROR && (nlp->nlmsg_flags & NLM_F_ACK) != NLM_F_ACK) { struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(nlp); if (err->error != 0) { -#ifdef ZT_TRACE - //fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error))); +#ifdef ZT_NETLINK_TRACE + fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error))); #endif } p = buf; @@ -145,7 +147,7 @@ int LinuxNetLink::_doRecv(int fd) } if (nlp->nlmsg_type == NLMSG_OVERRUN) { -//#ifdef ZT_TRACE +//#ifdef ZT_NETLINK_TRACE fprintf(stderr, "NLMSG_OVERRUN: Data lost\n"); //#endif p = buf; @@ -242,8 +244,8 @@ void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp) } } -#ifdef ZT_TRACE - //fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); +#ifdef ZT_NETLINK_TRACE + fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); #endif } @@ -276,8 +278,8 @@ void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp) } } -#ifdef ZT_TRACE - //fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); +#ifdef ZT_NETLINK_TRACE + fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); #endif } @@ -313,8 +315,8 @@ void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) } sprintf(ms, "%d", rtp->rtm_dst_len); -#ifdef ZT_TRACE - //fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); +#ifdef ZT_NETLINK_TRACE + fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); #endif } @@ -350,8 +352,8 @@ void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) } sprintf(ms, "%d", rtp->rtm_dst_len); -#ifdef ZT_TRACE - //fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); +#ifdef ZT_NETLINK_TRACE + fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); #endif } @@ -605,11 +607,11 @@ void LinuxNetLink::addRoute(const InetAddress &target, const InetAddress &via, c return; } -#ifdef ZT_TRACE - //char tmp[64]; - //char tmp2[64]; - //char tmp3[64]; - //fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName); +#ifdef ZT_NETLINK_TRACE + char tmp[64]; + char tmp2[64]; + char tmp3[64]; + fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName); #endif int rtl = sizeof(struct rtmsg); @@ -720,11 +722,11 @@ void LinuxNetLink::delRoute(const InetAddress &target, const InetAddress &via, c return; } -#ifdef ZT_TRACE - //char tmp[64]; - //char tmp2[64]; - //char tmp3[64]; - //fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName); +#ifdef ZT_NETLINK_TRACE + char tmp[64]; + char tmp2[64]; + char tmp3[64]; + fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName); #endif int rtl = sizeof(struct rtmsg); @@ -839,9 +841,9 @@ void LinuxNetLink::addAddress(const InetAddress &addr, const char *iface) return; } -#ifdef ZT_TRACE - //char tmp[128]; - //fprintf(stderr, "Adding IP address %s to interface %s", addr.toString(tmp), iface); +#ifdef ZT_NETLINK_TRACE + char tmp[128]; + fprintf(stderr, "Adding IP address %s to interface %s\n", addr.toString(tmp), iface); #endif int interface_index = _indexForInterface(iface); @@ -955,9 +957,9 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface) return; } -#ifdef ZT_TRACE - //char tmp[128]; - //fprintf(stderr, "Removing IP address %s from interface %s", addr.toString(tmp), iface); +#ifdef ZT_NETLINK_TRACE + char tmp[128]; + fprintf(stderr, "Removing IP address %s from interface %s\n", addr.toString(tmp), iface); #endif int interface_index = _indexForInterface(iface); diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index 8bab2174c..d64f4279e 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -520,13 +520,13 @@ bool ManagedRoute::sync() if (!_applied.count(leftt)) { _applied[leftt] = false; // boolean unused - LinuxNetLink::getInstance().delRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); + //LinuxNetLink::getInstance().delRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); LinuxNetLink::getInstance().addRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); //_routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device); } if ((rightt)&&(!_applied.count(rightt))) { _applied[rightt] = false; // boolean unused - LinuxNetLink::getInstance().delRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); + //LinuxNetLink::getInstance().delRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); LinuxNetLink::getInstance().addRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); //_routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device); } diff --git a/service/OneService.cpp b/service/OneService.cpp index 3e899358c..4a6669c26 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -1989,7 +1989,6 @@ public: SharedPtr &mr = n.managedRoutes[*target]; if (!mr) mr.set(new ManagedRoute(*target, *via, *src, tapdev.c_str())); - mr->sync(); #endif } @@ -1998,6 +1997,18 @@ public: n.managedRoutes.erase(r++); else ++r; } + + // Sync device-local managed routes first, then indirect results. That way + // we don't get destination unreachable for routes that are via things + // that do not yet have routes in the system. + for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { + if (!r->second->via()) + r->second->sync(); + } + for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { + if (r->second->via()) + r->second->sync(); + } } if (syncDns) { From bb62279a203bf02d7ff17c2fcc64fc08a152ce52 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 20:25:03 -0500 Subject: [PATCH 29/35] Queue up managed route syncs and do them periodically to avoid dependency problems. --- service/OneService.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/service/OneService.cpp b/service/OneService.cpp index 4a6669c26..f94b3af63 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -43,7 +43,6 @@ #include "../node/Peer.hpp" #include "../osdep/Phy.hpp" -#include "../osdep/Thread.hpp" #include "../osdep/OSUtils.hpp" #include "../osdep/Http.hpp" #include "../osdep/PortMapper.hpp" @@ -530,6 +529,7 @@ public: ZT_VirtualNetworkConfig config; // memcpy() of raw config from core std::vector managedIps; std::map< InetAddress, SharedPtr > managedRoutes; + std::list< InetAddress > routeSyncQueue; NetworkSettings settings; }; std::map _nets; @@ -918,8 +918,23 @@ public: OSUtils::cleanDirectory((_homePath + ZT_PATH_SEPARATOR_S "peers.d").c_str(),now - 2592000000LL); // delete older than 30 days } - const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 100; - clockShouldBe = now + (uint64_t)delay; + // Check to see if we have to sync any managed routes, and if so do it every 100ms to + // avoid route dependency problems. + { + Mutex::Lock nl(_nets_m); + for(std::map::iterator n(_nets.begin());n!=_nets.end();++n) { + if (!n->second.routeSyncQueue.empty()) { + std::map< InetAddress, SharedPtr >::const_iterator mr(n->second.managedRoutes.find(n->second.routeSyncQueue.front())); + if (mr != n->second.managedRoutes.end()) + mr->second->sync(); + n->second.routeSyncQueue.pop_front(); + dl = now + 100; + } + } + } + + const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 500; + clockShouldBe = now + (int64_t)delay; _phy.poll(delay); } } catch (std::exception &e) { @@ -2003,11 +2018,11 @@ public: // that do not yet have routes in the system. for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { if (!r->second->via()) - r->second->sync(); + n.routeSyncQueue.push_back(r->first); } for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { if (r->second->via()) - r->second->sync(); + n.routeSyncQueue.push_back(r->first); } } From 57516cfbe6f1b00cdd52ece9e861a8f6189232bf Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 21:01:55 -0500 Subject: [PATCH 30/35] revert queue... not quite what we need --- service/OneService.cpp | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/service/OneService.cpp b/service/OneService.cpp index f94b3af63..aef2584f7 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -529,7 +529,6 @@ public: ZT_VirtualNetworkConfig config; // memcpy() of raw config from core std::vector managedIps; std::map< InetAddress, SharedPtr > managedRoutes; - std::list< InetAddress > routeSyncQueue; NetworkSettings settings; }; std::map _nets; @@ -918,21 +917,6 @@ public: OSUtils::cleanDirectory((_homePath + ZT_PATH_SEPARATOR_S "peers.d").c_str(),now - 2592000000LL); // delete older than 30 days } - // Check to see if we have to sync any managed routes, and if so do it every 100ms to - // avoid route dependency problems. - { - Mutex::Lock nl(_nets_m); - for(std::map::iterator n(_nets.begin());n!=_nets.end();++n) { - if (!n->second.routeSyncQueue.empty()) { - std::map< InetAddress, SharedPtr >::const_iterator mr(n->second.managedRoutes.find(n->second.routeSyncQueue.front())); - if (mr != n->second.managedRoutes.end()) - mr->second->sync(); - n->second.routeSyncQueue.pop_front(); - dl = now + 100; - } - } - } - const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 500; clockShouldBe = now + (int64_t)delay; _phy.poll(delay); @@ -2018,11 +2002,11 @@ public: // that do not yet have routes in the system. for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { if (!r->second->via()) - n.routeSyncQueue.push_back(r->first); + n->second->sync(); } for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { if (r->second->via()) - n.routeSyncQueue.push_back(r->first); + n->second->sync(); } } From 3730917ddaa1303c7f62ef2240908da816f54242 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 21:06:34 -0500 Subject: [PATCH 31/35] LinuxNetLink cleanup --- osdep/LinuxNetLink.cpp | 29 +++---------- osdep/LinuxNetLink.hpp | 99 ++++++++++++++++++------------------------ 2 files changed, 49 insertions(+), 79 deletions(-) diff --git a/osdep/LinuxNetLink.cpp b/osdep/LinuxNetLink.cpp index 8eff09229..e101b407d 100644 --- a/osdep/LinuxNetLink.cpp +++ b/osdep/LinuxNetLink.cpp @@ -45,10 +45,6 @@ struct nl_adr_req { LinuxNetLink::LinuxNetLink() : _t() , _running(false) - , _routes_ipv4() - , _rv4_m() - , _routes_ipv6() - , _rv6_m() , _seq(0) , _interfaces() , _if_m() @@ -147,9 +143,9 @@ int LinuxNetLink::_doRecv(int fd) } if (nlp->nlmsg_type == NLMSG_OVERRUN) { -//#ifdef ZT_NETLINK_TRACE +#ifdef ZT_NETLINK_TRACE fprintf(stderr, "NLMSG_OVERRUN: Data lost\n"); -//#endif +#endif p = buf; nll = 0; break; @@ -175,11 +171,10 @@ int LinuxNetLink::_doRecv(int fd) void LinuxNetLink::threadMain() throw() { int rtn = 0; - while(_running) { rtn = _doRecv(_fd); if (rtn <= 0) { - Thread::sleep(100); + Thread::sleep(250); continue; } } @@ -217,6 +212,7 @@ void LinuxNetLink::_processMessage(struct nlmsghdr *nlp, int nll) void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp) { +#ifdef ZT_NETLINK_TRACE struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp); struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap); int ifal = IFA_PAYLOAD(nlp); @@ -244,13 +240,13 @@ void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp) } } -#ifdef ZT_NETLINK_TRACE fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); #endif } void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp) { +#ifdef ZT_NETLINK_TRACE struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp); struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap); int ifal = IFA_PAYLOAD(nlp); @@ -278,13 +274,13 @@ void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp) } } -#ifdef ZT_NETLINK_TRACE fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast); #endif } void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) { +#ifdef ZT_NETLINK_TRACE char dsts[40] = {0}; char gws[40] = {0}; char srcs[40] = {0}; @@ -315,13 +311,13 @@ void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) } sprintf(ms, "%d", rtp->rtm_dst_len); -#ifdef ZT_NETLINK_TRACE fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); #endif } void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) { +#ifdef ZT_NETLINK_TRACE char dsts[40] = {0}; char gws[40] = {0}; char srcs[40] = {0}; @@ -352,7 +348,6 @@ void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) } sprintf(ms, "%d", rtp->rtm_dst_len); -#ifdef ZT_NETLINK_TRACE fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); #endif } @@ -1045,16 +1040,6 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface) close(fd); } -RouteList LinuxNetLink::getIPV4Routes() const -{ - return _routes_ipv4; -} - -RouteList LinuxNetLink::getIPV6Routes() const -{ - return _routes_ipv6; -} - int LinuxNetLink::_indexForInterface(const char *iface) { Mutex::Lock l(_if_m); diff --git a/osdep/LinuxNetLink.hpp b/osdep/LinuxNetLink.hpp index 73c017736..c6c4be00a 100644 --- a/osdep/LinuxNetLink.hpp +++ b/osdep/LinuxNetLink.hpp @@ -35,84 +35,69 @@ namespace ZeroTier { -struct route_entry { - InetAddress target; - InetAddress via; - int if_index; - char iface[IFNAMSIZ]; -}; -typedef std::vector RouteList; - /** * Interface with Linux's RTNETLINK */ class LinuxNetLink { private: - LinuxNetLink(); - ~LinuxNetLink(); + LinuxNetLink(); + ~LinuxNetLink(); public: - static LinuxNetLink& getInstance() - { - static LinuxNetLink instance; - return instance; - } + static LinuxNetLink& getInstance() + { + static LinuxNetLink instance; + return instance; + } - LinuxNetLink(LinuxNetLink const&) = delete; - void operator=(LinuxNetLink const&) = delete; + LinuxNetLink(LinuxNetLink const&) = delete; + void operator=(LinuxNetLink const&) = delete; - void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName); - void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName); - RouteList getIPV4Routes() const; - RouteList getIPV6Routes() const; + void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName); + void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName); - void addAddress(const InetAddress &addr, const char *iface); - void removeAddress(const InetAddress &addr, const char *iface); + void addAddress(const InetAddress &addr, const char *iface); + void removeAddress(const InetAddress &addr, const char *iface); - void threadMain() throw(); + void threadMain() throw(); private: - int _doRecv(int fd); + int _doRecv(int fd); - void _processMessage(struct nlmsghdr *nlp, int nll); - void _routeAdded(struct nlmsghdr *nlp); - void _routeDeleted(struct nlmsghdr *nlp); - void _linkAdded(struct nlmsghdr *nlp); - void _linkDeleted(struct nlmsghdr *nlp); - void _ipAddressAdded(struct nlmsghdr *nlp); - void _ipAddressDeleted(struct nlmsghdr *nlp); + void _processMessage(struct nlmsghdr *nlp, int nll); + void _routeAdded(struct nlmsghdr *nlp); + void _routeDeleted(struct nlmsghdr *nlp); + void _linkAdded(struct nlmsghdr *nlp); + void _linkDeleted(struct nlmsghdr *nlp); + void _ipAddressAdded(struct nlmsghdr *nlp); + void _ipAddressDeleted(struct nlmsghdr *nlp); - void _requestInterfaceList(); - void _requestIPv4Routes(); - void _requestIPv6Routes(); + void _requestInterfaceList(); + void _requestIPv4Routes(); + void _requestIPv6Routes(); - int _indexForInterface(const char *iface); + int _indexForInterface(const char *iface); - void _setSocketTimeout(int fd, int seconds = 1); + void _setSocketTimeout(int fd, int seconds = 1); - Thread _t; - bool _running; + Thread _t; + bool _running; - RouteList _routes_ipv4; - Mutex _rv4_m; - RouteList _routes_ipv6; - Mutex _rv6_m; + uint32_t _seq; - uint32_t _seq; + struct iface_entry { + int index; + char ifacename[IFNAMSIZ]; + char mac[18]; + char mac_bin[6]; + unsigned int mtu; + }; + Hashtable _interfaces; + Mutex _if_m; - struct iface_entry { - int index; - char ifacename[IFNAMSIZ]; - char mac[18]; - char mac_bin[6]; - unsigned int mtu; - }; - Hashtable _interfaces; - Mutex _if_m; - - // socket communication vars; - int _fd; - struct sockaddr_nl _la; + // socket communication vars; + int _fd; + struct sockaddr_nl _la; }; } From 4e3a59f3293a5290ab17d30f619d32f45276800e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 21:21:28 -0500 Subject: [PATCH 32/35] Build fix. --- service/OneService.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/OneService.cpp b/service/OneService.cpp index aef2584f7..4c977c6f7 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -2002,11 +2002,11 @@ public: // that do not yet have routes in the system. for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { if (!r->second->via()) - n->second->sync(); + r->second->sync(); } for(std::map< InetAddress, SharedPtr >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) { if (r->second->via()) - n->second->sync(); + r->second->sync(); } } From a3875f996551c2d6d815254566f9b29565e6f71f Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 21:55:31 -0500 Subject: [PATCH 33/35] Add a route DB to LinuxNetLink to make route sync robust. --- osdep/LinuxNetLink.cpp | 141 ++++++++++++++++++++++++++++++++++++++--- osdep/LinuxNetLink.hpp | 43 +++++++++++++ 2 files changed, 174 insertions(+), 10 deletions(-) diff --git a/osdep/LinuxNetLink.cpp b/osdep/LinuxNetLink.cpp index e101b407d..f9defdcb3 100644 --- a/osdep/LinuxNetLink.cpp +++ b/osdep/LinuxNetLink.cpp @@ -280,7 +280,6 @@ void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp) void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) { -#ifdef ZT_NETLINK_TRACE char dsts[40] = {0}; char gws[40] = {0}; char srcs[40] = {0}; @@ -291,33 +290,84 @@ void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp); int rtl = RTM_PAYLOAD(nlp); + Route r; + bool wecare = false; + for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl)) { switch(rtap->rta_type) { case RTA_DST: - inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40); + switch(rtp->rtm_family) { + case AF_INET: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24); + r.target.set(RTA_DATA(rtap), 4, 0); + wecare = true; + break; + case AF_INET6: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24); + r.target.set(RTA_DATA(rtap), 16, 0); + wecare = true; + break; + } break; case RTA_SRC: - inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24: 40); + switch(rtp->rtm_family) { + case AF_INET: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24); + r.src.set(RTA_DATA(rtap), 4, 0); + wecare = true; + break; + case AF_INET6: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24); + r.src.set(RTA_DATA(rtap), 16, 0); + wecare = true; + break; + } break; case RTA_GATEWAY: - inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40); + switch(rtp->rtm_family) { + case AF_INET: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24); + r.via.set(RTA_DATA(rtap), 4, 0); + wecare = true; + break; + case AF_INET6: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24); + r.via.set(RTA_DATA(rtap), 16, 0); + wecare = true; + break; + } break; case RTA_OIF: + switch(rtp->rtm_family) { + case AF_INET: + r.ifidx = *((int*)RTA_DATA(rtap)); + wecare = true; + break; + case AF_INET6: + r.ifidx = *((int*)RTA_DATA(rtap)); + wecare = true; + break; + } sprintf(ifs, "%d", *((int*)RTA_DATA(rtap))); break; } } - sprintf(ms, "%d", rtp->rtm_dst_len); + if (wecare) { + Mutex::Lock rl(_routes_m); + _routes[target].insert(r); + } + +#ifdef ZT_NETLINK_TRACE + sprintf(ms, "%d", rtp->rtm_dst_len); fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); #endif } void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) { -#ifdef ZT_NETLINK_TRACE char dsts[40] = {0}; char gws[40] = {0}; char srcs[40] = {0}; @@ -328,26 +378,78 @@ void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp); int rtl = RTM_PAYLOAD(nlp); + Route r; + bool wecare = false; + for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl)) { switch(rtap->rta_type) { case RTA_DST: - inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40); + switch(rtp->rtm_family) { + case AF_INET: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24); + r.target.set(RTA_DATA(rtap), 4, 0); + wecare = true; + break; + case AF_INET6: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24); + r.target.set(RTA_DATA(rtap), 16, 0); + wecare = true; + break; + } break; case RTA_SRC: - inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24 : 40); + switch(rtp->rtm_family) { + case AF_INET: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24); + r.src.set(RTA_DATA(rtap), 4, 0); + wecare = true; + break; + case AF_INET6: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24); + r.src.set(RTA_DATA(rtap), 16, 0); + wecare = true; + break; + } break; case RTA_GATEWAY: - inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40); + switch(rtp->rtm_family) { + case AF_INET: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24); + r.via.set(RTA_DATA(rtap), 4, 0); + wecare = true; + break; + case AF_INET6: + inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24); + r.via.set(RTA_DATA(rtap), 16, 0); + wecare = true; + break; + } break; case RTA_OIF: + switch(rtp->rtm_family) { + case AF_INET: + r.ifidx = *((int*)RTA_DATA(rtap)); + wecare = true; + break; + case AF_INET6: + r.ifidx = *((int*)RTA_DATA(rtap)); + wecare = true; + break; + } sprintf(ifs, "%d", *((int*)RTA_DATA(rtap))); break; } } - sprintf(ms, "%d", rtp->rtm_dst_len); + if (wecare) { + Mutex::Lock rl(_routes_m); + _routes[target].erase(r); + } + +#ifdef ZT_NETLINK_TRACE + sprintf(ms, "%d", rtp->rtm_dst_len); fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs); #endif } @@ -1040,6 +1142,25 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface) close(fd); } +bool LinuxNetLink::routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname) +{ + Mutex::Lock rl(_routes_m); + const std::set &rs = _routes[target]; + for(std::set::const_iterator ri(rs.begin());ri!=rs.end();++ri) { + if ((ri->via == via)&&(ri->src == src)) { + if (ifname) { + Mutex::Lock ifl(_if_m); + const iface_entry *ife = _interfaces.get(rs->ifidx); + if ((ife)&&(!strncmp(ife->ifacename,ifname,IFNAMSIZ))) + return true; + } else { + return true; + } + } + } + return false; +} + int LinuxNetLink::_indexForInterface(const char *iface) { Mutex::Lock l(_if_m); diff --git a/osdep/LinuxNetLink.hpp b/osdep/LinuxNetLink.hpp index c6c4be00a..7f49fd2ed 100644 --- a/osdep/LinuxNetLink.hpp +++ b/osdep/LinuxNetLink.hpp @@ -45,6 +45,41 @@ private: ~LinuxNetLink(); public: + struct Route { + InetAddress target; + InetAddress via; + InetAddress src; + int ifidx; + + inline bool operator==(const Route &r) const + { return ((target == r.target)&&(via == r.via)&&(src == r.src)&&(ifidx == r.ifidx)); } + inline bool operator!=(const Route &r) const + { return (!(*this == r)); } + inline bool operator<(const Route &r) const + { + if (target < r.target) { + return true; + } else if (target == r.target) { + if (via < r.via) { + return true; + } else if (via == r.via) { + if (src < r.src) { + return true; + } else if (src == r.src) { + return (ifidx < r.ifidx); + } + } + } + return false; + } + inline bool operator>(const Route &r) const + { return (r < *this); } + inline bool operator<=(const Route &r) const + { return !(r < *this); } + inline bool operator>=(const Route &r) const + { return !(*this < r); } + }; + static LinuxNetLink& getInstance() { static LinuxNetLink instance; @@ -60,7 +95,10 @@ public: void addAddress(const InetAddress &addr, const char *iface); void removeAddress(const InetAddress &addr, const char *iface); + bool routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname); + void threadMain() throw(); + private: int _doRecv(int fd); @@ -85,7 +123,12 @@ private: uint32_t _seq; + std::map< InetAddress,std::set > _routes; + Mutex _routes_m; + struct iface_entry { + iface_entry() + { memset(this,0,sizeof(iface_entry)); } int index; char ifacename[IFNAMSIZ]; char mac[18]; From 5ea7ef6b659abc4e3002e24b456b71a1974893ce Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 22:06:58 -0500 Subject: [PATCH 34/35] Rework Linux route setting to actually consult netlink interface. --- osdep/ManagedRoute.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index d64f4279e..d0a892617 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -518,17 +518,14 @@ bool ManagedRoute::sync() #ifdef __LINUX__ // ---------------------------------------------------------- - if (!_applied.count(leftt)) { + const char *const devptr = (_via) ? (const char *)0 : _device; + if (!LinuxNetLink::getInstance().routeIsSet(leftt,_via,_src,devptr)) { _applied[leftt] = false; // boolean unused - //LinuxNetLink::getInstance().delRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); - LinuxNetLink::getInstance().addRoute(leftt, _via, _src, (_via) ? (const char *)0 : _device); - //_routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().addRoute(leftt, _via, _src, devptr); } - if ((rightt)&&(!_applied.count(rightt))) { + if ((rightt)&&(!LinuxNetLink::getInstance().routeIsSet(rightt,_via,_src,devptr))) { _applied[rightt] = false; // boolean unused - //LinuxNetLink::getInstance().delRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); - LinuxNetLink::getInstance().addRoute(rightt, _via, _src, (_via) ? (const char *)0 : _device); - //_routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device); + LinuxNetLink::getInstance().addRoute(rightt, _via, _src, devptr); } #endif // __LINUX__ ---------------------------------------------------------- From 4aab9121d9d1cc1deb3f315b4f22470491ac8cc1 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 13 Nov 2020 22:25:12 -0500 Subject: [PATCH 35/35] Linux build fixes, and new route code seems to work. --- osdep/LinuxNetLink.cpp | 12 ++++++------ osdep/LinuxNetLink.hpp | 4 +++- osdep/ManagedRoute.cpp | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/osdep/LinuxNetLink.cpp b/osdep/LinuxNetLink.cpp index f9defdcb3..8ee477278 100644 --- a/osdep/LinuxNetLink.cpp +++ b/osdep/LinuxNetLink.cpp @@ -13,7 +13,7 @@ #include "../node/Constants.hpp" -#define ZT_NETLINK_TRACE +//#define ZT_NETLINK_TRACE #ifdef __LINUX__ @@ -357,7 +357,7 @@ void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp) if (wecare) { Mutex::Lock rl(_routes_m); - _routes[target].insert(r); + _routes[r.target].insert(r); } #ifdef ZT_NETLINK_TRACE @@ -445,7 +445,7 @@ void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp) if (wecare) { Mutex::Lock rl(_routes_m); - _routes[target].erase(r); + _routes[r.target].erase(r); } #ifdef ZT_NETLINK_TRACE @@ -1145,12 +1145,12 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface) bool LinuxNetLink::routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname) { Mutex::Lock rl(_routes_m); - const std::set &rs = _routes[target]; - for(std::set::const_iterator ri(rs.begin());ri!=rs.end();++ri) { + const std::set &rs = _routes[target]; + for(std::set::const_iterator ri(rs.begin());ri!=rs.end();++ri) { if ((ri->via == via)&&(ri->src == src)) { if (ifname) { Mutex::Lock ifl(_if_m); - const iface_entry *ife = _interfaces.get(rs->ifidx); + const iface_entry *ife = _interfaces.get(ri->ifidx); if ((ife)&&(!strncmp(ife->ifacename,ifname,IFNAMSIZ))) return true; } else { diff --git a/osdep/LinuxNetLink.hpp b/osdep/LinuxNetLink.hpp index 7f49fd2ed..c5e08c232 100644 --- a/osdep/LinuxNetLink.hpp +++ b/osdep/LinuxNetLink.hpp @@ -19,6 +19,8 @@ #ifdef __LINUX__ #include +#include +#include #include #include @@ -123,7 +125,7 @@ private: uint32_t _seq; - std::map< InetAddress,std::set > _routes; + std::map< InetAddress,std::set > _routes; Mutex _routes_m; struct iface_entry { diff --git a/osdep/ManagedRoute.cpp b/osdep/ManagedRoute.cpp index d0a892617..a91f9f6da 100644 --- a/osdep/ManagedRoute.cpp +++ b/osdep/ManagedRoute.cpp @@ -519,7 +519,7 @@ bool ManagedRoute::sync() #ifdef __LINUX__ // ---------------------------------------------------------- const char *const devptr = (_via) ? (const char *)0 : _device; - if (!LinuxNetLink::getInstance().routeIsSet(leftt,_via,_src,devptr)) { + if ((leftt)&&(!LinuxNetLink::getInstance().routeIsSet(leftt,_via,_src,devptr))) { _applied[leftt] = false; // boolean unused LinuxNetLink::getInstance().addRoute(leftt, _via, _src, devptr); }