From 312310221102b8472ab294e70b14317bce7d6ec7 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Fri, 6 Sep 2019 07:37:44 -0700 Subject: [PATCH] root cleanup and add nifty geoip thing --- root/geoip-html.h | 53 ++++++++ root/root.cpp | 319 +++++++++++++++++++++++----------------------- 2 files changed, 215 insertions(+), 157 deletions(-) create mode 100644 root/geoip-html.h diff --git a/root/geoip-html.h b/root/geoip-html.h new file mode 100644 index 000000000..8bc21795b --- /dev/null +++ b/root/geoip-html.h @@ -0,0 +1,53 @@ +#ifndef ZT_ROOT_GEOIP_HTML_H +#define ZT_ROOT_GEOIP_HTML_H + +#define ZT_GEOIP_HTML_HEAD \ +"\n" \ +"\n" \ +" \n" \ +" \n" \ +" \n" \ +" GeoIP Map\n" \ +" \n" \ +" \n" \ +" \n" \ +"
\n" \ +" \n" \ +" \n" \ +" \n" \ +" \n" \ +"" + +#endif diff --git a/root/root.cpp b/root/root.cpp index 2b5a2f876..6f6625c06 100644 --- a/root/root.cpp +++ b/root/root.cpp @@ -99,6 +99,8 @@ #include #include +#include "geoip-html.h" + using namespace ZeroTier; using json = nlohmann::json; @@ -111,6 +113,32 @@ using json = nlohmann::json; ////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////// +/** + * RootPeer is a normal peer known to this root + * + * This can also be a sibling root, which is itself a peer. Sibling roots + * are sent HELLO while for other peers we only listen for HELLO. + */ +struct RootPeer +{ + ZT_ALWAYS_INLINE RootPeer() : lastSend(0),lastReceive(0),lastSync(0),lastEcho(0),lastHello(0),vProto(-1),vMajor(-1),vMinor(-1),vRev(-1) {} + ZT_ALWAYS_INLINE ~RootPeer() { Utils::burn(key,sizeof(key)); } + + Identity id; // Identity + uint8_t key[32]; // Shared secret key + InetAddress ip4,ip6; // IPv4 and IPv6 addresses + int64_t lastSend; // Time of last send (any packet) + int64_t lastReceive; // Time of last receive (any packet) + int64_t lastSync; // Time of last data synchronization with LF or other root state backend (currently unused) + int64_t lastEcho; // Time of last received ECHO + int64_t lastHello; // Time of last received HELLO + int vProto; // Protocol version + int vMajor,vMinor,vRev; // Peer version or -1,-1,-1 if unknown + std::mutex lock; + + AtomicCounter __refCount; +}; + // Hashers for std::unordered_map struct IdentityHasher { ZT_ALWAYS_INLINE std::size_t operator()(const Identity &id) const { return (std::size_t)id.hashCode(); } }; struct AddressHasher { ZT_ALWAYS_INLINE std::size_t operator()(const Address &a) const { return (std::size_t)a.toInt(); } }; @@ -136,33 +164,6 @@ struct RendezvousKey struct Hasher { ZT_ALWAYS_INLINE std::size_t operator()(const RendezvousKey &k) const { return (std::size_t)(k.a.toInt() ^ k.b.toInt()); } }; }; -/** - * RootPeer is a normal peer known to this root - * - * This can also be a sibling root, which is itself a peer. Sibling roots - * are sent HELLO while for other peers we only listen for HELLO. - */ -struct RootPeer -{ - ZT_ALWAYS_INLINE RootPeer() : lastSend(0),lastReceive(0),lastSync(0),lastEcho(0),lastHello(0),vProto(-1),vMajor(-1),vMinor(-1),vRev(-1),sibling(false) {} - ZT_ALWAYS_INLINE ~RootPeer() { Utils::burn(key,sizeof(key)); } - - Identity id; // Identity - uint8_t key[32]; // Shared secret key - InetAddress ip4,ip6; // IPv4 and IPv6 addresses - int64_t lastSend; // Time of last send (any packet) - int64_t lastReceive; // Time of last receive (any packet) - int64_t lastSync; // Time of last data synchronization with LF or other root state backend (currently unused) - int64_t lastEcho; // Time of last received ECHO - int64_t lastHello; // Time of last received HELLO - int vProto; // Protocol version - int vMajor,vMinor,vRev; // Peer version or -1,-1,-1 if unknown - bool sibling; // If true, this is a sibling root that will get forwards we don't know where to send - std::mutex lock; - - AtomicCounter __refCount; -}; - struct RendezvousStats { RendezvousStats() : count(0),ts(0) {} @@ -178,28 +179,29 @@ struct ForwardingStats Meter bps; }; -static int64_t s_startTime; // Time service was started -static std::vector s_ports; // Ports to bind for UDP traffic -static int s_relayMaxHops = 0; // Max relay hops -static Identity s_self; // My identity (including secret) -static std::atomic_bool s_run; // Remains true until shutdown is ordered -static json s_config; // JSON config file contents -static std::string s_statsRoot; // Root to write stats, peers, etc. +static int64_t s_startTime; // Time service was started +static std::vector s_ports; // Ports to bind for UDP traffic +static int s_relayMaxHops = 0; // Max relay hops +static Identity s_self; // My identity (including secret) +static std::atomic_bool s_run; // Remains true until shutdown is ordered +static json s_config; // JSON config file contents +static std::string s_statsRoot; // Root to write stats, peers, etc. +static std::atomic_bool s_geoInit; // True if geoIP data is initialized static Meter s_inputRate; static Meter s_outputRate; static Meter s_forwardRate; -static Meter s_siblingForwardRate; static Meter s_discardedForwardRate; static std::string s_planet; -static std::vector< SharedPtr > s_siblings; static std::unordered_map< uint64_t,std::unordered_map< MulticastGroup,std::unordered_map< Address,int64_t,AddressHasher >,MulticastGroupHasher > > s_multicastSubscriptions; static std::unordered_map< Identity,SharedPtr,IdentityHasher > s_peersByIdentity; static std::unordered_map< Address,std::set< SharedPtr >,AddressHasher > s_peersByVirtAddr; static std::unordered_map< InetAddress,std::set< SharedPtr >,InetAddressHasher > s_peersByPhysAddr; -static std::unordered_map< RendezvousKey,RendezvousStats,RendezvousKey::Hasher > s_lastSentRendezvous; +static std::unordered_map< RendezvousKey,RendezvousStats,RendezvousKey::Hasher > s_rendezvousTracking; static std::unordered_map< Address,ForwardingStats,AddressHasher > s_lastForwardedTo; +static std::map< std::pair< uint32_t,uint32_t >,std::pair< float,float > > s_geoIp4; +static std::map< std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > >,std::pair< float,float > > s_geoIp6; static std::mutex s_planet_l; static std::mutex s_siblings_l; @@ -207,12 +209,26 @@ static std::mutex s_multicastSubscriptions_l; static std::mutex s_peersByIdentity_l; static std::mutex s_peersByVirtAddr_l; static std::mutex s_peersByPhysAddr_l; -static std::mutex s_lastSentRendezvous_l; +static std::mutex s_rendezvousTracking_l; static std::mutex s_lastForwardedTo_l; ////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////// +static uint32_t ip4ToH32(const InetAddress &ip) +{ + return Utils::ntoh((uint32_t)(((const struct sockaddr_in *)&ip)->sin_addr.s_addr)); +} + +static std::array< uint64_t,2 > ip6ToH128(const InetAddress &ip) +{ + std::array i128; + memcpy(i128.data(),ip.rawIpData(),16); + i128[0] = Utils::ntoh(i128[0]); + i128[1] = Utils::ntoh(i128[1]); + return i128; +} + static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip,Packet &pkt) { char ipstr[128],ipstr2[128],astr[32],astr2[32],tmpstr[256]; @@ -498,8 +514,8 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip if (hops == 1) { RendezvousKey rk(source,dest); - std::lock_guard l(s_lastSentRendezvous_l); - RendezvousStats &lr = s_lastSentRendezvous[rk]; + std::lock_guard l(s_rendezvousTracking_l); + RendezvousStats &lr = s_rendezvousTracking[rk]; if ((now - lr.ts) >= 30000) { ++lr.count; lr.ts = now; @@ -522,18 +538,6 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip } } } - if (toAddrs.empty()) { - std::lock_guard sib_l(s_siblings_l); - for(auto s=s_siblings.begin();s!=s_siblings.end();++s) { - if (((now - (*s)->lastReceive) < (ZT_PEER_PING_PERIOD * 2))&&((*s)->sibling)) { - if ((*s)->ip4) { - toAddrs.push_back(std::pair< InetAddress *,SharedPtr >(&((*s)->ip4),*s)); - } else if ((*s)->ip6) { - toAddrs.push_back(std::pair< InetAddress *,SharedPtr >(&((*s)->ip6),*s)); - } - } - } - } if (toAddrs.empty()) { s_discardedForwardRate.log(now,pkt.size()); return; @@ -621,8 +625,6 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip if (sendto(i->first->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)i->first,(socklen_t)(i->first->isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) > 0) { s_outputRate.log(now,pkt.size()); s_forwardRate.log(now,pkt.size()); - if (i->second->sibling) - s_siblingForwardRate.log(now,pkt.size()); i->second->lastSend = now; } } @@ -639,17 +641,17 @@ static int bindSocket(struct sockaddr *const bindAddr) return -1; } - int f = 1048576; - while (f > 16384) { + int f = 4194304; + while (f > 131072) { if (setsockopt(s,SOL_SOCKET,SO_RCVBUF,(const char *)&f,sizeof(f)) == 0) break; - f -= 16384; + f -= 131072; } - f = 1048576; - while (f > 16384) { + f = 4194304; + while (f > 131072) { if (setsockopt(s,SOL_SOCKET,SO_SNDBUF,(const char *)&f,sizeof(f)) == 0) break; - f -= 16384; + f -= 131072; } if (bindAddr->sa_family == AF_INET6) { @@ -694,6 +696,10 @@ static void shutdownSigHandler(int sig) { s_run = false; } int main(int argc,char **argv) { + std::vector threads; + std::vector sockets; + int v4Sock = -1,v6Sock = -1; + signal(SIGTERM,shutdownSigHandler); signal(SIGINT,shutdownSigHandler); signal(SIGQUIT,shutdownSigHandler); @@ -703,6 +709,7 @@ int main(int argc,char **argv) signal(SIGCHLD,SIG_IGN); s_startTime = OSUtils::now(); + s_geoInit = false; if (argc < 3) { printf("Usage: zerotier-root " ZT_EOL_S); @@ -799,7 +806,7 @@ int main(int argc,char **argv) s_relayMaxHops = ZT_RELAY_MAX_HOPS; try { - s_relayMaxHops = s_config["s_relayMaxHops"]; + s_relayMaxHops = s_config["relayMaxHops"]; if (s_relayMaxHops > ZT_PROTO_MAX_HOPS) s_relayMaxHops = ZT_PROTO_MAX_HOPS; else if (s_relayMaxHops < 0) @@ -809,68 +816,54 @@ int main(int argc,char **argv) } try { - if (s_config.count("s_siblings") > 0) { - auto sibs = s_config["s_siblings"]; - if (sibs.is_array()) { - for(long i=0;i<(long)sibs.size();++i) { - auto sib = sibs[i]; - if (sib.is_object()) { - std::string idStr = sib["id"]; - std::string ipStr = sib["ip"]; - Identity id; - if (!id.fromString(idStr.c_str())) { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: invalid identity in sibling entry" ZT_EOL_S); - return 1; + std::string geoIpPath = s_config["geoIp"]; + if (geoIpPath.length() > 0) { + FILE *gf = fopen(geoIpPath.c_str(),"rb"); + if (gf) { + threads.emplace_back(std::thread([gf]() { + try { + char line[1024]; + while (fgets(line,sizeof(line),gf)) { + InetAddress start,end; + float lat,lon; + int field = 0; + for(char *saveptr=nullptr,*f=Utils::stok(line,",\r\n",&saveptr);(f);f=Utils::stok(nullptr,",\r\n",&saveptr)) { + switch(field++) { + case 0: + start.fromString(f); + break; + case 1: + end.fromString(f); + break; + case 2: + lat = strtof(f,nullptr); + break; + case 3: + lon = strtof(f,nullptr); + break; + } + } + if ((start)&&(end)&&(start.ss_family == end.ss_family)&&(lat >= -90.0F)&&(lat <= 90.0F)&&(lon >= -180.0F)&&(lon <= 180.0F)) { + if (start.ss_family == AF_INET) { + s_geoIp4[std::pair< uint32_t,uint32_t >(ip4ToH32(start),ip4ToH32(end))] = std::pair< float,float >(lat,lon); + } else if (start.ss_family == AF_INET6) { + s_geoIp6[std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > >(ip6ToH128(start),ip6ToH128(end))] = std::pair< float,float >(lat,lon); + } + } } - InetAddress ip; - if (!ip.fromString(ipStr.c_str())) { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: invalid IP address in sibling entry" ZT_EOL_S); - return 1; - } - ip.setPort((unsigned int)sib["port"]); - SharedPtr rp(new RootPeer); - rp->id = id; - if (!s_self.agree(id,rp->key)) { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: invalid identity in sibling entry (unable to execute key agreement)" ZT_EOL_S); - return 1; - } - if (ip.isV4()) { - rp->ip4 = ip; - } else if (ip.isV6()) { - rp->ip6 = ip; - } else { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: invalid IP address in sibling entry" ZT_EOL_S); - return 1; - } - rp->sibling = true; - s_siblings.push_back(rp); - s_peersByIdentity[id] = rp; - s_peersByVirtAddr[id.address()].insert(rp); - s_peersByPhysAddr[ip].insert(rp); - } else { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: sibling entry is not a JSON object" ZT_EOL_S); - return 1; - } - } - } else { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: s_siblings is not a JSON array" ZT_EOL_S); - return 1; + s_geoInit = true; + } catch ( ... ) {} + fclose(gf); + })); } } - } catch ( ... ) { - printf("FATAL: invalid JSON while parsing s_siblings section in config file: parse error" ZT_EOL_S); - return 1; - } + } catch ( ... ) {} unsigned int ncores = std::thread::hardware_concurrency(); if (ncores == 0) ncores = 1; s_run = true; - std::vector threads; - std::vector sockets; - int v4Sock = -1,v6Sock = -1; - for(auto port=s_ports.begin();port!=s_ports.end();++port) { for(unsigned int tn=0;tn k4(0,0); + std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > > k6; + k6.second[0] = 0; k6.second[1] = 0; + std::lock_guard l(s_peersByPhysAddr_l); + for(auto p=s_peersByPhysAddr.begin();p!=s_peersByPhysAddr.end();++p) { + if (!p->second.empty()) { + k4.first = ip4ToH32(p->first); + auto geo = s_geoIp4.lower_bound(k4); + while ((geo != s_geoIp4.end())&&(geo->first.first == k4.first)) { + if (geo->first.second >= k4.first) { + if (!firstCoord) + o << ','; + firstCoord = false; + o << "{lat:" << geo->second.first << ",lng:" << geo->second.second << ",_l:\""; + bool firstAddr = true; + for(auto a=p->second.begin();a!=p->second.end();++a) { + if (!firstAddr) + o << ','; + o << (*a)->id.address().toString(tmp); + firstAddr = false; + } + o << "\"}"; + break; + } + } + } + } + } + o << ZT_GEOIP_HTML_TAIL; + res.set_content(o.str(),"text/html"); + }); + apiServ.listen("127.0.0.1",httpPort,0); })); // In the main thread periodically clean stuff up int64_t lastCleaned = 0; int64_t lastWroteStats = 0; - int64_t lastPingedSiblings = 0; while (s_run) { //s_peersByIdentity_l.lock(); //s_peersByPhysAddr_l.lock(); @@ -1024,36 +1060,6 @@ int main(int argc,char **argv) const int64_t now = OSUtils::now(); - // Send HELLO to sibling roots - if ((now - lastPingedSiblings) >= ZT_PEER_PING_PERIOD) { - lastPingedSiblings = now; - std::lock_guard l(s_siblings_l); - for(auto s=s_siblings.begin();s!=s_siblings.end();++s) { - const InetAddress *ip = nullptr; - socklen_t sl = 0; - Packet outp((*s)->id.address(),s_self.address(),Packet::VERB_HELLO); - outp.append((uint8_t)ZT_PROTO_VERSION); - outp.append((uint8_t)0); - outp.append((uint8_t)0); - outp.append((uint16_t)0); - outp.append((uint64_t)now); - s_self.serialize(outp,false); - if ((*s)->ip4) { - (*s)->ip4.serialize(outp); - ip = &((*s)->ip4); - sl = sizeof(struct sockaddr_in); - } else if ((*s)->ip6) { - (*s)->ip6.serialize(outp); - ip = &((*s)->ip6); - sl = sizeof(struct sockaddr_in6); - } - if (ip) { - outp.armor((*s)->key,false); - sendto(ip->isV4() ? v4Sock : v6Sock,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)ip,sl); - } - } - } - if ((now - lastCleaned) > 120000) { lastCleaned = now; @@ -1081,7 +1087,7 @@ int main(int argc,char **argv) { std::lock_guard pbi_l(s_peersByIdentity_l); for(auto p=s_peersByIdentity.begin();p!=s_peersByIdentity.end();) { - if (((now - p->second->lastReceive) > ZT_PEER_ACTIVITY_TIMEOUT)&&(!p->second->sibling)) { + if ((now - p->second->lastReceive) > ZT_PEER_ACTIVITY_TIMEOUT) { std::lock_guard pbv_l(s_peersByVirtAddr_l); std::lock_guard pbp_l(s_peersByPhysAddr_l); @@ -1116,10 +1122,10 @@ int main(int argc,char **argv) // Remove old rendezvous and last forwarded tracking entries { - std::lock_guard l(s_lastSentRendezvous_l); - for(auto lr=s_lastSentRendezvous.begin();lr!=s_lastSentRendezvous.end();) { + std::lock_guard l(s_rendezvousTracking_l); + for(auto lr=s_rendezvousTracking.begin();lr!=s_rendezvousTracking.end();) { if ((now - lr->second.ts) > ZT_PEER_ACTIVITY_TIMEOUT) - s_lastSentRendezvous.erase(lr++); + s_rendezvousTracking.erase(lr++); else ++lr; } } @@ -1219,23 +1225,22 @@ int main(int argc,char **argv) s_peersByPhysAddr_l.lock(); fprintf(sf,"Physical Endpoints : %llu" ZT_EOL_S,(unsigned long long)s_peersByPhysAddr.size()); s_peersByPhysAddr_l.unlock(); - s_lastSentRendezvous_l.lock(); + s_rendezvousTracking_l.lock(); uint64_t unsuccessfulp2p = 0; - for(auto lr=s_lastSentRendezvous.begin();lr!=s_lastSentRendezvous.end();++lr) { + for(auto lr=s_rendezvousTracking.begin();lr!=s_rendezvousTracking.end();++lr) { if (lr->second.count > 6) // 6 == two attempts per edge, one for each direction ++unsuccessfulp2p; } - fprintf(sf,"Recent P2P Graph Edges : %llu" ZT_EOL_S,(unsigned long long)s_lastSentRendezvous.size()); - if (s_lastSentRendezvous.empty()) { - fprintf(sf,"Estimated P2P Success Rate : 100.0000%%" ZT_EOL_S); + fprintf(sf,"Recent P2P Graph Edges : %llu" ZT_EOL_S,(unsigned long long)s_rendezvousTracking.size()); + if (s_rendezvousTracking.empty()) { + fprintf(sf,"Recent P2P Success Rate : 100.0000%%" ZT_EOL_S); } else { - fprintf(sf,"Estimated P2P Success Rate : %.4f%%" ZT_EOL_S,(1.0 - ((double)unsuccessfulp2p / (double)s_lastSentRendezvous.size())) * 100.0); + fprintf(sf,"Recent P2P Success Rate : %.4f%%" ZT_EOL_S,(1.0 - ((double)unsuccessfulp2p / (double)s_rendezvousTracking.size())) * 100.0); } - s_lastSentRendezvous_l.unlock(); + s_rendezvousTracking_l.unlock(); fprintf(sf,"Input (MiB/s) : %.4f" ZT_EOL_S,s_inputRate.perSecond(now) / 1048576.0); fprintf(sf,"Output (MiB/s) : %.4f" ZT_EOL_S,s_outputRate.perSecond(now) / 1048576.0); fprintf(sf,"Forwarded (MiB/s) : %.4f" ZT_EOL_S,s_forwardRate.perSecond(now) / 1048576.0); - fprintf(sf,"Sibling Forwarded (MiB/s) : %.4f" ZT_EOL_S,s_siblingForwardRate.perSecond(now) / 1048576.0); fprintf(sf,"Discarded Forward (MiB/s) : %.4f" ZT_EOL_S,s_discardedForwardRate.perSecond(now) / 1048576.0); fclose(sf);