From 96aa1c30a6ed56a866561445313df2db19f8bf6c Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Thu, 17 Feb 2022 15:39:17 -0800 Subject: [PATCH 1/9] Proactively seek, enumerate, and distribute external surface addresses --- node/Peer.cpp | 4 +++- node/SelfAwareness.cpp | 15 +++++++++++++++ node/SelfAwareness.hpp | 7 +++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/node/Peer.cpp b/node/Peer.cpp index 8ab51687d..6e5182260 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -224,6 +224,8 @@ void Peer::received( if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) { _lastDirectPathPushSent = now; std::vector pathsToPush(RR->node->directPaths()); + std::vector ma = RR->sa->whoami(); + pathsToPush.insert(pathsToPush.end(), ma.begin(), ma.end()); if (!pathsToPush.empty()) { std::vector::const_iterator p(pathsToPush.begin()); while (p != pathsToPush.end()) { @@ -453,7 +455,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA if (atAddress) { outp.armor(_key,false,nullptr); // false == don't encrypt full payload, but add MAC RR->node->expectReplyTo(outp.packetId()); - RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,-1,atAddress,outp.data(),outp.size()); } else { RR->node->expectReplyTo(outp.packetId()); RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 57b409c12..45bc25410 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -99,6 +99,21 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receive } } +std::vector SelfAwareness::whoami() +{ + std::vector surfaceAddresses; + Mutex::Lock _l(_phy_m); + Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); + PhySurfaceKey *k = (PhySurfaceKey *)0; + PhySurfaceEntry *e = (PhySurfaceEntry *)0; + while (i.next(k,e)) { + if (std::find(surfaceAddresses.begin(), surfaceAddresses.end(), e->mySurface) == surfaceAddresses.end()) { + surfaceAddresses.push_back(e->mySurface); + } + } + return surfaceAddresses; +} + void SelfAwareness::clean(int64_t now) { Mutex::Lock _l(_phy_m); diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index f34d5778e..e0fbf91d0 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -44,6 +44,13 @@ public: */ void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,int64_t now); + /** + * Return all known external surface addresses reported by peers + * + * @return A vector of InetAddress(es) + */ + std::vector whoami(); + /** * Clean up database periodically * From 5e13b42abc7b1cfddd8924a420e8f01622cb8100 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Mon, 21 Feb 2022 14:37:39 -0800 Subject: [PATCH 2/9] Rate gate ECHO per Path instead of per Peer --- node/IncomingPacket.cpp | 2 +- node/Path.hpp | 17 +++++++++++++++++ node/Peer.cpp | 1 - node/Peer.hpp | 13 ------------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 6441b6ad7..347c0649d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -860,7 +860,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { uint64_t now = RR->node->now(); - if (!peer->rateGateEchoRequest(now)) { + if (!_path->rateGateEchoRequest(now)) { return true; } diff --git a/node/Path.hpp b/node/Path.hpp index 753bf0ab2..fa55c2f99 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -83,6 +83,7 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _lastEchoRequestReceived(0), _localSocket(-1), _latency(0xffff), _addr(), @@ -93,6 +94,7 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _lastEchoRequestReceived(0), _localSocket(localSocket), _latency(0xffff), _addr(addr), @@ -266,6 +268,18 @@ public: */ inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } + /** + * Rate limit gate for inbound ECHO requests + */ + inline bool rateGateEchoRequest(const int64_t now) + { + if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT / 16)) { + _lastEchoRequestReceived = now; + return true; + } + return false; + } + void *_bondingMetricPtr; private: @@ -273,6 +287,9 @@ private: volatile int64_t _lastOut; volatile int64_t _lastIn; volatile int64_t _lastTrustEstablishedPacketReceived; + + int64_t _lastEchoRequestReceived; + int64_t _localSocket; volatile unsigned int _latency; InetAddress _addr; diff --git a/node/Peer.cpp b/node/Peer.cpp index 8ab51687d..76f83940f 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -34,7 +34,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastTriedMemorizedPath(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), - _lastEchoRequestReceived(0), _lastCredentialRequestSent(0), _lastWhoisRequestReceived(0), _lastCredentialsReceived(0), diff --git a/node/Peer.hpp b/node/Peer.hpp index 3bfe31950..77e35b786 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -418,18 +418,6 @@ public: return false; } - /** - * Rate limit gate for inbound ECHO requests - */ - inline bool rateGateEchoRequest(const int64_t now) - { - if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { - _lastEchoRequestReceived = now; - return true; - } - return false; - } - /** * Serialize a peer for storage in local cache * @@ -546,7 +534,6 @@ private: int64_t _lastTriedMemorizedPath; int64_t _lastDirectPathPushSent; int64_t _lastDirectPathPushReceive; - int64_t _lastEchoRequestReceived; int64_t _lastCredentialRequestSent; int64_t _lastWhoisRequestReceived; int64_t _lastCredentialsReceived; From d1335dca115600289503ca0e7d88963533ca0fe8 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Mon, 21 Feb 2022 16:22:33 -0800 Subject: [PATCH 3/9] Change ECHO rate-limit divsor from 16 to 20 --- node/Path.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/Path.hpp b/node/Path.hpp index fa55c2f99..ae12bf4bc 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -273,7 +273,7 @@ public: */ inline bool rateGateEchoRequest(const int64_t now) { - if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT / 16)) { + if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT / 20)) { _lastEchoRequestReceived = now; return true; } From 618202d4264ff803a8d0fe08b06484b57125f9d8 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Fri, 25 Feb 2022 10:52:39 -0800 Subject: [PATCH 4/9] Increase min failover to 500 ms and probe period to 1/3rd of failover --- node/Constants.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/Constants.hpp b/node/Constants.hpp index 930c91299..d9a3e7964 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -571,13 +571,13 @@ * Anything below this value gets into thrashing territory since we divide * this value by ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL to send ECHOs often. */ -#define ZT_BOND_FAILOVER_MIN_INTERVAL 250 +#define ZT_BOND_FAILOVER_MIN_INTERVAL 500 /** * How many times per failover interval that an ECHO is sent. This should be * at least 2. Anything more then 4 starts to increase overhead significantly. */ -#define ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL 4 +#define ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL 3 /** * A defensive timer to prevent path quality metrics from being From 1918c29fd7ee8f82e8f0428747b8f3ee4d913704 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Fri, 25 Feb 2022 11:29:07 -0800 Subject: [PATCH 5/9] Change ECHO divisor from 20 to 6 --- node/Path.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/Path.hpp b/node/Path.hpp index ae12bf4bc..3471e2d66 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -273,7 +273,7 @@ public: */ inline bool rateGateEchoRequest(const int64_t now) { - if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT / 20)) { + if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT / 6)) { _lastEchoRequestReceived = now; return true; } From 1cf8a1f4933add1df99a12d77e13661f5cbcaa71 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Fri, 25 Feb 2022 15:04:48 -0800 Subject: [PATCH 6/9] More accurate accounting of bond layer overhead traffic --- node/Bond.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/Bond.cpp b/node/Bond.cpp index 14d1e1023..3e3cd5b2a 100644 --- a/node/Bond.cpp +++ b/node/Bond.cpp @@ -717,6 +717,7 @@ void Bond::sendPATH_NEGOTIATION_REQUEST(void* tPtr, int pathIdx) if (_paths[pathIdx].p->address()) { outp.armor(_peer->key(), false, _peer->aesKeysIfSupported()); RR->node->putPacket(tPtr, _paths[pathIdx].p->localSocket(), _paths[pathIdx].p->address(), outp.data(), outp.size()); + _overheadBytes += outp.size(); } } @@ -726,7 +727,6 @@ void Bond::sendQOS_MEASUREMENT(void* tPtr, int pathIdx, int64_t localSocket, con Packet outp(_peer->_id.address(), RR->identity.address(), Packet::VERB_QOS_MEASUREMENT); char qosData[ZT_QOS_MAX_PACKET_SIZE]; int16_t len = generateQoSPacket(pathIdx, _now, qosData); - _overheadBytes += len; if (len) { outp.append(qosData, len); if (atAddress) { @@ -738,6 +738,7 @@ void Bond::sendQOS_MEASUREMENT(void* tPtr, int pathIdx, int64_t localSocket, con } _paths[pathIdx].packetsReceivedSinceLastQoS = 0; _paths[pathIdx].lastQoSMeasurement = now; + _overheadBytes += outp.size(); } // debug("send QOS via link %s (len=%d)", pathToStr(_paths[pathIdx].p).c_str(), len); } From f8e24f4629f0edde11c3e03c884c19a4fb69aaad Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Mon, 28 Feb 2022 12:26:32 -0800 Subject: [PATCH 7/9] Fix issue where restarting a controller causes a DB write for each network member --- controller/PostgreSQL.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index c6623d556..f83ebc9b9 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -777,6 +777,7 @@ void PostgreSQL::initializeMembers() std::string assignedAddresses = std::get<20>(row); config["id"] = memberId; + config["address"] = memberId; config["nwid"] = networkId; config["activeBridge"] = activeBridge.value_or(false); config["authorized"] = authorized.value_or(false); From 8148c658cfaf58f926a8fd2e0350b7f4a5e4baaa Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Wed, 2 Mar 2022 09:55:23 -0800 Subject: [PATCH 8/9] Remove bonds for peers that have fully expired. Remove notion of bond health --- include/ZeroTierOne.h | 5 --- node/Bond.cpp | 77 +++++++++++++++++++++--------------------- node/Bond.hpp | 20 +++++------ node/Node.cpp | 1 - node/Peer.cpp | 2 +- node/Peer.hpp | 5 ++- one.cpp | 33 +++--------------- service/OneService.cpp | 2 -- 8 files changed, 57 insertions(+), 88 deletions(-) diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e88fbda1f..e16d23f87 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1404,11 +1404,6 @@ typedef struct */ int bondingPolicy; - /** - * The health status of the bond to this peer - */ - bool isHealthy; - /** * The number of links that comprise the bond to this peer that are considered alive */ diff --git a/node/Bond.cpp b/node/Bond.cpp index 3e3cd5b2a..f1488a8ea 100644 --- a/node/Bond.cpp +++ b/node/Bond.cpp @@ -90,7 +90,7 @@ SharedPtr Bond::getBondByPeerId(int64_t identity) return _bonds.count(identity) ? _bonds[identity] : SharedPtr(); } -SharedPtr Bond::createTransportTriggeredBond(const RuntimeEnvironment* renv, const SharedPtr& peer) +SharedPtr Bond::createBond(const RuntimeEnvironment* renv, const SharedPtr& peer) { Mutex::Lock _l(_bonds_m); int64_t identity = peer->identity().address().toInt(); @@ -145,6 +145,12 @@ SharedPtr Bond::createTransportTriggeredBond(const RuntimeEnvironment* ren return SharedPtr(); } +void Bond::destroyBond(uint64_t peerId) +{ + Mutex::Lock _l(_bonds_m); + _bonds.erase(peerId); +} + SharedPtr Bond::getLinkBySocket(const std::string& policyAlias, uint64_t localSocket) { Mutex::Lock _l(_links_m); @@ -816,6 +822,17 @@ void Bond::curateBond(int64_t now, bool rebuildBond) if (! _paths[i].p) { continue; } + + /** + * Remove expired links from bond + */ + if ((now - _paths[i].p->_lastIn) > (ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD)) { + log("link %s has expired, removing from bond", pathToStr(_paths[i].p).c_str()); + _paths[i] = NominatedPath(); + _paths[i].p = SharedPtr(); + continue; + } + tmpNumTotalLinks++; if (_paths[i].eligible) { tmpNumAliveLinks++; @@ -876,42 +893,18 @@ void Bond::curateBond(int64_t now, bool rebuildBond) } /** - * Determine health status to report to user + * Trigger status report if number of links change */ _numAliveLinks = tmpNumAliveLinks; _numTotalLinks = tmpNumTotalLinks; - bool tmpHealthStatus = true; - - if (_policy == ZT_BOND_POLICY_BROADCAST) { - if (_numAliveLinks < 1) { - // Considered healthy if we're able to send frames at all - tmpHealthStatus = false; - } - } - if ((_policy == ZT_BOND_POLICY_BALANCE_RR) || (_policy == ZT_BOND_POLICY_BALANCE_XOR) || (_policy == ZT_BOND_POLICY_BALANCE_AWARE || (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP))) { - if (_numAliveLinks < _numTotalLinks) { - tmpHealthStatus = false; - } - } - if (tmpHealthStatus != _isHealthy) { - std::string healthStatusStr; - if (tmpHealthStatus == true) { - healthStatusStr = "HEALTHY"; - } - else { - healthStatusStr = "DEGRADED"; - } - log("bond is %s (%d/%d links)", healthStatusStr.c_str(), _numAliveLinks, _numTotalLinks); + if ((_numAliveLinks != tmpNumAliveLinks) || (_numTotalLinks != tmpNumTotalLinks)) { dumpInfo(now, true); } - _isHealthy = tmpHealthStatus; - /** * Curate the set of paths that are part of the bond proper. Select a set of paths * per logical link according to eligibility and user-specified constraints. */ - if ((_policy == ZT_BOND_POLICY_BALANCE_RR) || (_policy == ZT_BOND_POLICY_BALANCE_XOR) || (_policy == ZT_BOND_POLICY_BALANCE_AWARE)) { if (! _numBondedPaths) { rebuildBond = true; @@ -1009,14 +1002,14 @@ void Bond::estimatePathQuality(int64_t now) uint32_t totUserSpecifiedLinkSpeed = 0; if (_numBondedPaths) { // Compute relative user-specified speeds of links for (unsigned int i = 0; i < _numBondedPaths; ++i) { - SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); if (_paths[i].p && _paths[i].allowed()) { + SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); totUserSpecifiedLinkSpeed += link->speed(); } } for (unsigned int i = 0; i < _numBondedPaths; ++i) { - SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); if (_paths[i].p && _paths[i].allowed()) { + SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); link->setRelativeSpeed((uint8_t)round(((float)link->speed() / (float)totUserSpecifiedLinkSpeed) * 255)); } } @@ -1213,6 +1206,11 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now) int nonPreferredPathIdx; bool bFoundPrimaryLink = false; + if (_abPathIdx != ZT_MAX_PEER_NETWORK_PATHS && !_paths[_abPathIdx].p) { + _abPathIdx = ZT_MAX_PEER_NETWORK_PATHS; + log("main active-backup path has been removed"); + } + /** * Generate periodic status report */ @@ -1242,7 +1240,6 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now) * simply find the next eligible path. */ if (! userHasSpecifiedLinks()) { - debug("no user-specified links"); for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) { if (_paths[i].p && _paths[i].eligible) { SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); @@ -1575,7 +1572,6 @@ void Bond::setBondParameters(int policy, SharedPtr templateBond, bool useT // Bond status - _isHealthy = false; _numAliveLinks = 0; _numTotalLinks = 0; _numBondedPaths = 0; @@ -1685,11 +1681,14 @@ SharedPtr Bond::getLink(const SharedPtr& path) std::string Bond::pathToStr(const SharedPtr& path) { #ifdef ZT_TRACE - char pathStr[64] = { 0 }; - char fullPathStr[384] = { 0 }; - path->address().toString(pathStr); - snprintf(fullPathStr, 384, "%.16llx-%s/%s", (unsigned long long)(path->localSocket()), getLink(path)->ifname().c_str(), pathStr); - return std::string(fullPathStr); + if (path) { + char pathStr[64] = { 0 }; + char fullPathStr[384] = { 0 }; + path->address().toString(pathStr); + snprintf(fullPathStr, 384, "%.16llx-%s/%s", (unsigned long long)(path->localSocket()), getLink(path)->ifname().c_str(), pathStr); + return std::string(fullPathStr); + } + return ""; #else return ""; #endif @@ -1728,7 +1727,7 @@ void Bond::dumpInfo(int64_t now, bool force) _lastSummaryDump = now; float overhead = (_overheadBytes / (timeSinceLastDump / 1000.0f) / 1000.0f); _overheadBytes = 0; - log("bond: bp=%d, fi=%d, mi=%d, ud=%d, dd=%d, flows=%lu, leaf=%d, overhead=%f KB/s", + log("bond: bp=%d, fi=%d, mi=%d, ud=%d, dd=%d, flows=%lu, leaf=%d, overhead=%f KB/s, links=(%d/%d)", _policy, _failoverInterval, _monitorInterval, @@ -1736,7 +1735,9 @@ void Bond::dumpInfo(int64_t now, bool force) _downDelay, (unsigned long)_flows.size(), _isLeaf, - overhead); + overhead, + _numAliveLinks, + _numTotalLinks); for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) { if (_paths[i].p) { dumpPathStatus(now, i); diff --git a/node/Bond.hpp b/node/Bond.hpp index b30d3a90f..3465db51c 100644 --- a/node/Bond.hpp +++ b/node/Bond.hpp @@ -436,7 +436,14 @@ class Bond { * @param peer Remote peer that this bond services * @return A pointer to the newly created Bond */ - static SharedPtr createTransportTriggeredBond(const RuntimeEnvironment* renv, const SharedPtr& peer); + static SharedPtr createBond(const RuntimeEnvironment* renv, const SharedPtr& peer); + + /** + * Remove a bond from the bond controller. + * + * @param peerId Remote peer that this bond services + */ + static void destroyBond(uint64_t peerId); /** * Periodically perform maintenance tasks for the bonding layer. @@ -1020,14 +1027,6 @@ class Bond { return _policy; } - /** - * @return the health status of the bond - */ - inline bool isHealthy() - { - return _isHealthy; - } - /** * @return the number of links comprising this bond which are considered alive */ @@ -1344,7 +1343,7 @@ class Bond { int packetsIn; int packetsOut; - AtomicCounter __refCount; + //AtomicCounter __refCount; SharedPtr p; void set(uint64_t now, const SharedPtr& path) @@ -1490,7 +1489,6 @@ class Bond { /** * Link state reporting */ - bool _isHealthy; uint8_t _numAliveLinks; uint8_t _numTotalLinks; diff --git a/node/Node.cpp b/node/Node.cpp index a0dd03fc1..3533a7238 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -509,7 +509,6 @@ ZT_PeerList *Node::peers() const if (pi->second->bond()) { p->isBonded = pi->second->bond(); p->bondingPolicy = pi->second->bond()->policy(); - p->isHealthy = pi->second->bond()->isHealthy(); p->numAliveLinks = pi->second->bond()->getNumAliveLinks(); p->numTotalLinks = pi->second->bond()->getNumTotalLinks(); } diff --git a/node/Peer.cpp b/node/Peer.cpp index 5a76eda3b..008ece176 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -503,7 +503,7 @@ void Peer::performMultipathStateCheck(void *tPtr, int64_t now) _localMultipathSupported = ((numAlivePaths >= 1) && (RR->bc->inUse()) && (ZT_PROTO_VERSION > 9)); if (_localMultipathSupported && !_bond) { if (RR->bc) { - _bond = RR->bc->createTransportTriggeredBond(RR, this); + _bond = RR->bc->createBond(RR, this); /** * Allow new bond to retroactively learn all paths known to this peer */ diff --git a/node/Peer.hpp b/node/Peer.hpp index 77e35b786..ad267acc8 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -53,7 +53,10 @@ private: Peer() {} // disabled to prevent bugs -- should not be constructed uninitialized public: - ~Peer() { Utils::burn(_key,sizeof(_key)); } + ~Peer() { + Utils::burn(_key,sizeof(_key)); + RR->bc->destroyBond(_id.address().toInt()); + } /** * Construct a new peer diff --git a/one.cpp b/one.cpp index cfd94fd52..524b1ff8f 100644 --- a/one.cpp +++ b/one.cpp @@ -523,31 +523,23 @@ static int cli(int argc,char **argv) printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { bool bFoundBond = false; - printf(" " ZT_EOL_S); + printf(" " ZT_EOL_S); if (j.is_array()) { for(unsigned long k=0;k= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) { policyStr = Bond::getPolicyStrByCode(bondingPolicy); } - printf("%10s %32s %8s %d/%d" ZT_EOL_S, + printf("%10s %32s %d/%d" ZT_EOL_S, OSUtils::jsonString(p ["address"],"-").c_str(), policyStr.c_str(), - healthStr.c_str(), numAliveLinks, numTotalLinks); } @@ -617,12 +609,6 @@ static int cli(int argc,char **argv) if (json) { printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { - std::string healthStr; - if (OSUtils::jsonInt(j["isHealthy"],0)) { - healthStr = "Healthy"; - } else { - healthStr = "Degraded"; - } int numAliveLinks = OSUtils::jsonInt(j["numAliveLinks"],0); int numTotalLinks = OSUtils::jsonInt(j["numTotalLinks"],0); printf("Peer : %s\n", arg1.c_str()); @@ -630,7 +616,6 @@ static int cli(int argc,char **argv) //if (bondingPolicy == ZT_BOND_POLICY_ACTIVE_BACKUP) { printf("Link Select Method : %d\n", (int)OSUtils::jsonInt(j["linkSelectMethod"],0)); //} - printf("Status : %s\n", healthStr.c_str()); printf("Links : %d/%d\n", numAliveLinks, numTotalLinks); printf("Failover Interval : %d (ms)\n", (int)OSUtils::jsonInt(j["failoverInterval"],0)); printf("Up Delay : %d (ms)\n", (int)OSUtils::jsonInt(j["upDelay"],0)); @@ -705,33 +690,23 @@ static int cli(int argc,char **argv) printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { bool bFoundBond = false; - printf(" " ZT_EOL_S); + printf(" " ZT_EOL_S); if (j.is_array()) { for(unsigned long k=0;k= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) { policyStr = Bond::getPolicyStrByCode(bondingPolicy); } - - printf("%10s %32s %8s %d/%d" ZT_EOL_S, + printf("%10s %32s %d/%d" ZT_EOL_S, OSUtils::jsonString(p["address"],"-").c_str(), policyStr.c_str(), - healthStr.c_str(), numAliveLinks, numTotalLinks); } diff --git a/service/OneService.cpp b/service/OneService.cpp index 73b3a9d56..1ae4ace90 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -510,7 +510,6 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) pj["isBonded"] = peer->isBonded; if (peer->isBonded) { pj["bondingPolicy"] = peer->bondingPolicy; - pj["isHealthy"] = peer->isHealthy; pj["numAliveLinks"] = peer->numAliveLinks; pj["numTotalLinks"] = peer->numTotalLinks; } @@ -542,7 +541,6 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr &bond) return; } - pj["isHealthy"] = bond->isHealthy(); pj["numAliveLinks"] = bond->getNumAliveLinks(); pj["numTotalLinks"] = bond->getNumTotalLinks(); pj["failoverInterval"] = bond->getFailoverInterval(); From 3ead6b67cd2cf540ce51add2a2d425a1662decf8 Mon Sep 17 00:00:00 2001 From: Joseph Henry Date: Wed, 2 Mar 2022 14:58:31 -0800 Subject: [PATCH 9/9] Dynamically scale bonded link monitor frequency according to aliveness --- node/Bond.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/Bond.cpp b/node/Bond.cpp index f1488a8ea..e58c67297 100644 --- a/node/Bond.cpp +++ b/node/Bond.cpp @@ -768,7 +768,7 @@ void Bond::processBackgroundBondTasks(void* tPtr, int64_t now) for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) { if (_paths[i].p && _paths[i].allowed()) { if (_isLeaf) { - if ((_monitorInterval > 0) && (((now - _paths[i].p->_lastIn) >= _monitorInterval) /*|| ((now - _paths[i].p->_lastOut) >= _monitorInterval)*/)) { + if ((_monitorInterval > 0) && (((now - _paths[i].p->_lastIn) >= (_paths[i].alive ? _monitorInterval : _failoverInterval)))) { if ((_peer->remoteVersionProtocol() >= 5) && (! ((_peer->remoteVersionMajor() == 1) && (_peer->remoteVersionMinor() == 1) && (_peer->remoteVersionRevision() == 0)))) { Packet outp(_peer->address(), RR->identity.address(), Packet::VERB_ECHO); // ECHO (this is our bond's heartbeat) outp.armor(_peer->key(), true, _peer->aesKeysIfSupported());