diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index c6623d556..f83ebc9b9 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -777,6 +777,7 @@ void PostgreSQL::initializeMembers() std::string assignedAddresses = std::get<20>(row); config["id"] = memberId; + config["address"] = memberId; config["nwid"] = networkId; config["activeBridge"] = activeBridge.value_or(false); config["authorized"] = authorized.value_or(false); diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h index e88fbda1f..e16d23f87 100644 --- a/include/ZeroTierOne.h +++ b/include/ZeroTierOne.h @@ -1404,11 +1404,6 @@ typedef struct */ int bondingPolicy; - /** - * The health status of the bond to this peer - */ - bool isHealthy; - /** * The number of links that comprise the bond to this peer that are considered alive */ diff --git a/node/Bond.cpp b/node/Bond.cpp index 14d1e1023..e58c67297 100644 --- a/node/Bond.cpp +++ b/node/Bond.cpp @@ -90,7 +90,7 @@ SharedPtr Bond::getBondByPeerId(int64_t identity) return _bonds.count(identity) ? _bonds[identity] : SharedPtr(); } -SharedPtr Bond::createTransportTriggeredBond(const RuntimeEnvironment* renv, const SharedPtr& peer) +SharedPtr Bond::createBond(const RuntimeEnvironment* renv, const SharedPtr& peer) { Mutex::Lock _l(_bonds_m); int64_t identity = peer->identity().address().toInt(); @@ -145,6 +145,12 @@ SharedPtr Bond::createTransportTriggeredBond(const RuntimeEnvironment* ren return SharedPtr(); } +void Bond::destroyBond(uint64_t peerId) +{ + Mutex::Lock _l(_bonds_m); + _bonds.erase(peerId); +} + SharedPtr Bond::getLinkBySocket(const std::string& policyAlias, uint64_t localSocket) { Mutex::Lock _l(_links_m); @@ -717,6 +723,7 @@ void Bond::sendPATH_NEGOTIATION_REQUEST(void* tPtr, int pathIdx) if (_paths[pathIdx].p->address()) { outp.armor(_peer->key(), false, _peer->aesKeysIfSupported()); RR->node->putPacket(tPtr, _paths[pathIdx].p->localSocket(), _paths[pathIdx].p->address(), outp.data(), outp.size()); + _overheadBytes += outp.size(); } } @@ -726,7 +733,6 @@ void Bond::sendQOS_MEASUREMENT(void* tPtr, int pathIdx, int64_t localSocket, con Packet outp(_peer->_id.address(), RR->identity.address(), Packet::VERB_QOS_MEASUREMENT); char qosData[ZT_QOS_MAX_PACKET_SIZE]; int16_t len = generateQoSPacket(pathIdx, _now, qosData); - _overheadBytes += len; if (len) { outp.append(qosData, len); if (atAddress) { @@ -738,6 +744,7 @@ void Bond::sendQOS_MEASUREMENT(void* tPtr, int pathIdx, int64_t localSocket, con } _paths[pathIdx].packetsReceivedSinceLastQoS = 0; _paths[pathIdx].lastQoSMeasurement = now; + _overheadBytes += outp.size(); } // debug("send QOS via link %s (len=%d)", pathToStr(_paths[pathIdx].p).c_str(), len); } @@ -761,7 +768,7 @@ void Bond::processBackgroundBondTasks(void* tPtr, int64_t now) for (unsigned int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) { if (_paths[i].p && _paths[i].allowed()) { if (_isLeaf) { - if ((_monitorInterval > 0) && (((now - _paths[i].p->_lastIn) >= _monitorInterval) /*|| ((now - _paths[i].p->_lastOut) >= _monitorInterval)*/)) { + if ((_monitorInterval > 0) && (((now - _paths[i].p->_lastIn) >= (_paths[i].alive ? _monitorInterval : _failoverInterval)))) { if ((_peer->remoteVersionProtocol() >= 5) && (! ((_peer->remoteVersionMajor() == 1) && (_peer->remoteVersionMinor() == 1) && (_peer->remoteVersionRevision() == 0)))) { Packet outp(_peer->address(), RR->identity.address(), Packet::VERB_ECHO); // ECHO (this is our bond's heartbeat) outp.armor(_peer->key(), true, _peer->aesKeysIfSupported()); @@ -815,6 +822,17 @@ void Bond::curateBond(int64_t now, bool rebuildBond) if (! _paths[i].p) { continue; } + + /** + * Remove expired links from bond + */ + if ((now - _paths[i].p->_lastIn) > (ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD)) { + log("link %s has expired, removing from bond", pathToStr(_paths[i].p).c_str()); + _paths[i] = NominatedPath(); + _paths[i].p = SharedPtr(); + continue; + } + tmpNumTotalLinks++; if (_paths[i].eligible) { tmpNumAliveLinks++; @@ -875,42 +893,18 @@ void Bond::curateBond(int64_t now, bool rebuildBond) } /** - * Determine health status to report to user + * Trigger status report if number of links change */ _numAliveLinks = tmpNumAliveLinks; _numTotalLinks = tmpNumTotalLinks; - bool tmpHealthStatus = true; - - if (_policy == ZT_BOND_POLICY_BROADCAST) { - if (_numAliveLinks < 1) { - // Considered healthy if we're able to send frames at all - tmpHealthStatus = false; - } - } - if ((_policy == ZT_BOND_POLICY_BALANCE_RR) || (_policy == ZT_BOND_POLICY_BALANCE_XOR) || (_policy == ZT_BOND_POLICY_BALANCE_AWARE || (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP))) { - if (_numAliveLinks < _numTotalLinks) { - tmpHealthStatus = false; - } - } - if (tmpHealthStatus != _isHealthy) { - std::string healthStatusStr; - if (tmpHealthStatus == true) { - healthStatusStr = "HEALTHY"; - } - else { - healthStatusStr = "DEGRADED"; - } - log("bond is %s (%d/%d links)", healthStatusStr.c_str(), _numAliveLinks, _numTotalLinks); + if ((_numAliveLinks != tmpNumAliveLinks) || (_numTotalLinks != tmpNumTotalLinks)) { dumpInfo(now, true); } - _isHealthy = tmpHealthStatus; - /** * Curate the set of paths that are part of the bond proper. Select a set of paths * per logical link according to eligibility and user-specified constraints. */ - if ((_policy == ZT_BOND_POLICY_BALANCE_RR) || (_policy == ZT_BOND_POLICY_BALANCE_XOR) || (_policy == ZT_BOND_POLICY_BALANCE_AWARE)) { if (! _numBondedPaths) { rebuildBond = true; @@ -1008,14 +1002,14 @@ void Bond::estimatePathQuality(int64_t now) uint32_t totUserSpecifiedLinkSpeed = 0; if (_numBondedPaths) { // Compute relative user-specified speeds of links for (unsigned int i = 0; i < _numBondedPaths; ++i) { - SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); if (_paths[i].p && _paths[i].allowed()) { + SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); totUserSpecifiedLinkSpeed += link->speed(); } } for (unsigned int i = 0; i < _numBondedPaths; ++i) { - SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); if (_paths[i].p && _paths[i].allowed()) { + SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); link->setRelativeSpeed((uint8_t)round(((float)link->speed() / (float)totUserSpecifiedLinkSpeed) * 255)); } } @@ -1212,6 +1206,11 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now) int nonPreferredPathIdx; bool bFoundPrimaryLink = false; + if (_abPathIdx != ZT_MAX_PEER_NETWORK_PATHS && !_paths[_abPathIdx].p) { + _abPathIdx = ZT_MAX_PEER_NETWORK_PATHS; + log("main active-backup path has been removed"); + } + /** * Generate periodic status report */ @@ -1241,7 +1240,6 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now) * simply find the next eligible path. */ if (! userHasSpecifiedLinks()) { - debug("no user-specified links"); for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) { if (_paths[i].p && _paths[i].eligible) { SharedPtr link = RR->bc->getLinkBySocket(_policyAlias, _paths[i].p->localSocket()); @@ -1574,7 +1572,6 @@ void Bond::setBondParameters(int policy, SharedPtr templateBond, bool useT // Bond status - _isHealthy = false; _numAliveLinks = 0; _numTotalLinks = 0; _numBondedPaths = 0; @@ -1684,11 +1681,14 @@ SharedPtr Bond::getLink(const SharedPtr& path) std::string Bond::pathToStr(const SharedPtr& path) { #ifdef ZT_TRACE - char pathStr[64] = { 0 }; - char fullPathStr[384] = { 0 }; - path->address().toString(pathStr); - snprintf(fullPathStr, 384, "%.16llx-%s/%s", (unsigned long long)(path->localSocket()), getLink(path)->ifname().c_str(), pathStr); - return std::string(fullPathStr); + if (path) { + char pathStr[64] = { 0 }; + char fullPathStr[384] = { 0 }; + path->address().toString(pathStr); + snprintf(fullPathStr, 384, "%.16llx-%s/%s", (unsigned long long)(path->localSocket()), getLink(path)->ifname().c_str(), pathStr); + return std::string(fullPathStr); + } + return ""; #else return ""; #endif @@ -1727,7 +1727,7 @@ void Bond::dumpInfo(int64_t now, bool force) _lastSummaryDump = now; float overhead = (_overheadBytes / (timeSinceLastDump / 1000.0f) / 1000.0f); _overheadBytes = 0; - log("bond: bp=%d, fi=%d, mi=%d, ud=%d, dd=%d, flows=%lu, leaf=%d, overhead=%f KB/s", + log("bond: bp=%d, fi=%d, mi=%d, ud=%d, dd=%d, flows=%lu, leaf=%d, overhead=%f KB/s, links=(%d/%d)", _policy, _failoverInterval, _monitorInterval, @@ -1735,7 +1735,9 @@ void Bond::dumpInfo(int64_t now, bool force) _downDelay, (unsigned long)_flows.size(), _isLeaf, - overhead); + overhead, + _numAliveLinks, + _numTotalLinks); for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) { if (_paths[i].p) { dumpPathStatus(now, i); diff --git a/node/Bond.hpp b/node/Bond.hpp index b30d3a90f..3465db51c 100644 --- a/node/Bond.hpp +++ b/node/Bond.hpp @@ -436,7 +436,14 @@ class Bond { * @param peer Remote peer that this bond services * @return A pointer to the newly created Bond */ - static SharedPtr createTransportTriggeredBond(const RuntimeEnvironment* renv, const SharedPtr& peer); + static SharedPtr createBond(const RuntimeEnvironment* renv, const SharedPtr& peer); + + /** + * Remove a bond from the bond controller. + * + * @param peerId Remote peer that this bond services + */ + static void destroyBond(uint64_t peerId); /** * Periodically perform maintenance tasks for the bonding layer. @@ -1020,14 +1027,6 @@ class Bond { return _policy; } - /** - * @return the health status of the bond - */ - inline bool isHealthy() - { - return _isHealthy; - } - /** * @return the number of links comprising this bond which are considered alive */ @@ -1344,7 +1343,7 @@ class Bond { int packetsIn; int packetsOut; - AtomicCounter __refCount; + //AtomicCounter __refCount; SharedPtr p; void set(uint64_t now, const SharedPtr& path) @@ -1490,7 +1489,6 @@ class Bond { /** * Link state reporting */ - bool _isHealthy; uint8_t _numAliveLinks; uint8_t _numTotalLinks; diff --git a/node/Constants.hpp b/node/Constants.hpp index 930c91299..d9a3e7964 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -571,13 +571,13 @@ * Anything below this value gets into thrashing territory since we divide * this value by ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL to send ECHOs often. */ -#define ZT_BOND_FAILOVER_MIN_INTERVAL 250 +#define ZT_BOND_FAILOVER_MIN_INTERVAL 500 /** * How many times per failover interval that an ECHO is sent. This should be * at least 2. Anything more then 4 starts to increase overhead significantly. */ -#define ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL 4 +#define ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL 3 /** * A defensive timer to prevent path quality metrics from being diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index 6441b6ad7..347c0649d 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -860,7 +860,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { uint64_t now = RR->node->now(); - if (!peer->rateGateEchoRequest(now)) { + if (!_path->rateGateEchoRequest(now)) { return true; } diff --git a/node/Node.cpp b/node/Node.cpp index a0dd03fc1..3533a7238 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -509,7 +509,6 @@ ZT_PeerList *Node::peers() const if (pi->second->bond()) { p->isBonded = pi->second->bond(); p->bondingPolicy = pi->second->bond()->policy(); - p->isHealthy = pi->second->bond()->isHealthy(); p->numAliveLinks = pi->second->bond()->getNumAliveLinks(); p->numTotalLinks = pi->second->bond()->getNumTotalLinks(); } diff --git a/node/Path.hpp b/node/Path.hpp index 753bf0ab2..3471e2d66 100644 --- a/node/Path.hpp +++ b/node/Path.hpp @@ -83,6 +83,7 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _lastEchoRequestReceived(0), _localSocket(-1), _latency(0xffff), _addr(), @@ -93,6 +94,7 @@ public: _lastOut(0), _lastIn(0), _lastTrustEstablishedPacketReceived(0), + _lastEchoRequestReceived(0), _localSocket(localSocket), _latency(0xffff), _addr(addr), @@ -266,6 +268,18 @@ public: */ inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; } + /** + * Rate limit gate for inbound ECHO requests + */ + inline bool rateGateEchoRequest(const int64_t now) + { + if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT / 6)) { + _lastEchoRequestReceived = now; + return true; + } + return false; + } + void *_bondingMetricPtr; private: @@ -273,6 +287,9 @@ private: volatile int64_t _lastOut; volatile int64_t _lastIn; volatile int64_t _lastTrustEstablishedPacketReceived; + + int64_t _lastEchoRequestReceived; + int64_t _localSocket; volatile unsigned int _latency; InetAddress _addr; diff --git a/node/Peer.cpp b/node/Peer.cpp index 8ab51687d..008ece176 100644 --- a/node/Peer.cpp +++ b/node/Peer.cpp @@ -34,7 +34,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident _lastTriedMemorizedPath(0), _lastDirectPathPushSent(0), _lastDirectPathPushReceive(0), - _lastEchoRequestReceived(0), _lastCredentialRequestSent(0), _lastWhoisRequestReceived(0), _lastCredentialsReceived(0), @@ -224,6 +223,8 @@ void Peer::received( if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) { _lastDirectPathPushSent = now; std::vector pathsToPush(RR->node->directPaths()); + std::vector ma = RR->sa->whoami(); + pathsToPush.insert(pathsToPush.end(), ma.begin(), ma.end()); if (!pathsToPush.empty()) { std::vector::const_iterator p(pathsToPush.begin()); while (p != pathsToPush.end()) { @@ -453,7 +454,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA if (atAddress) { outp.armor(_key,false,nullptr); // false == don't encrypt full payload, but add MAC RR->node->expectReplyTo(outp.packetId()); - RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size()); + RR->node->putPacket(tPtr,-1,atAddress,outp.data(),outp.size()); } else { RR->node->expectReplyTo(outp.packetId()); RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC @@ -502,7 +503,7 @@ void Peer::performMultipathStateCheck(void *tPtr, int64_t now) _localMultipathSupported = ((numAlivePaths >= 1) && (RR->bc->inUse()) && (ZT_PROTO_VERSION > 9)); if (_localMultipathSupported && !_bond) { if (RR->bc) { - _bond = RR->bc->createTransportTriggeredBond(RR, this); + _bond = RR->bc->createBond(RR, this); /** * Allow new bond to retroactively learn all paths known to this peer */ diff --git a/node/Peer.hpp b/node/Peer.hpp index 3bfe31950..ad267acc8 100644 --- a/node/Peer.hpp +++ b/node/Peer.hpp @@ -53,7 +53,10 @@ private: Peer() {} // disabled to prevent bugs -- should not be constructed uninitialized public: - ~Peer() { Utils::burn(_key,sizeof(_key)); } + ~Peer() { + Utils::burn(_key,sizeof(_key)); + RR->bc->destroyBond(_id.address().toInt()); + } /** * Construct a new peer @@ -418,18 +421,6 @@ public: return false; } - /** - * Rate limit gate for inbound ECHO requests - */ - inline bool rateGateEchoRequest(const int64_t now) - { - if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) { - _lastEchoRequestReceived = now; - return true; - } - return false; - } - /** * Serialize a peer for storage in local cache * @@ -546,7 +537,6 @@ private: int64_t _lastTriedMemorizedPath; int64_t _lastDirectPathPushSent; int64_t _lastDirectPathPushReceive; - int64_t _lastEchoRequestReceived; int64_t _lastCredentialRequestSent; int64_t _lastWhoisRequestReceived; int64_t _lastCredentialsReceived; diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp index 57b409c12..45bc25410 100644 --- a/node/SelfAwareness.cpp +++ b/node/SelfAwareness.cpp @@ -99,6 +99,21 @@ void SelfAwareness::iam(void *tPtr,const Address &reporter,const int64_t receive } } +std::vector SelfAwareness::whoami() +{ + std::vector surfaceAddresses; + Mutex::Lock _l(_phy_m); + Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy); + PhySurfaceKey *k = (PhySurfaceKey *)0; + PhySurfaceEntry *e = (PhySurfaceEntry *)0; + while (i.next(k,e)) { + if (std::find(surfaceAddresses.begin(), surfaceAddresses.end(), e->mySurface) == surfaceAddresses.end()) { + surfaceAddresses.push_back(e->mySurface); + } + } + return surfaceAddresses; +} + void SelfAwareness::clean(int64_t now) { Mutex::Lock _l(_phy_m); diff --git a/node/SelfAwareness.hpp b/node/SelfAwareness.hpp index f34d5778e..e0fbf91d0 100644 --- a/node/SelfAwareness.hpp +++ b/node/SelfAwareness.hpp @@ -44,6 +44,13 @@ public: */ void iam(void *tPtr,const Address &reporter,const int64_t receivedOnLocalSocket,const InetAddress &reporterPhysicalAddress,const InetAddress &myPhysicalAddress,bool trusted,int64_t now); + /** + * Return all known external surface addresses reported by peers + * + * @return A vector of InetAddress(es) + */ + std::vector whoami(); + /** * Clean up database periodically * diff --git a/one.cpp b/one.cpp index cfd94fd52..524b1ff8f 100644 --- a/one.cpp +++ b/one.cpp @@ -523,31 +523,23 @@ static int cli(int argc,char **argv) printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { bool bFoundBond = false; - printf(" " ZT_EOL_S); + printf(" " ZT_EOL_S); if (j.is_array()) { for(unsigned long k=0;k= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) { policyStr = Bond::getPolicyStrByCode(bondingPolicy); } - printf("%10s %32s %8s %d/%d" ZT_EOL_S, + printf("%10s %32s %d/%d" ZT_EOL_S, OSUtils::jsonString(p ["address"],"-").c_str(), policyStr.c_str(), - healthStr.c_str(), numAliveLinks, numTotalLinks); } @@ -617,12 +609,6 @@ static int cli(int argc,char **argv) if (json) { printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { - std::string healthStr; - if (OSUtils::jsonInt(j["isHealthy"],0)) { - healthStr = "Healthy"; - } else { - healthStr = "Degraded"; - } int numAliveLinks = OSUtils::jsonInt(j["numAliveLinks"],0); int numTotalLinks = OSUtils::jsonInt(j["numTotalLinks"],0); printf("Peer : %s\n", arg1.c_str()); @@ -630,7 +616,6 @@ static int cli(int argc,char **argv) //if (bondingPolicy == ZT_BOND_POLICY_ACTIVE_BACKUP) { printf("Link Select Method : %d\n", (int)OSUtils::jsonInt(j["linkSelectMethod"],0)); //} - printf("Status : %s\n", healthStr.c_str()); printf("Links : %d/%d\n", numAliveLinks, numTotalLinks); printf("Failover Interval : %d (ms)\n", (int)OSUtils::jsonInt(j["failoverInterval"],0)); printf("Up Delay : %d (ms)\n", (int)OSUtils::jsonInt(j["upDelay"],0)); @@ -705,33 +690,23 @@ static int cli(int argc,char **argv) printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str()); } else { bool bFoundBond = false; - printf(" " ZT_EOL_S); + printf(" " ZT_EOL_S); if (j.is_array()) { for(unsigned long k=0;k= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) { policyStr = Bond::getPolicyStrByCode(bondingPolicy); } - - printf("%10s %32s %8s %d/%d" ZT_EOL_S, + printf("%10s %32s %d/%d" ZT_EOL_S, OSUtils::jsonString(p["address"],"-").c_str(), policyStr.c_str(), - healthStr.c_str(), numAliveLinks, numTotalLinks); } diff --git a/service/OneService.cpp b/service/OneService.cpp index 73b3a9d56..1ae4ace90 100644 --- a/service/OneService.cpp +++ b/service/OneService.cpp @@ -510,7 +510,6 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer) pj["isBonded"] = peer->isBonded; if (peer->isBonded) { pj["bondingPolicy"] = peer->bondingPolicy; - pj["isHealthy"] = peer->isHealthy; pj["numAliveLinks"] = peer->numAliveLinks; pj["numTotalLinks"] = peer->numTotalLinks; } @@ -542,7 +541,6 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr &bond) return; } - pj["isHealthy"] = bond->isHealthy(); pj["numAliveLinks"] = bond->getNumAliveLinks(); pj["numTotalLinks"] = bond->getNumTotalLinks(); pj["failoverInterval"] = bond->getFailoverInterval();