ExternalVendorCode/ZeroTierOne
1
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2024-12-19 04:57:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Adjust terminology

Browse Source
This commit is contained in:
Joseph Henry 2020-06-17 14:54:13 -07:00
parent 5f0ee4fc78
commit a33a494d60
13 changed files with 440 additions and 442 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
include/ZeroTierOne.h
View File

@ -455,39 +455,39 @@ enum ZT_MultipathBondingPolicy
}; };
/** /**
* Multipath active re-selection policy (slaveSelectMethod) * Multipath active re-selection policy (linkSelectMethod)
*/ */
enum ZT_MultipathSlaveSelectMethod enum ZT_MultipathLinkSelectMethod
{ {
/** /**
* Primary slave regains status as active slave whenever it comes back up * Primary link regains status as active link whenever it comes back up
* (default when slaves are explicitly specified) * (default when links are explicitly specified)
*/ */
ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS = 0, ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS = 0,
/** /**
* Primary slave regains status as active slave when it comes back up and * Primary link regains status as active link when it comes back up and
* (if) it is better than the currently-active slave. * (if) it is better than the currently-active link.
*/ */
ZT_MULTIPATH_RESELECTION_POLICY_BETTER = 1, ZT_MULTIPATH_RESELECTION_POLICY_BETTER = 1,
/** /**
* Primary slave regains status as active slave only if the currently-active * Primary link regains status as active link only if the currently-active
* slave fails. * link fails.
*/ */
ZT_MULTIPATH_RESELECTION_POLICY_FAILURE = 2, ZT_MULTIPATH_RESELECTION_POLICY_FAILURE = 2,
/** /**
* The primary slave can change if a superior path is detected. * The primary link can change if a superior path is detected.
* (default if user provides no fail-over guidance) * (default if user provides no fail-over guidance)
*/ */
ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE = 3 ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE = 3
}; };
/** /**
* Mode of multipath slave interface * Mode of multipath link interface
*/ */
enum ZT_MultipathSlaveMode enum ZT_MultipathLinkMode
{ {
ZT_MULTIPATH_SLAVE_MODE_PRIMARY = 0, ZT_MULTIPATH_SLAVE_MODE_PRIMARY = 0,
ZT_MULTIPATH_SLAVE_MODE_SPARE = 1 ZT_MULTIPATH_SLAVE_MODE_SPARE = 1
@ -527,7 +527,7 @@ enum ZT_MultipathMonitorStrategy
enum ZT_MultipathFlowRebalanceStrategy enum ZT_MultipathFlowRebalanceStrategy
{ {
/** /**
* Flows will only be re-balanced among slaves during * Flows will only be re-balanced among links during
* assignment or failover. This minimizes the possibility * assignment or failover. This minimizes the possibility
* of sequence reordering and is thus the default setting. * of sequence reordering and is thus the default setting.
*/ */
@ -535,13 +535,13 @@ enum ZT_MultipathFlowRebalanceStrategy
/** /**
* Flows that are active may be re-assigned to a new more * Flows that are active may be re-assigned to a new more
* suitable slave if it can be done without disrupting the flow. * suitable link if it can be done without disrupting the flow.
* This setting can sometimes cause sequence re-ordering. * This setting can sometimes cause sequence re-ordering.
*/ */
ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_OPPORTUNISTIC = 0, ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_OPPORTUNISTIC = 0,
/** /**
* Flows will be continuously re-assigned the most suitable slave * Flows will be continuously re-assigned the most suitable link
* in order to maximize "balance". This can often cause sequence * in order to maximize "balance". This can often cause sequence
* reordering and is thus only reccomended for protocols like UDP. * reordering and is thus only reccomended for protocols like UDP.
*/ */

364
node/Bond.cpp
View File

@ -54,9 +54,9 @@ Bond::Bond(const RuntimeEnvironment *renv, SharedPtr<Bond> originalBond, const S
void Bond::nominatePath(const SharedPtr<Path>& path, int64_t now) void Bond::nominatePath(const SharedPtr<Path>& path, int64_t now)
{ {
char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "nominatePath: %s %s\n", getSlave(path)->ifname().c_str(), pathStr); char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "nominatePath: %s %s\n", getLink(path)->ifname().c_str(), pathStr);
Mutex::Lock _l(_paths_m); Mutex::Lock _l(_paths_m);
if (!RR->bc->slaveAllowed(_policyAlias, getSlave(path))) { if (!RR->bc->linkAllowed(_policyAlias, getLink(path))) {
return; return;
} }
bool alreadyPresent = false; bool alreadyPresent = false;
@ -72,7 +72,7 @@ void Bond::nominatePath(const SharedPtr<Path>& path, int64_t now)
if (!_paths[i]) { if (!_paths[i]) {
fprintf(stderr, "notifyOfNewPath(): Setting path %s to idx=%d\n", pathStr, i); fprintf(stderr, "notifyOfNewPath(): Setting path %s to idx=%d\n", pathStr, i);
_paths[i] = path; _paths[i] = path;
//_paths[i]->slave = RR->bc->getSlaveBySocket(_policyAlias, path->localSocket()); //_paths[i]->link = RR->bc->getLinkBySocket(_policyAlias, path->localSocket());
_paths[i]->startTrial(now); _paths[i]->startTrial(now);
break; break;
} }
@ -107,18 +107,18 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
*/ */
if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) { if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) {
if (!_allowFlowHashing) { if (!_allowFlowHashing) {
//fprintf(stderr, "_rrPacketsSentOnCurrSlave=%d, _numBondedPaths=%d, _rrIdx=%d\n", _rrPacketsSentOnCurrSlave, _numBondedPaths, _rrIdx); //fprintf(stderr, "_rrPacketsSentOnCurrLink=%d, _numBondedPaths=%d, _rrIdx=%d\n", _rrPacketsSentOnCurrLink, _numBondedPaths, _rrIdx);
if (_packetsPerSlave == 0) { if (_packetsPerLink == 0) {
// Randomly select a path // Randomly select a path
return _paths[_bondedIdx[_freeRandomByte % _numBondedPaths]]; // TODO: Optimize return _paths[_bondedIdx[_freeRandomByte % _numBondedPaths]]; // TODO: Optimize
} }
if (_rrPacketsSentOnCurrSlave < _packetsPerSlave) { if (_rrPacketsSentOnCurrLink < _packetsPerLink) {
// Continue to use this slave // Continue to use this link
++_rrPacketsSentOnCurrSlave; ++_rrPacketsSentOnCurrLink;
return _paths[_bondedIdx[_rrIdx]]; return _paths[_bondedIdx[_rrIdx]];
} }
// Reset striping counter // Reset striping counter
_rrPacketsSentOnCurrSlave = 0; _rrPacketsSentOnCurrLink = 0;
if (_numBondedPaths == 1) { if (_numBondedPaths == 1) {
_rrIdx = 0; _rrIdx = 0;
} }
@ -170,7 +170,7 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
void Bond::recordIncomingInvalidPacket(const SharedPtr<Path>& path) void Bond::recordIncomingInvalidPacket(const SharedPtr<Path>& path)
{ {
// char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "recordIncomingInvalidPacket() %s %s\n", getSlave(path)->ifname().c_str(), pathStr); // char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "recordIncomingInvalidPacket() %s %s\n", getLink(path)->ifname().c_str(), pathStr);
Mutex::Lock _l(_paths_m); Mutex::Lock _l(_paths_m);
for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) { for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {
if (_paths[i] == path) { if (_paths[i] == path) {
@ -182,7 +182,7 @@ void Bond::recordIncomingInvalidPacket(const SharedPtr<Path>& path)
void Bond::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId, void Bond::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now) uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
{ {
// char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "recordOutgoingPacket() %s %s, packetId=%llx, payloadLength=%d, verb=%x, flowId=%lx\n", getSlave(path)->ifname().c_str(), pathStr, packetId, payloadLength, verb, flowId); // char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "recordOutgoingPacket() %s %s, packetId=%llx, payloadLength=%d, verb=%x, flowId=%lx\n", getLink(path)->ifname().c_str(), pathStr, packetId, payloadLength, verb, flowId);
_freeRandomByte += (unsigned char)(packetId >> 8); // Grab entropy to use in path selection logic _freeRandomByte += (unsigned char)(packetId >> 8); // Grab entropy to use in path selection logic
if (!_shouldCollectPathStatistics) { if (!_shouldCollectPathStatistics) {
return; return;
@ -218,7 +218,7 @@ void Bond::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t pack
void Bond::recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId, uint16_t payloadLength, void Bond::recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId, uint16_t payloadLength,
Packet::Verb verb, int32_t flowId, int64_t now) Packet::Verb verb, int32_t flowId, int64_t now)
{ {
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "recordIncomingPacket() %s %s, packetId=%llx, payloadLength=%d, verb=%x, flowId=%lx\n", getSlave(path)->ifname().c_str(), pathStr, packetId, payloadLength, verb, flowId); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "recordIncomingPacket() %s %s, packetId=%llx, payloadLength=%d, verb=%x, flowId=%lx\n", getLink(path)->ifname().c_str(), pathStr, packetId, payloadLength, verb, flowId);
bool isFrame = (verb == Packet::VERB_FRAME || verb == Packet::VERB_EXT_FRAME); bool isFrame = (verb == Packet::VERB_FRAME || verb == Packet::VERB_EXT_FRAME);
bool shouldRecord = (packetId & (ZT_QOS_ACK_DIVISOR - 1) bool shouldRecord = (packetId & (ZT_QOS_ACK_DIVISOR - 1)
&& (verb != Packet::VERB_ACK) && (verb != Packet::VERB_ACK)
@ -261,7 +261,7 @@ void Bond::recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId,
void Bond::receivedQoS(const SharedPtr<Path>& path, int64_t now, int count, uint64_t *rx_id, uint16_t *rx_ts) void Bond::receivedQoS(const SharedPtr<Path>& path, int64_t now, int count, uint64_t *rx_id, uint16_t *rx_ts)
{ {
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "receivedQoS() %s %s\n", getSlave(path)->ifname().c_str(), pathStr); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "receivedQoS() %s %s\n", getLink(path)->ifname().c_str(), pathStr);
Mutex::Lock _l(_paths_m); Mutex::Lock _l(_paths_m);
// Look up egress times and compute latency values for each record // Look up egress times and compute latency values for each record
std::map<uint64_t,uint64_t>::iterator it; std::map<uint64_t,uint64_t>::iterator it;
@ -273,13 +273,13 @@ void Bond::receivedQoS(const SharedPtr<Path>& path, int64_t now, int count, uint
} }
} }
path->qosRecordSize.push(count); path->qosRecordSize.push(count);
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "receivedQoS() on path %s %s, count=%d, successful=%d, qosStatsOut.size()=%d\n", getSlave(path)->ifname().c_str(), pathStr, count, path->aknowledgedQoSRecordCountSinceLastCheck, path->qosStatsOut.size()); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "receivedQoS() on path %s %s, count=%d, successful=%d, qosStatsOut.size()=%d\n", getLink(path)->ifname().c_str(), pathStr, count, path->aknowledgedQoSRecordCountSinceLastCheck, path->qosStatsOut.size());
} }
void Bond::receivedAck(const SharedPtr<Path>& path, int64_t now, int32_t ackedBytes) void Bond::receivedAck(const SharedPtr<Path>& path, int64_t now, int32_t ackedBytes)
{ {
Mutex::Lock _l(_paths_m); Mutex::Lock _l(_paths_m);
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "receivedAck() %s %s, (ackedBytes=%d, lastAckReceived=%lld, ackAge=%lld)\n", getSlave(path)->ifname().c_str(), pathStr, ackedBytes, path->lastAckReceived, path->ackAge(now)); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "receivedAck() %s %s, (ackedBytes=%d, lastAckReceived=%lld, ackAge=%lld)\n", getLink(path)->ifname().c_str(), pathStr, ackedBytes, path->lastAckReceived, path->ackAge(now));
path->_lastAckReceived = now; path->_lastAckReceived = now;
path->_unackedBytes = (ackedBytes > path->_unackedBytes) ? 0 : path->_unackedBytes - ackedBytes; path->_unackedBytes = (ackedBytes > path->_unackedBytes) ? 0 : path->_unackedBytes - ackedBytes;
int64_t timeSinceThroughputEstimate = (now - path->_lastThroughputEstimation); int64_t timeSinceThroughputEstimate = (now - path->_lastThroughputEstimation);
@ -300,7 +300,7 @@ void Bond::receivedAck(const SharedPtr<Path>& path, int64_t now, int32_t ackedBy
int32_t Bond::generateQoSPacket(const SharedPtr<Path>& path, int64_t now, char *qosBuffer) int32_t Bond::generateQoSPacket(const SharedPtr<Path>& path, int64_t now, char *qosBuffer)
{ {
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "generateQoSPacket() %s %s\n", getSlave(path)->ifname().c_str(), pathStr); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "generateQoSPacket() %s %s\n", getLink(path)->ifname().c_str(), pathStr);
int32_t len = 0; int32_t len = 0;
std::map<uint64_t,uint64_t>::iterator it = path->qosStatsIn.begin(); std::map<uint64_t,uint64_t>::iterator it = path->qosStatsIn.begin();
int i=0; int i=0;
@ -355,10 +355,10 @@ bool Bond::assignFlowToBondedPath(SharedPtr<Flow> &flow, int64_t now)
//fprintf(stderr, "new entropy = %d\n", entropy); //fprintf(stderr, "new entropy = %d\n", entropy);
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
if (_paths[i] && _paths[i]->bonded()) { if (_paths[i] && _paths[i]->bonded()) {
SharedPtr<Slave> slave = RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link = RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
uint8_t probabilitySegment = (_totalBondUnderload > 0) ? _paths[i]->_affinity : _paths[i]->_allocation; uint8_t probabilitySegment = (_totalBondUnderload > 0) ? _paths[i]->_affinity : _paths[i]->_allocation;
//fprintf(stderr, "i=%2d, entropy=%3d, alloc=%3d, byteload=%4d, segment=%3d, _totalBondUnderload=%3d, ifname=%s, path=%20s\n", i, entropy, _paths[i]->_allocation, _paths[i]->_relativeByteLoad, probabilitySegment, _totalBondUnderload, slave->ifname().c_str(), curPathStr); //fprintf(stderr, "i=%2d, entropy=%3d, alloc=%3d, byteload=%4d, segment=%3d, _totalBondUnderload=%3d, ifname=%s, path=%20s\n", i, entropy, _paths[i]->_allocation, _paths[i]->_relativeByteLoad, probabilitySegment, _totalBondUnderload, link->ifname().c_str(), curPathStr);
if (entropy <= probabilitySegment) { if (entropy <= probabilitySegment) {
idx = i; idx = i;
//fprintf(stderr, "\t is best path\n"); //fprintf(stderr, "\t is best path\n");
@ -380,8 +380,8 @@ bool Bond::assignFlowToBondedPath(SharedPtr<Flow> &flow, int64_t now)
} }
} }
flow->assignedPath()->address().toString(curPathStr); flow->assignedPath()->address().toString(curPathStr);
SharedPtr<Slave> slave = RR->bc->getSlaveBySocket(_policyAlias, flow->assignedPath()->localSocket()); SharedPtr<Link> link = RR->bc->getLinkBySocket(_policyAlias, flow->assignedPath()->localSocket());
fprintf(stderr, "assigned (tx) flow %x with peer %llx to path %s on %s (idx=%d)\n", flow->id(), _peer->_id.address().toInt(), curPathStr, slave->ifname().c_str(), idx); fprintf(stderr, "assigned (tx) flow %x with peer %llx to path %s on %s (idx=%d)\n", flow->id(), _peer->_id.address().toInt(), curPathStr, link->ifname().c_str(), idx);
return true; return true;
} }
@ -410,8 +410,8 @@ SharedPtr<Flow> Bond::createFlow(const SharedPtr<Path> &path, int32_t flowId, un
flow->assignPath(path,now); flow->assignPath(path,now);
path->address().toString(curPathStr); path->address().toString(curPathStr);
path->_assignedFlowCount++; path->_assignedFlowCount++;
SharedPtr<Slave> slave = RR->bc->getSlaveBySocket(_policyAlias, flow->assignedPath()->localSocket()); SharedPtr<Link> link = RR->bc->getLinkBySocket(_policyAlias, flow->assignedPath()->localSocket());
fprintf(stderr, "assigned (rx) flow %x with peer %llx to path %s on %s\n", flow->id(), _peer->_id.address().toInt(), curPathStr, slave->ifname().c_str()); fprintf(stderr, "assigned (rx) flow %x with peer %llx to path %s on %s\n", flow->id(), _peer->_id.address().toInt(), curPathStr, link->ifname().c_str());
} }
/** /**
* Add a flow when no path was provided. This means that it is an outgoing packet * Add a flow when no path was provided. This means that it is an outgoing packet
@ -460,7 +460,7 @@ void Bond::forgetFlowsWhenNecessary(uint64_t age, bool oldest, int64_t now)
void Bond::processIncomingPathNegotiationRequest(uint64_t now, SharedPtr<Path> &path, int16_t remoteUtility) void Bond::processIncomingPathNegotiationRequest(uint64_t now, SharedPtr<Path> &path, int16_t remoteUtility)
{ {
//fprintf(stderr, "processIncomingPathNegotiationRequest\n"); //fprintf(stderr, "processIncomingPathNegotiationRequest\n");
if (_abSlaveSelectMethod != ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) { if (_abLinkSelectMethod != ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) {
return; return;
} }
Mutex::Lock _l(_paths_m); Mutex::Lock _l(_paths_m);
@ -469,18 +469,18 @@ void Bond::processIncomingPathNegotiationRequest(uint64_t now, SharedPtr<Path> &
if (!_lastPathNegotiationCheck) { if (!_lastPathNegotiationCheck) {
return; return;
} }
SharedPtr<Slave> slave = RR->bc->getSlaveBySocket(_policyAlias, path->localSocket()); SharedPtr<Link> link = RR->bc->getLinkBySocket(_policyAlias, path->localSocket());
if (remoteUtility > _localUtility) { if (remoteUtility > _localUtility) {
fprintf(stderr, "peer suggests path, its utility (%d) is greater than ours (%d), we will switch to %s on %s (ls=%llx)\n", remoteUtility, _localUtility, pathStr, slave->ifname().c_str(), path->localSocket()); fprintf(stderr, "peer suggests path, its utility (%d) is greater than ours (%d), we will switch to %s on %s (ls=%llx)\n", remoteUtility, _localUtility, pathStr, link->ifname().c_str(), path->localSocket());
negotiatedPath = path; negotiatedPath = path;
} }
if (remoteUtility < _localUtility) { if (remoteUtility < _localUtility) {
fprintf(stderr, "peer suggests path, its utility (%d) is less than ours (%d), we will NOT switch to %s on %s (ls=%llx)\n", remoteUtility, _localUtility, pathStr, slave->ifname().c_str(), path->localSocket()); fprintf(stderr, "peer suggests path, its utility (%d) is less than ours (%d), we will NOT switch to %s on %s (ls=%llx)\n", remoteUtility, _localUtility, pathStr, link->ifname().c_str(), path->localSocket());
} }
if (remoteUtility == _localUtility) { if (remoteUtility == _localUtility) {
fprintf(stderr, "peer suggest path, but utility is equal, picking choice made by peer with greater identity.\n"); fprintf(stderr, "peer suggest path, but utility is equal, picking choice made by peer with greater identity.\n");
if (_peer->_id.address().toInt() > RR->node->identity().address().toInt()) { if (_peer->_id.address().toInt() > RR->node->identity().address().toInt()) {
fprintf(stderr, "peer identity was greater, going with their choice of %s on %s (ls=%llx)\n", pathStr, slave->ifname().c_str(), path->localSocket()); fprintf(stderr, "peer identity was greater, going with their choice of %s on %s (ls=%llx)\n", pathStr, link->ifname().c_str(), path->localSocket());
negotiatedPath = path; negotiatedPath = path;
} else { } else {
fprintf(stderr, "our identity was greater, no change\n"); fprintf(stderr, "our identity was greater, no change\n");
@ -532,8 +532,8 @@ void Bond::pathNegotiationCheck(void *tPtr, const int64_t now)
++_numSentPathNegotiationRequests; ++_numSentPathNegotiationRequests;
_lastSentPathNegotiationRequest = now; _lastSentPathNegotiationRequest = now;
_paths[maxOutPathIdx]->address().toString(pathStr); _paths[maxOutPathIdx]->address().toString(pathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[maxOutPathIdx]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[maxOutPathIdx]->localSocket());
fprintf(stderr, "sending request to use %s on %s, ls=%llx, utility=%d\n", pathStr, slave->ifname().c_str(), _paths[maxOutPathIdx]->localSocket(), _localUtility); fprintf(stderr, "sending request to use %s on %s, ls=%llx, utility=%d\n", pathStr, link->ifname().c_str(), _paths[maxOutPathIdx]->localSocket(), _localUtility);
} }
} }
/** /**
@ -551,8 +551,8 @@ void Bond::pathNegotiationCheck(void *tPtr, const int64_t now)
void Bond::sendPATH_NEGOTIATION_REQUEST(void *tPtr, const SharedPtr<Path> &path) void Bond::sendPATH_NEGOTIATION_REQUEST(void *tPtr, const SharedPtr<Path> &path)
{ {
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "sendPATH_NEGOTIATION_REQUEST() %s %s\n", getSlave(path)->ifname().c_str(), pathStr); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "sendPATH_NEGOTIATION_REQUEST() %s %s\n", getLink(path)->ifname().c_str(), pathStr);
if (_abSlaveSelectMethod != ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) { if (_abLinkSelectMethod != ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) {
return; return;
} }
Packet outp(_peer->_id.address(),RR->identity.address(),Packet::VERB_PATH_NEGOTIATION_REQUEST); Packet outp(_peer->_id.address(),RR->identity.address(),Packet::VERB_PATH_NEGOTIATION_REQUEST);
@ -566,7 +566,7 @@ void Bond::sendPATH_NEGOTIATION_REQUEST(void *tPtr, const SharedPtr<Path> &path)
void Bond::sendACK(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket, void Bond::sendACK(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,
const InetAddress &atAddress,int64_t now) const InetAddress &atAddress,int64_t now)
{ {
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "sendACK() %s %s\n", getSlave(path)->ifname().c_str(), pathStr); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "sendACK() %s %s\n", getLink(path)->ifname().c_str(), pathStr);
Packet outp(_peer->_id.address(),RR->identity.address(),Packet::VERB_ACK); Packet outp(_peer->_id.address(),RR->identity.address(),Packet::VERB_ACK);
int32_t bytesToAck = 0; int32_t bytesToAck = 0;
std::map<uint64_t,uint16_t>::iterator it = path->ackStatsIn.begin(); std::map<uint64_t,uint16_t>::iterator it = path->ackStatsIn.begin();
@ -589,7 +589,7 @@ void Bond::sendACK(void *tPtr,const SharedPtr<Path> &path,const int64_t localSoc
void Bond::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket, void Bond::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int64_t localSocket,
const InetAddress &atAddress,int64_t now) const InetAddress &atAddress,int64_t now)
{ {
//char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "sendQOS() %s %s\n", getSlave(path)->ifname().c_str(), pathStr); //char pathStr[128];path->address().toString(pathStr);fprintf(stderr, "sendQOS() %s %s\n", getLink(path)->ifname().c_str(), pathStr);
const int64_t _now = RR->node->now(); const int64_t _now = RR->node->now();
Packet outp(_peer->_id.address(),RR->identity.address(),Packet::VERB_QOS_MEASUREMENT); Packet outp(_peer->_id.address(),RR->identity.address(),Packet::VERB_QOS_MEASUREMENT);
char qosData[ZT_QOS_MAX_PACKET_SIZE]; char qosData[ZT_QOS_MAX_PACKET_SIZE];
@ -615,14 +615,14 @@ void Bond::processBackgroundTasks(void *tPtr, const int64_t now)
_lastBackgroundTaskCheck = now; _lastBackgroundTaskCheck = now;
// Compute dynamic path monitor timer interval // Compute dynamic path monitor timer interval
if (_slaveMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC) { if (_linkMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC) {
int suggestedMonitorInterval = (now - _lastFrame) / 100; int suggestedMonitorInterval = (now - _lastFrame) / 100;
_dynamicPathMonitorInterval = std::min(ZT_PATH_HEARTBEAT_PERIOD, ((suggestedMonitorInterval > _bondMonitorInterval) ? suggestedMonitorInterval : _bondMonitorInterval)); _dynamicPathMonitorInterval = std::min(ZT_PATH_HEARTBEAT_PERIOD, ((suggestedMonitorInterval > _bondMonitorInterval) ? suggestedMonitorInterval : _bondMonitorInterval));
//fprintf(stderr, "_lastFrame=%llu, suggestedMonitorInterval=%d, _dynamicPathMonitorInterval=%d\n", //fprintf(stderr, "_lastFrame=%llu, suggestedMonitorInterval=%d, _dynamicPathMonitorInterval=%d\n",
// (now-_lastFrame), suggestedMonitorInterval, _dynamicPathMonitorInterval); // (now-_lastFrame), suggestedMonitorInterval, _dynamicPathMonitorInterval);
} }
// TODO: Clarify and generalize this logic // TODO: Clarify and generalize this logic
if (_slaveMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC) { if (_linkMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC) {
_shouldCollectPathStatistics = true; _shouldCollectPathStatistics = true;
} }
@ -632,11 +632,11 @@ void Bond::processBackgroundTasks(void *tPtr, const int64_t now)
_shouldCollectPathStatistics = true; _shouldCollectPathStatistics = true;
} }
if (_bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) { if (_bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
if (_abSlaveSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_BETTER) { if (_abLinkSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_BETTER) {
// Required for judging suitability of primary slave after recovery // Required for judging suitability of primary link after recovery
_shouldCollectPathStatistics = true; _shouldCollectPathStatistics = true;
} }
if (_abSlaveSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) { if (_abLinkSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) {
// Required for judging suitability of new candidate primary // Required for judging suitability of new candidate primary
_shouldCollectPathStatistics = true; _shouldCollectPathStatistics = true;
} }
@ -696,18 +696,18 @@ void Bond::applyUserPrefs()
if (!_paths[i]) { if (!_paths[i]) {
continue; continue;
} }
SharedPtr<Slave> sl = getSlave(_paths[i]); SharedPtr<Link> sl = getLink(_paths[i]);
if (sl) { if (sl) {
if (sl->monitorInterval() == 0) { // If no interval was specified for this slave, use more generic bond-wide interval if (sl->monitorInterval() == 0) { // If no interval was specified for this link, use more generic bond-wide interval
sl->setMonitorInterval(_bondMonitorInterval); sl->setMonitorInterval(_bondMonitorInterval);
} }
RR->bc->setMinReqPathMonitorInterval((sl->monitorInterval() < RR->bc->minReqPathMonitorInterval()) ? sl->monitorInterval() : RR->bc->minReqPathMonitorInterval()); RR->bc->setMinReqPathMonitorInterval((sl->monitorInterval() < RR->bc->minReqPathMonitorInterval()) ? sl->monitorInterval() : RR->bc->minReqPathMonitorInterval());
bool bFoundCommonSlave = false; bool bFoundCommonLink = false;
SharedPtr<Slave> commonSlave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> commonLink =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
for(unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) { for(unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
if (_paths[j] && _paths[j].ptr() != _paths[i].ptr()) { if (_paths[j] && _paths[j].ptr() != _paths[i].ptr()) {
if (RR->bc->getSlaveBySocket(_policyAlias, _paths[j]->localSocket()) == commonSlave) { if (RR->bc->getLinkBySocket(_policyAlias, _paths[j]->localSocket()) == commonLink) {
bFoundCommonSlave = true; bFoundCommonLink = true;
} }
} }
} }
@ -717,7 +717,7 @@ void Bond::applyUserPrefs()
_paths[i]->_ipvPref = sl->ipvPref(); _paths[i]->_ipvPref = sl->ipvPref();
_paths[i]->_mode = sl->mode(); _paths[i]->_mode = sl->mode();
_paths[i]->_enabled = sl->enabled(); _paths[i]->_enabled = sl->enabled();
_paths[i]->_onlyPathOnSlave = !bFoundCommonSlave; _paths[i]->_onlyPathOnLink = !bFoundCommonLink;
} }
} }
if (_peer) { if (_peer) {
@ -739,11 +739,11 @@ void Bond::curateBond(const int64_t now, bool rebuildBond)
} }
bool currEligibility = _paths[i]->eligible(now,_ackSendInterval); bool currEligibility = _paths[i]->eligible(now,_ackSendInterval);
//_paths[i]->address().toString(pathStr); //_paths[i]->address().toString(pathStr);
//fprintf(stderr, "\n\n%ld path eligibility (for %s, %s):\n", (RR->node->now() - RR->bc->getBondStartTime()), getSlave(_paths[i])->ifname().c_str(), pathStr); //fprintf(stderr, "\n\n%ld path eligibility (for %s, %s):\n", (RR->node->now() - RR->bc->getBondStartTime()), getLink(_paths[i])->ifname().c_str(), pathStr);
//_paths[i]->printEligible(now,_ackSendInterval); //_paths[i]->printEligible(now,_ackSendInterval);
if (currEligibility != _paths[i]->_lastEligibilityState) { if (currEligibility != _paths[i]->_lastEligibilityState) {
_paths[i]->address().toString(pathStr); _paths[i]->address().toString(pathStr);
//fprintf(stderr, "\n\n%ld path eligibility (for %s, %s) has changed (from %d to %d)\n", (RR->node->now() - RR->bc->getBondStartTime()), getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->lastCheckedEligibility, _paths[i]->eligible(now,_ackSendInterval)); //fprintf(stderr, "\n\n%ld path eligibility (for %s, %s) has changed (from %d to %d)\n", (RR->node->now() - RR->bc->getBondStartTime()), getLink(_paths[i])->ifname().c_str(), pathStr, _paths[i]->lastCheckedEligibility, _paths[i]->eligible(now,_ackSendInterval));
if (currEligibility) { if (currEligibility) {
rebuildBond = true; rebuildBond = true;
} }
@ -766,7 +766,7 @@ void Bond::curateBond(const int64_t now, bool rebuildBond)
} }
/** /**
* Curate the set of paths that are part of the bond proper. Selects a single path * Curate the set of paths that are part of the bond proper. Selects a single path
* per logical slave according to eligibility and user-specified constraints. * per logical link according to eligibility and user-specified constraints.
*/ */
if ((_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) if ((_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR)
|| (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_XOR) || (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_XOR)
@ -777,68 +777,68 @@ void Bond::curateBond(const int64_t now, bool rebuildBond)
// TODO: Optimize // TODO: Optimize
if (rebuildBond) { if (rebuildBond) {
int updatedBondedPathCount = 0; int updatedBondedPathCount = 0;
std::map<SharedPtr<Slave>,int> slaveMap; std::map<SharedPtr<Link>,int> linkMap;
for (int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { for (int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
if (_paths[i] && _paths[i]->allowed() && (_paths[i]->eligible(now,_ackSendInterval) || !_numBondedPaths)) { if (_paths[i] && _paths[i]->allowed() && (_paths[i]->eligible(now,_ackSendInterval) || !_numBondedPaths)) {
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (!slaveMap.count(slave)) { if (!linkMap.count(link)) {
slaveMap[slave] = i; linkMap[link] = i;
} }
else { else {
bool overriden = false; bool overriden = false;
_paths[i]->address().toString(pathStr); _paths[i]->address().toString(pathStr);
//fprintf(stderr, " slave representative path already exists! (%s %s)\n", getSlave(_paths[i])->ifname().c_str(), pathStr); //fprintf(stderr, " link representative path already exists! (%s %s)\n", getLink(_paths[i])->ifname().c_str(), pathStr);
if (_paths[i]->preferred() && !_paths[slaveMap[slave]]->preferred()) { if (_paths[i]->preferred() && !_paths[linkMap[link]]->preferred()) {
// Override previous choice if preferred // Override previous choice if preferred
//fprintf(stderr, "overriding since its preferred!\n"); //fprintf(stderr, "overriding since its preferred!\n");
if (_paths[slaveMap[slave]]->_assignedFlowCount) { if (_paths[linkMap[link]]->_assignedFlowCount) {
_paths[slaveMap[slave]]->_deprecated = true; _paths[linkMap[link]]->_deprecated = true;
} }
else { else {
_paths[slaveMap[slave]]->_deprecated = true; _paths[linkMap[link]]->_deprecated = true;
_paths[slaveMap[slave]]->setBonded(false); _paths[linkMap[link]]->setBonded(false);
} }
slaveMap[slave] = i; linkMap[link] = i;
overriden = true; overriden = true;
} }
if ((_paths[i]->preferred() && _paths[slaveMap[slave]]->preferred()) if ((_paths[i]->preferred() && _paths[linkMap[link]]->preferred())
|| (!_paths[i]->preferred() && !_paths[slaveMap[slave]]->preferred())) { || (!_paths[i]->preferred() && !_paths[linkMap[link]]->preferred())) {
if (_paths[i]->preferenceRank() > _paths[slaveMap[slave]]->preferenceRank()) { if (_paths[i]->preferenceRank() > _paths[linkMap[link]]->preferenceRank()) {
// Override if higher preference // Override if higher preference
//fprintf(stderr, "overriding according to preference preferenceRank!\n"); //fprintf(stderr, "overriding according to preference preferenceRank!\n");
if (_paths[slaveMap[slave]]->_assignedFlowCount) { if (_paths[linkMap[link]]->_assignedFlowCount) {
_paths[slaveMap[slave]]->_deprecated = true; _paths[linkMap[link]]->_deprecated = true;
} }
else { else {
_paths[slaveMap[slave]]->_deprecated = true; _paths[linkMap[link]]->_deprecated = true;
_paths[slaveMap[slave]]->setBonded(false); _paths[linkMap[link]]->setBonded(false);
} }
slaveMap[slave] = i; linkMap[link] = i;
} }
} }
} }
} }
} }
std::map<SharedPtr<Slave>,int>::iterator it = slaveMap.begin(); std::map<SharedPtr<Link>,int>::iterator it = linkMap.begin();
for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) { for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {
if (!_paths[i]) { if (!_paths[i]) {
continue; continue;
} }
_bondedIdx[i] = ZT_MAX_PEER_NETWORK_PATHS; _bondedIdx[i] = ZT_MAX_PEER_NETWORK_PATHS;
if (it != slaveMap.end()) { if (it != linkMap.end()) {
_bondedIdx[i] = it->second; _bondedIdx[i] = it->second;
_paths[_bondedIdx[i]]->setBonded(true); _paths[_bondedIdx[i]]->setBonded(true);
++it; ++it;
++updatedBondedPathCount; ++updatedBondedPathCount;
_paths[_bondedIdx[i]]->address().toString(pathStr); _paths[_bondedIdx[i]]->address().toString(pathStr);
//fprintf(stderr, "setting i=%d, _bondedIdx[%d]=%d to bonded (%s %s)\n", i, i, _bondedIdx[i], getSlave(_paths[_bondedIdx[i]])->ifname().c_str(), pathStr); //fprintf(stderr, "setting i=%d, _bondedIdx[%d]=%d to bonded (%s %s)\n", i, i, _bondedIdx[i], getLink(_paths[_bondedIdx[i]])->ifname().c_str(), pathStr);
} }
} }
_numBondedPaths = updatedBondedPathCount; _numBondedPaths = updatedBondedPathCount;
if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) { if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) {
// Cause a RR reset since the currently used index might no longer be valid // Cause a RR reset since the currently used index might no longer be valid
_rrPacketsSentOnCurrSlave = _packetsPerSlave; _rrPacketsSentOnCurrLink = _packetsPerLink;
} }
} }
} }
@ -847,18 +847,18 @@ void Bond::curateBond(const int64_t now, bool rebuildBond)
void Bond::estimatePathQuality(const int64_t now) void Bond::estimatePathQuality(const int64_t now)
{ {
char pathStr[128]; char pathStr[128];
uint32_t totUserSpecifiedSlaveSpeed = 0; uint32_t totUserSpecifiedLinkSpeed = 0;
if (_numBondedPaths) { // Compute relative user-specified speeds of slaves if (_numBondedPaths) { // Compute relative user-specified speeds of links
for(unsigned int i=0;i<_numBondedPaths;++i) { for(unsigned int i=0;i<_numBondedPaths;++i) {
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (_paths[i] && _paths[i]->allowed()) { if (_paths[i] && _paths[i]->allowed()) {
totUserSpecifiedSlaveSpeed += slave->speed(); totUserSpecifiedLinkSpeed += link->speed();
} }
} }
for(unsigned int i=0;i<_numBondedPaths;++i) { for(unsigned int i=0;i<_numBondedPaths;++i) {
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (_paths[i] && _paths[i]->allowed()) { if (_paths[i] && _paths[i]->allowed()) {
slave->setRelativeSpeed(round( ((float)slave->speed() / (float)totUserSpecifiedSlaveSpeed) * 255)); link->setRelativeSpeed(round( ((float)link->speed() / (float)totUserSpecifiedLinkSpeed) * 255));
} }
} }
} }
@ -895,11 +895,11 @@ void Bond::estimatePathQuality(const int64_t now)
_paths[i]->_latencyVariance = _paths[i]->latencySamples.stddev(); _paths[i]->_latencyVariance = _paths[i]->latencySamples.stddev();
_paths[i]->_packetErrorRatio = 1.0 - (_paths[i]->packetValiditySamples.count() ? _paths[i]->packetValiditySamples.mean() : 1.0); _paths[i]->_packetErrorRatio = 1.0 - (_paths[i]->packetValiditySamples.count() ? _paths[i]->packetValiditySamples.mean() : 1.0);
if (userHasSpecifiedSlaveSpeeds()) { if (userHasSpecifiedLinkSpeeds()) {
// Use user-reported metrics // Use user-reported metrics
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (slave) { if (link) {
_paths[i]->_throughputMean = slave->speed(); _paths[i]->_throughputMean = link->speed();
_paths[i]->_throughputVariance = 0; _paths[i]->_throughputVariance = 0;
} }
} }
@ -929,7 +929,7 @@ void Bond::estimatePathQuality(const int64_t now)
maxPLR = plr[i] > maxPLR ? plr[i] : maxPLR; maxPLR = plr[i] > maxPLR ? plr[i] : maxPLR;
maxPER = per[i] > maxPER ? per[i] : maxPER; maxPER = per[i] > maxPER ? per[i] : maxPER;
//fprintf(stdout, "EH %d: lat=%8.3f, ltm=%8.3f, pdv=%8.3f, plr=%5.3f, per=%5.3f, thr=%8f, thm=%5.3f, thv=%5.3f, avl=%5.3f, age=%8.2f, scp=%4d, q=%5.3f, qtot=%5.3f, ac=%d if=%s, path=%s\n", //fprintf(stdout, "EH %d: lat=%8.3f, ltm=%8.3f, pdv=%8.3f, plr=%5.3f, per=%5.3f, thr=%8f, thm=%5.3f, thv=%5.3f, avl=%5.3f, age=%8.2f, scp=%4d, q=%5.3f, qtot=%5.3f, ac=%d if=%s, path=%s\n",
// i, lat[i], ltm[i], pdv[i], plr[i], per[i], thr[i], thm[i], thv[i], avl[i], age[i], scp[i], quality[i], totQuality, alloc[i], getSlave(_paths[i])->ifname().c_str(), pathStr); // i, lat[i], ltm[i], pdv[i], plr[i], per[i], thr[i], thm[i], thv[i], avl[i], age[i], scp[i], quality[i], totQuality, alloc[i], getLink(_paths[i])->ifname().c_str(), pathStr);
} }
// Convert metrics to relative quantities and apply contribution weights // Convert metrics to relative quantities and apply contribution weights
@ -962,7 +962,7 @@ void Bond::estimatePathQuality(const int64_t now)
//fprintf(stderr, "%lu FIN [%d/%d]: pmi=%5d, lat=%4.3f, ltm=%4.3f, pdv=%4.3f, plr=%4.3f, per=%4.3f, thr=%4.3f, thm=%4.3f, thv=%4.3f, age=%4.3f, scp=%4d, q=%4.3f, qtot=%4.3f, ac=%4d, asf=%3d, if=%s, path=%20s, bond=%d, qosout=%d, plrraw=%d\n", //fprintf(stderr, "%lu FIN [%d/%d]: pmi=%5d, lat=%4.3f, ltm=%4.3f, pdv=%4.3f, plr=%4.3f, per=%4.3f, thr=%4.3f, thm=%4.3f, thv=%4.3f, age=%4.3f, scp=%4d, q=%4.3f, qtot=%4.3f, ac=%4d, asf=%3d, if=%s, path=%20s, bond=%d, qosout=%d, plrraw=%d\n",
// ((now - RR->bc->getBondStartTime())), i, _numBondedPaths, _paths[i]->monitorInterval, // ((now - RR->bc->getBondStartTime())), i, _numBondedPaths, _paths[i]->monitorInterval,
// lat[i], ltm[i], pdv[i], plr[i], per[i], thr[i], thm[i], thv[i], age[i], scp[i], // lat[i], ltm[i], pdv[i], plr[i], per[i], thr[i], thm[i], thv[i], age[i], scp[i],
// quality[i], totQuality, alloc[i], _paths[i]->assignedFlowCount, getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->bonded(), _paths[i]->qosStatsOut.size(), _paths[i]->packetLossRatio); // quality[i], totQuality, alloc[i], _paths[i]->assignedFlowCount, getLink(_paths[i])->ifname().c_str(), pathStr, _paths[i]->bonded(), _paths[i]->qosStatsOut.size(), _paths[i]->packetLossRatio);
} }
} }
if (numPlottablePaths < 2) { if (numPlottablePaths < 2) {
@ -973,7 +973,7 @@ void Bond::estimatePathQuality(const int64_t now)
for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
if (_paths[i]) { if (_paths[i]) {
_paths[i]->address().toString(pathStr); _paths[i]->address().toString(pathStr);
std::string label = std::string((pathStr)) + " " + getSlave(_paths[i])->ifname(); std::string label = std::string((pathStr)) + " " + getLink(_paths[i])->ifname();
for (int i=0; i<19; ++i) { for (int i=0; i<19; ++i) {
fprintf(stdout, "%s, ", label.c_str()); fprintf(stdout, "%s, ", label.c_str());
} }
@ -987,7 +987,7 @@ void Bond::estimatePathQuality(const int64_t now)
if (_paths[i]) { if (_paths[i]) {
_paths[i]->address().toString(pathStr); _paths[i]->address().toString(pathStr);
fprintf(stdout, "%s, %s, %8.3f, %8.3f, %8.3f, %5.3f, %5.3f, %5.3f, %8f, %5.3f, %5.3f, %d, %5.3f, %d, %d, %d, %d, %d, %d, ", fprintf(stdout, "%s, %s, %8.3f, %8.3f, %8.3f, %5.3f, %5.3f, %5.3f, %8f, %5.3f, %5.3f, %d, %5.3f, %d, %d, %d, %d, %d, %d, ",
getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->_latencyMean, lat[i],pdv[i], _paths[i]->_packetLossRatio, plr[i],per[i],thr[i],thm[i],thv[i],(now - _paths[i]->lastIn()),quality[i],alloc[i], getLink(_paths[i])->ifname().c_str(), pathStr, _paths[i]->_latencyMean, lat[i],pdv[i], _paths[i]->_packetLossRatio, plr[i],per[i],thr[i],thm[i],thv[i],(now - _paths[i]->lastIn()),quality[i],alloc[i],
_paths[i]->_relativeByteLoad, _paths[i]->_assignedFlowCount, _paths[i]->alive(now, true), _paths[i]->eligible(now,_ackSendInterval), _paths[i]->qosStatsOut.size()); _paths[i]->_relativeByteLoad, _paths[i]->_assignedFlowCount, _paths[i]->alive(now, true), _paths[i]->eligible(now,_ackSendInterval), _paths[i]->qosStatsOut.size());
} }
}*/ }*/
@ -1040,7 +1040,7 @@ void Bond::processBalanceTasks(const int64_t now)
} }
if (!_paths[i]->eligible(now,_ackSendInterval) && _paths[i]->_shouldReallocateFlows) { if (!_paths[i]->eligible(now,_ackSendInterval) && _paths[i]->_shouldReallocateFlows) {
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
fprintf(stderr, "%d reallocating flows from dead path %s on %s\n", (RR->node->now() - RR->bc->getBondStartTime()), curPathStr, getSlave(_paths[i])->ifname().c_str()); fprintf(stderr, "%d reallocating flows from dead path %s on %s\n", (RR->node->now() - RR->bc->getBondStartTime()), curPathStr, getLink(_paths[i])->ifname().c_str());
std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin(); std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin();
while (flow_it != _flows.end()) { while (flow_it != _flows.end()) {
if (flow_it->second->assignedPath() == _paths[i]) { if (flow_it->second->assignedPath() == _paths[i]) {
@ -1066,7 +1066,7 @@ void Bond::processBalanceTasks(const int64_t now)
} }
if (_paths[i] && _paths[i]->bonded() && _paths[i]->eligible(now,_ackSendInterval) && (_paths[i]->_allocation < minimumAllocationValue) && _paths[i]->_assignedFlowCount) { if (_paths[i] && _paths[i]->bonded() && _paths[i]->eligible(now,_ackSendInterval) && (_paths[i]->_allocation < minimumAllocationValue) && _paths[i]->_assignedFlowCount) {
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
fprintf(stderr, "%d reallocating flows from under-performing path %s on %s\n", (RR->node->now() - RR->bc->getBondStartTime()), curPathStr, getSlave(_paths[i])->ifname().c_str()); fprintf(stderr, "%d reallocating flows from under-performing path %s on %s\n", (RR->node->now() - RR->bc->getBondStartTime()), curPathStr, getLink(_paths[i])->ifname().c_str());
std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin(); std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin();
while (flow_it != _flows.end()) { while (flow_it != _flows.end()) {
if (flow_it->second->assignedPath() == _paths[i]) { if (flow_it->second->assignedPath() == _paths[i]) {
@ -1086,7 +1086,7 @@ void Bond::processBalanceTasks(const int64_t now)
*/ */
if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) { if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) {
if (_allowFlowHashing) { if (_allowFlowHashing) {
// TODO: Should ideally failover from (idx) to a random slave, this is so that (idx+1) isn't overloaded // TODO: Should ideally failover from (idx) to a random link, this is so that (idx+1) isn't overloaded
} }
else if (!_allowFlowHashing) { else if (!_allowFlowHashing) {
// Nothing // Nothing
@ -1176,29 +1176,29 @@ void Bond::processActiveBackupTasks(const int64_t now)
SharedPtr<Path> prevActiveBackupPath = _abPath; SharedPtr<Path> prevActiveBackupPath = _abPath;
SharedPtr<Path> nonPreferredPath; SharedPtr<Path> nonPreferredPath;
bool bFoundPrimarySlave = false; bool bFoundPrimaryLink = false;
/** /**
* Select initial "active" active-backup slave * Select initial "active" active-backup link
*/ */
if (!_abPath) { if (!_abPath) {
fprintf(stderr, "%llu no active backup path yet...\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu no active backup path yet...\n", ((now - RR->bc->getBondStartTime())));
/** /**
* [Automatic mode] * [Automatic mode]
* The user has not explicitly specified slaves or their failover schedule, * The user has not explicitly specified links or their failover schedule,
* the bonding policy will now select the first eligible path and set it as * the bonding policy will now select the first eligible path and set it as
* its active backup path, if a substantially better path is detected the bonding * its active backup path, if a substantially better path is detected the bonding
* policy will assign it as the new active backup path. If the path fails it will * policy will assign it as the new active backup path. If the path fails it will
* simply find the next eligible path. * simply find the next eligible path.
*/ */
if (!userHasSpecifiedSlaves()) { if (!userHasSpecifiedLinks()) {
fprintf(stderr, "%llu AB: (auto) user did not specify any slaves. waiting until we know more\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (auto) user did not specify any links. waiting until we know more\n", ((now - RR->bc->getBondStartTime())));
for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) { for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {
if (_paths[i] && _paths[i]->eligible(now,_ackSendInterval)) { if (_paths[i] && _paths[i]->eligible(now,_ackSendInterval)) {
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (slave) { if (link) {
fprintf(stderr, "%llu AB: (initial) [%d] found eligible path %s on: %s\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, slave->ifname().c_str()); fprintf(stderr, "%llu AB: (initial) [%d] found eligible path %s on: %s\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, link->ifname().c_str());
} }
_abPath = _paths[i]; _abPath = _paths[i];
break; break;
@ -1207,57 +1207,57 @@ void Bond::processActiveBackupTasks(const int64_t now)
} }
/** /**
* [Manual mode] * [Manual mode]
* The user has specified slaves or failover rules that the bonding policy should adhere to. * The user has specified links or failover rules that the bonding policy should adhere to.
*/ */
else if (userHasSpecifiedSlaves()) { else if (userHasSpecifiedLinks()) {
fprintf(stderr, "%llu AB: (manual) no active backup slave, checking local.conf\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (manual) no active backup link, checking local.conf\n", ((now - RR->bc->getBondStartTime())));
if (userHasSpecifiedPrimarySlave()) { if (userHasSpecifiedPrimaryLink()) {
fprintf(stderr, "%llu AB: (manual) user has specified primary slave, looking for it.\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (manual) user has specified primary link, looking for it.\n", ((now - RR->bc->getBondStartTime())));
for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) { for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {
if (!_paths[i]) { if (!_paths[i]) {
continue; continue;
} }
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (_paths[i]->eligible(now,_ackSendInterval) && slave->primary()) { if (_paths[i]->eligible(now,_ackSendInterval) && link->primary()) {
if (!_paths[i]->preferred()) { if (!_paths[i]->preferred()) {
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
fprintf(stderr, "%llu AB: (initial) [%d] found path on primary slave, taking note in case we don't find a preferred path\n", ((now - RR->bc->getBondStartTime())), i); fprintf(stderr, "%llu AB: (initial) [%d] found path on primary link, taking note in case we don't find a preferred path\n", ((now - RR->bc->getBondStartTime())), i);
nonPreferredPath = _paths[i]; nonPreferredPath = _paths[i];
bFoundPrimarySlave = true; bFoundPrimaryLink = true;
} }
if (_paths[i]->preferred()) { if (_paths[i]->preferred()) {
_abPath = _paths[i]; _abPath = _paths[i];
_abPath->address().toString(curPathStr); _abPath->address().toString(curPathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
if (slave) { if (link) {
fprintf(stderr, "%llu AB: (initial) [%d] found preferred path %s on primary slave: %s\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, slave->ifname().c_str()); fprintf(stderr, "%llu AB: (initial) [%d] found preferred path %s on primary link: %s\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, link->ifname().c_str());
} }
bFoundPrimarySlave = true; bFoundPrimaryLink = true;
break; break;
} }
} }
} }
if (_abPath) { if (_abPath) {
_abPath->address().toString(curPathStr); _abPath->address().toString(curPathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _abPath->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _abPath->localSocket());
if (slave) { if (link) {
fprintf(stderr, "%llu AB: (initial) found preferred primary path: %s on %s\n", ((now - RR->bc->getBondStartTime())), curPathStr, slave->ifname().c_str()); fprintf(stderr, "%llu AB: (initial) found preferred primary path: %s on %s\n", ((now - RR->bc->getBondStartTime())), curPathStr, link->ifname().c_str());
} }
} }
else { else {
if (bFoundPrimarySlave && nonPreferredPath) { if (bFoundPrimaryLink && nonPreferredPath) {
fprintf(stderr, "%llu AB: (initial) found a non-preferred primary path\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (initial) found a non-preferred primary path\n", ((now - RR->bc->getBondStartTime())));
_abPath = nonPreferredPath; _abPath = nonPreferredPath;
} }
} }
if (!_abPath) { if (!_abPath) {
fprintf(stderr, "%llu AB: (initial) designated primary slave is not yet ready\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (initial) designated primary link is not yet ready\n", ((now - RR->bc->getBondStartTime())));
// TODO: Should fail-over to specified backup or just wait? // TODO: Should fail-over to specified backup or just wait?
} }
} }
else if (!userHasSpecifiedPrimarySlave()) { else if (!userHasSpecifiedPrimaryLink()) {
int _abIdx = ZT_MAX_PEER_NETWORK_PATHS; int _abIdx = ZT_MAX_PEER_NETWORK_PATHS;
fprintf(stderr, "%llu AB: (initial) user did not specify primary slave, just picking something\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (initial) user did not specify primary link, just picking something\n", ((now - RR->bc->getBondStartTime())));
for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) { for (int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {
if (_paths[i] && _paths[i]->eligible(now,_ackSendInterval)) { if (_paths[i] && _paths[i]->eligible(now,_ackSendInterval)) {
_abIdx = i; _abIdx = i;
@ -1269,9 +1269,9 @@ void Bond::processActiveBackupTasks(const int64_t now)
} }
else { else {
_abPath = _paths[_abIdx]; _abPath = _paths[_abIdx];
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _abPath->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _abPath->localSocket());
if (slave) { if (link) {
fprintf(stderr, "%llu AB: (initial) selected non-primary slave idx=%d, %s on %s\n", ((now - RR->bc->getBondStartTime())), _abIdx, pathStr, slave->ifname().c_str()); fprintf(stderr, "%llu AB: (initial) selected non-primary link idx=%d, %s on %s\n", ((now - RR->bc->getBondStartTime())), _abIdx, pathStr, link->ifname().c_str());
} }
} }
} }
@ -1281,14 +1281,14 @@ void Bond::processActiveBackupTasks(const int64_t now)
* Update and maintain the active-backup failover queue * Update and maintain the active-backup failover queue
*/ */
if (_abPath) { if (_abPath) {
// Don't worry about the failover queue until we have an active slave // Don't worry about the failover queue until we have an active link
// Remove ineligible paths from the failover slave queue // Remove ineligible paths from the failover link queue
for (std::list<SharedPtr<Path> >::iterator it(_abFailoverQueue.begin()); it!=_abFailoverQueue.end();) { for (std::list<SharedPtr<Path> >::iterator it(_abFailoverQueue.begin()); it!=_abFailoverQueue.end();) {
if ((*it) && !(*it)->eligible(now,_ackSendInterval)) { if ((*it) && !(*it)->eligible(now,_ackSendInterval)) {
(*it)->address().toString(curPathStr); (*it)->address().toString(curPathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, (*it)->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, (*it)->localSocket());
if (slave) { if (link) {
fprintf(stderr, "%llu AB: (fq) %s on %s is now ineligible, removing from failover queue\n", ((now - RR->bc->getBondStartTime())), curPathStr, slave->ifname().c_str()); fprintf(stderr, "%llu AB: (fq) %s on %s is now ineligible, removing from failover queue\n", ((now - RR->bc->getBondStartTime())), curPathStr, link->ifname().c_str());
} }
it = _abFailoverQueue.erase(it); it = _abFailoverQueue.erase(it);
} else { } else {
@ -1313,7 +1313,7 @@ void Bond::processActiveBackupTasks(const int64_t now)
if (!_paths[i] || !_paths[i]->allowed() || !_paths[i]->eligible(now,_ackSendInterval)) { if (!_paths[i] || !_paths[i]->allowed() || !_paths[i]->eligible(now,_ackSendInterval)) {
continue; continue;
} }
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
_paths[i]->address().toString(pathStr); _paths[i]->address().toString(pathStr);
int failoverScoreHandicap = _paths[i]->_failoverScore; int failoverScoreHandicap = _paths[i]->_failoverScore;
@ -1322,8 +1322,8 @@ void Bond::processActiveBackupTasks(const int64_t now)
failoverScoreHandicap += ZT_MULTIPATH_FAILOVER_HANDICAP_PREFERRED; failoverScoreHandicap += ZT_MULTIPATH_FAILOVER_HANDICAP_PREFERRED;
//fprintf(stderr, "%s on %s ----> %d for preferred\n", pathStr, _paths[i]->ifname().c_str(), failoverScoreHandicap); //fprintf(stderr, "%s on %s ----> %d for preferred\n", pathStr, _paths[i]->ifname().c_str(), failoverScoreHandicap);
} }
if (slave->primary()) { if (link->primary()) {
// If using "optimize" primary reselect mode, ignore user slave designations // If using "optimize" primary reselect mode, ignore user link designations
failoverScoreHandicap += ZT_MULTIPATH_FAILOVER_HANDICAP_PRIMARY; failoverScoreHandicap += ZT_MULTIPATH_FAILOVER_HANDICAP_PRIMARY;
//fprintf(stderr, "%s on %s ----> %d for primary\n", pathStr, _paths[i]->ifname().c_str(), failoverScoreHandicap); //fprintf(stderr, "%s on %s ----> %d for primary\n", pathStr, _paths[i]->ifname().c_str(), failoverScoreHandicap);
} }
@ -1333,17 +1333,17 @@ void Bond::processActiveBackupTasks(const int64_t now)
_paths[i]->_failoverScore = newHandicap; _paths[i]->_failoverScore = newHandicap;
//fprintf(stderr, "%s on %s ----> %d for allocation\n", pathStr, _paths[i]->ifname().c_str(), newHandicap); //fprintf(stderr, "%s on %s ----> %d for allocation\n", pathStr, _paths[i]->ifname().c_str(), newHandicap);
} }
SharedPtr<Slave> failoverSlave; SharedPtr<Link> failoverLink;
if (slave->failoverToSlave().length()) { if (link->failoverToLink().length()) {
failoverSlave = RR->bc->getSlaveByName(_policyAlias, slave->failoverToSlave()); failoverLink = RR->bc->getLinkByName(_policyAlias, link->failoverToLink());
} }
if (failoverSlave) { if (failoverLink) {
for (int j=0; j<ZT_MAX_PEER_NETWORK_PATHS; j++) { for (int j=0; j<ZT_MAX_PEER_NETWORK_PATHS; j++) {
if (_paths[j] && getSlave(_paths[j]) == failoverSlave.ptr()) { if (_paths[j] && getLink(_paths[j]) == failoverLink.ptr()) {
_paths[j]->address().toString(pathStr); _paths[j]->address().toString(pathStr);
int inheritedHandicap = failoverScoreHandicap - 10; int inheritedHandicap = failoverScoreHandicap - 10;
int newHandicap = _paths[j]->_failoverScore > inheritedHandicap ? _paths[j]->_failoverScore : inheritedHandicap; int newHandicap = _paths[j]->_failoverScore > inheritedHandicap ? _paths[j]->_failoverScore : inheritedHandicap;
//fprintf(stderr, "\thanding down %s on %s ----> %d\n", pathStr, getSlave(_paths[j])->ifname().c_str(), newHandicap); //fprintf(stderr, "\thanding down %s on %s ----> %d\n", pathStr, getLink(_paths[j])->ifname().c_str(), newHandicap);
if (!_paths[j]->preferred()) { if (!_paths[j]->preferred()) {
newHandicap--; newHandicap--;
} }
@ -1360,7 +1360,7 @@ void Bond::processActiveBackupTasks(const int64_t now)
} }
if (!bFoundPathInQueue) { if (!bFoundPathInQueue) {
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
fprintf(stderr, "%llu AB: (fq) [%d] added %s on %s to queue\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, getSlave(_paths[i])->ifname().c_str()); fprintf(stderr, "%llu AB: (fq) [%d] added %s on %s to queue\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, getLink(_paths[i])->ifname().c_str());
_abFailoverQueue.push_front(_paths[i]); _abFailoverQueue.push_front(_paths[i]);
} }
} }
@ -1385,8 +1385,8 @@ void Bond::processActiveBackupTasks(const int64_t now)
if (!_paths[i]->eligible(now,includeRefractoryPeriod)) { if (!_paths[i]->eligible(now,includeRefractoryPeriod)) {
failoverScoreHandicap = -10000; failoverScoreHandicap = -10000;
} }
if (getSlave(_paths[i])->primary() && _abSlaveSelectMethod != ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) { if (getLink(_paths[i])->primary() && _abLinkSelectMethod != ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE) {
// If using "optimize" primary reselect mode, ignore user slave designations // If using "optimize" primary reselect mode, ignore user link designations
failoverScoreHandicap = ZT_MULTIPATH_FAILOVER_HANDICAP_PRIMARY; failoverScoreHandicap = ZT_MULTIPATH_FAILOVER_HANDICAP_PRIMARY;
} }
if (_paths[i].ptr() == negotiatedPath.ptr()) { if (_paths[i].ptr() == negotiatedPath.ptr()) {
@ -1405,7 +1405,7 @@ void Bond::processActiveBackupTasks(const int64_t now)
} }
if (!bFoundPathInQueue) { if (!bFoundPathInQueue) {
_paths[i]->address().toString(curPathStr); _paths[i]->address().toString(curPathStr);
fprintf(stderr, "%llu AB: (fq) [%d] added %s on %s to queue\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, getSlave(_paths[i])->ifname().c_str()); fprintf(stderr, "%llu AB: (fq) [%d] added %s on %s to queue\n", ((now - RR->bc->getBondStartTime())), i, curPathStr, getLink(_paths[i])->ifname().c_str());
_abFailoverQueue.push_front(_paths[i]); _abFailoverQueue.push_front(_paths[i]);
} }
} }
@ -1428,11 +1428,11 @@ void Bond::processActiveBackupTasks(const int64_t now)
if (_abPath && !_abPath->eligible(now,_ackSendInterval)) { // Implicit ZT_MULTIPATH_RESELECTION_POLICY_FAILURE if (_abPath && !_abPath->eligible(now,_ackSendInterval)) { // Implicit ZT_MULTIPATH_RESELECTION_POLICY_FAILURE
_abPath->address().toString(curPathStr); fprintf(stderr, "%llu AB: (failure) failover event!, active backup path (%s) is no-longer eligible\n", ((now - RR->bc->getBondStartTime())), curPathStr); _abPath->address().toString(curPathStr); fprintf(stderr, "%llu AB: (failure) failover event!, active backup path (%s) is no-longer eligible\n", ((now - RR->bc->getBondStartTime())), curPathStr);
if (!_abFailoverQueue.empty()) { if (!_abFailoverQueue.empty()) {
fprintf(stderr, "%llu AB: (failure) there are (%lu) slaves in queue to choose from...\n", ((now - RR->bc->getBondStartTime())), _abFailoverQueue.size()); fprintf(stderr, "%llu AB: (failure) there are (%lu) links in queue to choose from...\n", ((now - RR->bc->getBondStartTime())), _abFailoverQueue.size());
dequeueNextActiveBackupPath(now); dequeueNextActiveBackupPath(now);
_abPath->address().toString(curPathStr); fprintf(stderr, "%llu AB: (failure) switched to %s on %s\n", ((now - RR->bc->getBondStartTime())), curPathStr, getSlave(_abPath)->ifname().c_str()); _abPath->address().toString(curPathStr); fprintf(stderr, "%llu AB: (failure) switched to %s on %s\n", ((now - RR->bc->getBondStartTime())), curPathStr, getLink(_abPath)->ifname().c_str());
} else { } else {
fprintf(stderr, "%llu AB: (failure) nothing available in the slave queue, doing nothing.\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (failure) nothing available in the link queue, doing nothing.\n", ((now - RR->bc->getBondStartTime())));
} }
} }
/** /**
@ -1441,38 +1441,38 @@ void Bond::processActiveBackupTasks(const int64_t now)
if (prevActiveBackupPath != _abPath) { if (prevActiveBackupPath != _abPath) {
_lastActiveBackupPathChange = now; _lastActiveBackupPathChange = now;
} }
if (_abSlaveSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS) { if (_abLinkSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS) {
if (_abPath && !getSlave(_abPath)->primary() if (_abPath && !getLink(_abPath)->primary()
&& getSlave(_abFailoverQueue.front())->primary()) { && getLink(_abFailoverQueue.front())->primary()) {
fprintf(stderr, "%llu AB: (always) switching to available primary\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (always) switching to available primary\n", ((now - RR->bc->getBondStartTime())));
dequeueNextActiveBackupPath(now); dequeueNextActiveBackupPath(now);
} }
} }
if (_abSlaveSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_BETTER) { if (_abLinkSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_BETTER) {
if (_abPath && !getSlave(_abPath)->primary()) { if (_abPath && !getLink(_abPath)->primary()) {
fprintf(stderr, "%llu AB: (better) active backup has switched to \"better\" primary slave according to re-select policy.\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (better) active backup has switched to \"better\" primary link according to re-select policy.\n", ((now - RR->bc->getBondStartTime())));
if (getSlave(_abFailoverQueue.front())->primary() if (getLink(_abFailoverQueue.front())->primary()
&& (_abFailoverQueue.front()->_failoverScore > _abPath->_failoverScore)) { && (_abFailoverQueue.front()->_failoverScore > _abPath->_failoverScore)) {
dequeueNextActiveBackupPath(now); dequeueNextActiveBackupPath(now);
fprintf(stderr, "%llu AB: (better) switched back to user-defined primary\n", ((now - RR->bc->getBondStartTime()))); fprintf(stderr, "%llu AB: (better) switched back to user-defined primary\n", ((now - RR->bc->getBondStartTime())));
} }
} }
} }
if (_abSlaveSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE && !_abFailoverQueue.empty()) { if (_abLinkSelectMethod == ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE && !_abFailoverQueue.empty()) {
/** /**
* Implement link negotiation that was previously-decided * Implement link negotiation that was previously-decided
*/ */
if (_abFailoverQueue.front()->_negotiated) { if (_abFailoverQueue.front()->_negotiated) {
dequeueNextActiveBackupPath(now); dequeueNextActiveBackupPath(now);
_abPath->address().toString(prevPathStr); _abPath->address().toString(prevPathStr);
fprintf(stderr, "%llu AB: (optimize) switched to negotiated path %s on %s\n", ((now - RR->bc->getBondStartTime())), prevPathStr, getSlave(_abPath)->ifname().c_str()); fprintf(stderr, "%llu AB: (optimize) switched to negotiated path %s on %s\n", ((now - RR->bc->getBondStartTime())), prevPathStr, getLink(_abPath)->ifname().c_str());
_lastPathNegotiationCheck = now; _lastPathNegotiationCheck = now;
} }
else { else {
// Try to find a better path and automatically switch to it -- not too often, though. // Try to find a better path and automatically switch to it -- not too often, though.
if ((now - _lastActiveBackupPathChange) > ZT_MULTIPATH_MIN_ACTIVE_BACKUP_AUTOFLOP_INTERVAL) { if ((now - _lastActiveBackupPathChange) > ZT_MULTIPATH_MIN_ACTIVE_BACKUP_AUTOFLOP_INTERVAL) {
if (!_abFailoverQueue.empty()) { if (!_abFailoverQueue.empty()) {
//fprintf(stderr, "AB: (optimize) there are (%d) slaves in queue to choose from...\n", _abFailoverQueue.size()); //fprintf(stderr, "AB: (optimize) there are (%d) links in queue to choose from...\n", _abFailoverQueue.size());
int newFScore = _abFailoverQueue.front()->_failoverScore; int newFScore = _abFailoverQueue.front()->_failoverScore;
int prevFScore = _abPath->_failoverScore; int prevFScore = _abPath->_failoverScore;
// Establish a minimum switch threshold to prevent flapping // Establish a minimum switch threshold to prevent flapping
@ -1483,7 +1483,7 @@ void Bond::processActiveBackupTasks(const int64_t now)
_abPath->address().toString(prevPathStr); _abPath->address().toString(prevPathStr);
dequeueNextActiveBackupPath(now); dequeueNextActiveBackupPath(now);
_abPath->address().toString(curPathStr); _abPath->address().toString(curPathStr);
fprintf(stderr, "%llu AB: (optimize) switched from %s on %s (fs=%d) to %s on %s (fs=%d)\n", ((now - RR->bc->getBondStartTime())), prevPathStr, getSlave(oldPath)->ifname().c_str(), prevFScore, curPathStr, getSlave(_abPath)->ifname().c_str(), newFScore); fprintf(stderr, "%llu AB: (optimize) switched from %s on %s (fs=%d) to %s on %s (fs=%d)\n", ((now - RR->bc->getBondStartTime())), prevPathStr, getLink(oldPath)->ifname().c_str(), prevFScore, curPathStr, getLink(_abPath)->ifname().c_str(), newFScore);
} }
} }
} }
@ -1527,7 +1527,7 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
_lastFlowExpirationCheck=0; _lastFlowExpirationCheck=0;
_numBondedPaths=0; _numBondedPaths=0;
_rrPacketsSentOnCurrSlave=0; _rrPacketsSentOnCurrLink=0;
_rrIdx=0; _rrIdx=0;
_lastFlowRebalance=0; _lastFlowRebalance=0;
@ -1537,7 +1537,7 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
_maxAcceptablePacketDelayVariance = 50; _maxAcceptablePacketDelayVariance = 50;
_maxAcceptablePacketLossRatio = 0.10; _maxAcceptablePacketLossRatio = 0.10;
_maxAcceptablePacketErrorRatio = 0.10; _maxAcceptablePacketErrorRatio = 0.10;
_userHasSpecifiedSlaveSpeeds=0; _userHasSpecifiedLinkSpeeds=0;
_lastFrame=0; _lastFrame=0;
@ -1553,8 +1553,8 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
switch (policy) { switch (policy) {
case ZT_BONDING_POLICY_ACTIVE_BACKUP: case ZT_BONDING_POLICY_ACTIVE_BACKUP:
_failoverInterval = 500; _failoverInterval = 500;
_abSlaveSelectMethod = ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE; _abLinkSelectMethod = ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE;
_slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; _linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC;
_qualityWeights[ZT_QOS_LAT_IDX] = 0.2f; _qualityWeights[ZT_QOS_LAT_IDX] = 0.2f;
_qualityWeights[ZT_QOS_LTM_IDX] = 0.0f; _qualityWeights[ZT_QOS_LTM_IDX] = 0.0f;
_qualityWeights[ZT_QOS_PDV_IDX] = 0.2f; _qualityWeights[ZT_QOS_PDV_IDX] = 0.2f;
@ -1578,8 +1578,8 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
case ZT_BONDING_POLICY_BALANCE_RR: case ZT_BONDING_POLICY_BALANCE_RR:
_failoverInterval = 500; _failoverInterval = 500;
_allowFlowHashing = false; _allowFlowHashing = false;
_packetsPerSlave = 1024; _packetsPerLink = 1024;
_slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; _linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC;
_qualityWeights[ZT_QOS_LAT_IDX] = 0.4f; _qualityWeights[ZT_QOS_LAT_IDX] = 0.4f;
_qualityWeights[ZT_QOS_LTM_IDX] = 0.0f; _qualityWeights[ZT_QOS_LTM_IDX] = 0.0f;
_qualityWeights[ZT_QOS_PDV_IDX] = 0.2f; _qualityWeights[ZT_QOS_PDV_IDX] = 0.2f;
@ -1598,7 +1598,7 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
_failoverInterval = 500; _failoverInterval = 500;
_upDelay = _bondMonitorInterval * 2; _upDelay = _bondMonitorInterval * 2;
_allowFlowHashing = true; _allowFlowHashing = true;
_slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; _linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC;
_qualityWeights[ZT_QOS_LAT_IDX] = 0.4f; _qualityWeights[ZT_QOS_LAT_IDX] = 0.4f;
_qualityWeights[ZT_QOS_LTM_IDX] = 0.0f; _qualityWeights[ZT_QOS_LTM_IDX] = 0.0f;
_qualityWeights[ZT_QOS_PDV_IDX] = 0.2f; _qualityWeights[ZT_QOS_PDV_IDX] = 0.2f;
@ -1617,7 +1617,7 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
case ZT_BONDING_POLICY_BALANCE_AWARE: case ZT_BONDING_POLICY_BALANCE_AWARE:
_failoverInterval = 3000; _failoverInterval = 3000;
_allowFlowHashing = true; _allowFlowHashing = true;
_slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; _linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC;
_qualityWeights[ZT_QOS_LAT_IDX] = 0.4f; _qualityWeights[ZT_QOS_LAT_IDX] = 0.4f;
_qualityWeights[ZT_QOS_LTM_IDX] = 0.0f; _qualityWeights[ZT_QOS_LTM_IDX] = 0.0f;
_qualityWeights[ZT_QOS_PDV_IDX] = 0.4f; _qualityWeights[ZT_QOS_PDV_IDX] = 0.4f;
@ -1641,7 +1641,7 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
_upDelay = templateBond->_upDelay; _upDelay = templateBond->_upDelay;
fprintf(stderr, "TIMERS: strat=%d, fi= %d, bmi= %d, qos= %d, ack= %d, estimateInt= %d, refractory= %d, ud= %d, dd= %d\n", fprintf(stderr, "TIMERS: strat=%d, fi= %d, bmi= %d, qos= %d, ack= %d, estimateInt= %d, refractory= %d, ud= %d, dd= %d\n",
_slaveMonitorStrategy, _linkMonitorStrategy,
_failoverInterval, _failoverInterval,
_bondMonitorInterval, _bondMonitorInterval,
_qosSendInterval, _qosSendInterval,
@ -1651,11 +1651,11 @@ void Bond::setReasonableDefaults(int policy, SharedPtr<Bond> templateBond, bool
_upDelay, _upDelay,
_downDelay); _downDelay);
if (templateBond->_slaveMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE if (templateBond->_linkMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE
&& templateBond->_failoverInterval != 0) { && templateBond->_failoverInterval != 0) {
fprintf(stderr, "warning: passive path monitoring was specified, this will prevent failovers from happening in a timely manner.\n"); fprintf(stderr, "warning: passive path monitoring was specified, this will prevent failovers from happening in a timely manner.\n");
} }
_abSlaveSelectMethod = templateBond->_abSlaveSelectMethod; _abLinkSelectMethod = templateBond->_abLinkSelectMethod;
memcpy(_qualityWeights, templateBond->_qualityWeights, ZT_QOS_WEIGHT_SIZE * sizeof(float)); memcpy(_qualityWeights, templateBond->_qualityWeights, ZT_QOS_WEIGHT_SIZE * sizeof(float));
} }
@ -1711,9 +1711,9 @@ bool Bond::relevant() {
|| _peer->identity().address().toInt() == 0x795cbf86fa; || _peer->identity().address().toInt() == 0x795cbf86fa;
} }
SharedPtr<Slave> Bond::getSlave(const SharedPtr<Path>& path) SharedPtr<Link> Bond::getLink(const SharedPtr<Path>& path)
{ {
return RR->bc->getSlaveBySocket(_policyAlias, path->localSocket()); return RR->bc->getLinkBySocket(_policyAlias, path->localSocket());
} }
void Bond::dumpInfo(const int64_t now) void Bond::dumpInfo(const int64_t now)
@ -1726,12 +1726,12 @@ void Bond::dumpInfo(const int64_t now)
return; return;
} }
/* /*
fprintf(stderr, "---[ bp=%d, id=%llx, dd=%d, up=%d, pmi=%d, specifiedSlaves=%d, _specifiedPrimarySlave=%d, _specifiedFailInst=%d ]\n", fprintf(stderr, "---[ bp=%d, id=%llx, dd=%d, up=%d, pmi=%d, specifiedLinks=%d, _specifiedPrimaryLink=%d, _specifiedFailInst=%d ]\n",
_policy, _peer->identity().address().toInt(), _downDelay, _upDelay, _monitorInterval, _userHasSpecifiedSlaves, _userHasSpecifiedPrimarySlave, _userHasSpecifiedFailoverInstructions); _policy, _peer->identity().address().toInt(), _downDelay, _upDelay, _monitorInterval, _userHasSpecifiedLinks, _userHasSpecifiedPrimaryLink, _userHasSpecifiedFailoverInstructions);
if (_bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) { if (_bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
fprintf(stderr, "Paths (bp=%d, stats=%d, primaryReselect=%d) :\n", fprintf(stderr, "Paths (bp=%d, stats=%d, primaryReselect=%d) :\n",
_policy, _shouldCollectPathStatistics, _abSlaveSelectMethod); _policy, _shouldCollectPathStatistics, _abLinkSelectMethod);
} }
if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR if (_bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR
|| _bondingPolicy == ZT_BONDING_POLICY_BALANCE_XOR || _bondingPolicy == ZT_BONDING_POLICY_BALANCE_XOR
@ -1748,13 +1748,13 @@ void Bond::dumpInfo(const int64_t now)
for(int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) { for(int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {
if (_paths[i]) { if (_paths[i]) {
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[i]->localSocket());
_paths[i]->address().toString(pathStr); _paths[i]->address().toString(pathStr);
fprintf(stderr, " %2d: lat=%8.3f, ac=%3d, fail%5s, fscore=%6d, in=%7d, out=%7d, age=%7ld, ack=%7ld, ref=%6d, ls=%llx", fprintf(stderr, " %2d: lat=%8.3f, ac=%3d, fail%5s, fscore=%6d, in=%7d, out=%7d, age=%7ld, ack=%7ld, ref=%6d, ls=%llx",
i, i,
_paths[i]->_latencyMean, _paths[i]->_latencyMean,
_paths[i]->_allocation, _paths[i]->_allocation,
slave->failoverToSlave().c_str(), link->failoverToLink().c_str(),
_paths[i]->_failoverScore, _paths[i]->_failoverScore,
_paths[i]->_packetsIn, _paths[i]->_packetsIn,
_paths[i]->_packetsOut, _paths[i]->_packetsOut,
@ -1763,12 +1763,12 @@ void Bond::dumpInfo(const int64_t now)
_paths[i]->_refractoryPeriod, _paths[i]->_refractoryPeriod,
_paths[i]->localSocket() _paths[i]->localSocket()
); );
if (slave->spare()) { if (link->spare()) {
fprintf(stderr, " SPR."); fprintf(stderr, " SPR.");
} else { } else {
fprintf(stderr, " "); fprintf(stderr, " ");
} }
if (slave->primary()) { if (link->primary()) {
fprintf(stderr, " PRIM."); fprintf(stderr, " PRIM.");
} else { } else {
fprintf(stderr, " "); fprintf(stderr, " ");
@ -1808,7 +1808,7 @@ void Bond::dumpInfo(const int64_t now)
} else if (_bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) { } else if (_bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
fprintf(stderr, " "); fprintf(stderr, " ");
} }
fprintf(stderr, "%5s %s\n", slave->ifname().c_str(), pathStr); fprintf(stderr, "%5s %s\n", link->ifname().c_str(), pathStr);
} }
} }
@ -1817,12 +1817,12 @@ void Bond::dumpInfo(const int64_t now)
fprintf(stderr, "\nFailover Queue:\n"); fprintf(stderr, "\nFailover Queue:\n");
for (std::list<SharedPtr<Path> >::iterator it(_abFailoverQueue.begin()); it!=_abFailoverQueue.end();++it) { for (std::list<SharedPtr<Path> >::iterator it(_abFailoverQueue.begin()); it!=_abFailoverQueue.end();++it) {
(*it)->address().toString(currPathStr); (*it)->address().toString(currPathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, (*it)->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, (*it)->localSocket());
fprintf(stderr, "\t%8s\tspeed=%7d\trelSpeed=%3d\tipvPref=%3d\tfscore=%9d\t\t%s\n", fprintf(stderr, "\t%8s\tspeed=%7d\trelSpeed=%3d\tipvPref=%3d\tfscore=%9d\t\t%s\n",
slave->ifname().c_str(), link->ifname().c_str(),
slave->speed(), link->speed(),
slave->relativeSpeed(), link->relativeSpeed(),
slave->ipvPref(), link->ipvPref(),
(*it)->_failoverScore, (*it)->_failoverScore,
currPathStr); currPathStr);
} }
@ -1840,15 +1840,15 @@ void Bond::dumpInfo(const int64_t now)
fprintf(stderr, "\nBonded Paths:\n"); fprintf(stderr, "\nBonded Paths:\n");
for (int i=0; i<_numBondedPaths; ++i) { for (int i=0; i<_numBondedPaths; ++i) {
_paths[_bondedIdx[i]]->address().toString(currPathStr); _paths[_bondedIdx[i]]->address().toString(currPathStr);
SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[_bondedIdx[i]]->localSocket()); SharedPtr<Link> link =RR->bc->getLinkBySocket(_policyAlias, _paths[_bondedIdx[i]]->localSocket());
fprintf(stderr, " [%d]\t%8s\tflows=%3d\tspeed=%7d\trelSpeed=%3d\tipvPref=%3d\tfscore=%9d\t\t%s\n", i, fprintf(stderr, " [%d]\t%8s\tflows=%3d\tspeed=%7d\trelSpeed=%3d\tipvPref=%3d\tfscore=%9d\t\t%s\n", i,
//fprintf(stderr, " [%d]\t%8s\tspeed=%7d\trelSpeed=%3d\tflowCount=%2d\tipvPref=%3d\tfscore=%9d\t\t%s\n", i, //fprintf(stderr, " [%d]\t%8s\tspeed=%7d\trelSpeed=%3d\tflowCount=%2d\tipvPref=%3d\tfscore=%9d\t\t%s\n", i,
slave->ifname().c_str(), link->ifname().c_str(),
_paths[_bondedIdx[i]]->_assignedFlowCount, _paths[_bondedIdx[i]]->_assignedFlowCount,
slave->speed(), link->speed(),
slave->relativeSpeed(), link->relativeSpeed(),
//_paths[_bondedIdx[i]].p->assignedFlows.size(), //_paths[_bondedIdx[i]].p->assignedFlows.size(),
slave->ipvPref(), link->ipvPref(),
_paths[_bondedIdx[i]]->_failoverScore, _paths[_bondedIdx[i]]->_failoverScore,
currPathStr); currPathStr);
} }

60
node/Bond.hpp
View File

@ -18,13 +18,13 @@
#include "Path.hpp" #include "Path.hpp"
#include "Peer.hpp" #include "Peer.hpp"
#include "../osdep/Slave.hpp" #include "../osdep/Link.hpp"
#include "Flow.hpp" #include "Flow.hpp"
namespace ZeroTier { namespace ZeroTier {
class RuntimeEnvironment; class RuntimeEnvironment;
class Slave; class Link;
class Bond class Bond
{ {
@ -52,7 +52,7 @@ public:
void dumpInfo(const int64_t now); void dumpInfo(const int64_t now);
bool relevant(); bool relevant();
SharedPtr<Slave> getSlave(const SharedPtr<Path>& path); SharedPtr<Link> getLink(const SharedPtr<Path>& path);
/** /**
* Constructor. Creates a bond based off of ZT defaults * Constructor. Creates a bond based off of ZT defaults
@ -281,7 +281,7 @@ public:
void processActiveBackupTasks(int64_t now); void processActiveBackupTasks(int64_t now);
/** /**
* Switches the active slave in an active-backup scenario to the next best during * Switches the active link in an active-backup scenario to the next best during
* a failover event. * a failover event.
* *
* @param now Current time * @param now Current time
@ -348,24 +348,24 @@ public:
} }
/** /**
* @return Whether the user has defined slaves for use on this bond * @return Whether the user has defined links for use on this bond
*/ */
inline bool userHasSpecifiedSlaves() { return _userHasSpecifiedSlaves; } inline bool userHasSpecifiedLinks() { return _userHasSpecifiedLinks; }
/** /**
* @return Whether the user has defined a set of failover slave(s) for this bond * @return Whether the user has defined a set of failover link(s) for this bond
*/ */
inline bool userHasSpecifiedFailoverInstructions() { return _userHasSpecifiedFailoverInstructions; }; inline bool userHasSpecifiedFailoverInstructions() { return _userHasSpecifiedFailoverInstructions; };
/** /**
* @return Whether the user has specified a primary slave * @return Whether the user has specified a primary link
*/ */
inline bool userHasSpecifiedPrimarySlave() { return _userHasSpecifiedPrimarySlave; } inline bool userHasSpecifiedPrimaryLink() { return _userHasSpecifiedPrimaryLink; }
/** /**
* @return Whether the user has specified slave speeds * @return Whether the user has specified link speeds
*/ */
inline bool userHasSpecifiedSlaveSpeeds() { return _userHasSpecifiedSlaveSpeeds; } inline bool userHasSpecifiedLinkSpeeds() { return _userHasSpecifiedLinkSpeeds; }
/** /**
* Periodically perform maintenance tasks for each active bond. * Periodically perform maintenance tasks for each active bond.
@ -441,7 +441,7 @@ public:
/** /**
* @param strategy Strategy that the bond uses to prob for path aliveness and quality * @param strategy Strategy that the bond uses to prob for path aliveness and quality
*/ */
inline void setSlaveMonitorStrategy(uint8_t strategy) { _slaveMonitorStrategy = strategy; } inline void setLinkMonitorStrategy(uint8_t strategy) { _linkMonitorStrategy = strategy; }
/** /**
* @return the current up delay parameter * @return the current up delay parameter
@ -464,12 +464,12 @@ public:
inline void setDownDelay(int downDelay) { if (downDelay >= 0) { _downDelay = downDelay; } } inline void setDownDelay(int downDelay) { if (downDelay >= 0) { _downDelay = downDelay; } }
/** /**
* @return the current monitoring interval for the bond (can be overridden with intervals specific to certain slaves.) * @return the current monitoring interval for the bond (can be overridden with intervals specific to certain links.)
*/ */
inline uint16_t getBondMonitorInterval() { return _bondMonitorInterval; } inline uint16_t getBondMonitorInterval() { return _bondMonitorInterval; }
/** /**
* Set the current monitoring interval for the bond (can be overridden with intervals specific to certain slaves.) * Set the current monitoring interval for the bond (can be overridden with intervals specific to certain links.)
* *
* @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer. * @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer.
*/ */
@ -498,21 +498,21 @@ public:
/** /**
* *
* @param packetsPerSlave * @param packetsPerLink
*/ */
inline void setPacketsPerSlave(int packetsPerSlave) { _packetsPerSlave = packetsPerSlave; } inline void setPacketsPerLink(int packetsPerLink) { _packetsPerLink = packetsPerLink; }
/** /**
* *
* @param slaveSelectMethod * @param linkSelectMethod
*/ */
inline void setSlaveSelectMethod(uint8_t method) { _abSlaveSelectMethod = method; } inline void setLinkSelectMethod(uint8_t method) { _abLinkSelectMethod = method; }
/** /**
* *
* @return * @return
*/ */
inline uint8_t getSlaveSelectMethod() { return _abSlaveSelectMethod; } inline uint8_t getLinkSelectMethod() { return _abLinkSelectMethod; }
/** /**
* *
@ -568,25 +568,25 @@ private:
// active-backup // active-backup
SharedPtr<Path> _abPath; // current active path SharedPtr<Path> _abPath; // current active path
std::list<SharedPtr<Path> > _abFailoverQueue; std::list<SharedPtr<Path> > _abFailoverQueue;
uint8_t _abSlaveSelectMethod; // slave re-selection policy for the primary slave in active-backup uint8_t _abLinkSelectMethod; // link re-selection policy for the primary link in active-backup
uint64_t _lastActiveBackupPathChange; uint64_t _lastActiveBackupPathChange;
// balance-rr // balance-rr
uint8_t _rrIdx; // index to path currently in use during Round Robin operation uint8_t _rrIdx; // index to path currently in use during Round Robin operation
uint16_t _rrPacketsSentOnCurrSlave; // number of packets sent on this slave since the most recent path switch. uint16_t _rrPacketsSentOnCurrLink; // number of packets sent on this link since the most recent path switch.
/** /**
* How many packets will be sent on a path before moving to the next path * How many packets will be sent on a path before moving to the next path
* in the round-robin sequence. A value of zero will cause a random path * in the round-robin sequence. A value of zero will cause a random path
* selection for each outgoing packet. * selection for each outgoing packet.
*/ */
int _packetsPerSlave; int _packetsPerLink;
// balance-aware // balance-aware
uint64_t _totalBondUnderload; uint64_t _totalBondUnderload;
uint8_t _flowRebalanceStrategy; uint8_t _flowRebalanceStrategy;
// dynamic slave monitoring // dynamic link monitoring
uint8_t _slaveMonitorStrategy; uint8_t _linkMonitorStrategy;
uint64_t _lastFrame; uint64_t _lastFrame;
uint32_t _dynamicPathMonitorInterval; uint32_t _dynamicPathMonitorInterval;
@ -651,14 +651,14 @@ private:
Mutex _flows_m; Mutex _flows_m;
/** /**
* Whether the user has specified slaves for this bond. * Whether the user has specified links for this bond.
*/ */
bool _userHasSpecifiedSlaves; bool _userHasSpecifiedLinks;
/** /**
* Whether the user has specified a primary slave for this bond. * Whether the user has specified a primary link for this bond.
*/ */
bool _userHasSpecifiedPrimarySlave; bool _userHasSpecifiedPrimaryLink;
/** /**
* Whether the user has specified failover instructions for this bond. * Whether the user has specified failover instructions for this bond.
@ -666,9 +666,9 @@ private:
bool _userHasSpecifiedFailoverInstructions; bool _userHasSpecifiedFailoverInstructions;
/** /**
* Whether the user has specified slaves speeds for this bond. * Whether the user has specified links speeds for this bond.
*/ */
bool _userHasSpecifiedSlaveSpeeds; bool _userHasSpecifiedLinkSpeeds;
/** /**
* How frequently (in ms) a VERB_ECHO is sent to a peer to verify that a * How frequently (in ms) a VERB_ECHO is sent to a peer to verify that a

80
node/BondController.cpp
View File

@ -27,33 +27,33 @@ BondController::BondController(const RuntimeEnvironment *renv) :
_defaultBondingPolicy = ZT_BONDING_POLICY_NONE; _defaultBondingPolicy = ZT_BONDING_POLICY_NONE;
} }
bool BondController::slaveAllowed(std::string &policyAlias, SharedPtr<Slave> slave) bool BondController::linkAllowed(std::string &policyAlias, SharedPtr<Link> link)
{ {
bool foundInDefinitions = false; bool foundInDefinitions = false;
if (_slaveDefinitions.count(policyAlias)) { if (_linkDefinitions.count(policyAlias)) {
auto it = _slaveDefinitions[policyAlias].begin(); auto it = _linkDefinitions[policyAlias].begin();
while (it != _slaveDefinitions[policyAlias].end()) { while (it != _linkDefinitions[policyAlias].end()) {
if (slave->ifname() == (*it)->ifname()) { if (link->ifname() == (*it)->ifname()) {
foundInDefinitions = true; foundInDefinitions = true;
break; break;
} }
++it; ++it;
} }
} }
return _slaveDefinitions[policyAlias].empty() || foundInDefinitions; return _linkDefinitions[policyAlias].empty() || foundInDefinitions;
} }
void BondController::addCustomSlave(std::string& policyAlias, SharedPtr<Slave> slave) void BondController::addCustomLink(std::string& policyAlias, SharedPtr<Link> link)
{ {
Mutex::Lock _l(_slaves_m); Mutex::Lock _l(_links_m);
_slaveDefinitions[policyAlias].push_back(slave); _linkDefinitions[policyAlias].push_back(link);
auto search = _interfaceToSlaveMap[policyAlias].find(slave->ifname()); auto search = _interfaceToLinkMap[policyAlias].find(link->ifname());
if (search == _interfaceToSlaveMap[policyAlias].end()) { if (search == _interfaceToLinkMap[policyAlias].end()) {
slave->setAsUserSpecified(true); link->setAsUserSpecified(true);
_interfaceToSlaveMap[policyAlias].insert(std::pair<std::string, SharedPtr<Slave>>(slave->ifname(), slave)); _interfaceToLinkMap[policyAlias].insert(std::pair<std::string, SharedPtr<Link>>(link->ifname(), link));
} else { } else {
fprintf(stderr, "slave already exists=%s\n", slave->ifname().c_str()); fprintf(stderr, "link already exists=%s\n", link->ifname().c_str());
// Slave is already defined, overlay user settings // Link is already defined, overlay user settings
} }
} }
@ -115,20 +115,20 @@ SharedPtr<Bond> BondController::createTransportTriggeredBond(const RuntimeEnviro
/** /**
* Determine if user has specified anything that could affect the bonding policy's decisions * Determine if user has specified anything that could affect the bonding policy's decisions
*/ */
if (_interfaceToSlaveMap.count(bond->policyAlias())) { if (_interfaceToLinkMap.count(bond->policyAlias())) {
std::map<std::string, SharedPtr<Slave> >::iterator it = _interfaceToSlaveMap[bond->policyAlias()].begin(); std::map<std::string, SharedPtr<Link> >::iterator it = _interfaceToLinkMap[bond->policyAlias()].begin();
while (it != _interfaceToSlaveMap[bond->policyAlias()].end()) { while (it != _interfaceToLinkMap[bond->policyAlias()].end()) {
if (it->second->isUserSpecified()) { if (it->second->isUserSpecified()) {
bond->_userHasSpecifiedSlaves = true; bond->_userHasSpecifiedLinks = true;
} }
if (it->second->isUserSpecified() && it->second->primary()) { if (it->second->isUserSpecified() && it->second->primary()) {
bond->_userHasSpecifiedPrimarySlave = true; bond->_userHasSpecifiedPrimaryLink = true;
} }
if (it->second->isUserSpecified() && it->second->userHasSpecifiedFailoverInstructions()) { if (it->second->isUserSpecified() && it->second->userHasSpecifiedFailoverInstructions()) {
bond->_userHasSpecifiedFailoverInstructions = true; bond->_userHasSpecifiedFailoverInstructions = true;
} }
if (it->second->isUserSpecified() && (it->second->speed() > 0)) { if (it->second->isUserSpecified() && (it->second->speed() > 0)) {
bond->_userHasSpecifiedSlaveSpeeds = true; bond->_userHasSpecifiedLinkSpeeds = true;
} }
++it; ++it;
} }
@ -138,16 +138,16 @@ SharedPtr<Bond> BondController::createTransportTriggeredBond(const RuntimeEnviro
return SharedPtr<Bond>(); return SharedPtr<Bond>();
} }
SharedPtr<Slave> BondController::getSlaveBySocket(const std::string& policyAlias, uint64_t localSocket) SharedPtr<Link> BondController::getLinkBySocket(const std::string& policyAlias, uint64_t localSocket)
{ {
Mutex::Lock _l(_slaves_m); Mutex::Lock _l(_links_m);
char ifname[16]; char ifname[16];
_phy->getIfName((PhySocket *) ((uintptr_t)localSocket), ifname, 16); _phy->getIfName((PhySocket *) ((uintptr_t)localSocket), ifname, 16);
std::string ifnameStr(ifname); std::string ifnameStr(ifname);
auto search = _interfaceToSlaveMap[policyAlias].find(ifnameStr); auto search = _interfaceToLinkMap[policyAlias].find(ifnameStr);
if (search == _interfaceToSlaveMap[policyAlias].end()) { if (search == _interfaceToLinkMap[policyAlias].end()) {
SharedPtr<Slave> s = new Slave(ifnameStr, 0, 0, 0, 0, 0, true, ZT_MULTIPATH_SLAVE_MODE_SPARE, "", 0.0); SharedPtr<Link> s = new Link(ifnameStr, 0, 0, 0, 0, 0, true, ZT_MULTIPATH_SLAVE_MODE_SPARE, "", 0.0);
_interfaceToSlaveMap[policyAlias].insert(std::pair<std::string,SharedPtr<Slave> >(ifnameStr, s)); _interfaceToLinkMap[policyAlias].insert(std::pair<std::string,SharedPtr<Link> >(ifnameStr, s));
return s; return s;
} }
else { else {
@ -155,14 +155,14 @@ SharedPtr<Slave> BondController::getSlaveBySocket(const std::string& policyAlias
} }
} }
SharedPtr<Slave> BondController::getSlaveByName(const std::string& policyAlias, const std::string& ifname) SharedPtr<Link> BondController::getLinkByName(const std::string& policyAlias, const std::string& ifname)
{ {
Mutex::Lock _l(_slaves_m); Mutex::Lock _l(_links_m);
auto search = _interfaceToSlaveMap[policyAlias].find(ifname); auto search = _interfaceToLinkMap[policyAlias].find(ifname);
if (search != _interfaceToSlaveMap[policyAlias].end()) { if (search != _interfaceToLinkMap[policyAlias].end()) {
return search->second; return search->second;
} }
return SharedPtr<Slave>(); return SharedPtr<Link>();
} }
bool BondController::allowedToBind(const std::string& ifname) bool BondController::allowedToBind(const std::string& ifname)
@ -172,18 +172,18 @@ bool BondController::allowedToBind(const std::string& ifname)
if (!_defaultBondingPolicy) { if (!_defaultBondingPolicy) {
return true; // no restrictions return true; // no restrictions
} }
Mutex::Lock _l(_slaves_m); Mutex::Lock _l(_links_m);
if (_interfaceToSlaveMap.empty()) { if (_interfaceToLinkMap.empty()) {
return true; // no restrictions return true; // no restrictions
} }
std::map<std::string, std::map<std::string, SharedPtr<Slave> > >::iterator policyItr = _interfaceToSlaveMap.begin(); std::map<std::string, std::map<std::string, SharedPtr<Link> > >::iterator policyItr = _interfaceToLinkMap.begin();
while (policyItr != _interfaceToSlaveMap.end()) { while (policyItr != _interfaceToLinkMap.end()) {
std::map<std::string, SharedPtr<Slave> >::iterator slaveItr = policyItr->second.begin(); std::map<std::string, SharedPtr<Link> >::iterator linkItr = policyItr->second.begin();
while (slaveItr != policyItr->second.end()) { while (linkItr != policyItr->second.end()) {
if (slaveItr->first == ifname) { if (linkItr->first == ifname) {
return true; return true;
} }
++slaveItr; ++linkItr;
} }
++policyItr; ++policyItr;
} }

34
node/BondController.hpp
View File

@ -19,7 +19,7 @@
#include "SharedPtr.hpp" #include "SharedPtr.hpp"
#include "../osdep/Phy.hpp" #include "../osdep/Phy.hpp"
#include "../osdep/Slave.hpp" #include "../osdep/Link.hpp"
namespace ZeroTier { namespace ZeroTier {
@ -36,9 +36,9 @@ public:
BondController(const RuntimeEnvironment *renv); BondController(const RuntimeEnvironment *renv);
/** /**
* @return Whether this slave is permitted to become a member of a bond. * @return Whether this link is permitted to become a member of a bond.
*/ */
bool slaveAllowed(std::string &policyAlias, SharedPtr<Slave> slave); bool linkAllowed(std::string &policyAlias, SharedPtr<Link> link);
/** /**
* @return The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements. * @return The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements.
@ -103,12 +103,12 @@ public:
static int defaultBondingPolicy() { return _defaultBondingPolicy; } static int defaultBondingPolicy() { return _defaultBondingPolicy; }
/** /**
* Add a user-defined slave to a given bonding policy. * Add a user-defined link to a given bonding policy.
* *
* @param policyAlias User-defined custom name for variant of bonding policy * @param policyAlias User-defined custom name for variant of bonding policy
* @param slave Pointer to new slave definition * @param link Pointer to new link definition
*/ */
void addCustomSlave(std::string& policyAlias, SharedPtr<Slave> slave); void addCustomLink(std::string& policyAlias, SharedPtr<Link> link);
/** /**
* Add a user-defined bonding policy that is based on one of the standard types. * Add a user-defined bonding policy that is based on one of the standard types.
@ -145,22 +145,22 @@ public:
void processBackgroundTasks(void *tPtr, int64_t now); void processBackgroundTasks(void *tPtr, int64_t now);
/** /**
* Gets a reference to a physical slave definition given a policy alias and a local socket. * Gets a reference to a physical link definition given a policy alias and a local socket.
* *
* @param policyAlias Policy in use * @param policyAlias Policy in use
* @param localSocket Local source socket * @param localSocket Local source socket
* @return Physical slave definition * @return Physical link definition
*/ */
SharedPtr<Slave> getSlaveBySocket(const std::string& policyAlias, uint64_t localSocket); SharedPtr<Link> getLinkBySocket(const std::string& policyAlias, uint64_t localSocket);
/** /**
* Gets a reference to a physical slave definition given its human-readable system name. * Gets a reference to a physical link definition given its human-readable system name.
* *
* @param policyAlias Policy in use * @param policyAlias Policy in use
* @param ifname Alphanumeric human-readable name * @param ifname Alphanumeric human-readable name
* @return Physical slave definition * @return Physical link definition
*/ */
SharedPtr<Slave> getSlaveByName(const std::string& policyAlias, const std::string& ifname); SharedPtr<Link> getLinkByName(const std::string& policyAlias, const std::string& ifname);
/** /**
* @param ifname Name of interface that we want to know if we can bind to * @param ifname Name of interface that we want to know if we can bind to
@ -175,7 +175,7 @@ private:
const RuntimeEnvironment *RR; const RuntimeEnvironment *RR;
Mutex _bonds_m; Mutex _bonds_m;
Mutex _slaves_m; Mutex _links_m;
/** /**
* The last time that the bond controller updated the set of bonds. * The last time that the bond controller updated the set of bonds.
@ -213,14 +213,14 @@ private:
std::map<std::string,SharedPtr<Bond> > _bondPolicyTemplates; std::map<std::string,SharedPtr<Bond> > _bondPolicyTemplates;
/** /**
* Set of slaves defined for a given bonding policy * Set of links defined for a given bonding policy
*/ */
std::map<std::string,std::vector<SharedPtr<Slave> > > _slaveDefinitions; std::map<std::string,std::vector<SharedPtr<Link> > > _linkDefinitions;
/** /**
* Set of slave objects mapped to their physical interfaces * Set of link objects mapped to their physical interfaces
*/ */
std::map<std::string, std::map<std::string, SharedPtr<Slave> > > _interfaceToSlaveMap; std::map<std::string, std::map<std::string, SharedPtr<Link> > > _interfaceToLinkMap;
// TODO: Remove // TODO: Remove
uint64_t bondStartTime; uint64_t bondStartTime;

6
node/Constants.hpp
View File

@ -405,12 +405,12 @@
#define ZT_FLOW_MAX_COUNT (1024*64) #define ZT_FLOW_MAX_COUNT (1024*64)
/** /**
* How often flows are rebalanced across slave interfaces (if at all) * How often flows are rebalanced across link (if at all)
*/ */
#define ZT_FLOW_MIN_REBALANCE_INTERVAL 5000 #define ZT_FLOW_MIN_REBALANCE_INTERVAL 5000
/** /**
* How often flows are rebalanced across slave interfaces (if at all) * How often flows are rebalanced across link (if at all)
*/ */
#define ZT_FLOW_REBALANCE_INTERVAL 5000 #define ZT_FLOW_REBALANCE_INTERVAL 5000
@ -428,7 +428,7 @@
/** /**
* Minimum amount of time (since a previous transition) before the active-backup bonding * Minimum amount of time (since a previous transition) before the active-backup bonding
* policy is allowed to transition to a different slave. Only valid for active-backup. * policy is allowed to transition to a different link. Only valid for active-backup.
*/ */
#define ZT_MULTIPATH_MIN_ACTIVE_BACKUP_AUTOFLOP_INTERVAL 10000 #define ZT_MULTIPATH_MIN_ACTIVE_BACKUP_AUTOFLOP_INTERVAL 10000

2
node/Node.cpp
View File

@ -501,14 +501,12 @@ ZT_PeerList *Node::peers() const
p->pathCount = 0; p->pathCount = 0;
for(std::vector< SharedPtr<Path> >::iterator path(paths.begin());path!=paths.end();++path) { for(std::vector< SharedPtr<Path> >::iterator path(paths.begin());path!=paths.end();++path) {
memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage)); memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage));
//memcpy(&(p->paths[p->pathCount].ifname,&((*path)->slave()),32);)
p->paths[p->pathCount].localSocket = (*path)->localSocket(); p->paths[p->pathCount].localSocket = (*path)->localSocket();
p->paths[p->pathCount].lastSend = (*path)->lastOut(); p->paths[p->pathCount].lastSend = (*path)->lastOut();
p->paths[p->pathCount].lastReceive = (*path)->lastIn(); p->paths[p->pathCount].lastReceive = (*path)->lastIn();
p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address()); p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address());
p->paths[p->pathCount].expired = 0; p->paths[p->pathCount].expired = 0;
p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0; p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0;
//p->paths[p->pathCount].age = (*path)->age(_now);
p->paths[p->pathCount].scope = (*path)->ipScope(); p->paths[p->pathCount].scope = (*path)->ipScope();
++p->pathCount; ++p->pathCount;
} }

20
node/Path.hpp
View File

@ -29,7 +29,7 @@
#include "Packet.hpp" #include "Packet.hpp"
#include "RingBuffer.hpp" #include "RingBuffer.hpp"
#include "../osdep/Slave.hpp" #include "../osdep/Link.hpp"
/** /**
* Maximum return value of preferenceRank() * Maximum return value of preferenceRank()
@ -103,7 +103,7 @@ public:
_downDelay(0), _downDelay(0),
_ipvPref(0), _ipvPref(0),
_mode(0), _mode(0),
_onlyPathOnSlave(false), _onlyPathOnLink(false),
_enabled(false), _enabled(false),
_bonded(false), _bonded(false),
_negotiated(false), _negotiated(false),
@ -152,7 +152,7 @@ public:
_downDelay(0), _downDelay(0),
_ipvPref(0), _ipvPref(0),
_mode(0), _mode(0),
_onlyPathOnSlave(false), _onlyPathOnLink(false),
_enabled(false), _enabled(false),
_bonded(false), _bonded(false),
_negotiated(false), _negotiated(false),
@ -431,10 +431,10 @@ public:
} }
/** /**
* @return True if a path is preferred over another on the same physical slave (according to user pref.) * @return True if a path is preferred over another on the same physical link (according to user pref.)
*/ */
inline bool preferred() { inline bool preferred() {
return _onlyPathOnSlave return _onlyPathOnLink
|| (_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46)) || (_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46))
|| (_addr.isV6() && (_ipvPref == 6 || _ipvPref == 64)); || (_addr.isV6() && (_ipvPref == 6 || _ipvPref == 64));
} }
@ -549,22 +549,22 @@ private:
uint32_t _downDelay; uint32_t _downDelay;
/** /**
* IP version preference inherited from the physical slave. * IP version preference inherited from the physical link.
*/ */
uint8_t _ipvPref; uint8_t _ipvPref;
/** /**
* Mode inherited from the physical slave. * Mode inherited from the physical link.
*/ */
uint8_t _mode; uint8_t _mode;
/** /**
* IP version preference inherited from the physical slave. * IP version preference inherited from the physical link.
*/ */
bool _onlyPathOnSlave; bool _onlyPathOnLink;
/** /**
* Enabled state inherited from the physical slave. * Enabled state inherited from the physical link.
*/ */
bool _enabled; bool _enabled;

4
node/Peer.cpp
View File

@ -55,8 +55,8 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
_remoteMultipathSupported(false), _remoteMultipathSupported(false),
_canUseMultipath(false), _canUseMultipath(false),
_shouldCollectPathStatistics(0), _shouldCollectPathStatistics(0),
_lastComputedAggregateMeanLatency(0), _bondingPolicy(0),
_bondingPolicy(0) _lastComputedAggregateMeanLatency(0)
{ {
if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) { if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH)) {
throw ZT_EXCEPTION_INVALID_ARGUMENT; throw ZT_EXCEPTION_INVALID_ARGUMENT;

14
osdep/Binder.hpp
View File

@ -347,11 +347,11 @@ public:
} }
} }
// Generate set of unique interface names (used for formation of logical slave set in multipath code) // Generate set of unique interface names (used for formation of logical link set in multipath code)
for(std::map<InetAddress,std::string>::const_iterator ii(localIfAddrs.begin());ii!=localIfAddrs.end();++ii) { for(std::map<InetAddress,std::string>::const_iterator ii(localIfAddrs.begin());ii!=localIfAddrs.end();++ii) {
slaveIfNames.insert(ii->second); linkIfNames.insert(ii->second);
} }
for (std::set<std::string>::iterator si(slaveIfNames.begin());si!=slaveIfNames.end();si++) { for (std::set<std::string>::iterator si(linkIfNames.begin());si!=linkIfNames.end();si++) {
bool bFoundMatch = false; bool bFoundMatch = false;
for(std::map<InetAddress,std::string>::const_iterator ii(localIfAddrs.begin());ii!=localIfAddrs.end();++ii) { for(std::map<InetAddress,std::string>::const_iterator ii(localIfAddrs.begin());ii!=localIfAddrs.end();++ii) {
if (ii->second == *si) { if (ii->second == *si) {
@ -360,7 +360,7 @@ public:
} }
} }
if (!bFoundMatch) { if (!bFoundMatch) {
slaveIfNames.erase(si); linkIfNames.erase(si);
} }
} }
@ -461,15 +461,15 @@ public:
return false; return false;
} }
inline std::set<std::string> getSlaveInterfaceNames() inline std::set<std::string> getLinkInterfaceNames()
{ {
Mutex::Lock _l(_lock); Mutex::Lock _l(_lock);
return slaveIfNames; return linkIfNames;
} }
private: private:
std::set<std::string> slaveIfNames; std::set<std::string> linkIfNames;
_Binding _bindings[ZT_BINDER_MAX_BINDINGS]; _Binding _bindings[ZT_BINDER_MAX_BINDINGS];
std::atomic<unsigned int> _bindingCount; std::atomic<unsigned int> _bindingCount;
Mutex _lock; Mutex _lock;

84
osdep/Slave.hpp → osdep/Link.hpp
View File

@ -11,8 +11,8 @@
*/ */
/****/ /****/
#ifndef ZT_SLAVE_HPP #ifndef ZT_LINK_HPP
#define ZT_SLAVE_HPP #define ZT_LINK_HPP
#include <string> #include <string>
@ -20,13 +20,13 @@
namespace ZeroTier { namespace ZeroTier {
class Slave class Link
{ {
friend class SharedPtr<Slave>; friend class SharedPtr<Link>;
public: public:
Slave() {} Link() {}
/** /**
* *
@ -35,60 +35,60 @@ public:
* @param speed * @param speed
* @param enabled * @param enabled
* @param mode * @param mode
* @param failoverToSlaveStr * @param failoverToLinkStr
* @param userSpecifiedAlloc * @param userSpecifiedAlloc
*/ */
Slave(std::string& ifnameStr, Link(std::string& ifnameStr,
uint8_t ipvPref, uint8_t ipvPref,
uint32_t speed, uint32_t speed,
uint32_t slaveMonitorInterval, uint32_t linkMonitorInterval,
uint32_t upDelay, uint32_t upDelay,
uint32_t downDelay, uint32_t downDelay,
bool enabled, bool enabled,
uint8_t mode, uint8_t mode,
std::string failoverToSlaveStr, std::string failoverToLinkStr,
float userSpecifiedAlloc) : float userSpecifiedAlloc) :
_ifnameStr(ifnameStr), _ifnameStr(ifnameStr),
_ipvPref(ipvPref), _ipvPref(ipvPref),
_speed(speed), _speed(speed),
_relativeSpeed(0), _relativeSpeed(0),
_slaveMonitorInterval(slaveMonitorInterval), _linkMonitorInterval(linkMonitorInterval),
_upDelay(upDelay), _upDelay(upDelay),
_downDelay(downDelay), _downDelay(downDelay),
_enabled(enabled), _enabled(enabled),
_mode(mode), _mode(mode),
_failoverToSlaveStr(failoverToSlaveStr), _failoverToLinkStr(failoverToLinkStr),
_userSpecifiedAlloc(userSpecifiedAlloc), _userSpecifiedAlloc(userSpecifiedAlloc),
_isUserSpecified(false) _isUserSpecified(false)
{} {}
/** /**
* @return The string representation of this slave's underlying interface's system name. * @return The string representation of this link's underlying interface's system name.
*/ */
inline std::string ifname() { return _ifnameStr; } inline std::string ifname() { return _ifnameStr; }
/** /**
* @return Whether this slave is designated as a primary. * @return Whether this link is designated as a primary.
*/ */
inline bool primary() { return _mode == ZT_MULTIPATH_SLAVE_MODE_PRIMARY; } inline bool primary() { return _mode == ZT_MULTIPATH_SLAVE_MODE_PRIMARY; }
/** /**
* @return Whether this slave is designated as a spare. * @return Whether this link is designated as a spare.
*/ */
inline bool spare() { return _mode == ZT_MULTIPATH_SLAVE_MODE_SPARE; } inline bool spare() { return _mode == ZT_MULTIPATH_SLAVE_MODE_SPARE; }
/** /**
* @return The name of the slave interface that should be used in the event of a failure. * @return The name of the link interface that should be used in the event of a failure.
*/ */
inline std::string failoverToSlave() { return _failoverToSlaveStr; } inline std::string failoverToLink() { return _failoverToLinkStr; }
/** /**
* @return Whether this slave interface was specified by the user or auto-detected. * @return Whether this link interface was specified by the user or auto-detected.
*/ */
inline bool isUserSpecified() { return _isUserSpecified; } inline bool isUserSpecified() { return _isUserSpecified; }
/** /**
* Signify that this slave was specified by the user and not the result of auto-detection. * Signify that this link was specified by the user and not the result of auto-detection.
* *
* @param isUserSpecified * @param isUserSpecified
*/ */
@ -97,59 +97,59 @@ public:
/** /**
* @return Whether or not the user has specified failover instructions. * @return Whether or not the user has specified failover instructions.
*/ */
inline bool userHasSpecifiedFailoverInstructions() { return _failoverToSlaveStr.length(); } inline bool userHasSpecifiedFailoverInstructions() { return _failoverToLinkStr.length(); }
/** /**
* @return The speed of the slave relative to others in the bond. * @return The speed of the link relative to others in the bond.
*/ */
inline uint8_t relativeSpeed() { return _relativeSpeed; } inline uint8_t relativeSpeed() { return _relativeSpeed; }
/** /**
* Sets the speed of the slave relative to others in the bond. * Sets the speed of the link relative to others in the bond.
* *
* @param relativeSpeed The speed relative to the rest of the slave interfaces. * @param relativeSpeed The speed relative to the rest of the link.
*/ */
inline void setRelativeSpeed(uint8_t relativeSpeed) { _relativeSpeed = relativeSpeed; } inline void setRelativeSpeed(uint8_t relativeSpeed) { _relativeSpeed = relativeSpeed; }
/** /**
* Sets the speed of the slave relative to others in the bond. * Sets the speed of the link relative to others in the bond.
* *
* @param relativeSpeed * @param relativeSpeed
*/ */
inline void setMonitorInterval(uint32_t interval) { _slaveMonitorInterval = interval; } inline void setMonitorInterval(uint32_t interval) { _linkMonitorInterval = interval; }
/** /**
* @return The absolute speed of the slave interface (as specified by the user.) * @return The absolute speed of the link (as specified by the user.)
*/ */
inline uint32_t monitorInterval() { return _slaveMonitorInterval; } inline uint32_t monitorInterval() { return _linkMonitorInterval; }
/** /**
* @return The absolute speed of the slave interface (as specified by the user.) * @return The absolute speed of the link (as specified by the user.)
*/ */
inline uint32_t speed() { return _speed; } inline uint32_t speed() { return _speed; }
/** /**
* @return The address preference for this slave interface (as specified by the user.) * @return The address preference for this link (as specified by the user.)
*/ */
inline uint8_t ipvPref() { return _ipvPref; } inline uint8_t ipvPref() { return _ipvPref; }
/** /**
* @return The mode (e.g. primary/spare) for this slave interface (as specified by the user.) * @return The mode (e.g. primary/spare) for this link (as specified by the user.)
*/ */
inline uint8_t mode() { return _mode; } inline uint8_t mode() { return _mode; }
/** /**
* @return The upDelay parameter for all paths on this slave interface. * @return The upDelay parameter for all paths on this link.
*/ */
inline uint32_t upDelay() { return _upDelay; } inline uint32_t upDelay() { return _upDelay; }
/** /**
* @return The downDelay parameter for all paths on this slave interface. * @return The downDelay parameter for all paths on this link.
*/ */
inline uint32_t downDelay() { return _downDelay; } inline uint32_t downDelay() { return _downDelay; }
/** /**
* @return Whether this slave is enabled or disabled * @return Whether this link is enabled or disabled
*/ */
inline uint8_t enabled() { return _enabled; } inline uint8_t enabled() { return _enabled; }
@ -173,21 +173,21 @@ private:
uint8_t _ipvPref; uint8_t _ipvPref;
/** /**
* User-specified speed of this slave/link * User-specified speed of this link
*/ */
uint32_t _speed; uint32_t _speed;
/** /**
* Speed relative to other specified slaves/links (computed by Bond) * Speed relative to other specified links (computed by Bond)
*/ */
uint8_t _relativeSpeed; uint8_t _relativeSpeed;
/** /**
* User-specified interval for monitoring paths on this specific slave * User-specified interval for monitoring paths on this specific link
* instead of using the more generic interval specified for the entire * instead of using the more generic interval specified for the entire
* bond. * bond.
*/ */
uint32_t _slaveMonitorInterval; uint32_t _linkMonitorInterval;
/** /**
* How long before a path is considered to be usable after coming online. (when using policies that * How long before a path is considered to be usable after coming online. (when using policies that
@ -202,20 +202,20 @@ private:
uint32_t _downDelay; uint32_t _downDelay;
/** /**
* Whether this slave is enabled, or (disabled (possibly bad config)) * Whether this link is enabled, or (disabled (possibly bad config))
*/ */
uint8_t _enabled; uint8_t _enabled;
/** /**
* Whether this slave is designated as a primary, a spare, or no preference. * Whether this link is designated as a primary, a spare, or no preference.
*/ */
uint8_t _mode; uint8_t _mode;
/** /**
* The specific name of the interface to be used in the event that this * The specific name of the link to be used in the event that this
* slave fails. * link fails.
*/ */
std::string _failoverToSlaveStr; std::string _failoverToLinkStr;
/** /**
* User-specified allocation * User-specified allocation
@ -223,7 +223,7 @@ private:
float _userSpecifiedAlloc; float _userSpecifiedAlloc;
/** /**
* Whether or not this slave was created as a result of manual user specification. This is * Whether or not this link was created as a result of manual user specification. This is
* important to know because certain policy decisions are dependent on whether the user * important to know because certain policy decisions are dependent on whether the user
* intents to use a specific set of interfaces. * intents to use a specific set of interfaces.
*/ */

103
service/MULTIPATH.md
View File

@ -1,17 +1,17 @@
### **2.1.5.** Link aggregation ### Bonding (link aggregation)
Link aggregation allows the simultaneous (or conditional) use of multiple physical links to enable increased throughput, load balancing, redundancy, and fault tolerance. There are a variety of standard policies available that can be used right out of the box with little to no configuration. These policies are directly inspired by [the policies offered by the Linux kernel](https://www.kernel.org/doc/Documentation/networking/bonding.txt). Link aggregation allows the simultaneous (or conditional) use of multiple physical links to enable increased throughput, load balancing, redundancy, and fault tolerance. There are a variety of standard policies available that can be used right out of the box with little to no configuration. These policies are directly inspired by [the policies offered by the Linux kernel](https://www.kernel.org/doc/Documentation/networking/bonding.txt) but are now offered in user-space and hence available on all platforms that ZeroTier supports.
#### Standard Policies #### Standard policies
| Policy name | Fault tolerance | Min. failover (sec.) | Default Failover (sec.) | Balancing | Aggregation efficiency | Redundancy | Sequence Reordering | | Policy name | Fault tolerance | Min. failover (sec.) | Default Failover (sec.)| Balancing | Aggregation efficiency | Redundancy | Sequence Reordering |
|--------------------|:---------------------:|---------------------:|---------------------:|----------------------:|-----------------------:|-----------:|--------------------:| |--------------------|:---------------------:|---------------------:|-----------------------:|----------------------:|-----------------------:|-----------:|--------------------:|
| `none` | None | `60+` | `60+` | none | `none` |1 | No | `none` | None | `60+` | `60+` | none | `none` |1 | No
| `active-backup` | Brief interruption | `0.25` | `10` | none | `low` |1 | Only during failover | `active-backup` | Brief interruption | `0.25` | `10` | none | `low` |1 | Only during failover
| `broadcast` | Fully tolerant | `N/A` | `N/A` | none | `very low` |N | Often | `broadcast` | Fully tolerant | `N/A` | `N/A` | none | `very low` |N | Often
| `balance-rr` | Self-healing | `0.25` | `10` | packet-based | `high` |1 | Often | `balance-rr` | Self-healing | `0.25` | `10` | packet-based | `high` |1 | Often
| `balance-xor` | Self-healing | `0.25` | `10` | flow-based | `very high` |1 | Only during failover | `balance-xor` | Self-healing | `0.25` | `10` | flow-based | `very high` |1 | Only during failover
| `balance-aware` | Self-healing | `0.25` | `10` | *adaptive* flow-based | `very high` |1 | Only during failover and re-balance | `balance-aware` | Self-healing | `0.25` | `10` | *adaptive* flow-based | `very high` |1 | Only during failover and re-balance
A policy can be used easily without specifying any additional parameters: A policy can be used easily without specifying any additional parameters:
@ -23,7 +23,7 @@ A policy can be used easily without specifying any additional parameters:
} }
``` ```
#### Custom Policies #### Custom policies
To customize a bonding policy for your use-case simply specify a `basePolicy` and override chosen parameters. For example, to create a more aggressive `active-backup` policy with low monitoring overhead that will failover `0.250` seconds after it detects a link failure, one could do the following: To customize a bonding policy for your use-case simply specify a `basePolicy` and override chosen parameters. For example, to create a more aggressive `active-backup` policy with low monitoring overhead that will failover `0.250` seconds after it detects a link failure, one could do the following:
@ -45,11 +45,11 @@ To customize a bonding policy for your use-case simply specify a `basePolicy` an
} }
``` ```
#### Specifying Slave interfaces #### Specifying links
Available system network interfaces are referred to as `slaves`. Different sets of slaves can be constructed for different bonding policies and used simultaneously. One can specify the links that ZeroTier should use in any given bonding policy simply by providing an array of slaves with names corresponding to interface names. If a user doesn't specify a set of interfaces to use, ZeroTier will assume every system interface is available for use. However, if the user **does** specify a set of interfaces, ZeroTier will only use what is specified. The same applies to failover rules, if none are specified, ZeroTier will failover to any operational slave. On the other hand, if the user does specify failover rules and there is ever a situation where a slave is available for usage but does not fit within the rules specified by the user, it will go unused. Bonds are composed of multiple `links`. Different sets of links can be constructed for different bonding policies and used simultaneously. One can specify the links that ZeroTier should use in any given bonding policy simply by providing an array of links with names corresponding to interface names. If a user doesn't specify a set of interfaces to use, ZeroTier will assume every system interface is available for use. However, if the user **does** specify a set of interfaces, ZeroTier will only use what is specified. The same applies to failover rules, if none are specified, ZeroTier will failover to any operational link. On the other hand, if the user does specify failover rules and there is ever a situation where a link is available for usage but does not fit within the rules specified by the user, it will go unused.
To specify that ZeroTier should only use `eth0` and `eth1` as primary slaves, and `eth2` as a backup spare and that it should prefer IPv4 over IPv6 except on `eth2` where only IPv6 is allowed: To specify that ZeroTier should only use `eth0` and `eth1` as primary links, and `eth2` as a backup spare and that it should prefer IPv4 over IPv6 except on `eth2` where only IPv6 is allowed:
``` ```
{ {
@ -57,7 +57,7 @@ To specify that ZeroTier should only use `eth0` and `eth1` as primary slaves, an
"defaultBondingPolicy": "aggressive-active-backup", "defaultBondingPolicy": "aggressive-active-backup",
"policies": { "policies": {
"aggressive-active-backup": { "aggressive-active-backup": {
"slaves": { "links": {
"eth0": { "eth0": {
"ipvPref": 46, "ipvPref": 46,
"failoverTo": "eth2", "failoverTo": "eth2",
@ -79,27 +79,27 @@ To specify that ZeroTier should only use `eth0` and `eth1` as primary slaves, an
} }
``` ```
Additional slave-specific parameters: Additional link-specific parameters:
``` ```
"slaves": "links":
{ {
"interfaceName": /* System-name of the network interface. */ "interfaceName": /* System-name of the network interface. */
{ {
"failoverInterval": 0-65535, /* (optional) How quickly a path on this slave should failover after a detected failure. */ "failoverInterval": 0-65535, /* (optional) How quickly a path on this link should failover after a detected failure. */
"ipvPref": [0,4,6,46,64], /* (optional) IP version preference for detected paths on a slave. */ "ipvPref": [0,4,6,46,64], /* (optional) IP version preference for detected paths on a link. */
"speed": 0-1000000, /* (optional) How fast this slave is (in arbitrary units). This is a useful way to manually allocate a bond. */ "speed": 0-1000000, /* (optional) How fast this link is (in arbitrary units). This is a useful way to manually allocate a bond. */
"alloc": 0-255, /* (optional) A relative value representing a desired allocation. */ "alloc": 0-255, /* (optional) A relative value representing a desired allocation. */
"upDelay": 0-65535, /* (optional) How long after a path becomes alive before it is added to the bond. */ "upDelay": 0-65535, /* (optional) How long after a path becomes alive before it is added to the bond. */
"downDelay": 0-65535, /* (optional) How long after a path fails before it is removed from the bond. */ "downDelay": 0-65535, /* (optional) How long after a path fails before it is removed from the bond. */
"failoverTo": "spareInterfaceName", /* (optional) Which slave should be used next after a failure of this slave. */ "failoverTo": "spareInterfaceName", /* (optional) Which link should be used next after a failure of this link. */
"enabled": true|false, /* (optional) Whether any paths on this slave are allowed to be used this bond. */ "enabled": true|false, /* (optional) Whether any paths on this link are allowed to be used this bond. */
"mode": "primary"|"spare" /* (optional) Whether this slave is used by default or only after failover events. */ "mode": "primary"|"spare" /* (optional) Whether this link is used by default or only after failover events. */
} }
} }
``` ```
#### Peer-specific Bonds #### Peer-specific bonds
It is possible to direct ZeroTier to form a certain type of bond with specific peers of your choice. For instance, if one were to want `active-backup` by default but for certain peers to be bonded with a custom load-balanced bond such as `my-custom-balance-aware` one could do the following: It is possible to direct ZeroTier to form a certain type of bond with specific peers of your choice. For instance, if one were to want `active-backup` by default but for certain peers to be bonded with a custom load-balanced bond such as `my-custom-balance-aware` one could do the following:
@ -127,17 +127,17 @@ It is possible to direct ZeroTier to form a certain type of bond with specific p
} }
``` ```
#### Active Backup (`active-backup`) #### Active backup (`active-backup`)
Traffic is sent only on (one) path at any given time. A different path becomes active if the current path fails. This mode provides fault tolerance with a nearly immediate fail-over. This mode **does not** increase total throughput. Traffic is sent only on (one) path at any given time. A different path becomes active if the current path fails. This mode provides fault tolerance with a nearly immediate fail-over. This mode **does not** increase total throughput.
- `mode`: `primary, spare` Slave option which specifies which slave is the primary device. The specified device is intended to always be the active slave while it is available. There are exceptions to this behavior when using different `slaveSelectMethod` modes. There can only be one `primary` slave in this bonding policy. - `mode`: `primary, spare` Link option which specifies which link is the primary device. The specified device is intended to always be the active link while it is available. There are exceptions to this behavior when using different `linkSelectMethod` modes. There can only be one `primary` link in this bonding policy.
- `slaveSelectMethod`: Specifies the selection policy for the active slave during failure and/or recovery events. This is similar to the Linux Kernel's `primary_reselect` option but with a minor extension: - `linkSelectMethod`: Specifies the selection policy for the active link during failure and/or recovery events. This is similar to the Linux Kernel's `primary_reselect` option but with a minor extension:
- `optimize`: **(default if user provides no failover guidance)** The primary slave can change periodically if a superior path is detected. - `optimize`: **(default if user provides no failover guidance)** The primary link can change periodically if a superior path is detected.
- `always`: **(default when slaves are explicitly specified)**: Primary slave regains status as active slave whenever it comes back up. - `always`: **(default when links are explicitly specified)**: Primary link regains status as active link whenever it comes back up.
- `better`: Primary slave regains status as active slave when it comes back up and (if) it is better than the currently-active slave. - `better`: Primary link regains status as active link when it comes back up and (if) it is better than the currently-active link.
- `failure`: Primary slave regains status as active slave only if the currently-active slave fails. - `failure`: Primary link regains status as active link only if the currently-active link fails.
``` ```
{ {
@ -146,8 +146,8 @@ Traffic is sent only on (one) path at any given time. A different path becomes a
"defaultBondingPolicy": "active-backup", "defaultBondingPolicy": "active-backup",
"active-backup": "active-backup":
{ {
"slaveSelectMethod": "always", "linkSelectMethod": "always",
"slaves": "links":
{ {
"eth0": { "failoverTo": "eth1", "mode": "primary" }, "eth0": { "failoverTo": "eth1", "mode": "primary" },
"eth1": { "mode": "spare" }, "eth1": { "mode": "spare" },
@ -163,17 +163,17 @@ Traffic is sent only on (one) path at any given time. A different path becomes a
Traffic is sent on (all) available paths simultaneously. This mode provides fault tolerance and effectively immediate failover due to transmission redundancy. This mode is a poor utilization of throughput resources and will **not** increase throughput but can prevent packet loss during a link failure. The only option available is `dedup` which will de-duplicate all packets on the receiving end if set to `true`. Traffic is sent on (all) available paths simultaneously. This mode provides fault tolerance and effectively immediate failover due to transmission redundancy. This mode is a poor utilization of throughput resources and will **not** increase throughput but can prevent packet loss during a link failure. The only option available is `dedup` which will de-duplicate all packets on the receiving end if set to `true`.
#### Balance Round Robin (`balance-rr`) #### Balance round robin (`balance-rr`)
Traffic is striped across multiple paths. Offers partial fault tolerance immediately, full fault tolerance eventually. This policy is unaware of protocols and is primarily intended for use with protocols that are not sensitive to reordering delays. The only option available for this policy is `packetsPerSlave` which specifies the number of packets to transmit via a path before moving to the next in the RR sequence. When set to `0` a path is chosen at random for each outgoing packet. The default value is `8`, low values can begin to add overhead to packet processing. Traffic is striped across multiple paths. Offers partial fault tolerance immediately, full fault tolerance eventually. This policy is unaware of protocols and is primarily intended for use with protocols that are not sensitive to reordering delays. The only option available for this policy is `packetsPerLink` which specifies the number of packets to transmit via a path before moving to the next in the RR sequence. When set to `0` a path is chosen at random for each outgoing packet. The default value is `8`, low values can begin to add overhead to packet processing.
#### Balance XOR (`balance-xor`, similar to the Linux kernel's [balance-xor](https://www.kernel.org/doc/Documentation/networking/bonding.txt) with `xmit_hash_policy=layer3+4`) #### Balance XOR (`balance-xor`, similar to the Linux kernel's [balance-xor](https://www.kernel.org/doc/Documentation/networking/bonding.txt) with `xmit_hash_policy=layer3+4`)
Traffic is categorized into *flows* based on *source port*, *destination port*, and *protocol type* these flows are then hashed onto available slaves. Each flow will persist on its assigned slave interface for its entire life-cycle. Traffic that does not have an assigned port (such as ICMP pings) will be randomly distributed across slaves. The hash function is simply: `src_port ^ dst_port ^ proto`. Traffic is categorized into *flows* based on *source port*, *destination port*, and *protocol type* these flows are then hashed onto available links. Each flow will persist on its assigned link interface for its entire life-cycle. Traffic that does not have an assigned port (such as ICMP pings) will be randomly distributed across links. The hash function is simply: `src_port ^ dst_port ^ proto`.
#### Balance Aware (`balance-aware`, similar to Linux kernel's [`balance-*lb`](https://www.kernel.org/doc/Documentation/networking/bonding.txt) modes) #### Balance aware (`balance-aware`, similar to Linux kernel's [`balance-*lb`](https://www.kernel.org/doc/Documentation/networking/bonding.txt) modes)
Traffic is dynamically allocated and balanced across multiple slaves simultaneously according to the target allocation. Options allow for *packet* or *flow-based* processing, and active-flow reassignment. Flows mediated over a recently failed slaves will be reassigned in a manner that respects the target allocation of the bond. An optional `balancePolicy` can be specified with the following effects: `flow-dynamic` (default) will hash flows onto slaves according to target allocation and may perform periodic re-assignments in order to preserve balance. `flow-static`, will hash flows onto slaves according to target allocation but will not re-assign flows unless a failure occurs or the slave is no longer operating within acceptable parameters. And lastly `packet` which simply load balances packets across slaves according to target allocation but with no concern for sequence reordering. Traffic is dynamically allocated and balanced across multiple links simultaneously according to the target allocation. Options allow for *packet* or *flow-based* processing, and active-flow reassignment. Flows mediated over a recently failed links will be reassigned in a manner that respects the target allocation of the bond. An optional `balancePolicy` can be specified with the following effects: `flow-dynamic` (default) will hash flows onto links according to target allocation and may perform periodic re-assignments in order to preserve balance. `flow-static`, will hash flows onto links according to target allocation but will not re-assign flows unless a failure occurs or the link is no longer operating within acceptable parameters. And lastly `packet` which simply load balances packets across links according to target allocation but with no concern for sequence reordering.
``` ```
{ {
@ -187,21 +187,21 @@ Traffic is dynamically allocated and balanced across multiple slaves simultaneou
} }
``` ```
#### Link Quality #### Link quality
ZeroTier measures various properties of a link (such as latency, throughput, jitter, packet loss ratio, etc) in order to arrive at a quality estimate. This estimate is used by bonding policies to make allocation and failover decisions: ZeroTier measures various properties of a link (such as latency, throughput, jitter, packet loss ratio, etc) in order to arrive at a quality estimate. This estimate is used by bonding policies to make allocation and failover decisions:
| Policy name | Role | | Policy name | Role |
|:---------------|:-----| |:---------------|:-----|
|`active-backup` | Determines the order of the failover queue. And if `activeReselect=optimize` whether a new active slave is selected. | |`active-backup` | Determines the order of the failover queue. And if `activeReselect=optimize` whether a new active link is selected. |
|`broadcast` | Does not use quality measurements. | |`broadcast` | Does not use quality measurements. |
|`balance-rr` | May trigger removal of slave from bond. | |`balance-rr` | May trigger removal of link from bond. |
|`balance-xor` | May trigger removal of slave from bond. | |`balance-xor` | May trigger removal of link from bond. |
|`balance-aware` | Informs flow assignments and (re-)assignments. May trigger removal of slave from bond. | |`balance-aware` | Informs flow assignments and (re-)assignments. May trigger removal of link from bond. |
A slave's eligibility for being included in a bond is dependent on more than perceived quality. If a path on a slave begins to exhibit disruptive behavior such as extremely high packet loss, corruption, or periodic inability to process traffic it will be removed from the bond, its traffic will be appropriately reallocated and it will be punished. Punishments gradually fade and a slave can be readmitted to the bond over time. However, punishments increase exponentially if applied more than once within a given window of time. A link's eligibility for being included in a bond is dependent on more than perceived quality. If a path on a link begins to exhibit disruptive behavior such as extremely high packet loss, corruption, or periodic inability to process traffic it will be removed from the bond, its traffic will be appropriately reallocated and it will be punished. Punishments gradually fade and a link can be readmitted to the bond over time. However, punishments increase exponentially if applied more than once within a given window of time.
#### Asymmetric Links #### Asymmetric links
In cases where it is necessary to bond physical links that vary radically in terms of cost, throughput, latency, and or reliability, there are a couple of ways to automatically (or manually) allocate traffic among them. Traffic distribution and balancing can be either `packet` or `flow` based. Where packet-based is suitable for protocols not susceptible to reordering penalties and flow-based is suitable for protocols such as TCP where it is desirable to keep a conversation on a single link unless we can't avoid having to re-assign it. Additionally, a *target allocation* of traffic used by the bonding policy can be derived/specified in the following ways: In cases where it is necessary to bond physical links that vary radically in terms of cost, throughput, latency, and or reliability, there are a couple of ways to automatically (or manually) allocate traffic among them. Traffic distribution and balancing can be either `packet` or `flow` based. Where packet-based is suitable for protocols not susceptible to reordering penalties and flow-based is suitable for protocols such as TCP where it is desirable to keep a conversation on a single link unless we can't avoid having to re-assign it. Additionally, a *target allocation* of traffic used by the bonding policy can be derived/specified in the following ways:
@ -215,9 +215,6 @@ In cases where it is necessary to bond physical links that vary radically in ter
"pdv": 0.3, /* Packet delay variance in milliseconds. Similar to jitter */ "pdv": 0.3, /* Packet delay variance in milliseconds. Similar to jitter */
"plr": 0.1, /* Packet loss ratio */ "plr": 0.1, /* Packet loss ratio */
"per": 0.1, /* Packet error ratio */ "per": 0.1, /* Packet error ratio */
"thr": 0.0, /* Mean throughput */
"thm": 0.0, /* Maximum observed throughput */
"thv": 0.0, /* Variance of throughput */
"avl": 0.0, /* Availability */ "avl": 0.0, /* Availability */
} }
} }
@ -225,24 +222,24 @@ In cases where it is necessary to bond physical links that vary radically in ter
In the absence of user guidance ZeroTier will attempt to form an understanding of each link's speed and capacity but this value can be inaccurate if the links are not routinely saturated. Therefore we provide a way to explicitly signal the capacity of each link in terms of arbitrary but relative values: In the absence of user guidance ZeroTier will attempt to form an understanding of each link's speed and capacity but this value can be inaccurate if the links are not routinely saturated. Therefore we provide a way to explicitly signal the capacity of each link in terms of arbitrary but relative values:
``` ```
"slaves": { "links": {
"eth0": { "speed": 10000 }, "eth0": { "speed": 10000 },
"eth1": { "speed": 1000 }, "eth1": { "speed": 1000 },
"eth2": { "speed": 100 } "eth2": { "speed": 100 }
} }
``` ```
The user specifies allocation percentages (totaling `1.0`). In this case quality measurements will only be used to determine a slave's eligibility to be a member of a bond, now how much traffic it will carry: The user specifies allocation percentages (totaling `1.0`). In this case quality measurements will only be used to determine a link's eligibility to be a member of a bond, now how much traffic it will carry:
``` ```
"slaves": { "links": {
"eth0": { "alloc": 0.50 }, "eth0": { "alloc": 0.50 },
"eth1": { "alloc": 0.25 }, "eth1": { "alloc": 0.25 },
"eth2": { "alloc": 0.25 } "eth2": { "alloc": 0.25 }
} }
``` ```
#### Performance and Overhead Considerations #### Performance and overhead considerations
- Only packets with internal IDs divisible by `16` are included in measurements, this amounts to about `6.25%` of all traffic. - Only packets with internal IDs divisible by `16` are included in measurements, this amounts to about `6.25%` of all traffic.
- `failoverInterval` specifies how quickly failover should occur during a link failure. In order to accomplish this a combination of active and passive measurement techniques are employed which may result in `VERB_HELLO` probes being sent every `failoverInterval / 4` time units. As a mitigation `monitorStrategy` may be set to `dynamic` so that probe frequency directly correlates with native application traffic. - `failoverInterval` specifies how quickly failover should occur during a link failure. In order to accomplish this a combination of active and passive measurement techniques are employed which may result in `VERB_HELLO` probes being sent every `failoverInterval / 4` time units. As a mitigation `monitorStrategy` may be set to `dynamic` so that probe frequency directly correlates with native application traffic.

83
service/OneService.cpp
View File

@ -50,7 +50,7 @@
#include "../osdep/Binder.hpp" #include "../osdep/Binder.hpp"
#include "../osdep/ManagedRoute.hpp" #include "../osdep/ManagedRoute.hpp"
#include "../osdep/BlockingQueue.hpp" #include "../osdep/BlockingQueue.hpp"
#include "../osdep/Slave.hpp" #include "../osdep/Link.hpp"
#include "OneService.hpp" #include "OneService.hpp"
#include "SoftwareUpdater.hpp" #include "SoftwareUpdater.hpp"
@ -307,7 +307,7 @@ static void _peerBondToJson(nlohmann::json &pj,const ZT_Peer *peer)
//j["ifname"] = peer->paths[i].ifname; //j["ifname"] = peer->paths[i].ifname;
pa.push_back(j); pa.push_back(j);
} }
pj["slaves"] = pa; pj["links"] = pa;
} }
static void _moonToJson(nlohmann::json &mj,const World &world) static void _moonToJson(nlohmann::json &mj,const World &world)
@ -1623,58 +1623,61 @@ public:
newTemplateBond->setDownDelay(OSUtils::jsonInt(customPolicy["downDelay"],-1)); newTemplateBond->setDownDelay(OSUtils::jsonInt(customPolicy["downDelay"],-1));
newTemplateBond->setFlowRebalanceStrategy(OSUtils::jsonInt(customPolicy["flowRebalanceStrategy"],(uint64_t)0)); newTemplateBond->setFlowRebalanceStrategy(OSUtils::jsonInt(customPolicy["flowRebalanceStrategy"],(uint64_t)0));
newTemplateBond->setFailoverInterval(OSUtils::jsonInt(customPolicy["failoverInterval"],(uint64_t)0)); newTemplateBond->setFailoverInterval(OSUtils::jsonInt(customPolicy["failoverInterval"],(uint64_t)0));
newTemplateBond->setPacketsPerSlave(OSUtils::jsonInt(customPolicy["packetsPerSlave"],-1)); newTemplateBond->setPacketsPerLink(OSUtils::jsonInt(customPolicy["packetsPerLink"],-1));
std::string slaveMonitorStrategyStr(OSUtils::jsonString(customPolicy["slaveMonitorStrategy"],""));
uint8_t slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DEFAULT;
if (slaveMonitorStrategyStr == "passive") { newTemplateBond->setSlaveMonitorStrategy(ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE); }
if (slaveMonitorStrategyStr == "active") { newTemplateBond->setSlaveMonitorStrategy(ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_ACTIVE); }
if (slaveMonitorStrategyStr == "dynamic") { newTemplateBond->setSlaveMonitorStrategy(ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC); }
// Policy-Specific slave set
json &slaves = customPolicy["slaves"];
for (json::iterator slaveItr = slaves.begin(); slaveItr != slaves.end();++slaveItr) {
fprintf(stderr, "\t--- slave (%s)\n", slaveItr.key().c_str());
std::string slaveNameStr(slaveItr.key());
json &slave = slaveItr.value();
bool enabled = OSUtils::jsonInt(slave["enabled"],true); std::string linkMonitorStrategyStr(OSUtils::jsonString(customPolicy["linkMonitorStrategy"],""));
uint32_t speed = OSUtils::jsonInt(slave["speed"],0); uint8_t linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DEFAULT;
float alloc = (float)OSUtils::jsonDouble(slave["alloc"],0); if (linkMonitorStrategyStr == "passive") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE; }
if (linkMonitorStrategyStr == "active") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_ACTIVE; }
if (linkMonitorStrategyStr == "dynamic") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; }
newTemplateBond->setLinkMonitorStrategy(linkMonitorStrategy);
// Policy-Specific link set
json &links = customPolicy["links"];
for (json::iterator linkItr = links.begin(); linkItr != links.end();++linkItr) {
fprintf(stderr, "\t--- link (%s)\n", linkItr.key().c_str());
std::string linkNameStr(linkItr.key());
json &link = linkItr.value();
bool enabled = OSUtils::jsonInt(link["enabled"],true);
uint32_t speed = OSUtils::jsonInt(link["speed"],0);
float alloc = (float)OSUtils::jsonDouble(link["alloc"],0);
if (speed && alloc) { if (speed && alloc) {
fprintf(stderr, "error: cannot specify both speed (%d) and alloc (%f) for slave (%s), pick one, slave disabled.\n", fprintf(stderr, "error: cannot specify both speed (%d) and alloc (%f) for link (%s), pick one, link disabled.\n",
speed, alloc, slaveNameStr.c_str()); speed, alloc, linkNameStr.c_str());
enabled = false; enabled = false;
} }
uint32_t upDelay = OSUtils::jsonInt(slave["upDelay"],-1); uint32_t upDelay = OSUtils::jsonInt(link["upDelay"],-1);
uint32_t downDelay = OSUtils::jsonInt(slave["downDelay"],-1); uint32_t downDelay = OSUtils::jsonInt(link["downDelay"],-1);
uint8_t ipvPref = OSUtils::jsonInt(slave["ipvPref"],0); uint8_t ipvPref = OSUtils::jsonInt(link["ipvPref"],0);
uint32_t slaveMonitorInterval = OSUtils::jsonInt(slave["monitorInterval"],(uint64_t)0); uint32_t linkMonitorInterval = OSUtils::jsonInt(link["monitorInterval"],(uint64_t)0);
std::string failoverToStr(OSUtils::jsonString(slave["failoverTo"],"")); std::string failoverToStr(OSUtils::jsonString(link["failoverTo"],""));
// Mode // Mode
std::string slaveModeStr(OSUtils::jsonString(slave["mode"],"spare")); std::string linkModeStr(OSUtils::jsonString(link["mode"],"spare"));
uint8_t slaveMode = ZT_MULTIPATH_SLAVE_MODE_SPARE; uint8_t linkMode = ZT_MULTIPATH_SLAVE_MODE_SPARE;
if (slaveModeStr == "primary") { slaveMode = ZT_MULTIPATH_SLAVE_MODE_PRIMARY; } if (linkModeStr == "primary") { linkMode = ZT_MULTIPATH_SLAVE_MODE_PRIMARY; }
if (slaveModeStr == "spare") { slaveMode = ZT_MULTIPATH_SLAVE_MODE_SPARE; } if (linkModeStr == "spare") { linkMode = ZT_MULTIPATH_SLAVE_MODE_SPARE; }
// ipvPref // ipvPref
if ((ipvPref != 0) && (ipvPref != 4) && (ipvPref != 6) && (ipvPref != 46) && (ipvPref != 64)) { if ((ipvPref != 0) && (ipvPref != 4) && (ipvPref != 6) && (ipvPref != 46) && (ipvPref != 64)) {
fprintf(stderr, "error: invalid ipvPref value (%d), slave disabled.\n", ipvPref); fprintf(stderr, "error: invalid ipvPref value (%d), link disabled.\n", ipvPref);
enabled = false; enabled = false;
} }
if (slaveMode == ZT_MULTIPATH_SLAVE_MODE_SPARE && failoverToStr.length()) { if (linkMode == ZT_MULTIPATH_SLAVE_MODE_SPARE && failoverToStr.length()) {
fprintf(stderr, "error: cannot specify failover slaves for spares, slave disabled.\n"); fprintf(stderr, "error: cannot specify failover links for spares, link disabled.\n");
failoverToStr = ""; failoverToStr = "";
enabled = false; enabled = false;
} }
_node->bondController()->addCustomSlave(customPolicyStr, new Slave(slaveNameStr,ipvPref,speed,slaveMonitorInterval,upDelay,downDelay,enabled,slaveMode,failoverToStr,alloc)); _node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,speed,linkMonitorInterval,upDelay,downDelay,enabled,linkMode,failoverToStr,alloc));
} }
// TODO: This is dumb // TODO: This is dumb
std::string slaveSelectMethodStr(OSUtils::jsonString(customPolicy["activeReselect"],"optimize")); std::string linkSelectMethodStr(OSUtils::jsonString(customPolicy["activeReselect"],"optimize"));
if (slaveSelectMethodStr == "always") { newTemplateBond->setSlaveSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS); } if (linkSelectMethodStr == "always") { newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS); }
if (slaveSelectMethodStr == "better") { newTemplateBond->setSlaveSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_BETTER); } if (linkSelectMethodStr == "better") { newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_BETTER); }
if (slaveSelectMethodStr == "failure") { newTemplateBond->setSlaveSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_FAILURE); } if (linkSelectMethodStr == "failure") { newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_FAILURE); }
if (slaveSelectMethodStr == "optimize") { newTemplateBond->setSlaveSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE); } if (linkSelectMethodStr == "optimize") { newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE); }
if (newTemplateBond->getSlaveSelectMethod() < 0 || newTemplateBond->getSlaveSelectMethod() > 3) { if (newTemplateBond->getLinkSelectMethod() < 0 || newTemplateBond->getLinkSelectMethod() > 3) {
fprintf(stderr, "warning: invalid value (%s) for slaveSelectMethod, assuming mode: always\n", slaveSelectMethodStr.c_str()); fprintf(stderr, "warning: invalid value (%s) for linkSelectMethod, assuming mode: always\n", linkSelectMethodStr.c_str());
} }
/* /*
newBond->setPolicy(_node->bondController()->getPolicyCodeByStr(basePolicyStr)); newBond->setPolicy(_node->bondController()->getPolicyCodeByStr(basePolicyStr));
@ -1693,7 +1696,7 @@ public:
} }
// Check settings // Check settings
if (defaultBondingPolicyStr.length() && !defaultBondingPolicy && !_node->bondController()->inUse()) { if (defaultBondingPolicyStr.length() && !defaultBondingPolicy && !_node->bondController()->inUse()) {
fprintf(stderr, "error: unknown policy (%s) specified by defaultBondingPolicy, slave disabled.\n", defaultBondingPolicyStr.c_str()); fprintf(stderr, "error: unknown policy (%s) specified by defaultBondingPolicy, link disabled.\n", defaultBondingPolicyStr.c_str());
} }
} }