Formatting

node/Bond.cpp

@@ -140,7 +140,7 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
 					}
 				}
 			}
-			//fprintf(stderr, "resultant _rrIdx=%d\n", _rrIdx);
+			fprintf(stderr, "_rrIdx=%d\n", _rrIdx);
 			if (_paths[_bondedIdx[_rrIdx]]) {
 				return _paths[_bondedIdx[_rrIdx]];
 			}
@@ -246,7 +246,7 @@ void Bond::recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId,
 	}
 	/**
 	 * Learn new flows and pro-actively create entries for them in the bond so
-	 * that the next time we send a packet out that is part of a flow we know 
+	 * that the next time we send a packet out that is part of a flow we know
 	 * which path to use.
 	 */
 	if ((flowId != ZT_QOS_NO_FLOW)
@@ -385,7 +385,7 @@ SharedPtr<Flow> Bond::createFlow(const SharedPtr<Path> &path, int32_t flowId, un
 	}
 	if (_flows.size() >= ZT_FLOW_MAX_COUNT) {
 		fprintf(stderr, "max number of flows reached (%d), forcibly forgetting oldest flow\n", ZT_FLOW_MAX_COUNT);
-        forgetFlowsWhenNecessary(0,true,now);
+		forgetFlowsWhenNecessary(0,true,now);
 	}
 	SharedPtr<Flow> flow = new Flow(flowId, now);
 	_flows[flowId] = flow;
@@ -588,7 +588,7 @@ void Bond::sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,const int6
 	} else {
 		RR->sw->send(tPtr,outp,false);
 	}
-    // Account for the fact that a VERB_QOS_MEASUREMENT was just sent. Reset timers.
+	// Account for the fact that a VERB_QOS_MEASUREMENT was just sent. Reset timers.
 	path->_packetsReceivedSinceLastQoS = 0;
 	path->_lastQoSMeasurement = now;
 }
@@ -608,7 +608,7 @@ void Bond::processBackgroundTasks(void *tPtr, const int64_t now)
 		//fprintf(stderr, "_lastFrame=%llu, suggestedMonitorInterval=%d, _dynamicPathMonitorInterval=%d\n",
 		//	(now-_lastFrame), suggestedMonitorInterval, _dynamicPathMonitorInterval);
 	}
-		
+
 	if (_slaveMonitorStrategy == ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC) {
 		_shouldCollectPathStatistics = true;
 	}
@@ -673,7 +673,7 @@ void Bond::processBackgroundTasks(void *tPtr, const int64_t now)
 	if (((now - _lastPathNegotiationCheck) > ZT_PATH_NEGOTIATION_CHECK_INTERVAL) && _allowPathNegotiation) {
 		_lastPathNegotiationCheck = now;
 		pathNegotiationCheck(tPtr, now);
-	}	
+	}
 }
 
 void Bond::applyUserPrefs()
@@ -854,8 +854,8 @@ void Bond::estimatePathQuality(const int64_t now)
 	float plr[ZT_MAX_PEER_NETWORK_PATHS];
 	float per[ZT_MAX_PEER_NETWORK_PATHS];
 	float thr[ZT_MAX_PEER_NETWORK_PATHS];
-    float thm[ZT_MAX_PEER_NETWORK_PATHS];
-    float thv[ZT_MAX_PEER_NETWORK_PATHS];
+	float thm[ZT_MAX_PEER_NETWORK_PATHS];
+	float thv[ZT_MAX_PEER_NETWORK_PATHS];
 
 	float maxLAT = 0;
 	float maxPDV = 0;
@@ -867,7 +867,7 @@ void Bond::estimatePathQuality(const int64_t now)
 
 	float quality[ZT_MAX_PEER_NETWORK_PATHS];
 	uint8_t alloc[ZT_MAX_PEER_NETWORK_PATHS];
-	
+
 	float totQuality = 0.0f;
 
 	memset(&lat, 0, sizeof(lat));
@@ -950,7 +950,7 @@ void Bond::estimatePathQuality(const int64_t now)
 
 		//fprintf(stdout, "EH   %d: lat=%8.3f,  ltm=%8.3f,  pdv=%8.3f,  plr=%5.3f,  per=%5.3f,  thr=%8f,  thm=%5.3f,  thv=%5.3f,  avl=%5.3f,  age=%8.2f,  scp=%4d,  q=%5.3f,  qtot=%5.3f,  ac=%d if=%s, path=%s\n",
 		//	              i,   lat[i],     ltm[i],     pdv[i],     plr[i],     per[i],     thr[i],     thm[i],     thv[i],     avl[i],     age[i],     scp[i], quality[i], totQuality, alloc[i], getSlave(_paths[i])->ifname().c_str(), pathStr);
-		
+
 	}
 	// Convert metrics to relative quantities and apply contribution weights
 	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
@@ -966,7 +966,7 @@ void Bond::estimatePathQuality(const int64_t now)
 			totQuality += quality[i];
 		}
 	}
-	// 
+	//
 	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 		if (_paths[i] && _paths[i]->bonded()) {
 			alloc[i] = std::ceil((quality[i] / totQuality) * (float)255);
@@ -1011,8 +1011,8 @@ void Bond::estimatePathQuality(const int64_t now)
 			if (_paths[i]) {
 				_paths[i]->address().toString(pathStr);
 				fprintf(stdout, "%s, %s, %8.3f, %8.3f, %8.3f, %5.3f, %5.3f, %5.3f, %8f, %5.3f, %5.3f, %d, %5.3f, %d, %d, %d, %d, %d, %d, ",
-					              getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->latencyMean, lat[i],pdv[i], _paths[i]->packetLossRatio, plr[i],per[i],thr[i],thm[i],thv[i],(now - _paths[i]->lastIn()),quality[i],alloc[i],
-					              _paths[i]->relativeByteLoad, _paths[i]->assignedFlowCount, _paths[i]->alive(now, true), _paths[i]->eligible(now,_ackSendInterval), _paths[i]->qosStatsOut.size());
+								  getSlave(_paths[i])->ifname().c_str(), pathStr, _paths[i]->latencyMean, lat[i],pdv[i], _paths[i]->packetLossRatio, plr[i],per[i],thr[i],thm[i],thv[i],(now - _paths[i]->lastIn()),quality[i],alloc[i],
+								  _paths[i]->relativeByteLoad, _paths[i]->assignedFlowCount, _paths[i]->alive(now, true), _paths[i]->eligible(now,_ackSendInterval), _paths[i]->qosStatsOut.size());
 			}
 		}
 		fprintf(stdout, "\n");
@@ -1022,7 +1022,144 @@ void Bond::estimatePathQuality(const int64_t now)
 
 void Bond::processBalanceTasks(const int64_t now)
 {
-	// Omitted
+	//fprintf(stderr, "processBalanceTasks\n");
+	char curPathStr[128];
+	if (_allowFlowHashing) {
+		/**
+		 * Clean up and reset flows if necessary
+		 */
+		if ((now - _lastFlowExpirationCheck) > ZT_MULTIPATH_FLOW_CHECK_INTERVAL) {
+			Mutex::Lock _l(_flows_m);
+			forgetFlowsWhenNecessary(ZT_MULTIPATH_FLOW_EXPIRATION_INTERVAL,false,now);
+			_lastFlowExpirationCheck = now;
+		}
+		if ((now - _lastFlowStatReset) > ZT_FLOW_STATS_RESET_INTERVAL) {
+			Mutex::Lock _l(_flows_m);
+			_lastFlowStatReset = now;
+			std::map<int32_t,SharedPtr<Flow> >::iterator it = _flows.begin();
+			while (it != _flows.end()) {
+				it->second->resetByteCounts();
+				++it;
+			}
+		}
+		/**
+		 * Re-allocate flows from dead paths
+		 */
+		if (_bondingPolicy== ZT_BONDING_POLICY_BALANCE_XOR || _bondingPolicy== ZT_BONDING_POLICY_BALANCE_AWARE) {
+			Mutex::Lock _l(_flows_m);
+			for (int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+				if (!_paths[i]) {
+					continue;
+				}
+				if (!_paths[i]->eligible(now,_ackSendInterval) && _paths[i]->_shouldReallocateFlows) {
+					_paths[i]->address().toString(curPathStr);
+					fprintf(stderr, "%d reallocating flows from dead path %s on %s\n", (RR->node->now() - RR->bc->getBondStartTime()), curPathStr, getSlave(_paths[i])->ifname().c_str());
+					std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin();
+					while (flow_it != _flows.end()) {
+						if (flow_it->second->assignedPath() == _paths[i]) {
+							if(assignFlowToBondedPath(flow_it->second, now)) {
+								_paths[i]->_assignedFlowCount--;
+							}
+						}
+						++flow_it;
+					}
+					_paths[i]->_shouldReallocateFlows = false;
+				}
+			}
+		}
+	}
+	/**
+	 * Tasks specific to (Balance Round Robin)
+	 */
+	if (_bondingPolicy== ZT_BONDING_POLICY_BALANCE_RR) {
+		if (_allowFlowHashing) {
+			// TODO: Should ideally failover from (idx) to a random slave, this is so that (idx+1) isn't overloaded
+		}
+		else if (!_allowFlowHashing) {
+			// Nothing
+		}
+	}
+	/**
+	 * Tasks specific to (Balance XOR)
+	 */
+	if (_bondingPolicy== ZT_BONDING_POLICY_BALANCE_XOR) {
+		// Nothing specific for XOR
+	}
+	/**
+	 * Tasks specific to (Balance Aware)
+	 */
+	if ((_bondingPolicy== ZT_BONDING_POLICY_BALANCE_AWARE)) {
+		if (_allowFlowHashing) {
+			Mutex::Lock _l(_flows_m);
+			/**
+			 * Re-balance flows in proportion to slave capacity (or when eligibility changes)
+			 */
+			if ((now - _lastFlowRebalance) > ZT_FLOW_REBALANCE_INTERVAL) {
+				/**
+				 * Determine "load" for bonded paths
+				 */
+				uint64_t totalBytes = 0;
+				for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { // first pass: compute absolute byte load and total
+					if (_paths[i] && _paths[i]->bonded()) {
+						_paths[i]->_byteLoad = 0;
+						std::map<int32_t,SharedPtr<Flow> >::iterator flow_it = _flows.begin();
+						while (flow_it != _flows.end()) {
+							if (flow_it->second->assignedPath() == _paths[i]) {
+								_paths[i]->_byteLoad += flow_it->second->totalBytes();
+							}
+							++flow_it;
+						}
+						totalBytes += _paths[i]->_byteLoad;
+					}
+				}
+				/**
+				 * Determine "affinity" for bonded path
+				 */
+				//fprintf(stderr, "\n\n");
+				_totalBondUnderload = 0;
+
+				for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) { // second pass: compute relative byte loads and total imbalance
+					if (_paths[i] && _paths[i]->bonded()) {
+						if (totalBytes) {
+							uint8_t relativeByteLoad = std::ceil(((float)_paths[i]->_byteLoad / (float)totalBytes) * (float)255);
+							//fprintf(stderr, "lastComputedAllocation  = %d\n", _paths[i]->allocation);
+							//fprintf(stderr, "  relativeByteLoad      = %d\n", relativeByteLoad);
+							_paths[i]->_relativeByteLoad = relativeByteLoad;
+							uint8_t relativeUnderload = std::max(0, (int)_paths[i]->_allocation - (int)relativeByteLoad);
+							//fprintf(stderr, "    relativeUnderload  = %d\n", relativeUnderload);
+							_totalBondUnderload += relativeUnderload;
+							//fprintf(stderr, "    _totalBondUnderload = %d\n\n", _totalBondUnderload);
+							//_paths[i]->affinity = (relativeUnderload > 0 ? relativeUnderload : _paths[i]->_allocation);
+						}
+						else { // set everything to base values
+							_totalBondUnderload = 0;
+							//_paths[i]->affinity = 0;
+						}
+					}
+				}
+
+				//fprintf(stderr, "_totalBondUnderload=%d (end)\n\n", _totalBondUnderload);
+
+				/**
+				 *
+				 */
+				//fprintf(stderr, "_lastFlowRebalance\n");
+				std::map<int32_t, SharedPtr<Flow> >::iterator it = _flows.begin();
+				while (it != _flows.end()) {
+					int32_t flowId = it->first;
+					SharedPtr<Flow> flow = it->second;
+					if ((now - flow->_lastPathReassignment) > ZT_FLOW_MIN_REBALANCE_INTERVAL) {
+						//fprintf(stdout, "  could move : %x\n", flowId);
+					}
+					++it;
+				}
+				_lastFlowRebalance = now;
+			}
+		}
+		else if (!_allowFlowHashing) {
+			// Nothing
+		}
+	}
 }
 
 void Bond::dequeueNextActiveBackupPath(const uint64_t now)
@@ -1042,7 +1179,7 @@ void Bond::dequeueNextActiveBackupPath(const uint64_t now)
 }
 
 void Bond::processActiveBackupTasks(const int64_t now)
-{	
+{
 	//fprintf(stderr, "%llu processActiveBackupTasks\n", (now - RR->bc->getBondStartTime()));
 	char pathStr[128]; char prevPathStr[128]; char curPathStr[128];
 
@@ -1058,7 +1195,7 @@ void Bond::processActiveBackupTasks(const int64_t now)
 		/**
 		 * [Automatic mode]
 		 * The user has not explicitly specified slaves or their failover schedule,
-		 * the bonding policy will now select the first eligible path and set it as 
+		 * the bonding policy will now select the first eligible path and set it as
 		 * its active backup path, if a substantially better path is detected the bonding
 		 * policy will assign it as the new active backup path. If the path fails it will
 		 * simply find the next eligible path.
@@ -1187,9 +1324,9 @@ void Bond::processActiveBackupTasks(const int64_t now)
 				}
 				SharedPtr<Slave> slave =RR->bc->getSlaveBySocket(_policyAlias, _paths[i]->localSocket());
 				_paths[i]->address().toString(pathStr);
-				
+
 				int failoverScoreHandicap = _paths[i]->_failoverScore;
-				if (_paths[i]->preferred()) 
+				if (_paths[i]->preferred())
 				{
 					failoverScoreHandicap += ZT_MULTIPATH_FAILOVER_HANDICAP_PREFERRED;
 					//fprintf(stderr, "%s on %s ----> %d for preferred\n", pathStr, _paths[i]->ifname().c_str(), failoverScoreHandicap);
@@ -1264,7 +1401,7 @@ void Bond::processActiveBackupTasks(const int64_t now)
 				if (_paths[i].ptr() == negotiatedPath.ptr()) {
 					_paths[i]->_negotiated = true;
 					failoverScoreHandicap = ZT_MULTIPATH_FAILOVER_HANDICAP_NEGOTIATED;
-				} else { 
+				} else {
 					_paths[i]->_negotiated = false;
 				}
 				_paths[i]->_failoverScore = _paths[i]->_allocation + failoverScoreHandicap;
@@ -1386,7 +1523,7 @@ void Bond::setReasonableDefaults(int policy)
 	_lastPathNegotiationReceived=0;
 	_lastBackgroundTaskCheck=0;
 	_lastPathNegotiationCheck=0;
-	
+
 	_lastFlowStatReset=0;
 	_lastFlowExpirationCheck=0;
 	_localUtility=0;
@@ -1397,7 +1534,7 @@ void Bond::setReasonableDefaults(int policy)
 	_pathNegotiationCutoffCount=0;
 	_lastFlowRebalance=0;
 	_totalBondUnderload = 0;
-	
+
 	//_maxAcceptableLatency
 	_maxAcceptablePacketDelayVariance = 50;
 	_maxAcceptablePacketLossRatio = 0.10;
@@ -1445,7 +1582,7 @@ void Bond::setReasonableDefaults(int policy)
 		case ZT_BONDING_POLICY_BALANCE_RR:
 			_failoverInterval = 5000;
 			_allowFlowHashing = false;
-			_packetsPerSlave = 8;
+			_packetsPerSlave = 512;
 			_slaveMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC;
 			_qualityWeights[ZT_QOS_LAT_IDX] = 0.4f;
 			_qualityWeights[ZT_QOS_LTM_IDX] = 0.0f;
@@ -1550,8 +1687,8 @@ void Bond::setUserQualityWeights(float weights[], int len)
 
 
 bool Bond::relevant() {
-	return _peer->identity().address().toInt() == 0x16a03a3d03 
-		|| _peer->identity().address().toInt() == 0x4410300d03 
+	return _peer->identity().address().toInt() == 0x16a03a3d03
+		|| _peer->identity().address().toInt() == 0x4410300d03
 		|| _peer->identity().address().toInt() == 0x795cbf86fa;
 }
 
@@ -1566,7 +1703,7 @@ void Bond::dumpInfo(const int64_t now)
 	//char oldPathStr[128];
 	char currPathStr[128];
 
-	if (!relevant()) {	
+	if (!relevant()) {
 		return;
 	}
 	/*
@@ -1589,7 +1726,7 @@ void Bond::dumpInfo(const int64_t now)
 	}
 	_lastPrintTS = now;
 	_lastLogTS = now;
-	
+
 	fprintf(stderr, "\n\n");
 
 	for(int i=0; i<ZT_MAX_PEER_NETWORK_PATHS; ++i) {

node/Bond.hpp

@@ -45,48 +45,48 @@ class Bond
 
 public:
 
-    // TODO: Remove
-    bool _header;
-    int64_t _lastLogTS;
-    int64_t _lastPrintTS;
-    void dumpInfo(const int64_t now);
-    bool relevant();
+	// TODO: Remove
+	bool _header;
+	int64_t _lastLogTS;
+	int64_t _lastPrintTS;
+	void dumpInfo(const int64_t now);
+	bool relevant();
 
-    SharedPtr<Slave> getSlave(const SharedPtr<Path>& path);
+	SharedPtr<Slave> getSlave(const SharedPtr<Path>& path);
 
-    /**
-     * Constructor. For use only in first initialization in Node
-     *
-     * @param renv Runtime environment
-     */
-    Bond(const RuntimeEnvironment *renv);
+	/**
+	 * Constructor. For use only in first initialization in Node
+	 *
+	 * @param renv Runtime environment
+	 */
+	Bond(const RuntimeEnvironment *renv);
 
-    /**
-     * Constructor. Creates a bond based off of ZT defaults
-     *
-     * @param renv Runtime environment
-     * @param policy Bonding policy
-     * @param peer
-     */
-    Bond(const RuntimeEnvironment *renv, int policy, const SharedPtr<Peer>& peer);
+	/**
+	 * Constructor. Creates a bond based off of ZT defaults
+	 *
+	 * @param renv Runtime environment
+	 * @param policy Bonding policy
+	 * @param peer
+	 */
+	Bond(const RuntimeEnvironment *renv, int policy, const SharedPtr<Peer>& peer);
 
-    /**
-     * Constructor. For use when user intends to manually specify parameters
-     *
-     * @param basePolicy
-     * @param policyAlias
-     * @param peer
-     */
-    Bond(std::string& basePolicy, std::string& policyAlias, const SharedPtr<Peer>& peer);
+	/**
+	 * Constructor. For use when user intends to manually specify parameters
+	 *
+	 * @param basePolicy
+	 * @param policyAlias
+	 * @param peer
+	 */
+	Bond(std::string& basePolicy, std::string& policyAlias, const SharedPtr<Peer>& peer);
 
-    /**
-     * Constructor. Creates a bond based off of a user-defined bond template
-     *
-     * @param renv Runtime environment
-     * @param original
-     * @param peer
-     */
-    Bond(const RuntimeEnvironment *renv, const Bond &original, const SharedPtr<Peer>& peer);
+	/**
+	 * Constructor. Creates a bond based off of a user-defined bond template
+	 *
+	 * @param renv Runtime environment
+	 * @param original
+	 * @param peer
+	 */
+	Bond(const RuntimeEnvironment *renv, const Bond &original, const SharedPtr<Peer>& peer);
 
 	/**
 	 *
@@ -101,7 +101,7 @@ public:
 	 * @param now Current time
 	 */
 	void nominatePath(const SharedPtr<Path>& path, int64_t now);
-	
+
 	/**
 	 * Propagate and memoize often-used bonding preferences for each path
 	 */
@@ -109,9 +109,9 @@ public:
 
 	/**
 	 * Check path states and perform bond rebuilds if needed.
-	 * 
+	 *
 	 * @param now Current time
-	 * @param rebuild Whether or not the bond should be reconstructed. 
+	 * @param rebuild Whether or not the bond should be reconstructed.
 	 */
 	void curateBond(const int64_t now, bool rebuild);
 
@@ -156,7 +156,7 @@ public:
 
 	/**
 	 * Process the contents of an inbound VERB_ACK to gather path quality observations.
-	 * 
+	 *
 	 * @param path Path over which packet was received
 	 * @param now Current time
 	 * @param ackedBytes Number of bytes ACKed by this VERB_ACK
@@ -174,7 +174,7 @@ public:
 
 	/**
 	 * Record statistics for an inbound packet.
-	 * 
+	 *
 	 * @param path Path over which packet was received
 	 * @param packetId Packet ID
 	 * @param payloadLength Packet data length
@@ -183,7 +183,7 @@ public:
 	 * @param now Current time
 	 */
 	void recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId, uint16_t payloadLength,
-	        Packet::Verb verb, int32_t flowId, int64_t now);
+			Packet::Verb verb, int32_t flowId, int64_t now);
 
 	/**
 	 * Determines the most appropriate path for packet and flow egress. This decision is made by
@@ -197,7 +197,7 @@ public:
 
 	/**
 	 * Creates a new flow record
-	 * 
+	 *
 	 * @param path Path over which flow shall be handled
 	 * @param flowId Flow ID
 	 * @param entropy A byte of entropy to be used by the bonding algorithm
@@ -208,7 +208,7 @@ public:
 
 	/**
 	 * Removes flow records that are past a certain age limit.
-	 * 
+	 *
 	 * @param age Age threshold to be forgotten
 	 * @param oldest Whether only the oldest shall be forgotten
 	 * @param now Current time
@@ -217,17 +217,17 @@ public:
 
 	/**
 	 * Assigns a new flow to a bonded path
-	 * 
+	 *
 	 * @param flow Flow to be assigned
 	 * @param now Current time
 	 */
 	bool assignFlowToBondedPath(SharedPtr<Flow> &flow, int64_t now);
 
-    /**
+	/**
 	 * Determine whether a path change should occur given the remote peer's reported utility and our
 	 * local peer's known utility. This has the effect of assigning inbound and outbound traffic to
-	 * the same path.  
-	 * 
+	 * the same path.
+	 *
 	 * @param now Current time
 	 * @param path Path over which the negotiation request was received
 	 * @param remoteUtility How much utility the remote peer claims to gain by using the declared path
@@ -245,7 +245,7 @@ public:
 
 	/**
 	 * Sends a VERB_ACK to the remote peer.
-	 * 
+	 *
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param path Path over which packet should be sent
 	 * @param localSocket Local source socket
@@ -253,11 +253,11 @@ public:
 	 * @param now Current time
 	 */
 	void sendACK(void *tPtr,const SharedPtr<Path> &path,int64_t localSocket,
-	        const InetAddress &atAddress,int64_t now);
+			const InetAddress &atAddress,int64_t now);
 
 	/**
 	 * Sends a VERB_QOS_MEASUREMENT to the remote peer.
-	 * 
+	 *
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param path Path over which packet should be sent
 	 * @param localSocket Local source socket
@@ -265,11 +265,11 @@ public:
 	 * @param now Current time
 	 */
 	void sendQOS_MEASUREMENT(void *tPtr,const SharedPtr<Path> &path,int64_t localSocket,
-	        const InetAddress &atAddress,int64_t now);
+			const InetAddress &atAddress,int64_t now);
 
 	/**
 	 * Sends a VERB_PATH_NEGOTIATION_REQUEST to the remote peer.
-	 * 
+	 *
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param path Path over which packet should be sent
 	 */
@@ -280,10 +280,10 @@ public:
 	 * @param now Current time
 	 */
 	void processBalanceTasks(int64_t now);
-	
+
 	/**
 	 * Perform periodic tasks unique to active-backup
-	 * 
+	 *
 	 * @param now Current time
 	 */
 	void processActiveBackupTasks(int64_t now);
@@ -296,12 +296,12 @@ public:
 	 */
 	void dequeueNextActiveBackupPath(uint64_t now);
 
-    /**
-     * Set bond parameters to reasonable defaults, these may later be overwritten by
+	/**
+	 * Set bond parameters to reasonable defaults, these may later be overwritten by
 	 * user-specified parameters.
-     *
-     * @param policy Bonding policy
-     */
+	 *
+	 * @param policy Bonding policy
+	 */
 	void setReasonableDefaults(int policy);
 
 	/**
@@ -450,19 +450,19 @@ public:
 	 */
 	inline uint16_t getUpDelay() { return _upDelay; }
 
-    /**
-     * @param upDelay Length of time before a newly-discovered path is admitted to the bond
-     */
+	/**
+	 * @param upDelay Length of time before a newly-discovered path is admitted to the bond
+	 */
 	inline void setUpDelay(int upDelay) { if (upDelay >= 0) { _upDelay = upDelay; } }
 
-    /**
-     * @return Length of time before a newly-failed path is removed from the bond
-     */
+	/**
+	 * @return Length of time before a newly-failed path is removed from the bond
+	 */
 	inline uint16_t getDownDelay() { return _downDelay; }
 
-    /**
-     * @param downDelay Length of time before a newly-failed path is removed from the bond
-     */
+	/**
+	 * @param downDelay Length of time before a newly-failed path is removed from the bond
+	 */
 	inline void setDownDelay(int downDelay) { if (downDelay >= 0) { _downDelay = downDelay; } }
 
 	/**
@@ -470,11 +470,11 @@ public:
 	 */
 	inline uint16_t getBondMonitorInterval() { return _bondMonitorInterval; }
 
-    /**
-     * Set the current monitoring interval for the bond (can be overridden with intervals specific to certain slaves.)
-     *
-     * @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer.
-     */
+	/**
+	 * Set the current monitoring interval for the bond (can be overridden with intervals specific to certain slaves.)
+	 *
+	 * @param monitorInterval How often gratuitous VERB_HELLO(s) are sent to remote peer.
+	 */
 	inline void setBondMonitorInterval(uint16_t interval) { _bondMonitorInterval = interval; }
 
 	/**
@@ -487,10 +487,10 @@ public:
 	 */
 	inline uint8_t getPolicy() { return _bondingPolicy; }
 
-    /**
-     *
-     * @param allowFlowHashing
-     */
+	/**
+	 *
+	 * @param allowFlowHashing
+	 */
 	inline void setFlowHashing(bool allowFlowHashing) { _allowFlowHashing = allowFlowHashing; }
 
 	/**
@@ -498,10 +498,10 @@ public:
 	 */
 	bool flowHashingEnabled() { return _allowFlowHashing; }
 
-    /**
-     *
-     * @param packetsPerSlave
-     */
+	/**
+	 *
+	 * @param packetsPerSlave
+	 */
 	inline void setPacketsPerSlave(int packetsPerSlave) { _packetsPerSlave = packetsPerSlave; }
 
 	/**
@@ -514,7 +514,7 @@ public:
 	 *
 	 * @return
 	 */
-    inline uint8_t getSlaveSelectMethod() { return _abSlaveSelectMethod; }
+	inline uint8_t getSlaveSelectMethod() { return _abSlaveSelectMethod; }
 
 	/**
 	 *

node/BondController.hpp

@@ -55,10 +55,10 @@ public:
 	 */
 	bool inUse() { return !_bondPolicyTemplates.empty() || _defaultBondingPolicy; }
 
-    /**
-     * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
-     * @return The bonding policy code for a given human-readable bonding policy name
-     */
+	/**
+	 * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
+	 * @return The bonding policy code for a given human-readable bonding policy name
+	 */
 	static int getPolicyCodeByStr(const std::string& basePolicyName)
 	{
 		if (basePolicyName == "active-backup") { return 1; }
@@ -83,18 +83,18 @@ public:
 		return "none";
 	}
 
-    /**
-     * Sets the default bonding policy for new or undefined bonds.
+	/**
+	 * Sets the default bonding policy for new or undefined bonds.
 	 *
-     * @param bp Bonding policy
-     */
+	 * @param bp Bonding policy
+	 */
 	void setBondingLayerDefaultPolicy(uint8_t bp) { _defaultBondingPolicy = bp; }
 
-    /**
-     * Sets the default (custom) bonding policy for new or undefined bonds.
+	/**
+	 * Sets the default (custom) bonding policy for new or undefined bonds.
 	 *
-     * @param alias Human-readable string alias for bonding policy
-     */
+	 * @param alias Human-readable string alias for bonding policy
+	 */
 	void setBondingLayerDefaultPolicyStr(std::string alias) { _defaultBondingPolicyStr = alias; }
 
 	/**
@@ -119,7 +119,7 @@ public:
 	bool addCustomPolicy(const SharedPtr<Bond>& newBond);
 
 	/**
-	 * Assigns a specific bonding policy  
+	 * Assigns a specific bonding policy
 	 *
 	 * @param identity
 	 * @param policyAlias

node/Flow.hpp

@@ -24,10 +24,10 @@ namespace ZeroTier {
  */
 struct Flow
 {
-    /**
-     * @param flowId Given flow ID
-     * @param now Current time
-     */
+	/**
+	 * @param flowId Given flow ID
+	 * @param now Current time
+	 */
 	Flow(int32_t flowId, int64_t now) :
 		_flowId(flowId),
 		_bytesInPerUnitTime(0),
@@ -50,12 +50,12 @@ struct Flow
 	 * @return The Flow's ID
 	 */
 	int32_t id() { return _flowId; }
-	
+
 	/**
 	 * @return Number of incoming bytes processed on this flow per unit time
 	 */
 	int64_t bytesInPerUnitTime() { return _bytesInPerUnitTime; }
-	
+
 	/**
 	 * Record number of incoming bytes on this flow
 	 *

node/Path.hpp

@@ -28,7 +28,6 @@
 #include "Utils.hpp"
 #include "Packet.hpp"
 #include "RingBuffer.hpp"
-//#include "Bond.hpp"
 
 #include "../osdep/Slave.hpp"
 
@@ -48,7 +47,6 @@ class Path
 {
 	friend class SharedPtr<Path>;
 	friend class Bond;
-	//friend class SharedPtr<Bond>;
 
 public:
 	/**
@@ -361,7 +359,7 @@ public:
 	 * @return the age of the path in terms of receiving packets
 	 */
 	inline int64_t age(int64_t now) { return (now - _lastIn); }
-		
+
 	/**
 	 * @return Time last trust-established packet was received
 	 */
@@ -634,7 +632,7 @@ private:
 	 * The variance in the estimated throughput of this path.
 	 */
 	float _throughputVariance;
-	
+
 	/**
 	 * The relative quality of this path to all others in the bond, [0-255].
 	 */

osdep/Slave.hpp

@@ -61,7 +61,7 @@ public:
 		_userSpecifiedAlloc(userSpecifiedAlloc),
 		_isUserSpecified(false)
 	{}
-	
+
 	/**
 	 * @return The string representation of this slave's underlying interface's system name.
 	 */
@@ -163,7 +163,7 @@ private:
 	/**
 	 * What preference (if any) a user has for IP protocol version used in
 	 * path aggregations. Preference is expressed in the order of the digits:
-	 * 
+	 *
 	 *  0: no preference
 	 *  4: IPv4 only
 	 *  6: IPv6 only
@@ -212,7 +212,7 @@ private:
 	uint8_t _mode;
 
 	/**
-	 * The specific name of the interface to be used in the event that this 
+	 * The specific name of the interface to be used in the event that this
 	 * slave fails.
 	 */
 	std::string _failoverToSlaveStr;
@@ -230,7 +230,6 @@ private:
 	bool _isUserSpecified;
 
 	AtomicCounter __refCount;
-
 };
 
 } // namespace ZeroTier