2020-05-12 08:35:48 +00:00
/*
2021-09-02 04:37:49 +00:00
* Copyright ( c ) 2013 - 2021 ZeroTier , Inc .
2020-05-12 08:35:48 +00:00
*
* Use of this software is governed by the Business Source License included
* in the LICENSE . TXT file in the project ' s root directory .
*
2021-09-02 04:37:49 +00:00
* Change Date : 2026 - 01 - 01
2020-05-12 08:35:48 +00:00
*
* On the date above , in accordance with the Business Source License , use
* of this software will be governed by version 2.0 of the Apache License .
*/
/****/
# ifndef ZT_BOND_HPP
# define ZT_BOND_HPP
2022-09-20 21:27:34 +00:00
# include "../osdep/Binder.hpp"
2021-09-02 04:37:49 +00:00
# include "../osdep/Phy.hpp"
2021-05-04 04:12:45 +00:00
# include "Packet.hpp"
2020-05-12 08:35:48 +00:00
# include "Path.hpp"
2021-09-02 04:37:49 +00:00
# include "RuntimeEnvironment.hpp"
# include "Trace.hpp"
2021-05-04 00:59:31 +00:00
2021-09-02 04:37:49 +00:00
# include <cstdarg>
# include <deque>
2021-05-04 00:59:31 +00:00
# include <list>
# include <map>
2020-05-12 08:35:48 +00:00
2021-09-02 04:37:49 +00:00
/**
* Indices for the path quality weight vector
*/
2022-10-10 06:07:16 +00:00
enum ZT_BondQualityWeightIndex { ZT_QOS_LAT_MAX_IDX , ZT_QOS_PDV_MAX_IDX , ZT_QOS_PLR_MAX_IDX , ZT_QOS_PER_MAX_IDX , ZT_QOS_LAT_WEIGHT_IDX , ZT_QOS_PDV_WEIGHT_IDX , ZT_QOS_PLR_WEIGHT_IDX , ZT_QOS_PER_WEIGHT_IDX , ZT_QOS_PARAMETER_SIZE } ;
2021-09-02 04:37:49 +00:00
/**
* Multipath bonding policy
*/
enum ZT_BondBondingPolicy {
/**
* Normal operation . No fault tolerance , no load balancing
*/
ZT_BOND_POLICY_NONE = 0 ,
/**
* Sends traffic out on only one path at a time . Configurable immediate
* fail - over .
*/
ZT_BOND_POLICY_ACTIVE_BACKUP = 1 ,
/**
* Sends traffic out on all paths
*/
ZT_BOND_POLICY_BROADCAST = 2 ,
/**
* Stripes packets across all paths
*/
ZT_BOND_POLICY_BALANCE_RR = 3 ,
/**
* Packets destined for specific peers will always be sent over the same
* path .
*/
ZT_BOND_POLICY_BALANCE_XOR = 4 ,
/**
* Balances flows among all paths according to path performance
*/
ZT_BOND_POLICY_BALANCE_AWARE = 5
} ;
/**
* Multipath active re - selection policy ( linkSelectMethod )
*/
enum ZT_BondLinkSelectMethod {
/**
* Primary link regains status as active link whenever it comes back up
* ( default when links are explicitly specified )
*/
ZT_BOND_RESELECTION_POLICY_ALWAYS = 0 ,
/**
* Primary link regains status as active link when it comes back up and
* ( if ) it is better than the currently - active link .
*/
ZT_BOND_RESELECTION_POLICY_BETTER = 1 ,
/**
* Primary link regains status as active link only if the currently - active
* link fails .
*/
ZT_BOND_RESELECTION_POLICY_FAILURE = 2 ,
/**
* The primary link can change if a superior path is detected .
* ( default if user provides no fail - over guidance )
*/
ZT_BOND_RESELECTION_POLICY_OPTIMIZE = 3
} ;
/**
* Mode of multipath link interface
*/
enum ZT_BondLinkMode { ZT_BOND_SLAVE_MODE_PRIMARY = 0 , ZT_BOND_SLAVE_MODE_SPARE = 1 } ;
# include "../node/AtomicCounter.hpp"
# include "../node/SharedPtr.hpp"
# include <string>
2020-05-12 08:35:48 +00:00
namespace ZeroTier {
2021-09-02 04:37:49 +00:00
class Link {
friend class SharedPtr < Link > ;
public :
/**
*
* @ param ifnameStr
* @ param ipvPref
2022-10-10 06:07:16 +00:00
* @ param capacity
2021-09-02 04:37:49 +00:00
* @ param enabled
* @ param mode
* @ param failoverToLinkStr
*/
2023-01-12 04:12:15 +00:00
Link ( std : : string ifnameStr , uint8_t ipvPref , uint16_t mtu , uint32_t capacity , bool enabled , uint8_t mode , std : : string failoverToLinkStr )
2021-09-02 04:37:49 +00:00
: _ifnameStr ( ifnameStr )
, _ipvPref ( ipvPref )
2023-01-12 04:12:15 +00:00
, _mtu ( mtu )
2022-10-10 06:07:16 +00:00
, _capacity ( capacity )
, _relativeCapacity ( 0.0 )
2021-09-02 04:37:49 +00:00
, _enabled ( enabled )
, _mode ( mode )
, _failoverToLinkStr ( failoverToLinkStr )
, _isUserSpecified ( false )
{
}
/**
* @ return The string representation of this link ' s underlying interface ' s system name .
*/
inline std : : string ifname ( )
{
return _ifnameStr ;
}
/**
* @ return Whether this link is designated as a primary .
*/
inline bool primary ( )
{
return _mode = = ZT_BOND_SLAVE_MODE_PRIMARY ;
}
/**
* @ return Whether this link is designated as a spare .
*/
inline bool spare ( )
{
return _mode = = ZT_BOND_SLAVE_MODE_SPARE ;
}
/**
* @ return The name of the link interface that should be used in the event of a failure .
*/
inline std : : string failoverToLink ( )
{
return _failoverToLinkStr ;
}
/**
* @ return Whether this link interface was specified by the user or auto - detected .
*/
inline bool isUserSpecified ( )
{
return _isUserSpecified ;
}
/**
* Signify that this link was specified by the user and not the result of auto - detection .
*
* @ param isUserSpecified
*/
inline void setAsUserSpecified ( bool isUserSpecified )
{
_isUserSpecified = isUserSpecified ;
}
/**
* @ return Whether or not the user has specified failover instructions .
*/
inline bool userHasSpecifiedFailoverInstructions ( )
{
return _failoverToLinkStr . length ( ) ;
}
/**
2022-10-10 06:07:16 +00:00
* @ return The capacity of the link relative to others in the bond .
2021-09-02 04:37:49 +00:00
*/
2022-10-10 06:07:16 +00:00
inline float relativeCapacity ( )
2021-09-02 04:37:49 +00:00
{
2022-10-10 06:07:16 +00:00
return _relativeCapacity ;
2021-09-02 04:37:49 +00:00
}
/**
2022-10-10 06:07:16 +00:00
* Sets the capacity of the link relative to others in the bond .
2021-09-02 04:37:49 +00:00
*
2022-10-10 06:07:16 +00:00
* @ param relativeCapacity The capacity relative to the rest of the link .
2021-09-02 04:37:49 +00:00
*/
2022-10-10 06:07:16 +00:00
inline void setRelativeCapacity ( float relativeCapacity )
2021-09-02 04:37:49 +00:00
{
2022-10-10 06:07:16 +00:00
_relativeCapacity = relativeCapacity ;
2021-09-02 04:37:49 +00:00
}
/**
2022-10-10 06:07:16 +00:00
* @ return The absolute capacity of the link ( as specified by the user . )
2021-09-02 04:37:49 +00:00
*/
2022-10-10 06:07:16 +00:00
inline uint32_t capacity ( )
2021-09-02 04:37:49 +00:00
{
2022-10-10 06:07:16 +00:00
return _capacity ;
2021-09-02 04:37:49 +00:00
}
/**
* @ return The address preference for this link ( as specified by the user . )
*/
inline uint8_t ipvPref ( )
{
return _ipvPref ;
}
2023-01-12 04:12:15 +00:00
/**
* @ return The MTU for this link ( as specified by the user . )
*/
inline uint16_t mtu ( )
{
return _mtu ;
}
2021-09-02 04:37:49 +00:00
/**
* @ return The mode ( e . g . primary / spare ) for this link ( as specified by the user . )
*/
inline uint8_t mode ( )
{
return _mode ;
}
/**
* @ return Whether this link is enabled or disabled
*/
inline uint8_t enabled ( )
{
return _enabled ;
}
private :
/**
* String representation of underlying interface ' s system name
*/
std : : string _ifnameStr ;
/**
* What preference ( if any ) a user has for IP protocol version used in
* path aggregations . Preference is expressed in the order of the digits :
*
* 0 : no preference
* 4 : IPv4 only
* 6 : IPv6 only
* 46 : IPv4 over IPv6
* 64 : IPv6 over IPv4
*/
uint8_t _ipvPref ;
2023-01-12 04:12:15 +00:00
/**
* The physical - layer MTU for this link
*/
uint16_t _mtu ;
2021-09-02 04:37:49 +00:00
/**
2022-10-10 06:07:16 +00:00
* User - specified capacity of this link
2021-09-02 04:37:49 +00:00
*/
2022-10-10 06:07:16 +00:00
uint32_t _capacity ;
2021-09-02 04:37:49 +00:00
/**
* Speed relative to other specified links ( computed by Bond )
*/
2022-10-10 06:07:16 +00:00
float _relativeCapacity ;
2021-09-02 04:37:49 +00:00
/**
* Whether this link is enabled , or ( disabled ( possibly bad config ) )
*/
uint8_t _enabled ;
/**
* Whether this link is designated as a primary , a spare , or no preference .
*/
uint8_t _mode ;
/**
* The specific name of the link to be used in the event that this
* link fails .
*/
std : : string _failoverToLinkStr ;
/**
* Whether or not this link was created as a result of manual user specification . This is
* important to know because certain policy decisions are dependent on whether the user
* intents to use a specific set of interfaces .
*/
bool _isUserSpecified ;
AtomicCounter __refCount ;
} ;
2020-06-17 21:54:13 +00:00
class Link ;
2021-05-04 00:59:31 +00:00
class Peer ;
2020-05-12 08:35:48 +00:00
2021-05-04 00:59:31 +00:00
class Bond {
2021-09-02 04:37:49 +00:00
public :
2022-10-10 06:07:16 +00:00
/**
* Stop bond ' s internal functions ( can be resumed )
*/
void stopBond ( ) ;
/**
* Start or resume a bond ' s internal functions
*/
void startBond ( ) ;
2021-09-02 04:37:49 +00:00
/**
* @ return Whether this link is permitted to become a member of a bond .
*/
static bool linkAllowed ( std : : string & policyAlias , SharedPtr < Link > link ) ;
2020-05-12 08:35:48 +00:00
2021-09-02 04:37:49 +00:00
/**
* @ return The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements .
*/
static int minReqMonitorInterval ( )
{
return _minReqMonitorInterval ;
}
/**
* @ return Whether the bonding layer is currently set up to be used .
*/
static bool inUse ( )
{
return ! _bondPolicyTemplates . empty ( ) | | _defaultPolicy ;
}
2022-09-20 21:27:34 +00:00
/**
* Sets a pointer to an instance of _binder used by the Bond to get interface data
*/
static void setBinder ( Binder * b )
{
_binder = b ;
}
2021-09-02 04:37:49 +00:00
/**
* @ param basePolicyName Bonding policy name ( See ZeroTierOne . h )
* @ return The bonding policy code for a given human - readable bonding policy name
*/
static int getPolicyCodeByStr ( const std : : string & basePolicyName )
{
if ( basePolicyName = = " active-backup " ) {
return 1 ;
2020-05-12 08:35:48 +00:00
}
2021-09-02 04:37:49 +00:00
if ( basePolicyName = = " broadcast " ) {
return 2 ;
}
if ( basePolicyName = = " balance-rr " ) {
return 3 ;
}
if ( basePolicyName = = " balance-xor " ) {
return 4 ;
}
if ( basePolicyName = = " balance-aware " ) {
return 5 ;
}
return 0 ; // "none"
}
/**
* @ param policy Bonding policy code ( See ZeroTierOne . h )
* @ return The human - readable name for the given bonding policy code
*/
static std : : string getPolicyStrByCode ( int policy )
{
if ( policy = = 1 ) {
return " active-backup " ;
}
if ( policy = = 2 ) {
return " broadcast " ;
}
if ( policy = = 3 ) {
return " balance-rr " ;
}
if ( policy = = 4 ) {
return " balance-xor " ;
}
if ( policy = = 5 ) {
return " balance-aware " ;
}
return " none " ;
}
/**
* Sets the default bonding policy for new or undefined bonds .
*
* @ param bp Bonding policy
*/
static void setBondingLayerDefaultPolicy ( uint8_t bp )
{
_defaultPolicy = bp ;
}
/**
* Sets the default ( custom ) bonding policy for new or undefined bonds .
*
* @ param alias Human - readable string alias for bonding policy
*/
static void setBondingLayerDefaultPolicyStr ( std : : string alias )
{
_defaultPolicyStr = alias ;
}
/**
* Add a user - defined link to a given bonding policy .
*
* @ param policyAlias User - defined custom name for variant of bonding policy
* @ param link Pointer to new link definition
*/
static void addCustomLink ( std : : string & policyAlias , SharedPtr < Link > link ) ;
/**
* Add a user - defined bonding policy that is based on one of the standard types .
*
* @ param newBond Pointer to custom Bond object
* @ return Whether a uniquely - named custom policy was successfully added
*/
static bool addCustomPolicy ( const SharedPtr < Bond > & newBond ) ;
/**
* Assigns a specific bonding policy
*
* @ param identity
* @ param policyAlias
* @ return
*/
static bool assignBondingPolicyToPeer ( int64_t identity , const std : : string & policyAlias ) ;
/**
* Get pointer to bond by a given peer ID
*
* @ param peer Remote peer ID
* @ return A pointer to the Bond
*/
static SharedPtr < Bond > getBondByPeerId ( int64_t identity ) ;
2023-08-15 19:59:58 +00:00
/**
* Set MTU for link by given interface name and IP address ( across all bonds )
*
* @ param mtu MTU to be used on this link
* @ param ifStr interface name to match
* @ param ipStr IP address to match
* @ return Whether the MTU was set
*/
static bool setAllMtuByTuple ( uint16_t mtu , const std : : string & ifStr , const std : : string & ipStr ) ;
/**
* Set MTU for link by given interface name and IP address
*
* @ param mtu MTU to be used on this link
* @ param ifStr interface name to match
* @ param ipStr IP address to match
* @ return Whether the MTU was set
*/
bool setMtuByTuple ( uint16_t mtu , const std : : string & ifStr , const std : : string & ipStr ) ;
2021-09-02 04:37:49 +00:00
/**
* Add a new bond to the bond controller .
*
* @ param renv Runtime environment
* @ param peer Remote peer that this bond services
* @ return A pointer to the newly created Bond
*/
2022-03-02 17:55:23 +00:00
static SharedPtr < Bond > createBond ( const RuntimeEnvironment * renv , const SharedPtr < Peer > & peer ) ;
/**
* Remove a bond from the bond controller .
*
* @ param peerId Remote peer that this bond services
*/
static void destroyBond ( uint64_t peerId ) ;
2021-09-02 04:37:49 +00:00
/**
* Periodically perform maintenance tasks for the bonding layer .
*
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @ param now Current time
*/
static void processBackgroundTasks ( void * tPtr , int64_t now ) ;
/**
* Gets a reference to a physical link definition given a policy alias and a local socket .
*
* @ param policyAlias Policy in use
* @ param localSocket Local source socket
2022-05-06 21:59:34 +00:00
* @ param createIfNeeded Whether a Link object is created if the name wasn ' t previously in the link map
2021-09-02 04:37:49 +00:00
* @ return Physical link definition
*/
2022-09-20 21:27:34 +00:00
SharedPtr < Link > getLinkBySocket ( const std : : string & policyAlias , uint64_t localSocket , bool createIfNeeded ) ;
2021-09-02 04:37:49 +00:00
/**
* Gets a reference to a physical link definition given its human - readable system name .
*
* @ param policyAlias Policy in use
* @ param ifname Alphanumeric human - readable name
* @ return Physical link definition
*/
static SharedPtr < Link > getLinkByName ( const std : : string & policyAlias , const std : : string & ifname ) ;
private :
static Phy < Bond * > * _phy ;
static Mutex _bonds_m ;
static Mutex _links_m ;
/**
* The minimum required monitoring interval among all bonds
*/
static int _minReqMonitorInterval ;
/**
* The default bonding policy used for new bonds unless otherwise specified .
*/
static uint8_t _defaultPolicy ;
/**
* The default bonding policy used for new bonds unless otherwise specified .
*/
static std : : string _defaultPolicyStr ;
/**
* All currently active bonds .
*/
static std : : map < int64_t , SharedPtr < Bond > > _bonds ;
/**
* Map of peers to custom bonding policies
*/
static std : : map < int64_t , std : : string > _policyTemplateAssignments ;
/**
* User - defined bonding policies ( can be assigned to a peer )
*/
static std : : map < std : : string , SharedPtr < Bond > > _bondPolicyTemplates ;
/**
* Set of links defined for a given bonding policy
*/
static std : : map < std : : string , std : : vector < SharedPtr < Link > > > _linkDefinitions ;
/**
* Set of link objects mapped to their physical interfaces
*/
static std : : map < std : : string , std : : map < std : : string , SharedPtr < Link > > > _interfaceToLinkMap ;
struct NominatedPath ;
struct Flow ;
friend class SharedPtr < Bond > ;
friend class Peer ;
2020-05-12 08:35:48 +00:00
2021-05-04 00:59:31 +00:00
public :
2021-09-02 04:37:49 +00:00
void dumpInfo ( int64_t now , bool force ) ;
2021-12-15 05:13:19 +00:00
std : : string pathToStr ( const SharedPtr < Path > & path ) ;
2021-09-02 04:37:49 +00:00
void dumpPathStatus ( int64_t now , int pathIdx ) ;
2020-05-15 03:09:25 +00:00
2020-06-17 21:54:13 +00:00
SharedPtr < Link > getLink ( const SharedPtr < Path > & path ) ;
2020-05-15 03:09:25 +00:00
2021-09-02 04:37:49 +00:00
/**
* Constructor
*
*
*/
Bond ( const RuntimeEnvironment * renv ) ;
2020-05-15 03:09:25 +00:00
/**
* Constructor . Creates a bond based off of ZT defaults
*
* @ param renv Runtime environment
* @ param policy Bonding policy
* @ param peer
*/
2021-05-04 00:59:31 +00:00
Bond ( const RuntimeEnvironment * renv , int policy , const SharedPtr < Peer > & peer ) ;
2020-05-15 03:09:25 +00:00
/**
* Constructor . For use when user intends to manually specify parameters
*
* @ param basePolicy
* @ param policyAlias
* @ param peer
*/
2021-05-04 00:59:31 +00:00
Bond ( const RuntimeEnvironment * renv , std : : string & basePolicy , std : : string & policyAlias , const SharedPtr < Peer > & peer ) ;
2020-05-15 03:09:25 +00:00
/**
* Constructor . Creates a bond based off of a user - defined bond template
*
* @ param renv Runtime environment
* @ param original
* @ param peer
*/
2021-05-04 00:59:31 +00:00
Bond ( const RuntimeEnvironment * renv , SharedPtr < Bond > originalBond , const SharedPtr < Peer > & peer ) ;
2020-05-12 08:35:48 +00:00
/**
2020-05-27 00:57:37 +00:00
* @ return The human - readable name of the bonding policy
2020-05-12 08:35:48 +00:00
*/
2021-05-04 00:59:31 +00:00
std : : string policyAlias ( )
{
return _policyAlias ;
}
2020-05-12 08:35:48 +00:00
2022-10-10 06:07:16 +00:00
/**
* Return whether this bond is able to properly process traffic
*/
bool isReady ( )
{
return _numBondedPaths ;
}
2020-05-12 08:35:48 +00:00
/**
2020-08-25 01:56:49 +00:00
* Inform the bond about the path that its peer ( owning object ) just learned about .
* If the path is allowed to be used , it will be inducted into the bond on a trial
* period where link statistics will be collected to judge its quality .
2020-05-12 08:35:48 +00:00
*
* @ param path Newly - learned Path which should now be handled by the Bond
* @ param now Current time
*/
2021-09-02 04:37:49 +00:00
void nominatePathToBond ( const SharedPtr < Path > & path , int64_t now ) ;
2020-05-15 03:09:25 +00:00
2020-05-12 08:35:48 +00:00
/**
2021-09-02 04:37:49 +00:00
* Add a nominated path to the bond . This merely maps the index from the nominated set
* to a smaller set and sets the path ' s bonded flag to true .
*
* @ param nominatedIdx The index in the nominated set
* @ param bondedIdx The index in the bonded set ( subset of nominated )
2020-05-12 08:35:48 +00:00
*/
2021-09-02 04:37:49 +00:00
void addPathToBond ( int nominatedIdx , int bondedIdx ) ;
2020-05-12 08:35:48 +00:00
/**
* Check path states and perform bond rebuilds if needed .
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param now Current time
2020-05-15 03:09:25 +00:00
* @ param rebuild Whether or not the bond should be reconstructed .
2020-05-12 08:35:48 +00:00
*/
2021-09-02 04:37:49 +00:00
void curateBond ( int64_t now , bool rebuild ) ;
2020-05-12 08:35:48 +00:00
/**
* Periodically perform statistical summaries of quality metrics for all paths .
*
* @ param now Current time
*/
void estimatePathQuality ( int64_t now ) ;
/**
* Record an invalid incoming packet . This packet failed
* MAC / compression / cipher checks and will now contribute to a
* Packet Error Ratio ( PER ) .
*
* @ param path Path over which packet was received
*/
void recordIncomingInvalidPacket ( const SharedPtr < Path > & path ) ;
/**
* Record statistics on outbound an packet .
*
* @ param path Path over which packet is being sent
* @ param packetId Packet ID
* @ param payloadLength Packet data length
* @ param verb Packet verb
* @ param flowId Flow ID
* @ param now Current time
*/
2021-05-04 00:59:31 +00:00
void recordOutgoingPacket ( const SharedPtr < Path > & path , uint64_t packetId , uint16_t payloadLength , Packet : : Verb verb , int32_t flowId , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
* Process the contents of an inbound VERB_QOS_MEASUREMENT to gather path quality observations .
*
* @ param now Current time
* @ param count Number of records
* @ param rx_id table of packet IDs
* @ param rx_ts table of holding times
*/
2021-05-04 00:59:31 +00:00
void receivedQoS ( const SharedPtr < Path > & path , int64_t now , int count , uint64_t * rx_id , uint16_t * rx_ts ) ;
2020-05-12 08:35:48 +00:00
2022-06-08 17:01:54 +00:00
/**
* Process the contents of an inbound VERB_ACK to gather path quality observations .
*
* @ param pathIdx Path over which packet was received
* @ param now Current time
* @ param ackedBytes Number of bytes ACKed by this VERB_ACK
*/
void receivedAck ( int pathIdx , int64_t now , int32_t ackedBytes ) ;
2020-05-12 08:35:48 +00:00
/**
* Generate the contents of a VERB_QOS_MEASUREMENT packet .
*
* @ param now Current time
* @ param qosBuffer destination buffer
* @ return Size of payload
*/
2021-09-02 04:37:49 +00:00
int32_t generateQoSPacket ( int pathIdx , int64_t now , char * qosBuffer ) ;
2020-05-12 08:35:48 +00:00
/**
* Record statistics for an inbound packet .
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param path Path over which packet was received
* @ param packetId Packet ID
* @ param payloadLength Packet data length
* @ param verb Packet verb
* @ param flowId Flow ID
* @ param now Current time
*/
2021-05-04 00:59:31 +00:00
void recordIncomingPacket ( const SharedPtr < Path > & path , uint64_t packetId , uint16_t payloadLength , Packet : : Verb verb , int32_t flowId , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
* Determines the most appropriate path for packet and flow egress . This decision is made by
* the underlying bonding policy as well as QoS - related statistical observations of path quality .
*
* @ param now Current time
* @ param flowId Flow ID
* @ return Pointer to suggested Path
*/
SharedPtr < Path > getAppropriatePath ( int64_t now , int32_t flowId ) ;
/**
* Creates a new flow record
2020-05-15 03:09:25 +00:00
*
2021-09-02 04:37:49 +00:00
* @ param np Path over which flow shall be handled
2020-05-12 08:35:48 +00:00
* @ param flowId Flow ID
* @ param entropy A byte of entropy to be used by the bonding algorithm
* @ param now Current time
* @ return Pointer to newly - created Flow
*/
2021-09-02 04:37:49 +00:00
SharedPtr < Flow > createFlow ( int pathIdx , int32_t flowId , unsigned char entropy , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
* Removes flow records that are past a certain age limit .
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param age Age threshold to be forgotten
* @ param oldest Whether only the oldest shall be forgotten
* @ param now Current time
*/
void forgetFlowsWhenNecessary ( uint64_t age , bool oldest , int64_t now ) ;
/**
* Assigns a new flow to a bonded path
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param flow Flow to be assigned
* @ param now Current time
2022-10-10 06:07:16 +00:00
* @ param reassign Whether this flow is being re - assigned to another path
2020-05-12 08:35:48 +00:00
*/
2022-10-10 06:07:16 +00:00
bool assignFlowToBondedPath ( SharedPtr < Flow > & flow , int64_t now , bool reassign ) ;
2020-05-12 08:35:48 +00:00
2020-05-15 03:09:25 +00:00
/**
2020-05-12 08:35:48 +00:00
* Determine whether a path change should occur given the remote peer ' s reported utility and our
* local peer ' s known utility . This has the effect of assigning inbound and outbound traffic to
2020-05-15 03:09:25 +00:00
* the same path .
*
2020-05-12 08:35:48 +00:00
* @ param now Current time
* @ param path Path over which the negotiation request was received
* @ param remoteUtility How much utility the remote peer claims to gain by using the declared path
*/
2021-05-04 00:59:31 +00:00
void processIncomingPathNegotiationRequest ( uint64_t now , SharedPtr < Path > & path , int16_t remoteUtility ) ;
2020-05-12 08:35:48 +00:00
/**
* Determine state of path synchronization and whether a negotiation request
* shall be sent to the peer .
*
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @ param now Current time
*/
2021-09-02 04:37:49 +00:00
void pathNegotiationCheck ( void * tPtr , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
* Sends a VERB_ACK to the remote peer .
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @ param path Path over which packet should be sent
* @ param localSocket Local source socket
* @ param atAddress
* @ param now Current time
*/
2021-09-02 04:37:49 +00:00
void sendACK ( void * tPtr , int pathIdx , int64_t localSocket , const InetAddress & atAddress , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
* Sends a VERB_QOS_MEASUREMENT to the remote peer .
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @ param path Path over which packet should be sent
* @ param localSocket Local source socket
* @ param atAddress
* @ param now Current time
*/
2021-09-02 04:37:49 +00:00
void sendQOS_MEASUREMENT ( void * tPtr , int pathIdx , int64_t localSocket , const InetAddress & atAddress , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
* Sends a VERB_PATH_NEGOTIATION_REQUEST to the remote peer .
2020-05-15 03:09:25 +00:00
*
2020-05-12 08:35:48 +00:00
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @ param path Path over which packet should be sent
*/
2021-09-02 04:37:49 +00:00
void sendPATH_NEGOTIATION_REQUEST ( void * tPtr , int pathIdx ) ;
2020-05-12 08:35:48 +00:00
/**
*
* @ param now Current time
*/
void processBalanceTasks ( int64_t now ) ;
2020-05-15 03:09:25 +00:00
2020-05-12 08:35:48 +00:00
/**
* Perform periodic tasks unique to active - backup
2020-05-15 03:09:25 +00:00
*
2020-08-07 01:10:40 +00:00
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
2020-05-12 08:35:48 +00:00
* @ param now Current time
*/
2021-05-04 00:59:31 +00:00
void processActiveBackupTasks ( void * tPtr , int64_t now ) ;
2020-05-12 08:35:48 +00:00
/**
2020-06-17 21:54:13 +00:00
* Switches the active link in an active - backup scenario to the next best during
2020-05-12 08:35:48 +00:00
* a failover event .
*
* @ param now Current time
*/
void dequeueNextActiveBackupPath ( uint64_t now ) ;
2022-02-17 23:16:33 +00:00
/**
* Zero all timers
*/
void initTimers ( ) ;
2020-05-15 03:09:25 +00:00
/**
* Set bond parameters to reasonable defaults , these may later be overwritten by
2020-05-12 08:35:48 +00:00
* user - specified parameters .
2020-05-15 03:09:25 +00:00
*
* @ param policy Bonding policy
2020-06-02 05:58:58 +00:00
* @ param templateBond
2020-05-15 03:09:25 +00:00
*/
2021-09-02 04:37:49 +00:00
void setBondParameters ( int policy , SharedPtr < Bond > templateBond , bool useTemplate ) ;
2020-05-12 08:35:48 +00:00
/**
2022-10-10 06:07:16 +00:00
* Check and assign user - specified link quality parameters to this bond .
2020-05-12 08:35:48 +00:00
*
2022-10-10 06:07:16 +00:00
* @ param weights Set of user - specified parameters
* @ param len Length of parameter vector
2020-05-12 08:35:48 +00:00
*/
2022-10-10 06:07:16 +00:00
void setUserLinkQualitySpec ( float weights [ ] , int len ) ;
2020-05-12 08:35:48 +00:00
/**
2020-06-17 21:54:13 +00:00
* @ return Whether the user has defined links for use on this bond
2020-05-12 08:35:48 +00:00
*/
2021-05-04 00:59:31 +00:00
inline bool userHasSpecifiedLinks ( )
{
return _userHasSpecifiedLinks ;
}
2020-05-12 08:35:48 +00:00
/**
2020-06-17 21:54:13 +00:00
* @ return Whether the user has defined a set of failover link ( s ) for this bond
2020-05-12 08:35:48 +00:00
*/
2021-05-04 00:59:31 +00:00
inline bool userHasSpecifiedFailoverInstructions ( )
{
return _userHasSpecifiedFailoverInstructions ;
} ;
2020-05-12 08:35:48 +00:00
/**
2020-06-17 21:54:13 +00:00
* @ return Whether the user has specified a primary link
2020-05-12 08:35:48 +00:00
*/
2021-05-04 00:59:31 +00:00
inline bool userHasSpecifiedPrimaryLink ( )
{
return _userHasSpecifiedPrimaryLink ;
}
2020-05-12 08:35:48 +00:00
/**
2022-10-10 06:07:16 +00:00
* @ return Whether the user has specified link capacities
2020-05-12 08:35:48 +00:00
*/
2022-10-10 06:07:16 +00:00
inline bool userHasSpecifiedLinkCapacities ( )
2021-05-04 00:59:31 +00:00
{
2022-10-10 06:07:16 +00:00
return _userHasSpecifiedLinkCapacities ;
2021-05-04 00:59:31 +00:00
}
2020-05-12 08:35:48 +00:00
/**
* Periodically perform maintenance tasks for each active bond .
*
* @ param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
* @ param now Current time
*/
2021-09-02 04:37:49 +00:00
void processBackgroundBondTasks ( void * tPtr , int64_t now ) ;
2020-05-12 08:35:48 +00:00
2022-06-08 17:01:54 +00:00
/**
* Rate limit gate for VERB_ACK
*
* @ param now Current time
* @ return Whether the incoming packet should be rate - gated
*/
inline bool rateGateACK ( const int64_t now )
{
_ackCutoffCount + + ;
int numToDrain = _lastAckRateCheck ? ( now - _lastAckRateCheck ) / ZT_ACK_DRAINAGE_DIVISOR : _ackCutoffCount ;
_lastAckRateCheck = now ;
if ( _ackCutoffCount > numToDrain ) {
_ackCutoffCount - = numToDrain ;
2024-03-01 23:43:28 +00:00
}
else {
2022-06-08 17:01:54 +00:00
_ackCutoffCount = 0 ;
}
return ( _ackCutoffCount < ZT_ACK_CUTOFF_LIMIT ) ;
}
2020-05-12 08:35:48 +00:00
/**
* Rate limit gate for VERB_QOS_MEASUREMENT
*
* @ param now Current time
* @ return Whether the incoming packet should be rate - gated
*/
2021-09-02 04:37:49 +00:00
inline bool rateGateQoS ( int64_t now , SharedPtr < Path > & path )
2020-05-12 08:35:48 +00:00
{
2021-09-02 04:37:49 +00:00
char pathStr [ 64 ] = { 0 } ;
path - > address ( ) . toString ( pathStr ) ;
2022-10-10 06:07:16 +00:00
uint64_t diff = now - _lastQoSRateCheck ;
2021-09-02 04:37:49 +00:00
if ( ( diff ) < = ( _qosSendInterval / ZT_MAX_PEER_NETWORK_PATHS ) ) {
+ + _qosCutoffCount ;
2024-03-01 23:43:28 +00:00
}
else {
2020-05-12 08:35:48 +00:00
_qosCutoffCount = 0 ;
}
2021-09-02 04:37:49 +00:00
_lastQoSRateCheck = now ;
return ( _qosCutoffCount < ( ZT_MAX_PEER_NETWORK_PATHS * 2 ) ) ;
2020-05-12 08:35:48 +00:00
}
/**
* Rate limit gate for VERB_PATH_NEGOTIATION_REQUEST
*
* @ param now Current time
* @ return Whether the incoming packet should be rate - gated
*/
2021-09-02 04:37:49 +00:00
inline bool rateGatePathNegotiation ( int64_t now , SharedPtr < Path > & path )
2020-05-12 08:35:48 +00:00
{
2021-09-02 04:37:49 +00:00
char pathStr [ 64 ] = { 0 } ;
path - > address ( ) . toString ( pathStr ) ;
int diff = now - _lastPathNegotiationReceived ;
if ( ( diff ) < = ( ZT_PATH_NEGOTIATION_CUTOFF_TIME / ZT_MAX_PEER_NETWORK_PATHS ) ) {
2020-05-12 08:35:48 +00:00
+ + _pathNegotiationCutoffCount ;
2024-03-01 23:43:28 +00:00
}
else {
2021-05-04 00:59:31 +00:00
_pathNegotiationCutoffCount = 0 ;
2021-09-02 04:37:49 +00:00
}
2020-05-12 08:35:48 +00:00
_lastPathNegotiationReceived = now ;
2021-09-02 04:37:49 +00:00
return ( _pathNegotiationCutoffCount < ( ZT_MAX_PEER_NETWORK_PATHS * 2 ) ) ;
2020-05-12 08:35:48 +00:00
}
/**
* @ param interval Maximum amount of time user expects a failover to take on this bond .
*/
2021-05-04 00:59:31 +00:00
inline void setFailoverInterval ( uint32_t interval )
{
_failoverInterval = interval ;
}
2020-05-12 08:35:48 +00:00
2020-11-23 17:59:28 +00:00
/**
* @ param interval Maximum amount of time user expects a failover to take on this bond .
*/
2021-05-04 00:59:31 +00:00
inline uint32_t getFailoverInterval ( )
{
return _failoverInterval ;
}
2020-11-23 17:59:28 +00:00
2020-06-16 19:30:21 +00:00
/**
* @ param strategy Strategy that the bond uses to prob for path aliveness and quality
2020-05-12 08:35:48 +00:00
*/
2021-05-04 00:59:31 +00:00
inline void setLinkMonitorStrategy ( uint8_t strategy )
{
_linkMonitorStrategy = strategy ;
}
2020-05-12 08:35:48 +00:00
/**
* @ return the current up delay parameter
*/
2021-05-04 00:59:31 +00:00
inline uint16_t getUpDelay ( )
{
return _upDelay ;
}
2020-05-12 08:35:48 +00:00
2020-05-15 03:09:25 +00:00
/**
* @ param upDelay Length of time before a newly - discovered path is admitted to the bond
*/
2021-05-04 00:59:31 +00:00
inline void setUpDelay ( int upDelay )
{
if ( upDelay > = 0 ) {
_upDelay = upDelay ;
}
}
2020-05-12 08:35:48 +00:00
2020-05-15 03:09:25 +00:00
/**
* @ return Length of time before a newly - failed path is removed from the bond
*/
2021-05-04 00:59:31 +00:00
inline uint16_t getDownDelay ( )
{
return _downDelay ;
}
2020-05-12 08:35:48 +00:00
2020-05-15 03:09:25 +00:00
/**
* @ param downDelay Length of time before a newly - failed path is removed from the bond
*/
2021-05-04 00:59:31 +00:00
inline void setDownDelay ( int downDelay )
{
if ( downDelay > = 0 ) {
_downDelay = downDelay ;
}
}
2020-05-12 08:35:48 +00:00
/**
2021-09-02 04:37:49 +00:00
* @ return The current monitoring interval for the bond
2020-05-12 08:35:48 +00:00
*/
2021-09-02 04:37:49 +00:00
inline int monitorInterval ( )
2021-05-04 00:59:31 +00:00
{
2021-09-02 04:37:49 +00:00
return _monitorInterval ;
2021-05-04 00:59:31 +00:00
}
2020-05-12 08:35:48 +00:00
2020-05-15 03:09:25 +00:00
/**
2020-06-17 21:54:13 +00:00
* Set the current monitoring interval for the bond ( can be overridden with intervals specific to certain links . )
2020-05-15 03:09:25 +00:00
*
* @ param monitorInterval How often gratuitous VERB_HELLO ( s ) are sent to remote peer .
*/
2021-05-04 00:59:31 +00:00
inline void setBondMonitorInterval ( uint16_t interval )
{
2021-09-02 04:37:49 +00:00
_monitorInterval = interval ;
2021-05-04 00:59:31 +00:00
}
2020-05-12 08:35:48 +00:00
/**
* @ param policy Bonding policy for this bond
*/
2021-12-14 19:49:43 +00:00
2021-05-04 00:59:31 +00:00
inline void setPolicy ( uint8_t policy )
{
2021-09-02 04:37:49 +00:00
_policy = policy ;
2021-05-04 00:59:31 +00:00
}
2021-12-14 19:49:43 +00:00
2020-05-12 08:35:48 +00:00
/**
* @ return the current bonding policy
*/
2021-09-02 04:37:49 +00:00
inline uint8_t policy ( )
2021-05-04 00:59:31 +00:00
{
2021-09-02 04:37:49 +00:00
return _policy ;
2021-05-04 00:59:31 +00:00
}
2020-05-12 08:35:48 +00:00
2020-07-28 06:01:12 +00:00
/**
2022-09-14 17:09:29 +00:00
* @ return the number of links in this bond which are considered alive
2020-07-28 06:01:12 +00:00
*/
2021-05-04 00:59:31 +00:00
inline uint8_t getNumAliveLinks ( )
{
return _numAliveLinks ;
} ;
2020-07-28 06:01:12 +00:00
/**
2022-09-14 17:09:29 +00:00
* @ return the number of links in this bond
2020-07-28 06:01:12 +00:00
*/
2021-05-04 00:59:31 +00:00
inline uint8_t getNumTotalLinks ( )
{
return _numTotalLinks ;
}
2020-07-28 06:01:12 +00:00
2020-05-15 03:09:25 +00:00
/**
2022-10-10 06:07:16 +00:00
* @ return Whether flow - hashing is currently supported for this bond .
2020-05-12 08:35:48 +00:00
*/
2022-10-10 06:07:16 +00:00
bool flowHashingSupported ( )
2021-05-04 00:59:31 +00:00
{
2022-10-10 06:07:16 +00:00
return _policy = = ZT_BOND_POLICY_BALANCE_XOR | | _policy = = ZT_BOND_POLICY_BALANCE_AWARE ;
2021-05-04 00:59:31 +00:00
}
2020-05-12 08:35:48 +00:00
2020-05-15 03:09:25 +00:00
/**
*
2020-06-17 21:54:13 +00:00
* @ param packetsPerLink
2020-05-15 03:09:25 +00:00
*/
2021-05-04 00:59:31 +00:00
inline void setPacketsPerLink ( int packetsPerLink )
{
_packetsPerLink = packetsPerLink ;
}
2020-05-12 08:35:48 +00:00
2020-11-23 17:59:28 +00:00
/**
* @ return Number of packets to be sent on each interface in a balance - rr bond
*/
2021-05-04 00:59:31 +00:00
inline int getPacketsPerLink ( )
{
return _packetsPerLink ;
}
2020-11-23 17:59:28 +00:00
2020-05-12 08:35:48 +00:00
/**
*
2020-06-17 21:54:13 +00:00
* @ param linkSelectMethod
2020-05-12 08:35:48 +00:00
*/
2021-05-04 00:59:31 +00:00
inline void setLinkSelectMethod ( uint8_t method )
{
_abLinkSelectMethod = method ;
}
2020-05-12 08:35:48 +00:00
/**
*
* @ return
*/
2021-05-04 00:59:31 +00:00
inline uint8_t getLinkSelectMethod ( )
{
return _abLinkSelectMethod ;
}
2020-05-12 08:35:48 +00:00
/**
*
* @ param allowPathNegotiation
*/
2021-05-04 00:59:31 +00:00
inline void setAllowPathNegotiation ( bool allowPathNegotiation )
{
_allowPathNegotiation = allowPathNegotiation ;
}
2020-05-12 08:35:48 +00:00
/**
*
* @ return
*/
2021-05-04 00:59:31 +00:00
inline bool allowPathNegotiation ( )
{
return _allowPathNegotiation ;
}
2020-05-12 08:35:48 +00:00
2020-11-23 17:59:28 +00:00
/**
* Forcibly rotates the currently active link used in an active - backup bond to the next link in the failover queue
*
* @ return True if this operation succeeded , false if otherwise
*/
bool abForciblyRotateLink ( ) ;
2021-09-02 04:37:49 +00:00
/**
* Emit message to tracing system but with added timestamp and subsystem info
*/
void log ( const char * fmt , . . . )
2022-05-07 00:25:15 +00:00
# ifdef __GNUC__
__attribute__ ( ( format ( printf , 2 , 3 ) ) )
# endif
2022-05-06 21:59:34 +00:00
{
2021-12-15 05:13:19 +00:00
# ifdef ZT_TRACE
2021-09-02 04:37:49 +00:00
time_t rawtime ;
struct tm * timeinfo ;
char timestamp [ 80 ] ;
time ( & rawtime ) ;
timeinfo = localtime ( & rawtime ) ;
strftime ( timestamp , 80 , " %F %T " , timeinfo ) ;
# define MAX_BOND_MSG_LEN 1024
char traceMsg [ MAX_BOND_MSG_LEN ] ;
char userMsg [ MAX_BOND_MSG_LEN ] ;
va_list args ;
va_start ( args , fmt ) ;
if ( vsnprintf ( userMsg , sizeof ( userMsg ) , fmt , args ) < 0 ) {
fprintf ( stderr , " Encountered format encoding error while writing to trace log \n " ) ;
return ;
}
snprintf ( traceMsg , MAX_BOND_MSG_LEN , " %s (%llx/%s) %s " , timestamp , _peerId , _policyAlias . c_str ( ) , userMsg ) ;
va_end ( args ) ;
RR - > t - > bondStateMessage ( NULL , traceMsg ) ;
2022-01-26 23:14:02 +00:00
# undef MAX_MSG_LEN
# endif
}
/**
* Emit message to tracing system but with added timestamp and subsystem info
*/
void debug ( const char * fmt , . . . )
2022-05-07 00:25:15 +00:00
# ifdef __GNUC__
__attribute__ ( ( format ( printf , 2 , 3 ) ) )
# endif
2022-05-06 21:59:34 +00:00
{
2022-01-26 23:14:02 +00:00
# ifdef ZT_DEBUG
time_t rawtime ;
struct tm * timeinfo ;
char timestamp [ 80 ] ;
time ( & rawtime ) ;
timeinfo = localtime ( & rawtime ) ;
strftime ( timestamp , 80 , " %F %T " , timeinfo ) ;
# define MAX_BOND_MSG_LEN 1024
char traceMsg [ MAX_BOND_MSG_LEN ] ;
char userMsg [ MAX_BOND_MSG_LEN ] ;
va_list args ;
va_start ( args , fmt ) ;
if ( vsnprintf ( userMsg , sizeof ( userMsg ) , fmt , args ) < 0 ) {
fprintf ( stderr , " Encountered format encoding error while writing to trace log \n " ) ;
return ;
}
snprintf ( traceMsg , MAX_BOND_MSG_LEN , " %s (%llx/%s) %s " , timestamp , _peerId , _policyAlias . c_str ( ) , userMsg ) ;
va_end ( args ) ;
RR - > t - > bondStateMessage ( NULL , traceMsg ) ;
2021-09-02 04:37:49 +00:00
# undef MAX_MSG_LEN
2021-12-15 05:13:19 +00:00
# endif
2021-05-04 00:59:31 +00:00
}
2020-05-12 08:35:48 +00:00
2021-05-04 00:59:31 +00:00
private :
2021-09-02 04:37:49 +00:00
struct NominatedPath {
NominatedPath ( )
2022-06-08 17:01:54 +00:00
: lastAckSent ( 0 )
, lastAckReceived ( 0 )
2022-09-07 16:08:13 +00:00
, lastQoSReceived ( 0 )
2022-06-08 17:01:54 +00:00
, unackedBytes ( 0 )
, packetsReceivedSinceLastAck ( 0 )
, lastQoSMeasurement ( 0 )
2021-09-02 04:37:49 +00:00
, lastThroughputEstimation ( 0 )
, lastRefractoryUpdate ( 0 )
, lastAliveToggle ( 0 )
, alive ( false )
, eligible ( true )
2022-09-07 16:08:13 +00:00
, lastEligibility ( 0 )
2021-09-02 04:37:49 +00:00
, whenNominated ( 0 )
, refractoryPeriod ( 0 )
, ipvPref ( 0 )
, mode ( 0 )
, onlyPathOnLink ( false )
, bonded ( false )
, negotiated ( false )
2022-10-10 06:07:16 +00:00
, shouldAvoid ( false )
2021-09-02 04:37:49 +00:00
, assignedFlowCount ( 0 )
2022-10-10 06:07:16 +00:00
, latency ( 0 )
2021-09-02 04:37:49 +00:00
, latencyVariance ( 0 )
, packetLossRatio ( 0 )
, packetErrorRatio ( 0 )
2022-10-10 06:07:16 +00:00
, relativeQuality ( 0 )
, relativeLinkCapacity ( 0 )
2021-09-02 04:37:49 +00:00
, failoverScore ( 0 )
, packetsReceivedSinceLastQoS ( 0 )
, packetsIn ( 0 )
, packetsOut ( 0 )
2024-03-01 23:43:28 +00:00
, localPort ( 0 )
2021-09-02 04:37:49 +00:00
{
}
/**
* Set or update a refractory period for the path .
*
* @ param punishment How much a path should be punished
* @ param pathFailure Whether this call is the result of a recent path failure
*/
inline void adjustRefractoryPeriod ( int64_t now , uint32_t punishment , bool pathFailure )
{
if ( pathFailure ) {
unsigned int suggestedRefractoryPeriod = refractoryPeriod ? punishment + ( refractoryPeriod * 2 ) : punishment ;
refractoryPeriod = std : : min ( suggestedRefractoryPeriod , ( unsigned int ) ZT_BOND_MAX_REFRACTORY_PERIOD ) ;
lastRefractoryUpdate = 0 ;
2024-03-01 23:43:28 +00:00
}
else {
2021-09-02 04:37:49 +00:00
uint32_t drainRefractory = 0 ;
if ( lastRefractoryUpdate ) {
drainRefractory = ( now - lastRefractoryUpdate ) ;
2024-03-01 23:43:28 +00:00
}
else {
2021-09-02 04:37:49 +00:00
drainRefractory = ( now - lastAliveToggle ) ;
}
lastRefractoryUpdate = now ;
if ( refractoryPeriod > drainRefractory ) {
refractoryPeriod - = drainRefractory ;
2024-03-01 23:43:28 +00:00
}
else {
2021-09-02 04:37:49 +00:00
refractoryPeriod = 0 ;
lastRefractoryUpdate = 0 ;
}
}
}
/**
* @ return True if a path is permitted to be used in a bond ( according to user pref . )
*/
inline bool allowed ( )
{
return ( ! ipvPref | | ( ( p - > _addr . isV4 ( ) & & ( ipvPref = = 4 | | ipvPref = = 46 | | ipvPref = = 64 ) ) | | ( ( p - > _addr . isV6 ( ) & & ( ipvPref = = 6 | | ipvPref = = 46 | | ipvPref = = 64 ) ) ) ) ) ;
}
2022-09-14 17:09:29 +00:00
/**
* @ return True if a path exists on a link marked as a spare
*/
inline bool isSpare ( )
{
return mode = = ZT_BOND_SLAVE_MODE_SPARE ;
}
2021-09-02 04:37:49 +00:00
/**
* @ return True if a path is preferred over another on the same physical link ( according to user pref . )
*/
inline bool preferred ( )
{
return onlyPathOnLink | | ( p - > _addr . isV4 ( ) & & ( ipvPref = = 4 | | ipvPref = = 46 ) ) | | ( p - > _addr . isV6 ( ) & & ( ipvPref = = 6 | | ipvPref = = 64 ) ) ;
}
/**
* @ param now Current time
* @ return Whether a QoS ( VERB_QOS_MEASUREMENT ) packet needs to be emitted at this time
*/
2022-10-10 06:07:16 +00:00
inline bool needsToSendQoS ( int64_t now , uint64_t qosSendInterval )
2021-09-02 04:37:49 +00:00
{
return ( ( packetsReceivedSinceLastQoS > = ZT_QOS_TABLE_SIZE ) | | ( ( now - lastQoSMeasurement ) > qosSendInterval ) ) & & packetsReceivedSinceLastQoS ;
}
2022-06-08 17:01:54 +00:00
/**
* @ param now Current time
* @ return Whether an ACK ( VERB_ACK ) packet needs to be emitted at this time
*/
2022-10-10 06:07:16 +00:00
inline bool needsToSendAck ( int64_t now , uint64_t ackSendInterval )
2022-06-08 17:01:54 +00:00
{
return ( ( now - lastAckSent ) > = ackSendInterval | | ( packetsReceivedSinceLastAck = = ZT_QOS_TABLE_SIZE ) ) & & packetsReceivedSinceLastAck ;
}
2021-09-02 04:37:49 +00:00
/**
* Reset packet counters
*/
inline void resetPacketCounts ( )
{
packetsIn = 0 ;
packetsOut = 0 ;
}
std : : map < uint64_t , uint64_t > qosStatsOut ; // id:egress_time
std : : map < uint64_t , uint64_t > qosStatsIn ; // id:now
2022-06-08 17:01:54 +00:00
std : : map < uint64_t , uint64_t > ackStatsIn ; // id:now
2021-09-02 04:37:49 +00:00
RingBuffer < int , ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE > qosRecordSize ;
RingBuffer < float , ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE > qosRecordLossSamples ;
RingBuffer < uint64_t , ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE > throughputSamples ;
RingBuffer < bool , ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE > packetValiditySamples ;
RingBuffer < float , ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE > throughputVarianceSamples ;
RingBuffer < uint16_t , ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE > latencySamples ;
2022-06-08 17:01:54 +00:00
uint64_t lastAckSent ;
uint64_t lastAckReceived ;
2022-09-07 16:08:13 +00:00
uint64_t lastQoSReceived ;
2022-06-08 17:01:54 +00:00
uint64_t unackedBytes ;
uint64_t packetsReceivedSinceLastAck ;
2021-09-02 04:37:49 +00:00
uint64_t lastQoSMeasurement ; // Last time that a VERB_QOS_MEASUREMENT was sent out on this path.
uint64_t lastThroughputEstimation ; // Last time that the path's throughput was estimated.
uint64_t lastRefractoryUpdate ; // The last time that the refractory period was updated.
uint64_t lastAliveToggle ; // The last time that the path was marked as "alive".
bool alive ;
2022-10-10 06:07:16 +00:00
bool eligible ; // State of eligibility at last check. Used for determining state changes.
uint64_t lastEligibility ; // The last time that this path was eligible
uint64_t whenNominated ; // Timestamp indicating when this path's trial period began.
uint32_t refractoryPeriod ; // Amount of time that this path will be prevented from becoming a member of a bond.
uint8_t ipvPref ; // IP version preference inherited from the physical link.
uint8_t mode ; // Mode inherited from the physical link.
bool onlyPathOnLink ; // IP version preference inherited from the physical link.
bool enabled ; // Enabled state inherited from the physical link.
bool bonded ; // Whether this path is currently part of a bond.
bool negotiated ; // Whether this path was intentionally negotiated by either peer.
bool shouldAvoid ; // Whether flows should be moved from this path. Current traffic flows will be re-allocated immediately.
uint16_t assignedFlowCount ; // The number of flows currently assigned to this path.
float latency ; // The mean latency (computed from a sliding window.)
float latencyVariance ; // Packet delay variance (computed from a sliding window.)
float packetLossRatio ; // The ratio of lost packets to received packets.
float packetErrorRatio ; // The ratio of packets that failed their MAC/CRC checks to those that did not.
float relativeQuality ; // The relative quality of the link.
float relativeLinkCapacity ; // The relative capacity of the link.
2021-09-02 04:37:49 +00:00
uint32_t failoverScore ; // Score that indicates to what degree this path is preferred over others that are available to the bonding policy. (specifically for active-backup)
int32_t packetsReceivedSinceLastQoS ; // Number of packets received since the last VERB_QOS_MEASUREMENT was sent to the remote peer.
/**
* Counters used for tracking path load .
*/
int packetsIn ;
int packetsOut ;
2024-03-01 23:43:28 +00:00
uint16_t localPort ;
2022-04-26 15:26:45 +00:00
// AtomicCounter __refCount;
2021-09-02 04:37:49 +00:00
SharedPtr < Path > p ;
void set ( uint64_t now , const SharedPtr < Path > & path )
{
p = path ;
whenNominated = now ;
}
} ;
2020-05-12 08:35:48 +00:00
/**
2021-09-02 04:37:49 +00:00
* Paths nominated to the bond ( may or may not actually be bonded )
2020-05-12 08:35:48 +00:00
*/
2021-09-02 04:37:49 +00:00
NominatedPath _paths [ ZT_MAX_PEER_NETWORK_PATHS ] ;
inline int getNominatedPathIdx ( const SharedPtr < Path > & path )
{
for ( int i = 0 ; i < ZT_MAX_PEER_NETWORK_PATHS ; + + i ) {
if ( _paths [ i ] . p = = path ) {
return i ;
}
}
return ZT_MAX_PEER_NETWORK_PATHS ;
}
2020-05-12 08:35:48 +00:00
/**
2021-09-02 04:37:49 +00:00
* A protocol flow that is identified by the origin and destination port .
2020-05-12 08:35:48 +00:00
*/
2021-09-02 04:37:49 +00:00
struct Flow {
/**
* @ param flowId Given flow ID
* @ param now Current time
*/
Flow ( int32_t flowId , int64_t now ) : id ( flowId ) , bytesIn ( 0 ) , bytesOut ( 0 ) , lastActivity ( now ) , lastPathReassignment ( 0 ) , assignedPath ( ZT_MAX_PEER_NETWORK_PATHS )
{
}
/**
* Reset flow statistics
*/
inline void resetByteCounts ( )
{
bytesIn = 0 ;
bytesOut = 0 ;
}
/**
* How long since a packet was sent or received in this flow
*
* @ param now Current time
* @ return The age of the flow in terms of last recorded activity
*/
int64_t age ( int64_t now )
{
return now - lastActivity ;
}
/**
* @ param path Assigned path over which this flow should be handled
*/
inline void assignPath ( int pathIdx , int64_t now )
{
assignedPath = pathIdx ;
lastPathReassignment = now ;
}
AtomicCounter __refCount ;
int32_t id ; // Flow ID used for hashing and path selection
uint64_t bytesIn ; // Used for tracking flow size
uint64_t bytesOut ; // Used for tracking flow size
int64_t lastActivity ; // The last time that this flow handled traffic
int64_t lastPathReassignment ; // Time of last path assignment. Used for anti-flapping
int assignedPath ; // Index of path to which this flow is assigned
} ;
const RuntimeEnvironment * RR ;
AtomicCounter __refCount ;
std : : string _policyAlias ; // Custom name given by the user to this bond type.
2020-05-12 08:35:48 +00:00
2022-09-20 21:27:34 +00:00
static Binder * _binder ;
2020-05-12 08:35:48 +00:00
/**
* Set of indices corresponding to paths currently included in the bond proper . This
* may only be updated during a call to curateBond ( ) . The reason for this is so that
* we can simplify the high frequency packet egress logic .
*/
2022-10-10 06:07:16 +00:00
int _realIdxMap [ ZT_MAX_PEER_NETWORK_PATHS ] = { ZT_MAX_PEER_NETWORK_PATHS } ;
int _numBondedPaths ; // Number of paths currently included in the _realIdxMap set.
std : : map < int16_t , SharedPtr < Flow > > _flows ; // Flows hashed according to port and protocol
float _qw [ ZT_QOS_PARAMETER_SIZE ] ; // Link quality specification (can be customized by user)
bool _run ;
2020-05-12 08:35:48 +00:00
2021-09-02 04:37:49 +00:00
uint8_t _policy ;
2020-05-12 08:35:48 +00:00
uint32_t _upDelay ;
uint32_t _downDelay ;
// active-backup
2021-09-02 04:37:49 +00:00
int _abPathIdx ; // current active path
std : : deque < int > _abFailoverQueue ;
2021-05-04 00:59:31 +00:00
uint8_t _abLinkSelectMethod ; // link re-selection policy for the primary link in active-backup
2020-05-12 08:35:48 +00:00
// balance-rr
2021-05-04 00:59:31 +00:00
uint8_t _rrIdx ; // index to path currently in use during Round Robin operation
uint16_t _rrPacketsSentOnCurrLink ; // number of packets sent on this link since the most recent path switch.
2020-05-12 08:35:48 +00:00
/**
* How many packets will be sent on a path before moving to the next path
* in the round - robin sequence . A value of zero will cause a random path
* selection for each outgoing packet .
*/
2020-06-17 21:54:13 +00:00
int _packetsPerLink ;
2020-05-12 08:35:48 +00:00
// balance-aware
uint64_t _totalBondUnderload ;
2020-06-17 21:54:13 +00:00
// dynamic link monitoring
uint8_t _linkMonitorStrategy ;
2020-05-12 08:35:48 +00:00
// path negotiation
int16_t _localUtility ;
2021-12-09 21:43:52 +00:00
int _negotiatedPathIdx ;
2020-05-12 08:35:48 +00:00
uint8_t _numSentPathNegotiationRequests ;
bool _allowPathNegotiation ;
2020-08-07 01:10:40 +00:00
/**
* Timers and intervals
*/
2022-10-10 06:07:16 +00:00
uint64_t _failoverInterval ;
uint64_t _qosSendInterval ;
uint64_t _ackSendInterval ;
uint64_t throughputMeasurementInterval ;
uint64_t _qualityEstimationInterval ;
2020-05-12 08:35:48 +00:00
2020-08-07 01:10:40 +00:00
/**
* Link state reporting
*/
2020-07-28 06:01:12 +00:00
uint8_t _numAliveLinks ;
uint8_t _numTotalLinks ;
2020-05-12 08:35:48 +00:00
/**
* Default initial punishment inflicted on misbehaving paths . Punishment slowly
* drains linearly . For each eligibility change the remaining punishment is doubled .
*/
uint32_t _defaultPathRefractoryPeriod ;
2021-09-02 04:37:49 +00:00
unsigned char _freeRandomByte ; // Free byte of entropy that is updated on every packet egress event.
SharedPtr < Peer > _peer ; // Remote peer that this bond services
unsigned long long _peerId ; // ID of the peer that this bond services
2022-02-17 23:16:33 +00:00
bool _isLeaf ;
2020-05-12 08:35:48 +00:00
/**
2021-09-02 04:37:49 +00:00
* Rate - limiting
2020-08-07 01:10:40 +00:00
*/
uint16_t _qosCutoffCount ;
2022-06-08 17:01:54 +00:00
uint16_t _ackCutoffCount ;
2021-09-02 04:37:49 +00:00
uint64_t _lastQoSRateCheck ;
2022-06-08 17:01:54 +00:00
uint64_t _lastAckRateCheck ;
2021-09-02 04:37:49 +00:00
uint16_t _pathNegotiationCutoffCount ;
uint64_t _lastPathNegotiationReceived ;
2020-08-07 01:10:40 +00:00
/**
* Recent event timestamps
*/
2021-09-02 04:37:49 +00:00
uint64_t _lastSummaryDump ;
2020-08-07 01:10:40 +00:00
uint64_t _lastQualityEstimation ;
uint64_t _lastBackgroundTaskCheck ;
uint64_t _lastBondStatusLog ;
uint64_t _lastPathNegotiationCheck ;
uint64_t _lastSentPathNegotiationRequest ;
uint64_t _lastFlowExpirationCheck ;
uint64_t _lastFlowRebalance ;
uint64_t _lastFrame ;
uint64_t _lastActiveBackupPathChange ;
2020-05-12 08:35:48 +00:00
Mutex _paths_m ;
2021-09-02 04:37:49 +00:00
Mutex _flows_m ;
2020-05-12 08:35:48 +00:00
2021-09-02 04:37:49 +00:00
bool _userHasSpecifiedLinks ; // Whether the user has specified links for this bond.
bool _userHasSpecifiedPrimaryLink ; // Whether the user has specified a primary link for this bond.
bool _userHasSpecifiedFailoverInstructions ; // Whether the user has specified failover instructions for this bond.
2022-10-10 06:07:16 +00:00
bool _userHasSpecifiedLinkCapacities ; // Whether the user has specified links capacities for this bond.
2020-05-12 08:35:48 +00:00
/**
* How frequently ( in ms ) a VERB_ECHO is sent to a peer to verify that a
* path is still active . A value of zero ( 0 ) will disable active path
* monitoring ; as result , all monitoring will be a function of traffic .
*/
2021-09-02 04:37:49 +00:00
int _monitorInterval ;
bool _allowFlowHashing ; // Whether or not flow hashing is allowed.
2020-05-12 08:35:48 +00:00
2021-09-02 04:37:49 +00:00
uint64_t _overheadBytes ;
2020-05-12 08:35:48 +00:00
} ;
2021-05-04 00:59:31 +00:00
} // namespace ZeroTier
2020-05-12 08:35:48 +00:00
2021-04-07 22:15:35 +00:00
# endif