2015-10-14 21:12:12 +00:00
|
|
|
/*
|
|
|
|
* ZeroTier One - Network Virtualization Everywhere
|
|
|
|
* Copyright (C) 2011-2015 ZeroTier, Inc.
|
|
|
|
*
|
|
|
|
* This program is free software: you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*
|
|
|
|
* --
|
|
|
|
*
|
|
|
|
* ZeroTier may be used and distributed under the terms of the GPLv3, which
|
|
|
|
* are available at: http://www.gnu.org/licenses/gpl-3.0.html
|
|
|
|
*
|
|
|
|
* If you would like to embed ZeroTier into a commercial application or
|
|
|
|
* redistribute it in a modified binary form, please contact ZeroTier Networks
|
|
|
|
* LLC. Start here: http://www.zerotier.com/
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef ZT_CLUSTER_HPP
|
|
|
|
#define ZT_CLUSTER_HPP
|
|
|
|
|
|
|
|
#ifdef ZT_ENABLE_CLUSTER
|
|
|
|
|
|
|
|
#include <vector>
|
|
|
|
#include <algorithm>
|
|
|
|
|
|
|
|
#include "Constants.hpp"
|
2015-10-20 22:27:53 +00:00
|
|
|
#include "../include/ZeroTierOne.h"
|
2015-10-14 21:12:12 +00:00
|
|
|
#include "Address.hpp"
|
|
|
|
#include "InetAddress.hpp"
|
|
|
|
#include "SHA512.hpp"
|
|
|
|
#include "Utils.hpp"
|
|
|
|
#include "Buffer.hpp"
|
|
|
|
#include "Mutex.hpp"
|
2015-10-20 22:27:53 +00:00
|
|
|
#include "SharedPtr.hpp"
|
2015-10-20 23:24:21 +00:00
|
|
|
#include "Hashtable.hpp"
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Timeout for cluster members being considered "alive"
|
2015-10-29 17:10:09 +00:00
|
|
|
*
|
|
|
|
* A cluster member is considered dead and will no longer have peers
|
|
|
|
* redirected to it if we have not heard a heartbeat in this long.
|
2015-10-14 21:12:12 +00:00
|
|
|
*/
|
2015-10-29 17:10:09 +00:00
|
|
|
#define ZT_CLUSTER_TIMEOUT 10000
|
2015-10-14 21:12:12 +00:00
|
|
|
|
2015-10-20 23:24:21 +00:00
|
|
|
/**
|
|
|
|
* How often should we announce that we have a peer?
|
|
|
|
*/
|
2015-11-02 23:15:20 +00:00
|
|
|
#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2)
|
2015-10-20 23:24:21 +00:00
|
|
|
|
2015-10-14 21:12:12 +00:00
|
|
|
/**
|
2015-10-20 22:27:53 +00:00
|
|
|
* Desired period between doPeriodicTasks() in milliseconds
|
2015-10-16 19:10:57 +00:00
|
|
|
*/
|
2015-10-29 17:10:09 +00:00
|
|
|
#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
|
2015-10-16 19:10:57 +00:00
|
|
|
|
2015-11-02 23:15:20 +00:00
|
|
|
/**
|
|
|
|
* How often to flush outgoing message queues (maximum interval)
|
|
|
|
*/
|
|
|
|
#define ZT_CLUSTER_FLUSH_PERIOD 500
|
|
|
|
|
2015-10-14 21:12:12 +00:00
|
|
|
namespace ZeroTier {
|
|
|
|
|
|
|
|
class RuntimeEnvironment;
|
|
|
|
class CertificateOfMembership;
|
|
|
|
class MulticastGroup;
|
2015-10-20 22:27:53 +00:00
|
|
|
class Peer;
|
|
|
|
class Identity;
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Multi-homing cluster state replication and packet relaying
|
|
|
|
*
|
|
|
|
* Multi-homing means more than one node sharing the same ZeroTier identity.
|
|
|
|
* There is nothing in the protocol to prevent this, but to make it work well
|
|
|
|
* requires the devices sharing an identity to cooperate and share some
|
|
|
|
* information.
|
|
|
|
*
|
|
|
|
* There are three use cases we want to fulfill:
|
|
|
|
*
|
|
|
|
* (1) Multi-homing of root servers with handoff for efficient routing,
|
|
|
|
* HA, and load balancing across many commodity nodes.
|
|
|
|
* (2) Multi-homing of network controllers for the same reason.
|
|
|
|
* (3) Multi-homing of nodes on virtual networks, such as domain servers
|
|
|
|
* and other important endpoints.
|
|
|
|
*
|
|
|
|
* These use cases are in order of escalating difficulty. The initial
|
|
|
|
* version of Cluster is aimed at satisfying the first, though you are
|
|
|
|
* free to try #2 and #3.
|
|
|
|
*/
|
|
|
|
class Cluster
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
/**
|
|
|
|
* State message types
|
|
|
|
*/
|
|
|
|
enum StateMessageType
|
|
|
|
{
|
|
|
|
STATE_MESSAGE_NOP = 0,
|
|
|
|
|
|
|
|
/**
|
|
|
|
* This cluster member is alive:
|
|
|
|
* <[2] version minor>
|
|
|
|
* <[2] version major>
|
|
|
|
* <[2] version revision>
|
|
|
|
* <[1] protocol version>
|
|
|
|
* <[4] X location (signed 32-bit)>
|
|
|
|
* <[4] Y location (signed 32-bit)>
|
|
|
|
* <[4] Z location (signed 32-bit)>
|
|
|
|
* <[8] local clock at this member>
|
|
|
|
* <[8] load average>
|
|
|
|
* <[8] flags (currently unused, must be zero)>
|
|
|
|
* <[1] number of preferred ZeroTier endpoints>
|
|
|
|
* <[...] InetAddress(es) of preferred ZeroTier endpoint(s)>
|
|
|
|
*/
|
|
|
|
STATE_MESSAGE_ALIVE = 1,
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Cluster member has this peer:
|
|
|
|
* <[...] binary serialized peer identity>
|
2015-10-27 23:47:13 +00:00
|
|
|
* <[...] binary serialized peer remote physical address>
|
|
|
|
*
|
|
|
|
* Clusters send this message when they learn a path to a peer. The
|
|
|
|
* replicated physical address is the one learned.
|
2015-10-14 21:12:12 +00:00
|
|
|
*/
|
|
|
|
STATE_MESSAGE_HAVE_PEER = 2,
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Peer subscription to multicast group:
|
|
|
|
* <[8] network ID>
|
|
|
|
* <[5] peer ZeroTier address>
|
|
|
|
* <[6] MAC address of multicast group>
|
|
|
|
* <[4] 32-bit multicast group ADI>
|
|
|
|
*/
|
|
|
|
STATE_MESSAGE_MULTICAST_LIKE = 3,
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Certificate of network membership for a peer:
|
|
|
|
* <[...] serialized COM>
|
|
|
|
*/
|
|
|
|
STATE_MESSAGE_COM = 4,
|
|
|
|
|
|
|
|
/**
|
2015-10-27 21:04:12 +00:00
|
|
|
* Request that VERB_RENDEZVOUS be sent to a peer that we have:
|
|
|
|
* <[5] ZeroTier address of peer on recipient's side>
|
|
|
|
* <[5] ZeroTier address of peer on sender's side>
|
|
|
|
* <[1] 8-bit number of sender's peer's active path addresses>
|
|
|
|
* <[...] series of serialized InetAddresses of sender's peer's paths>
|
|
|
|
*
|
|
|
|
* This requests that we perform NAT-t introduction between a peer that
|
|
|
|
* we have and one on the sender's side. The sender furnishes contact
|
|
|
|
* info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours
|
|
|
|
* directly and with PROXY_SEND to theirs.
|
2015-10-14 21:12:12 +00:00
|
|
|
*/
|
2015-10-27 21:04:12 +00:00
|
|
|
STATE_MESSAGE_PROXY_UNITE = 5,
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
/**
|
2015-10-23 20:03:34 +00:00
|
|
|
* Request that a cluster member send a packet to a locally-known peer:
|
2015-10-14 21:12:12 +00:00
|
|
|
* <[5] ZeroTier address of recipient>
|
|
|
|
* <[1] packet verb>
|
|
|
|
* <[2] length of packet payload>
|
|
|
|
* <[...] packet payload>
|
|
|
|
*
|
|
|
|
* This differs from RELAY in that it requests the receiving cluster
|
|
|
|
* member to actually compose a ZeroTier Packet from itself to the
|
|
|
|
* provided recipient. RELAY simply says "please forward this blob."
|
|
|
|
* RELAY is used to implement peer-to-peer relaying with RENDEZVOUS,
|
|
|
|
* while PROXY_SEND is used to implement proxy sending (which right
|
|
|
|
* now is only used to send RENDEZVOUS).
|
|
|
|
*/
|
2015-10-27 17:37:39 +00:00
|
|
|
STATE_MESSAGE_PROXY_SEND = 6,
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Replicate a network config for a network we belong to:
|
|
|
|
* <[8] 64-bit network ID>
|
|
|
|
* <[2] 16-bit length of network config>
|
|
|
|
* <[...] serialized network config>
|
|
|
|
*
|
|
|
|
* This is used by clusters to avoid every member having to query
|
|
|
|
* for the same netconf for networks all members belong to.
|
|
|
|
*
|
|
|
|
* TODO: not implemented yet!
|
|
|
|
*/
|
|
|
|
STATE_MESSAGE_NETWORK_CONFIG = 7
|
2015-10-14 21:12:12 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Construct a new cluster
|
|
|
|
*/
|
|
|
|
Cluster(
|
|
|
|
const RuntimeEnvironment *renv,
|
|
|
|
uint16_t id,
|
2015-10-20 22:27:53 +00:00
|
|
|
const std::vector<InetAddress> &zeroTierPhysicalEndpoints,
|
2015-10-14 21:12:12 +00:00
|
|
|
int32_t x,
|
|
|
|
int32_t y,
|
|
|
|
int32_t z,
|
2015-10-20 22:27:53 +00:00
|
|
|
void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
|
|
|
|
void *sendFunctionArg,
|
|
|
|
int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
|
|
|
|
void *addressToLocationFunctionArg);
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
~Cluster();
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @return This cluster member's ID
|
|
|
|
*/
|
|
|
|
inline uint16_t id() const throw() { return _id; }
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Handle an incoming intra-cluster message
|
|
|
|
*
|
|
|
|
* @param data Message data
|
|
|
|
* @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH)
|
|
|
|
*/
|
|
|
|
void handleIncomingStateMessage(const void *msg,unsigned int len);
|
|
|
|
|
2015-10-21 00:22:53 +00:00
|
|
|
/**
|
|
|
|
* Send this packet via another node in this cluster if another node has this peer
|
|
|
|
*
|
|
|
|
* @param fromPeerAddress Source peer address (if known, should be NULL for fragments)
|
|
|
|
* @param toPeerAddress Destination peer address
|
|
|
|
* @param data Packet or packet fragment data
|
|
|
|
* @param len Length of packet or fragment
|
2015-10-27 21:04:12 +00:00
|
|
|
* @param unite If true, also request proxy unite across cluster
|
2015-10-21 00:22:53 +00:00
|
|
|
* @return True if this data was sent via another cluster member, false if none have this peer
|
|
|
|
*/
|
2015-10-27 21:04:12 +00:00
|
|
|
bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
|
2015-10-21 00:22:53 +00:00
|
|
|
|
2015-10-14 21:12:12 +00:00
|
|
|
/**
|
|
|
|
* Advertise to the cluster that we have this peer
|
|
|
|
*
|
2015-10-20 22:27:53 +00:00
|
|
|
* @param peerId Identity of peer that we have
|
2015-10-27 23:47:13 +00:00
|
|
|
* @param physicalAddress Physical address of peer (from our POV)
|
2015-10-14 21:12:12 +00:00
|
|
|
*/
|
2015-10-27 23:47:13 +00:00
|
|
|
void replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress);
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Advertise a multicast LIKE to the cluster
|
|
|
|
*
|
|
|
|
* @param nwid Network ID
|
|
|
|
* @param peerAddress Peer address that sent LIKE
|
|
|
|
* @param group Multicast group
|
|
|
|
*/
|
|
|
|
void replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Advertise a network COM to the cluster
|
|
|
|
*
|
|
|
|
* @param com Certificate of network membership (contains peer and network ID)
|
|
|
|
*/
|
|
|
|
void replicateCertificateOfNetworkMembership(const CertificateOfMembership &com);
|
|
|
|
|
|
|
|
/**
|
2015-10-20 22:27:53 +00:00
|
|
|
* Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds.
|
2015-10-14 21:12:12 +00:00
|
|
|
*/
|
|
|
|
void doPeriodicTasks();
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Add a member ID to this cluster
|
|
|
|
*
|
|
|
|
* @param memberId Member ID
|
|
|
|
*/
|
|
|
|
void addMember(uint16_t memberId);
|
|
|
|
|
2015-10-20 22:27:53 +00:00
|
|
|
/**
|
|
|
|
* Remove a member ID from this cluster
|
|
|
|
*
|
|
|
|
* @param memberId Member ID to remove
|
|
|
|
*/
|
|
|
|
void removeMember(uint16_t memberId);
|
|
|
|
|
|
|
|
/**
|
2015-10-27 21:37:38 +00:00
|
|
|
* Find a better cluster endpoint for this peer (if any)
|
2015-10-27 16:36:48 +00:00
|
|
|
*
|
2015-10-27 21:37:38 +00:00
|
|
|
* @param redirectTo InetAddress to be set to a better endpoint (if there is one)
|
2015-10-27 16:36:48 +00:00
|
|
|
* @param peerAddress Address of peer to (possibly) redirect
|
2015-10-20 22:27:53 +00:00
|
|
|
* @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
|
|
|
|
* @param offload Always redirect if possible -- can be used to offload peers during shutdown
|
2015-10-27 21:37:38 +00:00
|
|
|
* @return True if redirectTo was set to a new address, false if redirectTo was not modified
|
2015-10-20 22:27:53 +00:00
|
|
|
*/
|
2015-10-27 21:37:38 +00:00
|
|
|
bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
|
2015-10-20 22:27:53 +00:00
|
|
|
|
2015-10-26 19:41:08 +00:00
|
|
|
/**
|
|
|
|
* Fill out ZT_ClusterStatus structure (from core API)
|
|
|
|
*
|
|
|
|
* @param status Reference to structure to hold result (anything there is replaced)
|
|
|
|
*/
|
|
|
|
void status(ZT_ClusterStatus &status) const;
|
|
|
|
|
2015-10-14 21:12:12 +00:00
|
|
|
private:
|
2015-10-20 23:24:21 +00:00
|
|
|
void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len);
|
2015-10-14 21:12:12 +00:00
|
|
|
void _flush(uint16_t memberId);
|
|
|
|
|
2015-10-27 22:57:26 +00:00
|
|
|
// These are initialized in the constructor and remain immutable
|
2015-10-14 21:12:12 +00:00
|
|
|
uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
|
|
|
|
unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
|
|
|
|
const RuntimeEnvironment *RR;
|
2015-10-20 22:27:53 +00:00
|
|
|
void (*_sendFunction)(void *,unsigned int,const void *,unsigned int);
|
|
|
|
void *_sendFunctionArg;
|
|
|
|
int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *);
|
|
|
|
void *_addressToLocationFunctionArg;
|
2015-10-14 21:12:12 +00:00
|
|
|
const int32_t _x;
|
|
|
|
const int32_t _y;
|
|
|
|
const int32_t _z;
|
|
|
|
const uint16_t _id;
|
2015-10-20 22:27:53 +00:00
|
|
|
const std::vector<InetAddress> _zeroTierPhysicalEndpoints;
|
2015-10-27 22:57:26 +00:00
|
|
|
// end immutable fields
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
struct _Member
|
|
|
|
{
|
|
|
|
unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
|
|
|
|
|
|
|
|
uint64_t lastReceivedAliveAnnouncement;
|
|
|
|
uint64_t lastAnnouncedAliveTo;
|
|
|
|
|
|
|
|
uint64_t load;
|
|
|
|
int32_t x,y,z;
|
|
|
|
|
2015-10-20 22:27:53 +00:00
|
|
|
std::vector<InetAddress> zeroTierPhysicalEndpoints;
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> q;
|
|
|
|
|
|
|
|
Mutex lock;
|
|
|
|
|
2015-10-20 22:27:53 +00:00
|
|
|
inline void clear()
|
|
|
|
{
|
|
|
|
lastReceivedAliveAnnouncement = 0;
|
|
|
|
lastAnnouncedAliveTo = 0;
|
|
|
|
load = 0;
|
|
|
|
x = 0;
|
|
|
|
y = 0;
|
|
|
|
z = 0;
|
|
|
|
zeroTierPhysicalEndpoints.clear();
|
|
|
|
q.clear();
|
|
|
|
}
|
|
|
|
|
|
|
|
_Member() { this->clear(); }
|
2015-10-14 21:12:12 +00:00
|
|
|
~_Member() { Utils::burn(key,sizeof(key)); }
|
|
|
|
};
|
2015-10-27 22:57:26 +00:00
|
|
|
_Member *const _members;
|
2015-10-14 21:12:12 +00:00
|
|
|
|
|
|
|
std::vector<uint16_t> _memberIds;
|
|
|
|
Mutex _memberIds_m;
|
|
|
|
|
2015-10-27 22:57:26 +00:00
|
|
|
struct _PA
|
2015-10-14 21:12:12 +00:00
|
|
|
{
|
2015-10-27 22:57:26 +00:00
|
|
|
_PA() : ts(0),mid(0xffff) {}
|
|
|
|
uint64_t ts;
|
|
|
|
uint16_t mid;
|
2015-10-14 21:12:12 +00:00
|
|
|
};
|
2015-10-27 22:57:26 +00:00
|
|
|
Hashtable< Address,_PA > _peerAffinities;
|
2015-10-14 21:12:12 +00:00
|
|
|
Mutex _peerAffinities_m;
|
2015-10-29 17:10:09 +00:00
|
|
|
|
2015-10-28 00:36:47 +00:00
|
|
|
uint64_t _lastCleanedPeerAffinities;
|
2015-10-29 17:10:09 +00:00
|
|
|
uint64_t _lastCheckedPeersForAnnounce;
|
2015-11-02 23:15:20 +00:00
|
|
|
uint64_t _lastFlushed;
|
2015-10-14 21:12:12 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace ZeroTier
|
|
|
|
|
|
|
|
#endif // ZT_ENABLE_CLUSTER
|
|
|
|
|
|
|
|
#endif
|