ExternalVendorCode/ZeroTierOne
1
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2024-12-21 05:53:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
404a0bbddd
ZeroTierOne/node/Cluster.cpp

914 lines
32 KiB
C++
Raw Blame History

/*
* ZeroTier One - Network Virtualization Everywhere
* Copyright (C) 2011-2016 ZeroTier, Inc. https://www.zerotier.com/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifdef ZT_ENABLE_CLUSTER
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <map>
#include <algorithm>
#include <set>
#include <utility>
#include <list>
#include <stdexcept>
#include "../version.h"
#include "Cluster.hpp"
#include "RuntimeEnvironment.hpp"
#include "MulticastGroup.hpp"
#include "CertificateOfMembership.hpp"
#include "Salsa20.hpp"
#include "Poly1305.hpp"
#include "Identity.hpp"
#include "Topology.hpp"
#include "Packet.hpp"
#include "Switch.hpp"
#include "Node.hpp"
#include "Array.hpp"
namespace ZeroTier {
static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2)
throw()
{
double dx = ((double)x2 - (double)x1);
double dy = ((double)y2 - (double)y1);
double dz = ((double)z2 - (double)z1);
return sqrt((dx * dx) + (dy * dy) + (dz * dz));
}
// An entry in _ClusterSendQueue
struct _ClusterSendQueueEntry
{
uint64_t timestamp;
Address fromPeerAddress;
Address toPeerAddress;
// if we ever support larger transport MTUs this must be increased
unsigned char data[ZT_CLUSTER_SEND_QUEUE_DATA_MAX];
unsigned int len;
bool unite;
};
// A multi-index map with entry memory pooling -- this allows our queue to
// be O(log(N)) and is complex enough that it makes the code a lot cleaner
// to break it out from Cluster.
class _ClusterSendQueue
{
public:
_ClusterSendQueue() :
_poolCount(0) {}
~_ClusterSendQueue() {} // memory is automatically freed when _chunks is destroyed
inline void enqueue(uint64_t now,const Address &from,const Address &to,const void *data,unsigned int len,bool unite)
{
if (len > ZT_CLUSTER_SEND_QUEUE_DATA_MAX)
return;
Mutex::Lock _l(_lock);
// Delete oldest queue entry for this sender if this enqueue() would take them over the per-sender limit
{
std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator qi(_bySrc.lower_bound(std::pair<Address,_ClusterSendQueueEntry *>(from,(_ClusterSendQueueEntry *)0)));
std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator oldest(qi);
unsigned long countForSender = 0;
while ((qi != _bySrc.end())&&(qi->first == from)) {
if (qi->second->timestamp < oldest->second->timestamp)
oldest = qi;
++countForSender;
++qi;
}
if (countForSender >= ZT_CLUSTER_MAX_QUEUE_PER_SENDER) {
_byDest.erase(std::pair<Address,_ClusterSendQueueEntry *>(oldest->second->toPeerAddress,oldest->second));
_pool[_poolCount++] = oldest->second;
_bySrc.erase(oldest);
}
}
_ClusterSendQueueEntry *e;
if (_poolCount > 0) {
e = _pool[--_poolCount];
} else {
if (_chunks.size() >= ZT_CLUSTER_MAX_QUEUE_CHUNKS)
return; // queue is totally full!
_chunks.push_back(Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE>());
e = &(_chunks.back().data[0]);
for(unsigned int i=1;i<ZT_CLUSTER_QUEUE_CHUNK_SIZE;++i)
_pool[_poolCount++] = &(_chunks.back().data[i]);
}
e->timestamp = now;
e->fromPeerAddress = from;
e->toPeerAddress = to;
memcpy(e->data,data,len);
e->len = len;
e->unite = unite;
_bySrc.insert(std::pair<Address,_ClusterSendQueueEntry *>(from,e));
_byDest.insert(std::pair<Address,_ClusterSendQueueEntry *>(to,e));
}
inline void expire(uint64_t now)
{
Mutex::Lock _l(_lock);
for(std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator qi(_bySrc.begin());qi!=_bySrc.end();) {
if ((now - qi->second->timestamp) > ZT_CLUSTER_QUEUE_EXPIRATION) {
_byDest.erase(std::pair<Address,_ClusterSendQueueEntry *>(qi->second->toPeerAddress,qi->second));
_pool[_poolCount++] = qi->second;
_bySrc.erase(qi++);
} else ++qi;
}
}
/**
* Get and dequeue entries for a given destination address
*
* After use these entries must be returned with returnToPool()!
*
* @param dest Destination address
* @param results Array to fill with results
* @param maxResults Size of results[] in pointers
* @return Number of actual results returned
*/
inline unsigned int getByDest(const Address &dest,_ClusterSendQueueEntry **results,unsigned int maxResults)
{
unsigned int count = 0;
Mutex::Lock _l(_lock);
std::set< std::pair<Address,_ClusterSendQueueEntry *> >::iterator qi(_byDest.lower_bound(std::pair<Address,_ClusterSendQueueEntry *>(dest,(_ClusterSendQueueEntry *)0)));
while ((qi != _byDest.end())&&(qi->first == dest)) {
_bySrc.erase(std::pair<Address,_ClusterSendQueueEntry *>(qi->second->fromPeerAddress,qi->second));
results[count++] = qi->second;
if (count == maxResults)
break;
_byDest.erase(qi++);
}
return count;
}
/**
* Return entries to pool after use
*
* @param entries Array of entries
* @param count Number of entries
*/
inline void returnToPool(_ClusterSendQueueEntry **entries,unsigned int count)
{
Mutex::Lock _l(_lock);
for(unsigned int i=0;i<count;++i)
_pool[_poolCount++] = entries[i];
}
private:
std::list< Array<_ClusterSendQueueEntry,ZT_CLUSTER_QUEUE_CHUNK_SIZE> > _chunks;
_ClusterSendQueueEntry *_pool[ZT_CLUSTER_QUEUE_CHUNK_SIZE * ZT_CLUSTER_MAX_QUEUE_CHUNKS];
unsigned long _poolCount;
std::set< std::pair<Address,_ClusterSendQueueEntry *> > _bySrc;
std::set< std::pair<Address,_ClusterSendQueueEntry *> > _byDest;
Mutex _lock;
};
Cluster::Cluster(
const RuntimeEnvironment *renv,
uint16_t id,
const std::vector<InetAddress> &zeroTierPhysicalEndpoints,
int32_t x,
int32_t y,
int32_t z,
void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
void *sendFunctionArg,
int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
void *addressToLocationFunctionArg) :
RR(renv),
_sendQueue(new _ClusterSendQueue()),
_sendFunction(sendFunction),
_sendFunctionArg(sendFunctionArg),
_addressToLocationFunction(addressToLocationFunction),
_addressToLocationFunctionArg(addressToLocationFunctionArg),
_x(x),
_y(y),
_z(z),
_id(id),
_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
_lastFlushed(0),
_lastCleanedRemotePeers(0),
_lastCleanedQueue(0)
{
uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
// Generate master secret by hashing the secret from our Identity key pair
RR->identity.sha512PrivateKey(_masterSecret);
// Generate our inbound message key, which is the master secret XORed with our ID and hashed twice
memcpy(stmp,_masterSecret,sizeof(stmp));
stmp[0] ^= Utils::hton(id);
SHA512::hash(stmp,stmp,sizeof(stmp));
SHA512::hash(stmp,stmp,sizeof(stmp));
memcpy(_key,stmp,sizeof(_key));
Utils::burn(stmp,sizeof(stmp));
}
Cluster::~Cluster()
{
Utils::burn(_masterSecret,sizeof(_masterSecret));
Utils::burn(_key,sizeof(_key));
delete [] _members;
delete _sendQueue;
}
void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
{
Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> dmsg;
{
// FORMAT: <[16] iv><[8] MAC><... data>
if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH))
return;
// 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV
char keytmp[32];
memcpy(keytmp,_key,32);
for(int i=0;i<8;++i)
keytmp[i] ^= reinterpret_cast<const char *>(msg)[i];
Salsa20 s20(keytmp,256,reinterpret_cast<const char *>(msg) + 8);
Utils::burn(keytmp,sizeof(keytmp));
// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
char polykey[ZT_POLY1305_KEY_LEN];
memset(polykey,0,sizeof(polykey));
s20.encrypt12(polykey,polykey,sizeof(polykey));
// Compute 16-byte MAC
char mac[ZT_POLY1305_MAC_LEN];
Poly1305::compute(mac,reinterpret_cast<const char *>(msg) + 24,len - 24,polykey);
// Check first 8 bytes of MAC against 64-bit MAC in stream
if (!Utils::secureEq(mac,reinterpret_cast<const char *>(msg) + 16,8))
return;
// Decrypt!
dmsg.setSize(len - 24);
s20.decrypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
}
if (dmsg.size() < 4)
return;
const uint16_t fromMemberId = dmsg.at<uint16_t>(0);
unsigned int ptr = 2;
if (fromMemberId == _id) // sanity check: we don't talk to ourselves
return;
const uint16_t toMemberId = dmsg.at<uint16_t>(ptr);
ptr += 2;
if (toMemberId != _id) // sanity check: message not for us?
return;
{ // make sure sender is actually considered a member
Mutex::Lock _l3(_memberIds_m);
if (std::find(_memberIds.begin(),_memberIds.end(),fromMemberId) == _memberIds.end())
return;
}
try {
while (ptr < dmsg.size()) {
const unsigned int mlen = dmsg.at<uint16_t>(ptr); ptr += 2;
const unsigned int nextPtr = ptr + mlen;
if (nextPtr > dmsg.size())
break;
int mtype = -1;
try {
switch((StateMessageType)(mtype = (int)dmsg[ptr++])) {
default:
break;
case CLUSTER_MESSAGE_ALIVE: {
_Member &m = _members[fromMemberId];
Mutex::Lock mlck(m.lock);
ptr += 7; // skip version stuff, not used yet
m.x = dmsg.at<int32_t>(ptr); ptr += 4;
m.y = dmsg.at<int32_t>(ptr); ptr += 4;
m.z = dmsg.at<int32_t>(ptr); ptr += 4;
ptr += 8; // skip local clock, not used
m.load = dmsg.at<uint64_t>(ptr); ptr += 8;
m.peers = dmsg.at<uint64_t>(ptr); ptr += 8;
ptr += 8; // skip flags, unused
#ifdef ZT_TRACE
std::string addrs;
#endif
unsigned int physicalAddressCount = dmsg[ptr++];
m.zeroTierPhysicalEndpoints.clear();
for(unsigned int i=0;i<physicalAddressCount;++i) {
m.zeroTierPhysicalEndpoints.push_back(InetAddress());
ptr += m.zeroTierPhysicalEndpoints.back().deserialize(dmsg,ptr);
if (!(m.zeroTierPhysicalEndpoints.back())) {
m.zeroTierPhysicalEndpoints.pop_back();
}
#ifdef ZT_TRACE
else {
if (addrs.length() > 0)
addrs.push_back(',');
addrs.append(m.zeroTierPhysicalEndpoints.back().toString());
}
#endif
}
#ifdef ZT_TRACE
if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) {
TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str());
}
#endif
m.lastReceivedAliveAnnouncement = RR->node->now();
} break;
case CLUSTER_MESSAGE_HAVE_PEER: {
Identity id;
ptr += id.deserialize(dmsg,ptr);
if (id) {
RR->topology->saveIdentity(id);
{
Mutex::Lock _l(_remotePeers_m);
_remotePeers[std::pair<Address,unsigned int>(id.address(),(unsigned int)fromMemberId)] = RR->node->now();
}
_ClusterSendQueueEntry *q[16384]; // 16384 is "tons"
unsigned int qc = _sendQueue->getByDest(id.address(),q,16384);
for(unsigned int i=0;i<qc;++i)
this->sendViaCluster(q[i]->fromPeerAddress,q[i]->toPeerAddress,q[i]->data,q[i]->len,q[i]->unite);
_sendQueue->returnToPool(q,qc);
TRACE("[%u] has %s (retried %u queued sends)",(unsigned int)fromMemberId,id.address().toString().c_str(),qc);
}
} break;
case CLUSTER_MESSAGE_WANT_PEER: {
const Address zeroTierAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
SharedPtr<Peer> peer(RR->topology->getPeerNoCache(zeroTierAddress));
if ( (peer) && (peer->hasClusterOptimalPath(RR->node->now())) ) {
Buffer<1024> buf;
peer->identity().serialize(buf);
Mutex::Lock _l2(_members[fromMemberId].lock);
_send(fromMemberId,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size());
}
} break;
case CLUSTER_MESSAGE_REMOTE_PACKET: {
const unsigned int plen = dmsg.at<uint16_t>(ptr); ptr += 2;
if (plen) {
Packet remotep(dmsg.field(ptr,plen),plen); ptr += plen;
//TRACE("remote %s from %s via %u (%u bytes)",Packet::verbString(remotep.verb()),remotep.source().toString().c_str(),fromMemberId,plen);
switch(remotep.verb()) {
case Packet::VERB_WHOIS: _doREMOTE_WHOIS(fromMemberId,remotep); break;
case Packet::VERB_MULTICAST_GATHER: _doREMOTE_MULTICAST_GATHER(fromMemberId,remotep); break;
default: break; // ignore things we don't care about across cluster
}
}
} break;
case CLUSTER_MESSAGE_PROXY_UNITE: {
const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
const unsigned int numRemotePeerPaths = dmsg[ptr++];
InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max
for(unsigned int i=0;i<numRemotePeerPaths;++i)
ptr += remotePeerPaths[i].deserialize(dmsg,ptr);
TRACE("[%u] requested that we unite local %s with remote %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
const uint64_t now = RR->node->now();
SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress));
if ((localPeer)&&(numRemotePeerPaths > 0)) {
InetAddress bestLocalV4,bestLocalV6;
localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6);
InetAddress bestRemoteV4,bestRemoteV6;
for(unsigned int i=0;i<numRemotePeerPaths;++i) {
if ((bestRemoteV4)&&(bestRemoteV6))
break;
switch(remotePeerPaths[i].ss_family) {
case AF_INET:
if (!bestRemoteV4)
bestRemoteV4 = remotePeerPaths[i];
break;
case AF_INET6:
if (!bestRemoteV6)
bestRemoteV6 = remotePeerPaths[i];
break;
}
}
Packet rendezvousForLocal(localPeerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
rendezvousForLocal.append((uint8_t)0);
remotePeerAddress.appendTo(rendezvousForLocal);
Buffer<2048> rendezvousForRemote;
remotePeerAddress.appendTo(rendezvousForRemote);
rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS);
rendezvousForRemote.addSize(2); // space for actual packet payload length
rendezvousForRemote.append((uint8_t)0); // flags == 0
localPeerAddress.appendTo(rendezvousForRemote);
bool haveMatch = false;
if ((bestLocalV6)&&(bestRemoteV6)) {
haveMatch = true;
rendezvousForLocal.append((uint16_t)bestRemoteV6.port());
rendezvousForLocal.append((uint8_t)16);
rendezvousForLocal.append(bestRemoteV6.rawIpData(),16);
rendezvousForRemote.append((uint16_t)bestLocalV6.port());
rendezvousForRemote.append((uint8_t)16);
rendezvousForRemote.append(bestLocalV6.rawIpData(),16);
rendezvousForRemote.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 16));
} else if ((bestLocalV4)&&(bestRemoteV4)) {
haveMatch = true;
rendezvousForLocal.append((uint16_t)bestRemoteV4.port());
rendezvousForLocal.append((uint8_t)4);
rendezvousForLocal.append(bestRemoteV4.rawIpData(),4);
rendezvousForRemote.append((uint16_t)bestLocalV4.port());
rendezvousForRemote.append((uint8_t)4);
rendezvousForRemote.append(bestLocalV4.rawIpData(),4);
rendezvousForRemote.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(9 + 4));
}
if (haveMatch) {
{
Mutex::Lock _l2(_members[fromMemberId].lock);
_send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
}
RR->sw->send(rendezvousForLocal,true,0);
}
}
} break;
case CLUSTER_MESSAGE_PROXY_SEND: {
const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
const Packet::Verb verb = (Packet::Verb)dmsg[ptr++];
const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
Packet outp(rcpt,RR->identity.address(),verb);
outp.append(dmsg.field(ptr,len),len); ptr += len;
RR->sw->send(outp,true,0);
//TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
} break;
}
} catch ( ... ) {
TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype);
// drop invalids
}
ptr = nextPtr;
}
} catch ( ... ) {
TRACE("invalid message (outer loop), discarding");
// drop invalids
}
}
void Cluster::broadcastHavePeer(const Identity &id)
{
Buffer<1024> buf;
id.serialize(buf);
Mutex::Lock _l(_memberIds_m);
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
Mutex::Lock _l2(_members[*mid].lock);
_send(*mid,CLUSTER_MESSAGE_HAVE_PEER,buf.data(),buf.size());
}
}
void Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
{
if (len > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check
return;
const uint64_t now = RR->node->now();
uint64_t mostRecentTs = 0;
unsigned int mostRecentMemberId = 0xffffffff;
{
Mutex::Lock _l2(_remotePeers_m);
std::map< std::pair<Address,unsigned int>,uint64_t >::const_iterator rpe(_remotePeers.lower_bound(std::pair<Address,unsigned int>(toPeerAddress,0)));
for(;;) {
if ((rpe == _remotePeers.end())||(rpe->first.first != toPeerAddress))
break;
else if (rpe->second > mostRecentTs) {
mostRecentTs = rpe->second;
mostRecentMemberId = rpe->first.second;
}
++rpe;
}
}
const uint64_t age = now - mostRecentTs;
if (age >= (ZT_PEER_ACTIVITY_TIMEOUT / 3)) {
const bool enqueueAndWait = ((age >= ZT_PEER_ACTIVITY_TIMEOUT)||(mostRecentMemberId > 0xffff));
// Poll everyone with WANT_PEER if the age of our most recent entry is
// approaching expiration (or has expired, or does not exist).
char tmp[ZT_ADDRESS_LENGTH];
toPeerAddress.copyTo(tmp,ZT_ADDRESS_LENGTH);
{
Mutex::Lock _l(_memberIds_m);
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
Mutex::Lock _l2(_members[*mid].lock);
_send(*mid,CLUSTER_MESSAGE_WANT_PEER,tmp,ZT_ADDRESS_LENGTH);
}
}
// If there isn't a good place to send via, then enqueue this for retrying
// later and return after having broadcasted a WANT_PEER.
if (enqueueAndWait) {
TRACE("sendViaCluster %s -> %s enqueueing to wait for HAVE_PEER",fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
_sendQueue->enqueue(now,fromPeerAddress,toPeerAddress,data,len,unite);
return;
}
}
Buffer<1024> buf;
if (unite) {
InetAddress v4,v6;
if (fromPeerAddress) {
SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress));
if (fromPeer)
fromPeer->getBestActiveAddresses(now,v4,v6);
}
uint8_t addrCount = 0;
if (v4)
++addrCount;
if (v6)
++addrCount;
if (addrCount) {
toPeerAddress.appendTo(buf);
fromPeerAddress.appendTo(buf);
buf.append(addrCount);
if (v4)
v4.serialize(buf);
if (v6)
v6.serialize(buf);
}
}
{
Mutex::Lock _l2(_members[mostRecentMemberId].lock);
if (buf.size() > 0)
_send(mostRecentMemberId,CLUSTER_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
for(std::vector<InetAddress>::const_iterator i1(_zeroTierPhysicalEndpoints.begin());i1!=_zeroTierPhysicalEndpoints.end();++i1) {
for(std::vector<InetAddress>::const_iterator i2(_members[mostRecentMemberId].zeroTierPhysicalEndpoints.begin());i2!=_members[mostRecentMemberId].zeroTierPhysicalEndpoints.end();++i2) {
if (i1->ss_family == i2->ss_family) {
TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u (%s->%s)",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId,i1->toString().c_str(),i2->toString().c_str());
RR->node->putPacket(*i1,*i2,data,len);
return;
}
}
}
TRACE("sendViaCluster relaying %u bytes from %s to %s by way of %u failed: no common endpoints with the same address family!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)mostRecentMemberId);
return;
}
}
void Cluster::sendDistributedQuery(const Packet &pkt)
{
Buffer<4096> buf;
buf.append((uint16_t)pkt.size());
buf.append(pkt.data(),pkt.size());
Mutex::Lock _l(_memberIds_m);
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
Mutex::Lock _l2(_members[*mid].lock);
_send(*mid,CLUSTER_MESSAGE_REMOTE_PACKET,buf.data(),buf.size());
}
}
void Cluster::doPeriodicTasks()
{
const uint64_t now = RR->node->now();
if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) {
_lastFlushed = now;
Mutex::Lock _l(_memberIds_m);
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
Mutex::Lock _l2(_members[*mid].lock);
if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) {
_members[*mid].lastAnnouncedAliveTo = now;
Buffer<2048> alive;
alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR);
alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR);
alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
alive.append((uint8_t)ZT_PROTO_VERSION);
if (_addressToLocationFunction) {
alive.append((int32_t)_x);
alive.append((int32_t)_y);
alive.append((int32_t)_z);
} else {
alive.append((int32_t)0);
alive.append((int32_t)0);
alive.append((int32_t)0);
}
alive.append((uint64_t)now);
alive.append((uint64_t)0); // TODO: compute and send load average
alive.append((uint64_t)RR->topology->countActive(now));
alive.append((uint64_t)0); // unused/reserved flags
alive.append((uint8_t)_zeroTierPhysicalEndpoints.size());
for(std::vector<InetAddress>::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe)
pe->serialize(alive);
_send(*mid,CLUSTER_MESSAGE_ALIVE,alive.data(),alive.size());
}
_flush(*mid);
}
}
if ((now - _lastCleanedRemotePeers) >= (ZT_PEER_ACTIVITY_TIMEOUT * 2)) {
_lastCleanedRemotePeers = now;
Mutex::Lock _l(_remotePeers_m);
for(std::map< std::pair<Address,unsigned int>,uint64_t >::iterator rp(_remotePeers.begin());rp!=_remotePeers.end();) {
if ((now - rp->second) >= ZT_PEER_ACTIVITY_TIMEOUT)
_remotePeers.erase(rp++);
else ++rp;
}
}
if ((now - _lastCleanedQueue) >= ZT_CLUSTER_QUEUE_EXPIRATION) {
_lastCleanedQueue = now;
_sendQueue->expire(now);
}
}
void Cluster::addMember(uint16_t memberId)
{
if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id))
return;
Mutex::Lock _l2(_members[memberId].lock);
{
Mutex::Lock _l(_memberIds_m);
if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end())
return;
_memberIds.push_back(memberId);
std::sort(_memberIds.begin(),_memberIds.end());
}
_members[memberId].clear();
// Generate this member's message key from the master and its ID
uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
memcpy(stmp,_masterSecret,sizeof(stmp));
stmp[0] ^= Utils::hton(memberId);
SHA512::hash(stmp,stmp,sizeof(stmp));
SHA512::hash(stmp,stmp,sizeof(stmp));
memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key));
Utils::burn(stmp,sizeof(stmp));
// Prepare q
_members[memberId].q.clear();
char iv[16];
Utils::getSecureRandom(iv,16);
_members[memberId].q.append(iv,16);
_members[memberId].q.addSize(8); // room for MAC
_members[memberId].q.append((uint16_t)_id);
_members[memberId].q.append((uint16_t)memberId);
}
void Cluster::removeMember(uint16_t memberId)
{
Mutex::Lock _l(_memberIds_m);
std::vector<uint16_t> newMemberIds;
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
if (*mid != memberId)
newMemberIds.push_back(*mid);
}
_memberIds = newMemberIds;
}
bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
{
if (_addressToLocationFunction) {
// Pick based on location if it can be determined
int px = 0,py = 0,pz = 0;
if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
TRACE("no geolocation data for %s",peerPhysicalAddress.toIpString().c_str());
return false;
}
// Find member closest to this peer
const uint64_t now = RR->node->now();
std::vector<InetAddress> best;
const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
double bestDistance = (offload ? 2147483648.0 : currentDistance);
unsigned int bestMember = _id;
{
Mutex::Lock _l(_memberIds_m);
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
_Member &m = _members[*mid];
Mutex::Lock _ml(m.lock);
// Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to
if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) {
const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
if (mdist < bestDistance) {
bestDistance = mdist;
bestMember = *mid;
best = m.zeroTierPhysicalEndpoints;
}
}
}
}
// Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family
for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
if (a->ss_family == peerPhysicalAddress.ss_family) {
TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
redirectTo = *a;
return true;
}
}
TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance);
return false;
} else {
// TODO: pick based on load if no location info?
return false;
}
}
void Cluster::status(ZT_ClusterStatus &status) const
{
const uint64_t now = RR->node->now();
memset(&status,0,sizeof(ZT_ClusterStatus));
status.myId = _id;
{
ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]);
s->id = _id;
s->alive = 1;
s->x = _x;
s->y = _y;
s->z = _z;
s->load = 0; // TODO
s->peers = RR->topology->countActive(now);
for(std::vector<InetAddress>::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) {
if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
break;
memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage));
}
}
{
Mutex::Lock _l1(_memberIds_m);
for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check
break;
_Member &m = _members[*mid];
Mutex::Lock ml(m.lock);
ZT_ClusterMemberStatus *const s = &(status.members[status.clusterSize++]);
s->id = *mid;
s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement));
s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0;
s->x = m.x;
s->y = m.y;
s->z = m.z;
s->load = m.load;
s->peers = m.peers;
for(std::vector<InetAddress>::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) {
if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
break;
memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage));
}
}
}
}
void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len)
{
if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check
return;
_Member &m = _members[memberId];
// assumes m.lock is locked!
if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH)
_flush(memberId);
m.q.append((uint16_t)(len + 1));
m.q.append((uint8_t)type);
m.q.append(msg,len);
}
void Cluster::_flush(uint16_t memberId)
{
_Member &m = _members[memberId];
// assumes m.lock is locked!
if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID
// Create key from member's key and IV
char keytmp[32];
memcpy(keytmp,m.key,32);
for(int i=0;i<8;++i)
keytmp[i] ^= m.q[i];
Salsa20 s20(keytmp,256,m.q.field(8,8));
Utils::burn(keytmp,sizeof(keytmp));
// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
char polykey[ZT_POLY1305_KEY_LEN];
memset(polykey,0,sizeof(polykey));
s20.encrypt12(polykey,polykey,sizeof(polykey));
// Encrypt m.q in place
s20.encrypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24);
// Add MAC for authentication (encrypt-then-MAC)
char mac[ZT_POLY1305_MAC_LEN];
Poly1305::compute(mac,reinterpret_cast<const char *>(m.q.data()) + 24,m.q.size() - 24,polykey);
memcpy(m.q.field(16,8),mac,8);
// Send!
_sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size());
// Prepare for more
m.q.clear();
char iv[16];
Utils::getSecureRandom(iv,16);
m.q.append(iv,16);
m.q.addSize(8); // room for MAC
m.q.append((uint16_t)_id); // from member ID
m.q.append((uint16_t)memberId); // to member ID
}
}
void Cluster::_doREMOTE_WHOIS(uint64_t fromMemberId,const Packet &remotep)
{
if (remotep.payloadLength() >= ZT_ADDRESS_LENGTH) {
Identity queried(RR->topology->getIdentity(Address(remotep.payload(),ZT_ADDRESS_LENGTH)));
if (queried) {
Buffer<1024> routp;
remotep.source().appendTo(routp);
routp.append((uint8_t)Packet::VERB_OK);
routp.addSize(2); // space for length
routp.append((uint8_t)Packet::VERB_WHOIS);
routp.append(remotep.packetId());
queried.serialize(routp);
routp.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3));
TRACE("responding to remote WHOIS from %s @ %u with identity of %s",remotep.source().toString().c_str(),(unsigned int)fromMemberId,queried.address().toString().c_str());
Mutex::Lock _l2(_members[fromMemberId].lock);
_send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size());
}
}
}
void Cluster::_doREMOTE_MULTICAST_GATHER(uint64_t fromMemberId,const Packet &remotep)
{
const uint64_t nwid = remotep.at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID);
const MulticastGroup mg(MAC(remotep.field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),remotep.at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI));
unsigned int gatherLimit = remotep.at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT);
const Address remotePeerAddress(remotep.source());
if (gatherLimit) {
Buffer<ZT_PROTO_MAX_PACKET_LENGTH> routp;
remotePeerAddress.appendTo(routp);
routp.append((uint8_t)Packet::VERB_OK);
routp.addSize(2); // space for length
routp.append((uint8_t)Packet::VERB_MULTICAST_GATHER);
routp.append(remotep.packetId());
routp.append(nwid);
mg.mac().appendTo(routp);
routp.append((uint32_t)mg.adi());
if (gatherLimit > ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5))
gatherLimit = ((ZT_CLUSTER_MAX_MESSAGE_LENGTH - 80) / 5);
if (RR->mc->gather(remotePeerAddress,nwid,mg,routp,gatherLimit)) {
routp.setAt<uint16_t>(ZT_ADDRESS_LENGTH + 1,(uint16_t)(routp.size() - ZT_ADDRESS_LENGTH - 3));
TRACE("responding to remote MULTICAST_GATHER from %s @ %u with %u bytes",remotePeerAddress.toString().c_str(),(unsigned int)fromMemberId,routp.size());
Mutex::Lock _l2(_members[fromMemberId].lock);
_send(fromMemberId,CLUSTER_MESSAGE_PROXY_SEND,routp.data(),routp.size());
}
}
}
} // namespace ZeroTier
#endif // ZT_ENABLE_CLUSTER
Reference in New Issue View Git Blame Copy Permalink