ExternalVendorCode/ZeroTierOne
1
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2024-12-24 23:26:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Clean out a bunch of old code related to the possibility of multiple peers sharing virtual addresses. Does not seem to be currently exploitable but we are never going back to that idea, so this simplifies things. May also speed up the root.

Browse Source
This commit is contained in:
Adam Ierymenko 2021-09-20 14:42:49 -04:00
parent 5028aca372
commit c2413fca4d
1 changed files with 122 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

236
root/root.cpp
View File

@ -208,15 +208,13 @@ static std::string s_planet;
static std::vector< SharedPtr<RootPeer> > s_peers; static std::vector< SharedPtr<RootPeer> > s_peers;
static std::vector< SharedPtr<RootPeer> > s_peersToValidate; static std::vector< SharedPtr<RootPeer> > s_peersToValidate;
static std::unordered_map< uint64_t,std::unordered_map< MulticastGroup,std::unordered_map< Address,int64_t,AddressHasher >,MulticastGroupHasher > > s_multicastSubscriptions; static std::unordered_map< uint64_t,std::unordered_map< MulticastGroup,std::unordered_map< Address,int64_t,AddressHasher >,MulticastGroupHasher > > s_multicastSubscriptions;
static std::unordered_map< Identity,SharedPtr<RootPeer>,IdentityHasher > s_peersByIdentity; static std::unordered_map< Address,SharedPtr<RootPeer>,AddressHasher > s_peersByVirtAddr;
static std::unordered_map< Address,std::set< SharedPtr<RootPeer> >,AddressHasher > s_peersByVirtAddr;
static std::unordered_map< RendezvousKey,RendezvousStats,RendezvousKey::Hasher > s_rendezvousTracking; static std::unordered_map< RendezvousKey,RendezvousStats,RendezvousKey::Hasher > s_rendezvousTracking;
static std::mutex s_planet_l; static std::mutex s_planet_l;
static std::mutex s_peers_l; static std::mutex s_peers_l;
static std::mutex s_peersToValidate_l; static std::mutex s_peersToValidate_l;
static std::mutex s_multicastSubscriptions_l; static std::mutex s_multicastSubscriptions_l;
static std::mutex s_peersByIdentity_l;
static std::mutex s_peersByVirtAddr_l; static std::mutex s_peersByVirtAddr_l;
static std::mutex s_rendezvousTracking_l; static std::mutex s_rendezvousTracking_l;
@ -261,38 +259,34 @@ static void handlePacket(const int sock,const InetAddress *const ip,Packet &pkt)
Identity id; Identity id;
if (id.deserialize(pkt,ZT_PROTO_VERB_HELLO_IDX_IDENTITY)) { if (id.deserialize(pkt,ZT_PROTO_VERB_HELLO_IDX_IDENTITY)) {
{ {
std::lock_guard<std::mutex> pbi_l(s_peersByIdentity_l); std::lock_guard<std::mutex> p_l(s_peersByVirtAddr_l);
auto pById = s_peersByIdentity.find(id); auto p = s_peersByVirtAddr.find(source);
if (pById != s_peersByIdentity.end()) { if (p != s_peersByVirtAddr.end()) {
peer = pById->second; peer = p->second;
//printf("%s has %s (known (1))" ZT_EOL_S,ip->toString(ipstr),source().toString(astr));
} }
} }
if (peer) {
// Peer found with this identity.
if (!pkt.dearmor(peer->key)) {
printf("%s HELLO rejected: packet authentication failed" ZT_EOL_S,ip->toString(ipstr));
return;
}
} else {
// Check to ensure that there is no peer with the same address as this identity. If there is,
// verify both identities to pick the one with this address.
bool needsValidation = false;
bool identityValidated = false;
{
std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);
needsValidation = s_peersByVirtAddr.find(id.address()) != s_peersByVirtAddr.end();
}
if (unlikely(needsValidation)) {
if (!id.locallyValidate()) {
printf("%s HELLO rejected: identity validate failed" ZT_EOL_S,ip->toString(ipstr));
return;
}
identityValidated = true;
}
if (peer) {
if (unlikely(peer->id != id)) {
if (!peer->identityValidated) {
peer->identityValidated = peer->id.locallyValidate();
if (peer->identityValidated) {
printf("%s HELLO rejected: identity address collision!" ZT_EOL_S,ip->toString(ipstr));
// TODO: send error
return;
} else {
printf("* invalid identity found and discarded: %s" ZT_EOL_S,id.toString(false, tmpstr));
std::lock_guard<std::mutex> p_l(s_peersByVirtAddr_l);
s_peersByVirtAddr.erase(source);
peer.zero();
}
}
}
}
if (!peer) {
peer.set(new RootPeer); peer.set(new RootPeer);
peer->identityValidated = identityValidated; peer->identityValidated = false;
if (!s_self.agree(id,peer->key)) { if (!s_self.agree(id,peer->key)) {
printf("%s HELLO rejected: key agreement failed" ZT_EOL_S,ip->toString(ipstr)); printf("%s HELLO rejected: key agreement failed" ZT_EOL_S,ip->toString(ipstr));
@ -310,64 +304,41 @@ static void handlePacket(const int sock,const InetAddress *const ip,Packet &pkt)
peer->id = id; peer->id = id;
peer->lastReceive = now; peer->lastReceive = now;
bool added = false;
{ {
std::lock_guard<std::mutex> pbi_l(s_peersByIdentity_l); std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);
auto existing = s_peersByIdentity.find(id); // make sure another thread didn't do this while we were s_peersByVirtAddr[id.address()] = peer;
if (likely(existing == s_peersByIdentity.end())) {
s_peersByIdentity.emplace(id,peer);
added = true;
} else {
peer = existing->second;
} }
}
if (likely(added)) {
{ {
std::lock_guard<std::mutex> pl(s_peers_l); std::lock_guard<std::mutex> pl(s_peers_l);
s_peers.emplace_back(peer); s_peers.emplace_back(peer);
} }
if (!peer->identityValidated) { {
std::lock_guard<std::mutex> pv(s_peersToValidate_l); std::lock_guard<std::mutex> pv(s_peersToValidate_l);
s_peersToValidate.emplace_back(peer); s_peersToValidate.emplace_back(peer);
} }
{
std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);
std::set< SharedPtr<RootPeer> > &byVirt = s_peersByVirtAddr[id.address()];
for(auto i=byVirt.begin();i!=byVirt.end();++i) {
if (!(*i)->identityValidated)
(*i)->identityInvalid = !(*i)->id.locallyValidate();
}
s_peersByVirtAddr[id.address()].emplace(peer);
}
}
} }
} }
} }
// If it wasn't a HELLO, check to see if any known identities for the sender's
// short ZT address successfully decrypt the packet.
if (!peer) { if (!peer) {
{
std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l); std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);
auto peers = s_peersByVirtAddr.find(source); auto p = s_peersByVirtAddr.find(source);
if (peers != s_peersByVirtAddr.end()) { if (p != s_peersByVirtAddr.end()) {
for(auto p=peers->second.begin();p!=peers->second.end();++p) { peer = p->second;
if (!(*p)->identityInvalid) { }
if (pkt.dearmor((*p)->key)) { }
if (!pkt.dearmor(peer->key)) {
printf("%s HELLO rejected: packet authentication failed" ZT_EOL_S,ip->toString(ipstr));
return;
}
if (!pkt.uncompress()) { if (!pkt.uncompress()) {
printf("%s packet rejected: decompression failed" ZT_EOL_S,ip->toString(ipstr)); printf("%s packet rejected: decompression failed" ZT_EOL_S,ip->toString(ipstr));
return; return;
} }
peer = (*p);
break;
}
}
}
}
} }
// If we found the peer, update IP and/or time and handle certain key packet types that the if (likely(peer)) {
// root must concern itself with.
if (peer) {
const int64_t now = OSUtils::now(); const int64_t now = OSUtils::now();
if (ip->isV4()) { if (ip->isV4()) {
@ -452,12 +423,9 @@ static void handlePacket(const int sock,const InetAddress *const ip,Packet &pkt)
{ {
std::lock_guard<std::mutex> l(s_peersByVirtAddr_l); std::lock_guard<std::mutex> l(s_peersByVirtAddr_l);
for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;(ptr+ZT_ADDRESS_LENGTH)<=pkt.size();ptr+=ZT_ADDRESS_LENGTH) { for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;(ptr+ZT_ADDRESS_LENGTH)<=pkt.size();ptr+=ZT_ADDRESS_LENGTH) {
auto peers = s_peersByVirtAddr.find(Address(pkt.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH)); auto p = s_peersByVirtAddr.find(Address(pkt.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH));
if (peers != s_peersByVirtAddr.end()) { if (p != s_peersByVirtAddr.end()) {
for(auto p=peers->second.begin();p!=peers->second.end();++p) { results.push_back(p->second);
if (!(*p)->identityInvalid)
results.push_back(*p);
}
} }
} }
} }
@ -581,109 +549,91 @@ static void handlePacket(const int sock,const InetAddress *const ip,Packet &pkt)
} }
} }
std::vector< std::pair< InetAddress *,SharedPtr<RootPeer> > > toAddrs; std::pair< InetAddress,SharedPtr<RootPeer> > forwardTo;
toAddrs.reserve(4);
{ {
std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l); std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);
auto peers = s_peersByVirtAddr.find(dest); auto p = s_peersByVirtAddr.find(dest);
if (peers != s_peersByVirtAddr.end()) { if (p != s_peersByVirtAddr.end()) {
for(auto p=peers->second.begin();p!=peers->second.end();++p) { if ((p->second->v4s >= 0)&&(p->second->v6s >= 0)) {
if (!(*p)->identityInvalid) { if (p->second->lastReceiveV4 > p->second->lastReceiveV6) {
if (((*p)->v4s >= 0)&&((*p)->v6s >= 0)) { forwardTo = std::pair< InetAddress,SharedPtr<RootPeer> >(p->second->ip4,p->second);
if ((*p)->lastReceiveV4 > (*p)->lastReceiveV6) {
toAddrs.emplace_back(std::pair< InetAddress *,SharedPtr<RootPeer> >(&((*p)->ip4),*p));
} else { } else {
toAddrs.emplace_back(std::pair< InetAddress *,SharedPtr<RootPeer> >(&((*p)->ip6),*p)); forwardTo = std::pair< InetAddress,SharedPtr<RootPeer> >(p->second->ip6,p->second);
} }
} else if ((*p)->v4s >= 0) { } else if (p->second->v4s >= 0) {
toAddrs.emplace_back(std::pair< InetAddress *,SharedPtr<RootPeer> >(&((*p)->ip4),*p)); forwardTo = std::pair< InetAddress,SharedPtr<RootPeer> >(p->second->ip4,p->second);
} else if ((*p)->v6s >= 0) { } else if (p->second->v6s >= 0) {
toAddrs.emplace_back(std::pair< InetAddress *,SharedPtr<RootPeer> >(&((*p)->ip6),*p)); forwardTo = std::pair< InetAddress,SharedPtr<RootPeer> >(p->second->ip6,p->second);
} }
} }
} }
}
} if (unlikely(!forwardTo.second)) {
if (toAddrs.empty()) {
s_discardedForwardRate.log(now,pkt.size()); s_discardedForwardRate.log(now,pkt.size());
return; return;
} }
if (introduce) { if (introduce) {
std::lock_guard<std::mutex> l(s_peersByVirtAddr_l); std::lock_guard<std::mutex> l(s_peersByVirtAddr_l);
auto sources = s_peersByVirtAddr.find(source); auto sp = s_peersByVirtAddr.find(source);
if (sources != s_peersByVirtAddr.end()) {
for(auto a=sources->second.begin();a!=sources->second.end();++a) {
for(auto b=toAddrs.begin();b!=toAddrs.end();++b) {
if (((*a)->v6s >= 0)&&(b->second->v6s >= 0)) {
//printf("* introducing %s(%s) to %s(%s)" ZT_EOL_S,ip->toString(ipstr),source.toString(astr),b->second->ip6.toString(ipstr2),dest.toString(astr2));
// Introduce source to destination (V6) if ((sp->second->v6s >= 0)&&(forwardTo.second->v6s >= 0)) {
Packet outp(source,s_self.address(),Packet::VERB_RENDEZVOUS); Packet outp(source,s_self.address(),Packet::VERB_RENDEZVOUS);
outp.append((uint8_t)0); outp.append((uint8_t)0);
dest.appendTo(outp); dest.appendTo(outp);
outp.append((uint16_t)b->second->ip6.port()); outp.append((uint16_t)sp->second->ip6.port());
outp.append((uint8_t)16); outp.append((uint8_t)16);
outp.append((const uint8_t *)(b->second->ip6.rawIpData()),16); outp.append((const uint8_t *)(sp->second->ip6.rawIpData()),16);
outp.armor((*a)->key,true); outp.armor(forwardTo.second->key,true);
sendto((*a)->v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&((*a)->ip6),(socklen_t)sizeof(struct sockaddr_in6)); sendto(forwardTo.second->v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&forwardTo.first,(socklen_t)sizeof(struct sockaddr_in6));
s_outputRate.log(now,outp.size()); s_outputRate.log(now,outp.size());
(*a)->lastSend = now; forwardTo.second->lastSend = now;
// Introduce destination to source (V6)
outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS); outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS);
outp.append((uint8_t)0); outp.append((uint8_t)0);
source.appendTo(outp); source.appendTo(outp);
outp.append((uint16_t)(*a)->ip6.port()); outp.append((uint16_t)forwardTo.first.port());
outp.append((uint8_t)16); outp.append((uint8_t)16);
outp.append((const uint8_t *)((*a)->ip6.rawIpData()),16); outp.append((const uint8_t *)(forwardTo.first.rawIpData()),16);
outp.armor(b->second->key,true); outp.armor(sp->second->key,true);
sendto(b->second->v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip6),(socklen_t)sizeof(struct sockaddr_in6)); sendto(sp->second->v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(sp->second->ip6),(socklen_t)sizeof(struct sockaddr_in6));
s_outputRate.log(now,outp.size()); s_outputRate.log(now,outp.size());
b->second->lastSend = now; sp->second->lastSend = now;
} }
if (((*a)->v4s >= 0)&&(b->second->v4s >= 0)) {
//printf("* introducing %s(%s) to %s(%s)" ZT_EOL_S,ip->toString(ipstr),source.toString(astr),b->second->ip4.toString(ipstr2),dest.toString(astr2));
// Introduce source to destination (V4) if ((sp->second->v4s >= 0)&&(forwardTo.second->v4s >= 0)) {
Packet outp(source,s_self.address(),Packet::VERB_RENDEZVOUS); Packet outp(source,s_self.address(),Packet::VERB_RENDEZVOUS);
outp.append((uint8_t)0); outp.append((uint8_t)0);
dest.appendTo(outp); dest.appendTo(outp);
outp.append((uint16_t)b->second->ip4.port()); outp.append((uint16_t)sp->second->ip4.port());
outp.append((uint8_t)4); outp.append((uint8_t)4);
outp.append((const uint8_t *)b->second->ip4.rawIpData(),4); outp.append((const uint8_t *)sp->second->ip4.rawIpData(),4);
outp.armor((*a)->key,true); outp.armor(forwardTo.second->key,true);
sendto((*a)->v4s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&((*a)->ip4),(socklen_t)sizeof(struct sockaddr_in)); sendto(forwardTo.second->v4s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&forwardTo.first,(socklen_t)sizeof(struct sockaddr_in));
s_outputRate.log(now,outp.size()); s_outputRate.log(now,outp.size());
(*a)->lastSend = now; forwardTo.second->lastSend = now;
// Introduce destination to source (V4)
outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS); outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS);
outp.append((uint8_t)0); outp.append((uint8_t)0);
source.appendTo(outp); source.appendTo(outp);
outp.append((uint16_t)(*a)->ip4.port()); outp.append((uint16_t)forwardTo.first.port());
outp.append((uint8_t)4); outp.append((uint8_t)4);
outp.append((const uint8_t *)((*a)->ip4.rawIpData()),4); outp.append((const uint8_t *)(forwardTo.first.rawIpData()),4);
outp.armor(b->second->key,true); outp.armor(sp->second->key,true);
sendto(b->second->v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip4),(socklen_t)sizeof(struct sockaddr_in)); sendto(sp->second->v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(sp->second->ip4),(socklen_t)sizeof(struct sockaddr_in));
s_outputRate.log(now,outp.size()); s_outputRate.log(now,outp.size());
b->second->lastSend = now; sp->second->lastSend = now;
}
}
}
} }
} }
for(auto i=toAddrs.begin();i!=toAddrs.end();++i) { if (sendto(forwardTo.first.isV4() ? forwardTo.second->v4s : forwardTo.second->v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)&forwardTo.first,(socklen_t)(forwardTo.first.isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) > 0) {
if (sendto(i->first->isV4() ? i->second->v4s : i->second->v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)i->first,(socklen_t)(i->first->isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) > 0) {
s_outputRate.log(now,pkt.size()); s_outputRate.log(now,pkt.size());
s_forwardRate.log(now,pkt.size()); s_forwardRate.log(now,pkt.size());
i->second->lastSend = now; forwardTo.second->lastSend = now;
}
} }
} }
@ -1046,9 +996,9 @@ int main(int argc,char **argv)
std::ostringstream o; std::ostringstream o;
o << "ZeroTier Root Server " << ZEROTIER_ONE_VERSION_MAJOR << '.' << ZEROTIER_ONE_VERSION_MINOR << '.' << ZEROTIER_ONE_VERSION_REVISION << ZT_EOL_S; o << "ZeroTier Root Server " << ZEROTIER_ONE_VERSION_MAJOR << '.' << ZEROTIER_ONE_VERSION_MINOR << '.' << ZEROTIER_ONE_VERSION_REVISION << ZT_EOL_S;
o << "(c)2019 ZeroTier, Inc." ZT_EOL_S "Licensed under the ZeroTier BSL 1.1" ZT_EOL_S ZT_EOL_S; o << "(c)2019 ZeroTier, Inc." ZT_EOL_S "Licensed under the ZeroTier BSL 1.1" ZT_EOL_S ZT_EOL_S;
s_peersByIdentity_l.lock(); s_peersByVirtAddr_l.lock();
o << "Peers Online: " << s_peersByIdentity.size() << ZT_EOL_S; o << "Peers Online: " << s_peersByVirtAddr.size() << ZT_EOL_S;
s_peersByIdentity_l.unlock(); s_peersByVirtAddr_l.unlock();
res.set_content(o.str(),"text/plain"); res.set_content(o.str(),"text/plain");
}); });
@ -1060,9 +1010,9 @@ int main(int argc,char **argv)
const char *root_id = s_self.address().toString(buf); const char *root_id = s_self.address().toString(buf);
o << "# HELP root_peers_online Number of active peers online" << ZT_EOL_S; o << "# HELP root_peers_online Number of active peers online" << ZT_EOL_S;
o << "# TYPE root_peers_online gauge" << ZT_EOL_S; o << "# TYPE root_peers_online gauge" << ZT_EOL_S;
s_peersByIdentity_l.lock(); s_peersByVirtAddr_l.lock();
o << "root_peers_online{root_id=\"" << root_id << "\"} " << s_peersByIdentity.size() << ZT_EOL_S; o << "root_peers_online{root_id=\"" << root_id << "\"} " << s_peersByVirtAddr.size() << ZT_EOL_S;
s_peersByIdentity_l.unlock(); s_peersByVirtAddr_l.unlock();
o << "# HELP root_input_rate Input rate MiB/s" << ZT_EOL_S; o << "# HELP root_input_rate Input rate MiB/s" << ZT_EOL_S;
o << "# TYPE root_input_rate gauge" << ZT_EOL_S; o << "# TYPE root_input_rate gauge" << ZT_EOL_S;
o << "root_input_rate{root_id=\"" << root_id << "\"} " << std::setprecision(5) << (s_inputRate.perSecond(now)/1048576.0) << ZT_EOL_S; o << "root_input_rate{root_id=\"" << root_id << "\"} " << std::setprecision(5) << (s_inputRate.perSecond(now)/1048576.0) << ZT_EOL_S;
@ -1279,16 +1229,10 @@ int main(int argc,char **argv)
newPeers.swap(s_peers); newPeers.swap(s_peers);
} }
for(auto p=toRemove.begin();p!=toRemove.end();++p) { for(auto p=toRemove.begin();p!=toRemove.end();++p) {
{
std::lock_guard<std::mutex> pbi_l(s_peersByIdentity_l);
s_peersByIdentity.erase((*p)->id);
}
{ {
std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l); std::lock_guard<std::mutex> pbv_l(s_peersByVirtAddr_l);
auto pbv = s_peersByVirtAddr.find((*p)->id.address()); auto pbv = s_peersByVirtAddr.find((*p)->id.address());
if (pbv != s_peersByVirtAddr.end()) { if ((pbv != s_peersByVirtAddr.end())&&(pbv->second == *p)) {
pbv->second.erase(*p);
if (pbv->second.empty())
s_peersByVirtAddr.erase(pbv); s_peersByVirtAddr.erase(pbv);
} }
} }
@ -1375,12 +1319,8 @@ int main(int argc,char **argv)
FILE *sf = fopen(statsFilePath.c_str(),"wb"); FILE *sf = fopen(statsFilePath.c_str(),"wb");
if (sf) { if (sf) {
fprintf(sf,"Uptime (seconds) : %ld" ZT_EOL_S,(long)((now - s_startTime) / 1000)); fprintf(sf,"Uptime (seconds) : %ld" ZT_EOL_S,(long)((now - s_startTime) / 1000));
s_peersByIdentity_l.lock();
auto peersByIdentitySize = s_peersByIdentity.size();
s_peersByIdentity_l.unlock();
fprintf(sf,"Peers : %llu" ZT_EOL_S,(unsigned long long)peersByIdentitySize);
s_peersByVirtAddr_l.lock(); s_peersByVirtAddr_l.lock();
fprintf(sf,"Virtual Address Collisions : %lld" ZT_EOL_S,(long long)peersByIdentitySize - (long long)s_peersByVirtAddr.size()); fprintf(sf,"Peers : %llu" ZT_EOL_S,(unsigned long long)s_peersByVirtAddr.size());
s_peersByVirtAddr_l.unlock(); s_peersByVirtAddr_l.unlock();
s_rendezvousTracking_l.lock(); s_rendezvousTracking_l.lock();
uint64_t unsuccessfulp2p = 0; uint64_t unsuccessfulp2p = 0;