ExternalVendorCode/ZeroTierOne
1
0
Fork 0
mirror of https://github.com/zerotier/ZeroTierOne.git synced 2024-12-19 13:07:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Track member status in Redis

Browse Source
This commit is contained in:
Grant Limberg 2020-05-13 17:23:27 -07:00
parent ee91c81799
commit b637ced1ee
No known key found for this signature in database
GPG Key ID: 2BA62CCABBB4095A
2 changed files with 123 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
controller/PostgreSQL.cpp
View File

@ -229,12 +229,14 @@ void PostgreSQL::eraseNetwork(const uint64_t networkId)
tmp.first["objtype"] = "_delete_network";
tmp.second = true;
_commitQueue.post(tmp);
nlohmann::json nullJson;
_networkChanged(tmp.first, nullJson, true);
}
void PostgreSQL::eraseMember(const uint64_t networkId, const uint64_t memberId)
{
char tmp2[24];
std::pair<nlohmann::json,bool> tmp;
std::pair<nlohmann::json,bool> tmp, nw;
Utils::hex(networkId, tmp2);
tmp.first["nwid"] = tmp2;
Utils::hex(memberId, tmp2);
@ -242,6 +244,8 @@ void PostgreSQL::eraseMember(const uint64_t networkId, const uint64_t memberId)
tmp.first["objtype"] = "_delete_member";
tmp.second = true;
_commitQueue.post(tmp);
nlohmann::json nullJson;
_memberChanged(tmp.first, nullJson, true);
}
void PostgreSQL::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress)
@ -630,7 +634,7 @@ void PostgreSQL::heartbeat()
};
PGresult *res = PQexecParams(conn,
"INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, v_rev, v_build, host_port,use_redis) "
"INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, v_rev, v_build, host_port, use_redis) "
"VALUES ($1, $2, TO_TIMESTAMP($3::double precision/1000), $4, $5, $6, $7, $8, $9, $10) "
"ON CONFLICT (id) DO UPDATE SET cluster_host = EXCLUDED.cluster_host, last_alive = EXCLUDED.last_alive, "
"public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = EXCLUDED.v_minor, "
@ -1401,6 +1405,15 @@ void PostgreSQL::commitThread()
}
void PostgreSQL::onlineNotificationThread()
{
if (_rc != NULL) {
onlineNotification_Redis();
} else {
onlineNotification_Postgres();
}
}
void PostgreSQL::onlineNotification_Postgres()
{
PGconn *conn = getPgConn();
if (PQstatus(conn) == CONNECTION_BAD) {
@ -1410,9 +1423,7 @@ void PostgreSQL::onlineNotificationThread()
}
_connected = 1;
//int64_t lastUpdatedNetworkStatus = 0;
std::unordered_map< std::pair<uint64_t,uint64_t>,int64_t,_PairHasher > lastOnlineCumulative;
nlohmann::json jtmp1, jtmp2;
while (_run == 1) {
if (PQstatus(conn) != CONNECTION_OK) {
fprintf(stderr, "ERROR: Online Notification thread lost connection to Postgres.");
@ -1420,9 +1431,6 @@ void PostgreSQL::onlineNotificationThread()
exit(5);
}
// map used to send notifications to front end
std::unordered_map<std::string, std::vector<std::string>> updateMap;
std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > lastOnline;
{
std::lock_guard<std::mutex> l(_lastOnline_l);
@ -1443,20 +1451,13 @@ void PostgreSQL::onlineNotificationThread()
OSUtils::ztsnprintf(nwidTmp,sizeof(nwidTmp), "%.16llx", nwid_i);
OSUtils::ztsnprintf(memTmp,sizeof(memTmp), "%.10llx", i->first.second);
auto found = _networks.find(nwid_i);
if (found == _networks.end()) {
continue; // skip members trying to join non-existant networks
if(!get(nwid_i, jtmp1, i->first.second, jtmp2)) {
continue; // skip non existent networks/members
}
std::string networkId(nwidTmp);
std::string memberId(memTmp);
std::vector<std::string> &members = updateMap[networkId];
members.push_back(memberId);
lastOnlineCumulative[i->first] = i->second.first;
const char *qvals[2] = {
networkId.c_str(),
memberId.c_str()
@ -1526,6 +1527,107 @@ void PostgreSQL::onlineNotificationThread()
}
}
void PostgreSQL::onlineNotification_Redis()
{
_connected = 1;
char buf[11] = {0};
std::string controllerId = std::string(_myAddress.toString(buf));
while (_run == 1) {
std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > lastOnline;
{
std::lock_guard<std::mutex> l(_lastOnline_l);
lastOnline.swap(_lastOnline);
}
if (_rc->clusterMode) {
auto tx = _cluster->redis(controllerId).transaction(true);
_doRedisUpdate(tx, controllerId, lastOnline);
} else {
auto tx = _redis->transaction(true);
_doRedisUpdate(tx, controllerId, lastOnline);
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
}
void PostgreSQL::_doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline)
{
nlohmann::json jtmp1, jtmp2;
for (auto i=lastOnline.begin(); i != lastOnline.end(); ++i) {
uint64_t nwid_i = i->first.first;
uint64_t memberid_i = i->first.second;
char nwidTmp[64];
char memTmp[64];
char ipTmp[64];
OSUtils::ztsnprintf(nwidTmp,sizeof(nwidTmp), "%.16llx", nwid_i);
OSUtils::ztsnprintf(memTmp,sizeof(memTmp), "%.10llx", memberid_i);
if (!get(nwid_i, jtmp1, memberid_i, jtmp2)){
continue; // skip non existent members/networks
}
auto found = _networks.find(nwid_i);
if (found == _networks.end()) {
continue; // skip members trying to join non-existant networks
}
std::string networkId(nwidTmp);
std::string memberId(memTmp);
int64_t ts = i->second.first;
std::string ipAddr = i->second.second.toIpString(ipTmp);
std::string timestamp = std::to_string(ts);
std::unordered_map<std::string, std::string> record = {
{"id", memberId},
{"address", ipAddr},
{"last_updated", std::to_string(ts)}
};
tx.zadd("nodes-online:{"+controllerId+"}", memberId, ts)
.zadd("network-nodes-online:{"+controllerId+"}:"+networkId, memberId, ts)
.sadd("network-nodes-all:{"+controllerId+"}:"+networkId, memberId)
.hmset("network:{"+controllerId+"}:"+networkId+":"+memberId, record.begin(), record.end());
}
tx.exec();
// expire records from all-nodes and network-nodes member list
uint64_t expireOld = OSUtils::now() - 300000;
auto cursor = 0LL;
std::unordered_set<std::string> keys;
// can't scan for keys in a transaction, so we need to fall back to _cluster or _redis
// to get all network-members keys
if(_rc->clusterMode) {
auto r = _cluster->redis(controllerId);
while(true) {
cursor = r.scan(cursor, "network-nodes-online:{"+controllerId+"}:*", INT_MAX, std::inserter(keys, keys.begin()));
if (cursor == 0) {
break;
}
}
} else {
while(true) {
cursor = _redis->scan(cursor, "network-nodes-online:"+controllerId+":*", INT_MAX, std::inserter(keys, keys.begin()));
if (cursor == 0) {
break;
}
}
}
tx.zremrangebyscore("nodes-online:{"+controllerId+"}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
for(const auto &k : keys) {
tx.zremrangebyscore(k, sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
}
tx.exec();
}
PGconn *PostgreSQL::getPgConn(OverrideMode m)
{
if (m == ALLOW_PGBOUNCER_OVERRIDE) {

4
controller/PostgreSQL.hpp
View File

@ -70,6 +70,10 @@ private:
void commitThread();
void onlineNotificationThread();
void onlineNotification_Postgres();
void onlineNotification_Redis();
void _doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline);
enum OverrideMode {
ALLOW_PGBOUNCER_OVERRIDE = 0,