From 576b4f03a5840fb65c65e9dad593032d819fe943 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Wed, 18 Aug 2021 12:17:40 -0400 Subject: [PATCH 01/10] Adjust deauth time window and send revocation when SSO members expire. --- controller/DB.cpp | 8 ++++++++ controller/DB.hpp | 2 ++ controller/DBMirrorSet.cpp | 8 ++++++++ controller/DBMirrorSet.hpp | 1 + controller/EmbeddedNetworkController.cpp | 5 +++++ make-mac.mk | 6 +++--- 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/controller/DB.cpp b/controller/DB.cpp index 2edcadbbe..27578bf77 100644 --- a/controller/DB.cpp +++ b/controller/DB.cpp @@ -196,6 +196,14 @@ void DB::networks(std::set &networks) networks.insert(n->first); } +void DB::networkMemberSSOHasExpired(uint64_t nwid, int64_t now) { + std::lock_guard l(_networks_l); + auto nw = _networks.find(nwid); + if (nw != _networks.end()) { + nw->second->mostRecentDeauthTime = now; + } +} + void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) { uint64_t memberId = 0; diff --git a/controller/DB.hpp b/controller/DB.hpp index e0cef2b76..0a5d784c2 100644 --- a/controller/DB.hpp +++ b/controller/DB.hpp @@ -107,7 +107,9 @@ public: virtual void eraseNetwork(const uint64_t networkId) = 0; virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0; virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0; + virtual std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL) { return ""; } + virtual void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts); inline void addListener(DB::ChangeListener *const listener) { diff --git a/controller/DBMirrorSet.cpp b/controller/DBMirrorSet.cpp index cf1f02194..de7ebefe1 100644 --- a/controller/DBMirrorSet.cpp +++ b/controller/DBMirrorSet.cpp @@ -137,6 +137,14 @@ std::string DBMirrorSet::getSSOAuthURL(const nlohmann::json &member, const std:: return ""; } +void DBMirrorSet::networkMemberSSOHasExpired(uint64_t nwid, int64_t ts) +{ + std::lock_guard l(_dbs_l); + for(auto d=_dbs.begin();d!=_dbs.end();++d) { + (*d)->networkMemberSSOHasExpired(nwid, ts); + } +} + void DBMirrorSet::networks(std::set &networks) { std::lock_guard l(_dbs_l); diff --git a/controller/DBMirrorSet.hpp b/controller/DBMirrorSet.hpp index 83dc228aa..4ce962740 100644 --- a/controller/DBMirrorSet.hpp +++ b/controller/DBMirrorSet.hpp @@ -52,6 +52,7 @@ public: virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId); std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL); + void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts); inline void addDB(const std::shared_ptr &db) { diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 9a4a09844..da0d7965e 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1369,11 +1369,16 @@ void EmbeddedNetworkController::_request( fprintf(stderr, "authExpiryTime: %lld\n", authenticationExpiryTime); if (authenticationExpiryTime < now) { if (!authenticationURL.empty()) { + _db.networkMemberSSOHasExpired(nwid, now); + onNetworkMemberDeauthorize(&_db, nwid, identity.address().toInt()); + Dictionary<3072> authInfo; authInfo.add("aU", authenticationURL.c_str()); fprintf(stderr, "sending auth URL: %s\n", authenticationURL.c_str()); + DB::cleanMember(member); _db.save(member,true); + _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes()); return; } diff --git a/make-mac.mk b/make-mac.mk index c43661570..ffc8b5c96 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -75,7 +75,7 @@ ifeq ($(ZT_DEBUG),1) node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else CFLAGS?=-Ofast -fstack-protector-strong - CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) + CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -mmacosx-version-min=10.13 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif @@ -93,10 +93,10 @@ CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ all: one ext/x64-salsa2012-asm/salsa2012.o: - as -arch x86_64 -mmacosx-version-min=10.7 -o ext/x64-salsa2012-asm/salsa2012.o ext/x64-salsa2012-asm/salsa2012.s + as -arch x86_64 -mmacosx-version-min=10.13 -o ext/x64-salsa2012-asm/salsa2012.o ext/x64-salsa2012-asm/salsa2012.s mac-agent: FORCE - $(CC) -Ofast $(ARCH_FLAGS) -mmacosx-version-min=10.7 -o MacEthernetTapAgent osdep/MacEthernetTapAgent.c + $(CC) -Ofast $(ARCH_FLAGS) -mmacosx-version-min=10.13 -o MacEthernetTapAgent osdep/MacEthernetTapAgent.c $(CODESIGN) -f --options=runtime -s $(CODESIGN_APP_CERT) MacEthernetTapAgent osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm From 9eae4441046306d8a3333e484bb29c5d39efeac0 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 19 Aug 2021 09:21:52 -0700 Subject: [PATCH 02/10] kill some verbose logs --- controller/EmbeddedNetworkController.cpp | 2 +- controller/PostgreSQL.cpp | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index da0d7965e..fceaa3a40 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1484,7 +1484,7 @@ void EmbeddedNetworkController::_request( json &memberTags = member["tags"]; json &dns = network["dns"]; - fprintf(stderr, "IP Assignment Pools for Network %s: %s\n", nwids, OSUtils::jsonDump(ipAssignmentPools, 2).c_str()); + //fprintf(stderr, "IP Assignment Pools for Network %s: %s\n", nwids, OSUtils::jsonDump(ipAssignmentPools, 2).c_str()); if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,0) <= 0) { // Old versions with no rules engine support get an allow everything rule. diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 0fc3f40d7..eda5b6841 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -232,7 +232,6 @@ bool PostgreSQL::isReady() bool PostgreSQL::save(nlohmann::json &record,bool notifyListeners) { - fprintf(stderr, "PostgreSQL::save\n"); bool modified = false; try { if (!record.is_object()) { @@ -241,7 +240,7 @@ bool PostgreSQL::save(nlohmann::json &record,bool notifyListeners) } const std::string objtype = record["objtype"]; if (objtype == "network") { - fprintf(stderr, "network save\n"); + //fprintf(stderr, "network save\n"); const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL); if (nwid) { nlohmann::json old; @@ -257,17 +256,17 @@ bool PostgreSQL::save(nlohmann::json &record,bool notifyListeners) std::string memberId = record["id"]; const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"],0ULL); const uint64_t id = OSUtils::jsonIntHex(record["id"],0ULL); - fprintf(stderr, "member save %s-%s\n", networkId.c_str(), memberId.c_str()); + //fprintf(stderr, "member save %s-%s\n", networkId.c_str(), memberId.c_str()); if ((id)&&(nwid)) { nlohmann::json network,old; get(nwid,network,id,old); if ((!old.is_object())||(!_compareRecords(old,record))) { - fprintf(stderr, "commit queue post\n"); + //fprintf(stderr, "commit queue post\n"); record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL; _commitQueue.post(std::pair(record,notifyListeners)); modified = true; } else { - fprintf(stderr, "no change\n"); + //fprintf(stderr, "no change\n"); } } } else { From eec46a137ec36dd204eaa1e4c1ad959e4de7cd29 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 19 Aug 2021 12:44:02 -0700 Subject: [PATCH 03/10] optimize data loading from psql on startup --- controller/PostgreSQL.cpp | 482 ++++++++++++----------- ext/central-controller-docker/Dockerfile | 22 +- make-linux.mk | 1 + make-mac.mk | 25 +- 4 files changed, 272 insertions(+), 258 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index eda5b6841..c6636b30a 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -429,149 +429,168 @@ void PostgreSQL::initializeNetworks() std::unordered_set networkSet; fprintf(stderr, "Initializing Networks...\n"); - auto c = _pool->borrow(); - pqxx::work w{*c->c}; - pqxx::result r = w.exec_params("SELECT id, (EXTRACT(EPOCH FROM creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, capabilities, " + + char qbuf[2048] = {0}; + sprintf(qbuf, "SELECT id, (EXTRACT(EPOCH FROM creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, capabilities, " "enable_broadcast, (EXTRACT(EPOCH FROM last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, mtu, multicast_limit, name, private, remote_trace_level, " "remote_trace_target, revision, rules, tags, v4_assign_mode, v6_assign_mode, sso_enabled FROM ztc_network " - "WHERE deleted = false AND controller_id = $1", _myAddressStr); + "WHERE deleted = false AND controller_id = '%s'", _myAddressStr.c_str()); + auto c = _pool->borrow(); + auto c2 = _pool->borrow(); + pqxx::work w{*c->c}; + + auto stream = pqxx::stream_from::query(w, qbuf); - for (auto row = r.begin(); row != r.end(); row++) { + std::tuple< + std::string // network ID + , std::optional // creationTime + , std::optional // capabilities + , std::optional // enableBroadcast + , std::optional // lastModified + , std::optional // mtu + , std::optional // multicastLimit + , std::optional // name + , bool // private + , std::optional // remoteTraceLevel + , std::optional // remoteTraceTarget + , std::optional // revision + , std::optional // rules + , std::optional // tags + , std::optional // v4AssignMode + , std::optional // v6AssignMode + , std::optional // ssoEnabled + > row; + + while (stream >> row) { json empty; json config; initNetwork(config); - std::string nwid = row[0].as(); + std::string nwid = std::get<0>(row); + std::optional creationTime = std::get<1>(row); + std::optional capabilities = std::get<2>(row); + std::optional enableBroadcast = std::get<3>(row); + std::optional lastModified = std::get<4>(row); + std::optional mtu = std::get<5>(row); + std::optional multicastLimit = std::get<6>(row); + std::optional name = std::get<7>(row); + bool isPrivate = std::get<8>(row); + std::optional remoteTraceLevel = std::get<9>(row); + std::optional remoteTraceTarget = std::get<10>(row); + std::optional revision = std::get<11>(row); + std::optional rules = std::get<12>(row); + std::optional tags = std::get<13>(row); + std::optional v4AssignMode = std::get<14>(row); + std::optional v6AssignMode = std::get<15>(row); + std::optional ssoEnabled = std::get<16>(row); - networkSet.insert(nwid); + networkSet.insert(nwid); - config["id"] = nwid; - config["nwid"] = nwid; - - if (!row[1].is_null()) { - config["creationTime"] = row[1].as(); - } else { - config["creationTime"] = 0ULL; - } - config["capabilities"] = row[2].as(); - config["enableBroadcast"] = row[3].as(); - if (!row[4].is_null()) { - config["lastModified"] = row[4].as(); - } else { - config["lastModified"] = 0ULL; - } - if (!row[5].is_null()) { - config["mtu"] = row[5].as(); - } else { - config["mtu"] = 2800; - } - if (!row[6].is_null()) { - config["multicastLimit"] = row[6].as(); - } else { - config["multicastLimit"] = 64; - } - config["name"] = row[7].as(); - config["private"] = row[8].as(); - if (!row[9].is_null()) { - config["remoteTraceLevel"] = row[9].as(); - } else { - config["remoteTraceLevel"] = 0; - } + config["id"] = nwid; + config["nwid"] = nwid; + config["creationTime"] = creationTime.value_or(0); + config["capabilities"] = json::parse(capabilities.value_or("[]")); + config["enableBroadcast"] = enableBroadcast.value_or(false); + config["lastModified"] = lastModified.value_or(0); + config["mtu"] = mtu.value_or(2800); + config["multicastLimit"] = multicastLimit.value_or(64); + config["name"] = name.value_or(""); + config["private"] = isPrivate; + config["remoteTraceLevel"] = remoteTraceLevel.value_or(0); + config["remoteTraceTarget"] = remoteTraceTarget.value_or(""); + config["revision"] = revision.value_or(0); + config["rules"] = json::parse(rules.value_or("[]")); + config["tags"] = json::parse(tags.value_or("[]")); + config["v4AssignMode"] = json::parse(v4AssignMode.value_or("{}")); + config["v6AssignMode"] = json::parse(v6AssignMode.value_or("{}")); + config["ssoEnabled"] = ssoEnabled.value_or(false); + config["objtype"] = "network"; + config["ipAssignmentPools"] = json::array(); + config["routes"] = json::array(); - if (!row[10].is_null()) { - config["remoteTraceTarget"] = row[10].as(); - } else { - config["remoteTraceTarget"] = nullptr; - } + { + pqxx::work w2{*c2->c}; + pqxx::result r2 = w2.exec_params("SELECT host(ip_range_start), host(ip_range_end) FROM ztc_network_assignment_pool WHERE network_id = $1", nwid); + for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { + json ip; + ip["ipRangeStart"] = row2[0].as(); + ip["ipRangeEnd"] = row2[1].as(); - if (!row[11].is_null()) { - config["revision"] = row[11].as(); - } else { - config["revision"] = 0ULL; - //fprintf(stderr, "Error converting revision: %s\n", PQgetvalue(res, i, 11)); - } - config["rules"] = json::parse(row[12].as()); - config["tags"] = json::parse(row[13].as()); - config["v4AssignMode"] = json::parse(row[14].as()); - config["v6AssignMode"] = json::parse(row[15].as()); - config["ssoEnabled"] = row[16].as(); - config["objtype"] = "network"; - config["ipAssignmentPools"] = json::array(); - config["routes"] = json::array(); - - - pqxx::result r2 = w.exec_params("SELECT host(ip_range_start), host(ip_range_end) FROM ztc_network_assignment_pool WHERE network_id = $1", nwid); - - for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { - json ip; - ip["ipRangeStart"] = row2[0].as(); - ip["ipRangeEnd"] = row2[1].as(); - - config["ipAssignmentPools"].push_back(ip); - } - - - - r2 = w.exec_params("SELECT host(address), bits, host(via) FROM ztc_network_route WHERE network_id = $1", nwid); - - for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { - std::string addr = row2[0].as(); - std::string bits = row2[1].as(); - - json route; - route["target"] = addr + "/" + bits; - - if (row[2].is_null()) { - route["via"] = nullptr; - } else { - route["via"] = row[2].as(); + config["ipAssignmentPools"].push_back(ip); } - - config["routes"].push_back(route); + w2.commit(); } - r2 = w.exec_params("SELECT domain, servers FROM ztc_network_dns WHERE network_id = $1", nwid); - - if (r2.size() > 1) { - fprintf(stderr, "ERROR: invalid number of DNS configurations for network %s. Must be 0 or 1\n", nwid.c_str()); - } else if (r2.size() == 1) { - auto dnsRow = r2.begin(); - json obj; - std::string domain = dnsRow[0].as(); - std::string serverList = dnsRow[1].as(); - auto servers = json::array(); - if (serverList.rfind("{",0) != std::string::npos) { - serverList = serverList.substr(1, serverList.size()-2); - std::stringstream ss(serverList); - while(ss.good()) { - std::string server; - std::getline(ss, server, ','); - servers.push_back(server); + { + pqxx::work w2{*c2->c}; + pqxx::result r2 = w2.exec_params("SELECT host(address), bits, host(via) FROM ztc_network_route WHERE network_id = $1", nwid); + for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { + std::string addr = row2[0].as(); + std::string bits = row2[1].as(); + json route; + route["target"] = addr + "/" + bits; + if (row2[2].is_null()) { + route["via"] = nullptr; + } else { + route["via"] = row2[2].as(); } + config["routes"].push_back(route); } - obj["domain"] = domain; - obj["servers"] = servers; - config["dns"] = obj; + w2.commit(); } - r2 = w.exec_params("SELECT org.client_id, org.authorization_endpoint " - "FROM ztc_network nw " - "INNER JOIN ztc_org org " - " ON org.owner_id = nw.owner_id " - "WHERE nw.id = $1 AND nw.sso_enabled = true", nwid); - - if (r2.size() == 1) { - // only one should exist - pqxx::row row = r.at(0); - config["clientId"] = row[0].as(); - config["authorizationEndpoint"] = row[1].as(); + { + pqxx::work w2{*c2->c}; + pqxx::result r2 = w2.exec_params("SELECT domain, servers FROM ztc_network_dns WHERE network_id = $1", nwid); + + if (r2.size() > 1) { + fprintf(stderr, "ERROR: invalid number of DNS configurations for network %s. Must be 0 or 1\n", nwid.c_str()); + } else if (r2.size() == 1) { + auto dnsRow = r2.begin(); + json obj; + std::string domain = dnsRow[0].as(); + std::string serverList = dnsRow[1].as(); + auto servers = json::array(); + if (serverList.rfind("{",0) != std::string::npos) { + serverList = serverList.substr(1, serverList.size()-2); + std::stringstream ss(serverList); + while(ss.good()) { + std::string server; + std::getline(ss, server, ','); + servers.push_back(server); + } + } + obj["domain"] = domain; + obj["servers"] = servers; + config["dns"] = obj; + } + w2.commit(); } - _networkChanged(empty, config, false); + { + pqxx::work w2{*c2->c}; + pqxx::result r2 = w2.exec_params("SELECT org.client_id, org.authorization_endpoint " + "FROM ztc_network nw " + "INNER JOIN ztc_org org " + " ON org.owner_id = nw.owner_id " + "WHERE nw.id = $1 AND nw.sso_enabled = true", nwid); + + if (r2.size() == 1) { + // only one should exist + pqxx::row row2 = r2.at(0); + config["clientId"] = row2[0].as(); + config["authorizationEndpoint"] = row2[1].as(); + } + w2.commit(); + } + + _networkChanged(empty, config, false); } + w.commit(); + _pool->unborrow(c2); _pool->unborrow(c); if (++this->_ready == 2) { @@ -595,12 +614,10 @@ void PostgreSQL::initializeMembers() std::string networkId; try { std::unordered_map networkMembers; - fprintf(stderr, "Initializing Members...\n"); - auto c = _pool->borrow(); - pqxx::work w{*c->c}; - pqxx::result r = w.exec_params( - "SELECT m.id, m.network_id, m.active_bridge, m.authorized, m.capabilities, (EXTRACT(EPOCH FROM m.creation_time AT TIME ZONE 'UTC')*1000)::bigint, m.identity, " + + char qbuf[2048]; + sprintf(qbuf, "SELECT m.id, m.network_id, m.active_bridge, m.authorized, m.capabilities, (EXTRACT(EPOCH FROM m.creation_time AT TIME ZONE 'UTC')*1000)::bigint, m.identity, " " (EXTRACT(EPOCH FROM m.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, " " (EXTRACT(EPOCH FROM m.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, " " m.remote_trace_level, m.remote_trace_target, m.tags, m.v_major, m.v_minor, m.v_rev, m.v_proto, " @@ -608,137 +625,132 @@ void PostgreSQL::initializeMembers() "FROM ztc_member m " "INNER JOIN ztc_network n " " ON n.id = m.network_id " - "WHERE n.controller_id = $1 AND m.deleted = false", _myAddressStr); + "WHERE n.controller_id = '%s' AND m.deleted = false", _myAddressStr.c_str()); + auto c = _pool->borrow(); + auto c2 = _pool->borrow(); + pqxx::work w{*c->c}; + + auto stream = pqxx::stream_from::query(w, qbuf); - for (auto row = r.begin(); row != r.end(); row++) { + std::tuple< + std::string // memberId + , std::string // memberId + , std::optional // activeBridge + , std::optional // authorized + , std::optional // capabilities + , std::optional // creationTime + , std::optional // identity + , std::optional // lastAuthorizedTime + , std::optional // lastDeauthorizedTime + , std::optional // remoteTraceLevel + , std::optional // remoteTraceTarget + , std::optional // tags + , std::optional // vMajor + , std::optional // vMinor + , std::optional // vRev + , std::optional // vProto + , std::optional // noAutoAssignIps + , std::optional // revision + , std::optional // ssoExempt + > row; + + while (stream >> row) { json empty; json config; - memberId = ""; - networkId = ""; - initMember(config); - if (row[0].is_null()) { - fprintf(stderr, "Null memberID?!?\n"); - continue; - } - if (row[1].is_null()) { - fprintf(stderr, "Null NetworkID?!?\n"); - } - memberId = row[0].as(); - networkId = row[1].as(); + memberId = std::get<0>(row); + networkId = std::get<1>(row); + std::optional activeBridge = std::get<2>(row); + std::optional authorized = std::get<3>(row); + std::optional capabilities = std::get<4>(row); + std::optional creationTime = std::get<5>(row); + std::optional identity = std::get<6>(row); + std::optional lastAuthorizedTime = std::get<7>(row); + std::optional lastDeauthorizedTime = std::get<8>(row); + std::optional remoteTraceLevel = std::get<9>(row); + std::optional remoteTraceTarget = std::get<10>(row); + std::optional tags = std::get<11>(row); + std::optional vMajor = std::get<12>(row); + std::optional vMinor = std::get<13>(row); + std::optional vRev = std::get<14>(row); + std::optional vProto = std::get<15>(row); + std::optional noAutoAssignIps = std::get<16>(row); + std::optional revision = std::get<17>(row); + std::optional ssoExempt = std::get<18>(row); + config["id"] = memberId; config["nwid"] = networkId; - config["activeBridge"] = row[2].as(); - config["authorized"] = row[3].as(); - if (row[4].is_null()) { - config["capabilities"] = json::array(); - } else { - try { - config["capabilities"] = json::parse(row[4].as()); - } catch (std::exception &e) { - config["capabilities"] = json::array(); - } - } - config["creationTime"] = row[5].as(); - config["identity"] = row[6].as(); - if (!row[7].is_null()) { - config["lastAuthorizedTime"] = row[7].as(); - } else { - config["lastAuthorizedTime"] = 0ULL; - //fprintf(stderr, "Error updating last auth time (member): %s\n", PQgetvalue(res, i, 7)); - } - if (!row[8].is_null()) { - config["lastDeauthorizedTime"] = row[8].as(); - } else { - config["lastDeauthorizedTime"] = 0ULL; - //fprintf(stderr, "Error updating last deauth time (member): %s\n", PQgetvalue(res, i, 8)); - } - if (!row[9].is_null()) { - config["remoteTraceLevel"] = row[9].as(); - } else { - config["remoteTraceLevel"] = 0; - } - if (!row[10].is_null()) { - config["remoteTraceTarget"] = row[10].as(); - } else { - config["remoteTraceTarget"] = ""; - } - if (row[11].is_null()) { - config["tags"] = json::array(); - } else { - try { - config["tags"] = json::parse(row[11].as()); - } catch (std::exception &e) { - config["tags"] = json::array(); - } - } - if (!row[12].is_null()) { - config["vMajor"] = row[12].as(); - } else { - config["vMajor"] = -1; - } - if (!row[13].is_null()) { - config["vMinor"] = row[13].as(); - } else { - config["vMinor"] = -1; - } - if (!row[14].is_null()) { - config["vRev"] = row[14].as(); - } else { - config["vRev"] = -1; - } - if (!row[15].is_null()) { - config["vProto"] = row[15].as(); - } else { - config["vProto"] = -1; - } - config["noAutoAssignIps"] = row[16].as(); - if (!row[17].is_null()) { - config["revision"] = row[17].as(); - } else { - config["revision"] = 0ULL; - } - config["ssoExempt"] = row[18].as(); + config["activeBridge"] = activeBridge.value_or(false); + config["authorized"] = authorized.value_or(false); + config["capabilities"] = json::parse(capabilities.value_or("[]")); + config["creationTime"] = creationTime.value_or(0); + config["identity"] = identity.value_or(""); + config["lastAuthorizedTime"] = lastAuthorizedTime.value_or(0); + config["lastDeauthorizedTime"] = lastDeauthorizedTime.value_or(0); + config["remoteTraceLevel"] = remoteTraceLevel.value_or(0); + config["remoteTraceTarget"] = remoteTraceTarget.value_or(""); + config["tags"] = json::parse(tags.value_or("[]")); + config["vMajor"] = vMajor.value_or(-1); + config["vMinor"] = vMinor.value_or(-1); + config["vRev"] = vRev.value_or(-1); + config["vProto"] = vProto.value_or(-1); + config["noAutoAssignIps"] = noAutoAssignIps.value_or(false); + config["revision"] = revision.value_or(0); + config["ssoExempt"] = ssoExempt.value_or(false); - config["authenticationExpiryTime"] = 0LL; - pqxx::result authRes = w.exec_params( - "SELECT (EXTRACT(EPOCH FROM e.authentication_expiry_time)*1000)::bigint " - "FROM ztc_sso_expiry e " - "INNER JOIN ztc_network n " - " ON n.id = e.network_id " - "WHERE e.network_id = $1 AND e.member_id = $2 AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL " - "ORDER BY e.authentication_expiry_time DESC LIMIT 1", networkId, memberId); - - if (authRes.size() == 1 && !authRes.at(0)[0].is_null()) { - // there is an expiry time record - config["authenticationExpiryTime"] = authRes.at(0)[0].as(); - } else { - config["authenticationExpiryTime"] = 0; + { + config["authenticationExpiryTime"] = 0LL; + + pqxx::work w2{*c2->c}; + pqxx::result authRes = w2.exec_params( + "SELECT (EXTRACT(EPOCH FROM e.authentication_expiry_time)*1000)::bigint " + "FROM ztc_sso_expiry e " + "INNER JOIN ztc_network n " + " ON n.id = e.network_id " + "WHERE e.network_id = $1 AND e.member_id = $2 AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL " + "ORDER BY e.authentication_expiry_time DESC LIMIT 1", networkId, memberId); + + if (authRes.size() == 1 && !authRes.at(0)[0].is_null()) { + // there is an expiry time record + config["authenticationExpiryTime"] = authRes.at(0)[0].as(); + } else { + config["authenticationExpiryTime"] = 0; + } + w2.commit(); } config["objtype"] = "member"; - config["ipAssignments"] = json::array(); - pqxx::result r2 = w.exec_params("SELECT DISTINCT address " - "FROM ztc_member_ip_assignment " - "WHERE member_id = $1 AND network_id = $2", memberId, networkId); + { + config["ipAssignments"] = json::array(); - for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { - std::string ipaddr = row2[0].as(); - std::size_t pos = ipaddr.find('/'); - if (pos != std::string::npos) { - ipaddr = ipaddr.substr(0, pos); + pqxx::work w2{*c2->c}; + pqxx::result r2 = w2.exec_params("SELECT DISTINCT address " + "FROM ztc_member_ip_assignment " + "WHERE member_id = $1 AND network_id = $2", memberId, networkId); + + for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { + std::string ipaddr = row2[0].as(); + std::size_t pos = ipaddr.find('/'); + if (pos != std::string::npos) { + ipaddr = ipaddr.substr(0, pos); + } + config["ipAssignments"].push_back(ipaddr); } - config["ipAssignments"].push_back(ipaddr); + w2.commit(); } _memberChanged(empty, config, false); + + memberId = ""; + networkId = ""; } w.commit(); + _pool->unborrow(c2); _pool->unborrow(c); if (++this->_ready == 2) { @@ -1313,7 +1325,7 @@ void PostgreSQL::onlineNotification_Postgres() // using pqxx::stream_to would be a really nice alternative here, but // unfortunately it doesn't support upserts. - fprintf(stderr, "online notification tick\n"); + // fprintf(stderr, "online notification tick\n"); std::stringstream memberUpdate; memberUpdate << "INSERT INTO ztc_member_status (network_id, member_id, address, last_updated) VALUES "; bool firstRun = true; @@ -1375,7 +1387,7 @@ void PostgreSQL::onlineNotification_Postgres() pqxx::result res = w.exec0(memberUpdate.str()); w.commit(); } - fprintf(stderr, "Updated online status of %d members\n", updateCount); + // fprintf(stderr, "Updated online status of %d members\n", updateCount); _pool->unborrow(c); } catch (std::exception &e) { fprintf(stderr, "%s: error in onlinenotification thread: %s\n", _myAddressStr.c_str(), e.what()); diff --git a/ext/central-controller-docker/Dockerfile b/ext/central-controller-docker/Dockerfile index 9669c037e..945562283 100644 --- a/ext/central-controller-docker/Dockerfile +++ b/ext/central-controller-docker/Dockerfile @@ -9,14 +9,14 @@ RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x RUN dnf -qy module disable postgresql RUN yum -y install epel-release && yum -y update && yum clean all RUN yum groupinstall -y "Development Tools" -RUN yum install -y bash cmake postgresql10 postgresql10-devel clang jemalloc jemalloc-devel -RUN curl -sL https://github.com/jtv/libpqxx/archive/refs/tags/6.4.5.tar.gz -o libpqxx.tar.gz -RUN tar -xzf libpqxx.tar.gz && \ - pushd libpqxx-6.4.5/ && \ - mkdir build && pushd build/ && \ - cmake .. && \ - make install -j8 && \ - popd && popd +RUN yum install -y bash cmake postgresql10 postgresql10-devel clang jemalloc jemalloc-devel libpqxx libpqxx-devel +# RUN curl -sL https://github.com/jtv/libpqxx/archive/refs/tags/6.4.5.tar.gz -o libpqxx.tar.gz +# RUN tar -xzf libpqxx.tar.gz && \ +# pushd libpqxx-6.4.5/ && \ +# mkdir build && pushd build/ && \ +# cmake .. && \ +# make install -j8 && \ +# popd && popd # RUN git clone http://git.int.zerotier.com/zerotier/ZeroTierOne.git @@ -28,10 +28,10 @@ FROM centos:8 RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm RUN dnf -qy module disable postgresql RUN yum -y install epel-release && yum -y update && yum clean all -RUN yum install -y jemalloc jemalloc-devel postgresql10 +RUN yum install -y jemalloc jemalloc-devel postgresql10 libpqxx -COPY --from=builder /usr/local/lib64/libpqxx.so /usr/local/lib64/libpqxx.so -COPY --from=builder /usr/local/lib64/libpqxx.a /usr/local/lib64/libpqxx.a +# COPY --from=builder /usr/local/lib64/libpqxx.so /usr/local/lib64/libpqxx.so +# COPY --from=builder /usr/local/lib64/libpqxx.a /usr/local/lib64/libpqxx.a COPY --from=builder /ZeroTierOne/zerotier-one /usr/local/bin/zerotier-one RUN chmod a+x /usr/local/bin/zerotier-one RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/usr-local-lib64.conf && ldconfig diff --git a/make-linux.mk b/make-linux.mk index 99d8390e2..824f36028 100644 --- a/make-linux.mk +++ b/make-linux.mk @@ -266,6 +266,7 @@ ifeq ($(ZT_OFFICIAL),1) endif ifeq ($(ZT_CONTROLLER),1) + override CXXFLAGS+=-Wall -Wno-deprecated -std=c++17 -pthread $(INCLUDES) -DNDEBUG $(DEFS) override LDLIBS+=-L/usr/pgsql-10/lib/ -lpqxx -lpq ext/hiredis-0.14.1/lib/centos8/libhiredis.a ext/redis-plus-plus-1.1.1/install/centos8/lib/libredis++.a override DEFS+=-DZT_CONTROLLER_USE_LIBPQ override INCLUDES+=-I/usr/pgsql-10/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/centos8/include/sw/ diff --git a/make-mac.mk b/make-mac.mk index ffc8b5c96..7438ad9ed 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -26,13 +26,6 @@ DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUIL include objects.mk ONE_OBJS+=osdep/MacEthernetTap.o osdep/MacKextEthernetTap.o osdep/MacDNSHelper.o ext/http-parser/http_parser.o - -ifeq ($(ZT_CONTROLLER),1) - LIBS+=-L/usr/local/opt/libpqxx@6/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a - DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER_USE_REDIS -DZT_CONTROLLER - INCLUDES+=-I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx@6/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/ -endif - LIBS+=-framework CoreServices -framework SystemConfiguration -framework CoreFoundation # Official releases are signed with our Apple cert and apply software updates by default @@ -52,10 +45,20 @@ endif # Use fast ASM Salsa20/12 for x64 processors DEFS+=-DZT_USE_X64_ASM_SALSA2012 CORE_OBJS+=ext/x64-salsa2012-asm/salsa2012.o +CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ # Build miniupnpc and nat-pmp as included libraries -- extra defs are required for these sources DEFS+=-DMACOSX -DZT_USE_MINIUPNPC -DMINIUPNP_STATICLIB -D_DARWIN_C_SOURCE -DMINIUPNPC_SET_SOCKET_TIMEOUT -DMINIUPNPC_GET_SRC_ADDR -D_BSD_SOURCE -D_DEFAULT_SOURCE -DOS_STRING=\"Darwin/15.0.0\" -DMINIUPNPC_VERSION_STRING=\"2.0\" -DUPNP_VERSION_STRING=\"UPnP/1.1\" -DENABLE_STRNATPMPERR ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connecthostport.o ext/miniupnpc/igd_desc_parse.o ext/miniupnpc/minisoap.o ext/miniupnpc/minissdpc.o ext/miniupnpc/miniupnpc.o ext/miniupnpc/miniwget.o ext/miniupnpc/minixml.o ext/miniupnpc/portlistingparse.o ext/miniupnpc/receivedata.o ext/miniupnpc/upnpcommands.o ext/miniupnpc/upnpdev.o ext/miniupnpc/upnperrors.o ext/miniupnpc/upnpreplyparse.o osdep/PortMapper.o +ifeq ($(ZT_CONTROLLER),1) + MACOS_VERSION_MIN=10.15 + override CXXFLAGS=$(CFLAGS) -std=c++17 -stdlib=libc++ + LIBS+=-L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a + DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER_USE_REDIS -DZT_CONTROLLER + INCLUDES+=-I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/ +else + MACOS_VERSION_MIN=10.13 +endif # Build with address sanitization library for advanced debugging (clang) ifeq ($(ZT_SANITIZE),1) @@ -75,7 +78,7 @@ ifeq ($(ZT_DEBUG),1) node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g $(INCLUDES) $(DEFS) else CFLAGS?=-Ofast -fstack-protector-strong - CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -mmacosx-version-min=10.13 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) + CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -mmacosx-version-min=$(MACOS_VERSION_MIN) -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS) STRIP=strip endif @@ -88,15 +91,13 @@ ifeq ($(ZT_VAULT_SUPPORT),1) LIBS+=-lcurl endif -CXXFLAGS=$(CFLAGS) -std=c++11 -stdlib=libc++ - all: one ext/x64-salsa2012-asm/salsa2012.o: - as -arch x86_64 -mmacosx-version-min=10.13 -o ext/x64-salsa2012-asm/salsa2012.o ext/x64-salsa2012-asm/salsa2012.s + as -arch x86_64 -mmacosx-version-min=$(MACOS_VERSION_MIN) -o ext/x64-salsa2012-asm/salsa2012.o ext/x64-salsa2012-asm/salsa2012.s mac-agent: FORCE - $(CC) -Ofast $(ARCH_FLAGS) -mmacosx-version-min=10.13 -o MacEthernetTapAgent osdep/MacEthernetTapAgent.c + $(CC) -Ofast $(ARCH_FLAGS) -mmacosx-version-min=$(MACOS_VERSION_MIN) -o MacEthernetTapAgent osdep/MacEthernetTapAgent.c $(CODESIGN) -f --options=runtime -s $(CODESIGN_APP_CERT) MacEthernetTapAgent osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm From 20721491e8039c8022ca96761e9c7bc09e31fbfc Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 19 Aug 2021 13:03:56 -0700 Subject: [PATCH 04/10] kill some noisy logs --- controller/EmbeddedNetworkController.cpp | 6 +++--- controller/PostgreSQL.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index fceaa3a40..99d59aeef 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -1364,9 +1364,9 @@ void EmbeddedNetworkController::_request( if (networkSSOEnabled && !memberSSOExempt) { authenticationURL = _db.getSSOAuthURL(member, _ssoRedirectURL); std::string memberId = member["id"]; - fprintf(stderr, "ssoEnabled && !ssoExempt %s-%s\n", nwids, memberId.c_str()); + //fprintf(stderr, "ssoEnabled && !ssoExempt %s-%s\n", nwids, memberId.c_str()); uint64_t authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0); - fprintf(stderr, "authExpiryTime: %lld\n", authenticationExpiryTime); + //fprintf(stderr, "authExpiryTime: %lld\n", authenticationExpiryTime); if (authenticationExpiryTime < now) { if (!authenticationURL.empty()) { _db.networkMemberSSOHasExpired(nwid, now); @@ -1374,7 +1374,7 @@ void EmbeddedNetworkController::_request( Dictionary<3072> authInfo; authInfo.add("aU", authenticationURL.c_str()); - fprintf(stderr, "sending auth URL: %s\n", authenticationURL.c_str()); + //fprintf(stderr, "sending auth URL: %s\n", authenticationURL.c_str()); DB::cleanMember(member); _db.save(member,true); diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index c6636b30a..6a5e0fe68 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -335,7 +335,7 @@ std::string PostgreSQL::getSSOAuthURL(const nlohmann::json &member, const std::s std::string memberId = member["id"]; char authenticationURL[4096] = {0}; - fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str()); + //fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str()); bool have_auth = false; try { auto c = _pool->borrow(); From 50b0b2e2e9600829f6af29c92d4e0e7877d53e21 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Thu, 19 Aug 2021 17:55:30 -0700 Subject: [PATCH 05/10] query optimization --- controller/PostgreSQL.cpp | 150 +++++++++++++++++++------------------- 1 file changed, 77 insertions(+), 73 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 6a5e0fe68..d369bf9b4 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -24,9 +24,10 @@ #include #include #include +#include -#define ZT_TRACE 1 +//#define ZT_TRACE 1 using json = nlohmann::json; @@ -426,14 +427,18 @@ void PostgreSQL::initializeNetworks() try { std::string setKey = "networks:{" + _myAddressStr + "}"; - std::unordered_set networkSet; - fprintf(stderr, "Initializing Networks...\n"); char qbuf[2048] = {0}; - sprintf(qbuf, "SELECT id, (EXTRACT(EPOCH FROM creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, capabilities, " - "enable_broadcast, (EXTRACT(EPOCH FROM last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, mtu, multicast_limit, name, private, remote_trace_level, " - "remote_trace_target, revision, rules, tags, v4_assign_mode, v6_assign_mode, sso_enabled FROM ztc_network " + sprintf(qbuf, "SELECT n.id, (EXTRACT(EPOCH FROM n.creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, n.capabilities, " + "n.enable_broadcast, (EXTRACT(EPOCH FROM n.last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, n.mtu, n.multicast_limit, n.name, n.private, n.remote_trace_level, " + "n.remote_trace_target, n.revision, n.rules, n.tags, n.v4_assign_mode, n.v6_assign_mode, n.sso_enabled, (CASE WHEN n.sso_enabled THEN o.client_id ELSE NULL END) as client_id, " + "(CASE WHEN n.sso_enabled THEN o.authorization_endpoint ELSE NULL END) as authorization_endpoint, d.domain, d.servers " + "FROM ztc_network n " + "LEFT OUTER JOIN ztc_org o " + " ON o.owner_id = n.owner_id " + "LEFT OUTER JOIN ztc_network_dns d " + " ON d.network_id = n.id " "WHERE deleted = false AND controller_id = '%s'", _myAddressStr.c_str()); auto c = _pool->borrow(); auto c2 = _pool->borrow(); @@ -459,9 +464,18 @@ void PostgreSQL::initializeNetworks() , std::optional // v4AssignMode , std::optional // v6AssignMode , std::optional // ssoEnabled + , std::optional // clientId + , std::optional // authorizationEndpoint + , std::optional // domain + , std::optional // servers > row; + uint64_t count = 0; + auto tmp = std::chrono::high_resolution_clock::now(); + uint64_t total = 0; while (stream >> row) { + auto start = std::chrono::high_resolution_clock::now(); + json empty; json config; @@ -484,9 +498,11 @@ void PostgreSQL::initializeNetworks() std::optional v4AssignMode = std::get<14>(row); std::optional v6AssignMode = std::get<15>(row); std::optional ssoEnabled = std::get<16>(row); + std::optional clientId = std::get<17>(row); + std::optional authorizationEndpoint = std::get<18>(row); + std::optional dnsDomain = std::get<19>(row); + std::optional dnsServers = std::get<20>(row); - networkSet.insert(nwid); - config["id"] = nwid; config["nwid"] = nwid; config["creationTime"] = creationTime.value_or(0); @@ -508,6 +524,26 @@ void PostgreSQL::initializeNetworks() config["objtype"] = "network"; config["ipAssignmentPools"] = json::array(); config["routes"] = json::array(); + config["clientId"] = clientId.value_or(""); + config["authorizationEndpoint"] = authorizationEndpoint.value_or(""); + + if (dnsDomain.has_value()) { + std::string serverList = dnsServers.value(); + json obj; + auto servers = json::array(); + if (serverList.rfind("{",0) != std::string::npos) { + serverList = serverList.substr(1, serverList.size()-2); + std::stringstream ss(serverList); + while(ss.good()) { + std::string server; + std::getline(ss, server, ','); + servers.push_back(server); + } + } + obj["domain"] = dnsDomain.value(); + obj["servers"] = servers; + config["dns"] = obj; + } { pqxx::work w2{*c2->c}; @@ -520,11 +556,8 @@ void PostgreSQL::initializeNetworks() config["ipAssignmentPools"].push_back(ip); } w2.commit(); - } - { - pqxx::work w2{*c2->c}; - pqxx::result r2 = w2.exec_params("SELECT host(address), bits, host(via) FROM ztc_network_route WHERE network_id = $1", nwid); + r2 = w2.exec_params("SELECT host(address), bits, host(via) FROM ztc_network_route WHERE network_id = $1", nwid); for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { std::string addr = row2[0].as(); std::string bits = row2[1].as(); @@ -537,57 +570,21 @@ void PostgreSQL::initializeNetworks() } config["routes"].push_back(route); } - w2.commit(); - } - - { - pqxx::work w2{*c2->c}; - pqxx::result r2 = w2.exec_params("SELECT domain, servers FROM ztc_network_dns WHERE network_id = $1", nwid); - - if (r2.size() > 1) { - fprintf(stderr, "ERROR: invalid number of DNS configurations for network %s. Must be 0 or 1\n", nwid.c_str()); - } else if (r2.size() == 1) { - auto dnsRow = r2.begin(); - json obj; - std::string domain = dnsRow[0].as(); - std::string serverList = dnsRow[1].as(); - auto servers = json::array(); - if (serverList.rfind("{",0) != std::string::npos) { - serverList = serverList.substr(1, serverList.size()-2); - std::stringstream ss(serverList); - while(ss.good()) { - std::string server; - std::getline(ss, server, ','); - servers.push_back(server); - } - } - obj["domain"] = domain; - obj["servers"] = servers; - config["dns"] = obj; - } - w2.commit(); - } - - { - pqxx::work w2{*c2->c}; - pqxx::result r2 = w2.exec_params("SELECT org.client_id, org.authorization_endpoint " - "FROM ztc_network nw " - "INNER JOIN ztc_org org " - " ON org.owner_id = nw.owner_id " - "WHERE nw.id = $1 AND nw.sso_enabled = true", nwid); - - if (r2.size() == 1) { - // only one should exist - pqxx::row row2 = r2.at(0); - config["clientId"] = row2[0].as(); - config["authorizationEndpoint"] = row2[1].as(); - } - w2.commit(); } _networkChanged(empty, config, false); + + auto end = std::chrono::high_resolution_clock::now(); + auto dur = std::chrono::duration_cast(end - start);; + total += dur.count(); + ++count; + if (count % 1000 == 0) { + fprintf(stderr, "Averaging %llu us per network\n", (total/count)); + } } + fprintf(stderr, "Took %llu us per network to load\n", (total/count)); + stream.complete(); w.commit(); _pool->unborrow(c2); @@ -654,7 +651,11 @@ void PostgreSQL::initializeMembers() , std::optional // ssoExempt > row; + uint64_t count = 0; + auto tmp = std::chrono::high_resolution_clock::now(); + uint64_t total = 0; while (stream >> row) { + auto start = std::chrono::high_resolution_clock::now(); json empty; json config; @@ -700,7 +701,7 @@ void PostgreSQL::initializeMembers() config["noAutoAssignIps"] = noAutoAssignIps.value_or(false); config["revision"] = revision.value_or(0); config["ssoExempt"] = ssoExempt.value_or(false); - + config["objtype"] = "member"; { config["authenticationExpiryTime"] = 0LL; @@ -719,15 +720,8 @@ void PostgreSQL::initializeMembers() } else { config["authenticationExpiryTime"] = 0; } - w2.commit(); - } - config["objtype"] = "member"; - - { config["ipAssignments"] = json::array(); - - pqxx::work w2{*c2->c}; pqxx::result r2 = w2.exec_params("SELECT DISTINCT address " "FROM ztc_member_ip_assignment " "WHERE member_id = $1 AND network_id = $2", memberId, networkId); @@ -740,14 +734,24 @@ void PostgreSQL::initializeMembers() } config["ipAssignments"].push_back(ipaddr); } - w2.commit(); } _memberChanged(empty, config, false); memberId = ""; networkId = ""; + + auto end = std::chrono::high_resolution_clock::now(); + auto dur = std::chrono::duration_cast(end - start);; + total += dur.count(); + ++count; + if (count % 1000 == 0) { + fprintf(stderr, "Averaging %llu us per member\n", (total/count)); + } } + fprintf(stderr, "Took %llu us per member to load\n", (total/count)); + + stream.complete(); w.commit(); _pool->unborrow(c2); @@ -1006,7 +1010,7 @@ void PostgreSQL::commitThread() fprintf(stderr, "commitThread start\n"); std::pair qitem; while(_commitQueue.get(qitem)&(_run == 1)) { - fprintf(stderr, "commitThread tick\n"); + //fprintf(stderr, "commitThread tick\n"); if (!qitem.first.is_object()) { fprintf(stderr, "not an object\n"); continue; @@ -1016,7 +1020,7 @@ void PostgreSQL::commitThread() nlohmann::json *config = &(qitem.first); const std::string objtype = (*config)["objtype"]; if (objtype == "member") { - fprintf(stderr, "commitThread: member\n"); + //fprintf(stderr, "commitThread: member\n"); try { auto c = _pool->borrow(); pqxx::work w(*c->c); @@ -1109,7 +1113,7 @@ void PostgreSQL::commitThread() } } else if (objtype == "network") { try { - fprintf(stderr, "commitThread: network\n"); + //fprintf(stderr, "commitThread: network\n"); auto c = _pool->borrow(); pqxx::work w(*c->c); @@ -1248,7 +1252,7 @@ void PostgreSQL::commitThread() fprintf(stderr, "ERROR: Error updating network: %s\n", e.what()); } } else if (objtype == "_delete_network") { - fprintf(stderr, "commitThread: delete network\n"); + //fprintf(stderr, "commitThread: delete network\n"); try { auto c = _pool->borrow(); pqxx::work w(*c->c); @@ -1265,7 +1269,7 @@ void PostgreSQL::commitThread() } } else if (objtype == "_delete_member") { - fprintf(stderr, "commitThread: delete member\n"); + //fprintf(stderr, "commitThread: delete member\n"); try { auto c = _pool->borrow(); pqxx::work w(*c->c); @@ -1383,7 +1387,7 @@ void PostgreSQL::onlineNotification_Postgres() memberUpdate << " ON CONFLICT (network_id, member_id) DO UPDATE SET address = EXCLUDED.address, last_updated = EXCLUDED.last_updated;"; if (memberAdded) { - fprintf(stderr, "%s\n", memberUpdate.str().c_str()); + //fprintf(stderr, "%s\n", memberUpdate.str().c_str()); pqxx::result res = w.exec0(memberUpdate.str()); w.commit(); } From 1d8b8d8e9c005e8c3014a466da057bcea333b8f8 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Fri, 20 Aug 2021 09:22:44 -0700 Subject: [PATCH 06/10] optimize the controller build & ship workflow a bit --- ext/central-controller-docker/Dockerfile | 31 ++----------------- .../Dockerfile.builder | 12 +++++++ .../Dockerfile.run_base | 5 +++ 3 files changed, 19 insertions(+), 29 deletions(-) create mode 100644 ext/central-controller-docker/Dockerfile.builder create mode 100644 ext/central-controller-docker/Dockerfile.run_base diff --git a/ext/central-controller-docker/Dockerfile b/ext/central-controller-docker/Dockerfile index 945562283..8435b1685 100644 --- a/ext/central-controller-docker/Dockerfile +++ b/ext/central-controller-docker/Dockerfile @@ -1,37 +1,10 @@ # Dockerfile for ZeroTier Central Controllers -FROM centos:8 as builder +FROM registry.zerotier.com/zerotier/controller-builder:latest as builder MAINTAINER Adam Ierymekno , Grant Limberg - -ARG git_branch=master - -RUN yum update -y -RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm -RUN dnf -qy module disable postgresql -RUN yum -y install epel-release && yum -y update && yum clean all -RUN yum groupinstall -y "Development Tools" -RUN yum install -y bash cmake postgresql10 postgresql10-devel clang jemalloc jemalloc-devel libpqxx libpqxx-devel -# RUN curl -sL https://github.com/jtv/libpqxx/archive/refs/tags/6.4.5.tar.gz -o libpqxx.tar.gz -# RUN tar -xzf libpqxx.tar.gz && \ -# pushd libpqxx-6.4.5/ && \ -# mkdir build && pushd build/ && \ -# cmake .. && \ -# make install -j8 && \ -# popd && popd - - -# RUN git clone http://git.int.zerotier.com/zerotier/ZeroTierOne.git -# RUN if [ "$git_branch" != "master" ]; then cd ZeroTierOne && git checkout -b $git_branch origin/$git_branch; fi ADD . /ZeroTierOne RUN cd ZeroTierOne && make clean && make central-controller -j8 -FROM centos:8 -RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm -RUN dnf -qy module disable postgresql -RUN yum -y install epel-release && yum -y update && yum clean all -RUN yum install -y jemalloc jemalloc-devel postgresql10 libpqxx - -# COPY --from=builder /usr/local/lib64/libpqxx.so /usr/local/lib64/libpqxx.so -# COPY --from=builder /usr/local/lib64/libpqxx.a /usr/local/lib64/libpqxx.a +FROM registry.zerotier.com/zerotier/controller-run:latest COPY --from=builder /ZeroTierOne/zerotier-one /usr/local/bin/zerotier-one RUN chmod a+x /usr/local/bin/zerotier-one RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/usr-local-lib64.conf && ldconfig diff --git a/ext/central-controller-docker/Dockerfile.builder b/ext/central-controller-docker/Dockerfile.builder new file mode 100644 index 000000000..69e69701f --- /dev/null +++ b/ext/central-controller-docker/Dockerfile.builder @@ -0,0 +1,12 @@ +# Dockerfile for building ZeroTier Central Controllers +FROM centos:8 as builder +MAINTAINER Adam Ierymekno , Grant Limberg + +ARG git_branch=master + +RUN yum update -y +RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm +RUN dnf -qy module disable postgresql +RUN yum -y install epel-release && yum -y update && yum clean all +RUN yum groupinstall -y "Development Tools" && yum clean all +RUN yum install -y bash cmake postgresql10 postgresql10-devel clang jemalloc jemalloc-devel libpqxx libpqxx-devel && yum clean all diff --git a/ext/central-controller-docker/Dockerfile.run_base b/ext/central-controller-docker/Dockerfile.run_base new file mode 100644 index 000000000..4db9cbd28 --- /dev/null +++ b/ext/central-controller-docker/Dockerfile.run_base @@ -0,0 +1,5 @@ +FROM centos:8 +RUN yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-8-x86_64/pgdg-redhat-repo-latest.noarch.rpm +RUN dnf -qy module disable postgresql +RUN yum -y install epel-release && yum -y update && yum clean all +RUN yum install -y jemalloc jemalloc-devel postgresql10 libpqxx && yum clean all From 6baac1b4e0ffa0c977f9bdc394edae392e19afb4 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Fri, 20 Aug 2021 10:27:45 -0700 Subject: [PATCH 07/10] more query optimizations --- controller/PostgreSQL.cpp | 104 +++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 51 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index d369bf9b4..46257d365 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -70,6 +70,17 @@ std::string join(const std::vector &elements, const char * const se } */ +std::vector split(std::string str, char delim){ + std::istringstream iss(str); + std::vector tokens; + std::string item; + while(std::getline(iss, item, delim)) { + tokens.push_back(item); + } + return tokens; +} + + } // anonymous namespace using namespace ZeroTier; @@ -433,7 +444,9 @@ void PostgreSQL::initializeNetworks() sprintf(qbuf, "SELECT n.id, (EXTRACT(EPOCH FROM n.creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, n.capabilities, " "n.enable_broadcast, (EXTRACT(EPOCH FROM n.last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, n.mtu, n.multicast_limit, n.name, n.private, n.remote_trace_level, " "n.remote_trace_target, n.revision, n.rules, n.tags, n.v4_assign_mode, n.v6_assign_mode, n.sso_enabled, (CASE WHEN n.sso_enabled THEN o.client_id ELSE NULL END) as client_id, " - "(CASE WHEN n.sso_enabled THEN o.authorization_endpoint ELSE NULL END) as authorization_endpoint, d.domain, d.servers " + "(CASE WHEN n.sso_enabled THEN o.authorization_endpoint ELSE NULL END) as authorization_endpoint, d.domain, d.servers, " + "ARRAY(SELECT CONCAT(host(ip_range_start),'|', host(ip_range_end)) FROM ztc_network_assignment_pool WHERE network_id = n.id) AS assignment_pool, " + "ARRAY(SELECT CONCAT(host(address),'/',bits::text,'|',COALESCE(host(via), 'NULL'))FROM ztc_network_route WHERE network_id = n.id) AS routes " "FROM ztc_network n " "LEFT OUTER JOIN ztc_org o " " ON o.owner_id = n.owner_id " @@ -468,6 +481,8 @@ void PostgreSQL::initializeNetworks() , std::optional // authorizationEndpoint , std::optional // domain , std::optional // servers + , std::string // assignmentPoolString + , std::string // routeString > row; uint64_t count = 0; @@ -502,6 +517,8 @@ void PostgreSQL::initializeNetworks() std::optional authorizationEndpoint = std::get<18>(row); std::optional dnsDomain = std::get<19>(row); std::optional dnsServers = std::get<20>(row); + std::string assignmentPoolString = std::get<21>(row); + std::string routesString = std::get<22>(row); config["id"] = nwid; config["nwid"] = nwid; @@ -545,29 +562,28 @@ void PostgreSQL::initializeNetworks() config["dns"] = obj; } - { - pqxx::work w2{*c2->c}; - pqxx::result r2 = w2.exec_params("SELECT host(ip_range_start), host(ip_range_end) FROM ztc_network_assignment_pool WHERE network_id = $1", nwid); - for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { + config["ipAssignmentPools"] = json::array(); + if (assignmentPoolString != "{}") { + std::string tmp = assignmentPoolString.substr(1, assignmentPoolString.size()-2); + std::vector assignmentPools = split(tmp, ','); + for (auto it = assignmentPools.begin(); it != assignmentPools.end(); ++it) { + std::vector r = split(*it, '|'); json ip; - ip["ipRangeStart"] = row2[0].as(); - ip["ipRangeEnd"] = row2[1].as(); - + ip["ipRangeStart"] = r[0]; + ip["ipRangeEnd"] = r[1]; config["ipAssignmentPools"].push_back(ip); } - w2.commit(); + } - r2 = w2.exec_params("SELECT host(address), bits, host(via) FROM ztc_network_route WHERE network_id = $1", nwid); - for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { - std::string addr = row2[0].as(); - std::string bits = row2[1].as(); + config["routes"] = json::array(); + if (routesString != "{}") { + std::string tmp = routesString.substr(1, routesString.size()-2); + std::vector routes = split(tmp, ','); + for (auto it = routes.begin(); it != routes.end(); ++it) { + std::vector r = split(*it, '|'); json route; - route["target"] = addr + "/" + bits; - if (row2[2].is_null()) { - route["via"] = nullptr; - } else { - route["via"] = row2[2].as(); - } + route["target"] = r[0]; + route["via"] = ((route["via"] == "NULL")? nullptr : r[1]); config["routes"].push_back(route); } } @@ -618,7 +634,13 @@ void PostgreSQL::initializeMembers() " (EXTRACT(EPOCH FROM m.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, " " (EXTRACT(EPOCH FROM m.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, " " m.remote_trace_level, m.remote_trace_target, m.tags, m.v_major, m.v_minor, m.v_rev, m.v_proto, " - " m.no_auto_assign_ips, m.revision, sso_exempt " + " m.no_auto_assign_ips, m.revision, sso_exempt, " + " (SELECT (EXTRACT(EPOCH FROM e.authentication_expiry_time)*1000)::bigint " + " FROM ztc_sso_expiry e " + " INNER JOIN ztc_network n1 " + " ON n.id = e.network_id " + " WHERE e.network_id = m.network_id AND e.member_id = m.id AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL " + " ORDER BY e.authentication_expiry_time DESC LIMIT 1) AS authentication_expiry_time " "FROM ztc_member m " "INNER JOIN ztc_network n " " ON n.id = m.network_id " @@ -649,6 +671,8 @@ void PostgreSQL::initializeMembers() , std::optional // noAutoAssignIps , std::optional // revision , std::optional // ssoExempt + , std::optional // authenticationExpiryTime + , std::string // assignedAddresses > row; uint64_t count = 0; @@ -680,7 +704,8 @@ void PostgreSQL::initializeMembers() std::optional noAutoAssignIps = std::get<16>(row); std::optional revision = std::get<17>(row); std::optional ssoExempt = std::get<18>(row); - + std::optional authenticationExpiryTime = std::get<19>(row); + std::string assignedAddresses = std::get<20>(row); config["id"] = memberId; config["nwid"] = networkId; @@ -701,38 +726,15 @@ void PostgreSQL::initializeMembers() config["noAutoAssignIps"] = noAutoAssignIps.value_or(false); config["revision"] = revision.value_or(0); config["ssoExempt"] = ssoExempt.value_or(false); + config["authenticationExpiryTime"] = authenticationExpiryTime.value_or(0); config["objtype"] = "member"; - { - config["authenticationExpiryTime"] = 0LL; + config["ipAssignments"] = json::array(); - pqxx::work w2{*c2->c}; - pqxx::result authRes = w2.exec_params( - "SELECT (EXTRACT(EPOCH FROM e.authentication_expiry_time)*1000)::bigint " - "FROM ztc_sso_expiry e " - "INNER JOIN ztc_network n " - " ON n.id = e.network_id " - "WHERE e.network_id = $1 AND e.member_id = $2 AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL " - "ORDER BY e.authentication_expiry_time DESC LIMIT 1", networkId, memberId); - - if (authRes.size() == 1 && !authRes.at(0)[0].is_null()) { - // there is an expiry time record - config["authenticationExpiryTime"] = authRes.at(0)[0].as(); - } else { - config["authenticationExpiryTime"] = 0; - } - - config["ipAssignments"] = json::array(); - pqxx::result r2 = w2.exec_params("SELECT DISTINCT address " - "FROM ztc_member_ip_assignment " - "WHERE member_id = $1 AND network_id = $2", memberId, networkId); - - for (auto row2 = r2.begin(); row2 != r2.end(); row2++) { - std::string ipaddr = row2[0].as(); - std::size_t pos = ipaddr.find('/'); - if (pos != std::string::npos) { - ipaddr = ipaddr.substr(0, pos); - } - config["ipAssignments"].push_back(ipaddr); + if (assignedAddresses != "{}") { + std::string tmp = assignedAddresses.substr(1, assignedAddresses.size()-2); + std::vector addrs = split(tmp, ','); + for (auto it = addrs.begin(); it != addrs.end(); ++it) { + config["ipAssignments"].push_back(*it); } } From 3ec23f92ec7a860d1fcd3754f5090fe64c571b85 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Fri, 20 Aug 2021 10:30:37 -0700 Subject: [PATCH 08/10] helps to add part of the query --- controller/PostgreSQL.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 46257d365..6d2e78a88 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -640,7 +640,8 @@ void PostgreSQL::initializeMembers() " INNER JOIN ztc_network n1 " " ON n.id = e.network_id " " WHERE e.network_id = m.network_id AND e.member_id = m.id AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL " - " ORDER BY e.authentication_expiry_time DESC LIMIT 1) AS authentication_expiry_time " + " ORDER BY e.authentication_expiry_time DESC LIMIT 1) AS authentication_expiry_time, " + " ARRAY(SELECT DISTINCT address FROM ztc_member_ip_assignment WHERE member_id = m.id AND network_id = m.network_id) AS assigned_addresses " "FROM ztc_member m " "INNER JOIN ztc_network n " " ON n.id = m.network_id " From d0f4cfe6b4536f1ae22c78997ae7b6673aa310d6 Mon Sep 17 00:00:00 2001 From: Grant Limberg Date: Fri, 20 Aug 2021 10:34:00 -0700 Subject: [PATCH 09/10] print load status messages a little less often now that things go brrrrrrrrr --- controller/PostgreSQL.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controller/PostgreSQL.cpp b/controller/PostgreSQL.cpp index 6d2e78a88..64b0767bf 100644 --- a/controller/PostgreSQL.cpp +++ b/controller/PostgreSQL.cpp @@ -594,7 +594,7 @@ void PostgreSQL::initializeNetworks() auto dur = std::chrono::duration_cast(end - start);; total += dur.count(); ++count; - if (count % 1000 == 0) { + if (count % 10000 == 0) { fprintf(stderr, "Averaging %llu us per network\n", (total/count)); } } @@ -748,7 +748,7 @@ void PostgreSQL::initializeMembers() auto dur = std::chrono::duration_cast(end - start);; total += dur.count(); ++count; - if (count % 1000 == 0) { + if (count % 10000 == 0) { fprintf(stderr, "Averaging %llu us per member\n", (total/count)); } } From 2d8a54f05d24843acb8674dfba34b8e6131a621e Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Mon, 23 Aug 2021 11:57:12 -0400 Subject: [PATCH 10/10] Version bump -- still pre1.8 --- OFFICIAL-RELEASE-STEPS.md | 1 - ext/installfiles/mac/ZeroTier One.pkgproj | 2 +- version.h | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/OFFICIAL-RELEASE-STEPS.md b/OFFICIAL-RELEASE-STEPS.md index 6de3526cf..ba7b556db 100644 --- a/OFFICIAL-RELEASE-STEPS.md +++ b/OFFICIAL-RELEASE-STEPS.md @@ -14,7 +14,6 @@ The version must be incremented in all of the following files: /debian/changelog /ext/installfiles/mac/ZeroTier One.pkgproj /ext/installfiles/windows/ZeroTier One.aip - /windows/WinUI/AboutView.xaml The final .AIP file can only be edited on Windows with [Advanced Installer Enterprise](http://www.advancedinstaller.com/). In addition to incrementing the version be sure that a new product code is generated. (The "upgrade code" GUID on the other hand must never change.) diff --git a/ext/installfiles/mac/ZeroTier One.pkgproj b/ext/installfiles/mac/ZeroTier One.pkgproj index 040a93e36..ba678ee97 100755 --- a/ext/installfiles/mac/ZeroTier One.pkgproj +++ b/ext/installfiles/mac/ZeroTier One.pkgproj @@ -701,7 +701,7 @@ USE_HFS+_COMPRESSION VERSION - 1.7.0 + 1.7.1 TYPE 0 diff --git a/version.h b/version.h index 11b0f7789..e0b591497 100644 --- a/version.h +++ b/version.h @@ -27,7 +27,7 @@ /** * Revision */ -#define ZEROTIER_ONE_VERSION_REVISION 0 +#define ZEROTIER_ONE_VERSION_REVISION 1 /** * Build version