diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index a41a4a943..61aa23ae3 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -719,6 +719,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["lastModified"] = now; json &revj = member["revision"]; member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); + _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,address,member); _pushMemberUpdate(now,nwid,member); } @@ -1017,10 +1018,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( network["id"] = nwids; network["nwid"] = nwids; // legacy - if (true) { + if (network != origNetwork) { json &revj = network["revision"]; network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); network["lastModified"] = now; + _removeNetworkNonPersistedFields(network); _db.saveNetwork(nwid,network); // Send an update to all members of the network @@ -1187,6 +1189,10 @@ void EmbeddedNetworkController::_request( const Identity &identity, const Dictionary &metaData) { + char nwids[24]; + JSONDB::NetworkSummaryInfo ns; + json network,member,origMember; + if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender)) return; @@ -1200,17 +1206,13 @@ void EmbeddedNetworkController::_request( lrt = now; } - char nwids[24]; Utils::snprintf(nwids,sizeof(nwids),"%.16llx",nwid); - json network,member; - JSONDB::NetworkSummaryInfo ns; if (!_db.getNetworkAndMember(nwid,identity.address().toInt(),network,member,ns)) { _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND); return; } - - const bool newMember = (member.size() == 0); - json origMember(member); // for detecting modification later + origMember = member; + const bool newMember = ((!member.is_object())||(member.size() == 0)); _initMember(member); { @@ -1235,10 +1237,12 @@ void EmbeddedNetworkController::_request( } // These are always the same, but make sure they are set - const std::string addrs(identity.address().toString()); - member["id"] = addrs; - member["address"] = addrs; - member["nwid"] = nwids; + { + const std::string addrs(identity.address().toString()); + member["id"] = addrs; + member["address"] = addrs; + member["nwid"] = nwids; + } // Determine whether and how member is authorized const char *authorizedBy = (const char *)0; @@ -1347,6 +1351,7 @@ void EmbeddedNetworkController::_request( if (!authorizedBy) { if (origMember != member) { member["lastModified"] = now; + _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,identity.address().toInt(),member); } _sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED); @@ -1697,6 +1702,7 @@ void EmbeddedNetworkController::_request( if (member != origMember) { member["lastModified"] = now; + _removeMemberNonPersistedFields(member); _db.saveNetworkMember(nwid,identity.address().toInt(),member); } diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 8a2201395..64ea68844 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -157,10 +157,21 @@ private: network["activeMemberCount"] = ns.activeMemberCount; network["totalMemberCount"] = ns.totalMemberCount; } + inline void _removeNetworkNonPersistedFields(nlohmann::json &network) + { + network.erase("clock"); + network.erase("authorizedMemberCount"); + network.erase("activeMemberCount"); + network.erase("totalMemberCount"); + } inline void _addMemberNonPersistedFields(nlohmann::json &member,uint64_t now) { member["clock"] = now; } + inline void _removeMemberNonPersistedFields(nlohmann::json &member) + { + member.erase("clock"); + } const uint64_t _startTime; diff --git a/controller/JSONDB.cpp b/controller/JSONDB.cpp index 01eb50cd3..8cd957b62 100644 --- a/controller/JSONDB.cpp +++ b/controller/JSONDB.cpp @@ -110,24 +110,23 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj) bool JSONDB::hasNetwork(const uint64_t networkId) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); - return (i != _networks.end()); + return (_networks.find(networkId) != _networks.end()); } bool JSONDB::getNetwork(const uint64_t networkId,nlohmann::json &config) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return false; - config = i->second.config; + config = nlohmann::json::from_msgpack(i->second.config); return true; } bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo &ns) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return false; ns = i->second.summaryInfo; @@ -137,14 +136,14 @@ bool JSONDB::getNetworkSummaryInfo(const uint64_t networkId,NetworkSummaryInfo & int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &networkConfig,nlohmann::json &memberConfig,NetworkSummaryInfo &ns) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return 0; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + const std::unordered_map< uint64_t,std::vector >::const_iterator j(i->second.members.find(nodeId)); if (j == i->second.members.end()) return 1; - networkConfig = i->second.config; - memberConfig = j->second; + networkConfig = nlohmann::json::from_msgpack(i->second.config); + memberConfig = nlohmann::json::from_msgpack(j->second); ns = i->second.summaryInfo; return 3; } @@ -152,13 +151,13 @@ int JSONDB::getNetworkAndMember(const uint64_t networkId,const uint64_t nodeId,n bool JSONDB::getNetworkMember(const uint64_t networkId,const uint64_t nodeId,nlohmann::json &memberConfig) const { Mutex::Lock _l(_networks_m); - std::unordered_map::const_iterator i(_networks.find(networkId)); + const std::unordered_map::const_iterator i(_networks.find(networkId)); if (i == _networks.end()) return false; - std::unordered_map::const_iterator j(i->second.members.find(nodeId)); + const std::unordered_map< uint64_t,std::vector >::const_iterator j(i->second.members.find(nodeId)); if (j == i->second.members.end()) return false; - memberConfig = j->second; + memberConfig = nlohmann::json::from_msgpack(j->second); return true; } @@ -169,7 +168,7 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC writeRaw(n,OSUtils::jsonDump(networkConfig)); { Mutex::Lock _l(_networks_m); - _networks[networkId].config = networkConfig; + _networks[networkId].config = nlohmann::json::to_msgpack(networkConfig); } _recomputeSummaryInfo(networkId); } @@ -181,7 +180,7 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co writeRaw(n,OSUtils::jsonDump(memberConfig)); { Mutex::Lock _l(_networks_m); - _networks[networkId].members[nodeId] = memberConfig; + _networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig); } _recomputeSummaryInfo(networkId); } @@ -192,10 +191,10 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) std::vector memberIds; { Mutex::Lock _l(_networks_m); - std::unordered_map::iterator i(_networks.find(networkId)); + const std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; - for(std::unordered_map::iterator m(i->second.members.begin());m!=i->second.members.end();++m) + for(std::unordered_map< uint64_t,std::vector >::iterator m(i->second.members.begin());m!=i->second.members.end();++m) memberIds.push_back(m->first); } for(std::vector::iterator m(memberIds.begin());m!=memberIds.end();++m) @@ -221,7 +220,7 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId) std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; // sanity check, shouldn't happen - nlohmann::json tmp(i->second.config); + nlohmann::json tmp(nlohmann::json::from_msgpack(i->second.config)); _networks.erase(i); return tmp; } @@ -248,7 +247,7 @@ nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_ std::unordered_map::iterator i(_networks.find(networkId)); if (i == _networks.end()) return _EMPTY_JSON; - std::unordered_map::iterator j(i->second.members.find(nodeId)); + std::unordered_map< uint64_t,std::vector >::iterator j(i->second.members.find(nodeId)); if (j == i->second.members.end()) return _EMPTY_JSON; nlohmann::json tmp(j->second); @@ -288,13 +287,15 @@ void JSONDB::threadMain() ns.totalMemberCount = 0; ns.mostRecentDeauthTime = 0; - for(std::unordered_map::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(n->second.members.begin());m!=n->second.members.end();++m) { try { - if (OSUtils::jsonBool(m->second["authorized"],false)) { + nlohmann::json member(nlohmann::json::from_msgpack(m->second)); + + if (OSUtils::jsonBool(member["authorized"],false)) { ++ns.authorizedMemberCount; try { - const nlohmann::json &mlog = m->second["recentLog"]; + const nlohmann::json &mlog = member["recentLog"]; if ((mlog.is_array())&&(mlog.size() > 0)) { const nlohmann::json &mlog1 = mlog[0]; if (mlog1.is_object()) { @@ -305,12 +306,12 @@ void JSONDB::threadMain() } catch ( ... ) {} try { - if (OSUtils::jsonBool(m->second["activeBridge"],false)) + if (OSUtils::jsonBool(member["activeBridge"],false)) ns.activeBridges.push_back(Address(m->first)); } catch ( ... ) {} try { - const nlohmann::json &mips = m->second["ipAssignments"]; + const nlohmann::json &mips = member["ipAssignments"]; if (mips.is_array()) { for(unsigned long i=0;isecond["lastDeauthorizedTime"],0ULL)); + ns.mostRecentDeauthTime = std::max(ns.mostRecentDeauthTime,OSUtils::jsonInt(member["lastDeauthorizedTime"],0ULL)); } catch ( ... ) {} } ++ns.totalMemberCount; @@ -342,6 +343,8 @@ void JSONDB::threadMain() bool JSONDB::_load(const std::string &p) { if (_httpAddr) { + // In HTTP harnessed mode we download our entire working data set on startup. + std::string body; std::map headers; const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body); @@ -360,12 +363,12 @@ bool JSONDB::_load(const std::string &p) if ((id.length() == 16)&&(objtype == "network")) { const uint64_t nwid = Utils::hexStrToU64(id.c_str()); if (nwid) - _networks[nwid].config = j; + _networks[nwid].config = nlohmann::json::to_msgpack(j); } else if ((id.length() == 10)&&(objtype == "member")) { const uint64_t mid = Utils::hexStrToU64(id.c_str()); const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); if ((mid)&&(nwid)) - _networks[nwid].members[mid] = j; + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); } } } @@ -374,7 +377,10 @@ bool JSONDB::_load(const std::string &p) } catch ( ... ) {} // invalid JSON, so maybe incomplete request } return false; + } else { + // In regular mode we recursively read it from controller.d/ on disk + std::vector dl(OSUtils::listDirectory(p.c_str(),true)); for(std::vector::const_iterator di(dl.begin());di!=dl.end();++di) { if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) { @@ -389,14 +395,14 @@ bool JSONDB::_load(const std::string &p) const uint64_t nwid = Utils::strToU64(id.c_str()); if (nwid) { Mutex::Lock _l(_networks_m); - _networks[nwid].config = j; + _networks[nwid].config = nlohmann::json::to_msgpack(j); } } else if ((id.length() == 10)&&(objtype == "member")) { const uint64_t mid = Utils::strToU64(id.c_str()); const uint64_t nwid = Utils::strToU64(OSUtils::jsonString(j["nwid"],"0").c_str()); if ((mid)&&(nwid)) { Mutex::Lock _l(_networks_m); - _networks[nwid].members[mid] = j; + _networks[nwid].members[mid] = nlohmann::json::to_msgpack(j); } } } catch ( ... ) {} @@ -406,6 +412,7 @@ bool JSONDB::_load(const std::string &p) } } return true; + } } diff --git a/controller/JSONDB.hpp b/controller/JSONDB.hpp index 8db5cc485..ba16d97b7 100644 --- a/controller/JSONDB.hpp +++ b/controller/JSONDB.hpp @@ -99,9 +99,9 @@ public: Mutex::Lock _l(_networks_m); std::unordered_map::const_iterator i(_networks.find(networkId)); if (i != _networks.end()) { - for(std::unordered_map::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { + for(std::unordered_map< uint64_t,std::vector >::const_iterator m(i->second.members.begin());m!=i->second.members.end();++m) { try { - func(networkId,m->first,m->second); + func(networkId,m->first,nlohmann::json::from_msgpack(m->second)); } catch ( ... ) {} } } @@ -126,10 +126,10 @@ private: struct _NW { _NW() : summaryInfoLastComputed(0) {} - nlohmann::json config; + std::vector config; NetworkSummaryInfo summaryInfo; uint64_t summaryInfoLastComputed; - std::unordered_map members; + std::unordered_map< uint64_t,std::vector > members; }; std::unordered_map _networks;