From f119c4a45610bf5774a3fc88d0883b2fe8db6e8d Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 12:59:48 -0700 Subject: [PATCH 1/3] Cache network members for performance, add network non-persisted fields. --- controller/EmbeddedNetworkController.cpp | 404 ++++++----------------- controller/EmbeddedNetworkController.hpp | 54 +-- controller/README.md | 7 +- 3 files changed, 137 insertions(+), 328 deletions(-) diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 649ff0947..c861e4d87 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -386,247 +386,7 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa _path(dbPath) { OSUtils::mkdir(dbPath); - /* - if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK) - throw std::runtime_error("SqliteNetworkController cannot open database file"); - sqlite3_busy_timeout(_db,10000); - - sqlite3_exec(_db,"PRAGMA synchronous = OFF",0,0,0); - sqlite3_exec(_db,"PRAGMA journal_mode = MEMORY",0,0,0); - - sqlite3_stmt *s = (sqlite3_stmt *)0; - if ((sqlite3_prepare_v2(_db,"SELECT v FROM Config WHERE k = 'schemaVersion';",-1,&s,(const char **)0) == SQLITE_OK)&&(s)) { - int schemaVersion = -1234; - if (sqlite3_step(s) == SQLITE_ROW) { - schemaVersion = sqlite3_column_int(s,0); - } - - sqlite3_finalize(s); - - if (schemaVersion == -1234) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController schemaVersion not found in Config table (init failure?)"); - } - - if (schemaVersion < 2) { - // Create NodeHistory table to upgrade from version 1 to version 2 - if (sqlite3_exec(_db, - "CREATE TABLE NodeHistory (\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " networkVisitCounter INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestAuthorized INTEGER NOT NULL DEFAULT(0),\n" - " requestTime INTEGER NOT NULL DEFAULT(0),\n" - " clientMajorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientMinorVersion INTEGER NOT NULL DEFAULT(0),\n" - " clientRevision INTEGER NOT NULL DEFAULT(0),\n" - " networkRequestMetaData VARCHAR(1024),\n" - " fromAddress VARCHAR(128)\n" - ");\n" - "CREATE INDEX NodeHistory_nodeId ON NodeHistory (nodeId);\n" - "CREATE INDEX NodeHistory_networkId ON NodeHistory (networkId);\n" - "CREATE INDEX NodeHistory_requestTime ON NodeHistory (requestTime);\n" - "UPDATE \"Config\" SET \"v\" = 2 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 2: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 2; - } - } - - if (schemaVersion < 3) { - // Create Route table to upgrade from version 2 to version 3 and migrate old - // data. Also delete obsolete Gateway table that was never actually used, and - // migrate Network flags to a bitwise flags field instead of ASCII cruft. - if (sqlite3_exec(_db, - "DROP TABLE Gateway;\n" - "CREATE TABLE Route (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " target blob(16) NOT NULL,\n" - " via blob(16),\n" - " targetNetmaskBits integer NOT NULL,\n" - " ipVersion integer NOT NULL,\n" - " flags integer NOT NULL,\n" - " metric integer NOT NULL\n" - ");\n" - "CREATE INDEX Route_networkId ON Route (networkId);\n" - "INSERT INTO Route SELECT DISTINCT networkId,\"ip\" AS \"target\",NULL AS \"via\",ipNetmaskBits AS targetNetmaskBits,ipVersion,0 AS \"flags\",0 AS \"metric\" FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "ALTER TABLE Network ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 1) WHERE v4AssignMode = 'zt';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 2) WHERE v6AssignMode = 'rfc4193';\n" - "UPDATE Network SET \"flags\" = (\"flags\" | 4) WHERE v6AssignMode = '6plane';\n" - "ALTER TABLE Member ADD COLUMN \"flags\" integer NOT NULL DEFAULT(0);\n" - "DELETE FROM IpAssignment WHERE nodeId IS NULL AND \"type\" = 1;\n" - "UPDATE \"Config\" SET \"v\" = 3 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 3; - } - } - - if (schemaVersion < 4) { - // Turns out this was overkill and a huge performance drag. Will be revisiting this - // more later but for now a brief snapshot of recent history stored in Member is fine. - // Also prepare for implementation of proof of work requests. - if (sqlite3_exec(_db, - "DROP TABLE NodeHistory;\n" - "ALTER TABLE Member ADD COLUMN lastRequestTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowDifficulty integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN lastPowTime integer NOT NULL DEFAULT(0);\n" - "ALTER TABLE Member ADD COLUMN recentHistory blob;\n" - "CREATE INDEX Member_networkId_lastRequestTime ON Member(networkId, lastRequestTime);\n" - "UPDATE \"Config\" SET \"v\" = 4 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 4; - } - } - - if (schemaVersion < 5) { - // Upgrade old rough draft Rule table to new release format - if (sqlite3_exec(_db, - "DROP TABLE Relay;\n" - "DROP INDEX Rule_networkId_ruleNo;\n" - "ALTER TABLE \"Rule\" RENAME TO RuleOld;\n" - "CREATE TABLE Rule (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " capId integer,\n" - " ruleNo integer NOT NULL,\n" - " ruleType integer NOT NULL DEFAULT(0),\n" - " \"addr\" blob(16),\n" - " \"int1\" integer,\n" - " \"int2\" integer,\n" - " \"int3\" integer,\n" - " \"int4\" integer\n" - ");\n" - "INSERT INTO \"Rule\" SELECT networkId,(ruleNo*2) AS ruleNo,37 AS \"ruleType\",etherType AS \"int1\" FROM RuleOld WHERE RuleOld.etherType IS NOT NULL AND RuleOld.etherType > 0;\n" - "INSERT INTO \"Rule\" SELECT networkId,((ruleNo*2)+1) AS ruleNo,1 AS \"ruleType\" FROM RuleOld;\n" - "DROP TABLE RuleOld;\n" - "CREATE INDEX Rule_networkId_capId ON Rule (networkId,capId);\n" - "CREATE TABLE MemberTC (\n" - " networkId char(16) NOT NULL REFERENCES Network(id) ON DELETE CASCADE,\n" - " nodeId char(10) NOT NULL REFERENCES Node(id) ON DELETE CASCADE,\n" - " tagId integer,\n" - " tagValue integer,\n" - " capId integer,\n" - " capMaxCustodyChainLength integer NOT NULL DEFAULT(1)\n" - ");\n" - "CREATE INDEX MemberTC_networkId_nodeId ON MemberTC (networkId,nodeId);\n" - "UPDATE \"Config\" SET \"v\" = 5 WHERE \"k\" = 'schemaVersion';\n" - ,0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot upgrade the database to version 3: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } else { - schemaVersion = 5; - } - } - - if (schemaVersion != ZT_NETCONF_SQLITE_SCHEMA_VERSION) { - sqlite3_close(_db); - throw std::runtime_error("SqliteNetworkController database schema version mismatch"); - } - } else { - // Prepare statement will fail if Config table doesn't exist, which means our DB - // needs to be initialized. - if (sqlite3_exec(_db,ZT_NETCONF_SCHEMA_SQL"INSERT INTO Config (k,v) VALUES ('schemaVersion',"ZT_NETCONF_SQLITE_SCHEMA_VERSION_STR");",0,0,0) != SQLITE_OK) { - char err[1024]; - Utils::snprintf(err,sizeof(err),"SqliteNetworkController cannot initialize database and/or insert schemaVersion into Config table: %s",sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - } - - if ( - - (sqlite3_prepare_v2(_db,"SELECT name,private,enableBroadcast,allowPassiveBridging,\"flags\",multicastLimit,creationTime,revision,memberRevisionCounter,(SELECT COUNT(1) FROM Member WHERE Member.networkId = Network.id AND Member.authorized > 0) FROM Network WHERE id = ?",-1,&_sGetNetworkById,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT revision FROM Network WHERE id = ?",-1,&_sGetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET revision = ? WHERE id = ?",-1,&_sSetNetworkRevision,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Network (id,name,creationTime,revision) VALUES (?,?,?,1)",-1,&_sCreateNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Network WHERE id = ?",-1,&_sDeleteNetwork,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT id FROM Network ORDER BY id ASC",-1,&_sListNetworks,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Network SET memberRevisionCounter = (memberRevisionCounter + 1) WHERE id = ?",-1,&_sIncrementMemberRevisionCounter,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT identity FROM Node WHERE id = ?",-1,&_sGetNodeIdentity,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO Node (id,identity) VALUES (?,?)",-1,&_sCreateOrReplaceNode,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Rule (networkId,ruleNo,nodeId,ztSource,ztDest,vlanId,vlanPcp,vlanDei,) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",-1,&_sCreateRule,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ruleNo,nodeId,sourcePort,destPort,vlanId,vlanPcp,etherType,macSource,macDest,ipSource,ipDest,ipTos,ipProtocol,ipSourcePort,ipDestPort,\"flags\",invFlags,\"action\" FROM Rule WHERE networkId = ? ORDER BY ruleNo ASC",-1,&_sListRules,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Rule WHERE networkId = ?",-1,&_sDeleteRulesForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd FROM IpAssignmentPool WHERE networkId = ? AND ipVersion = ?",-1,&_sGetIpAssignmentPools,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT ipRangeStart,ipRangeEnd,ipVersion FROM IpAssignmentPool WHERE networkId = ? ORDER BY ipRangeStart ASC",-1,&_sGetIpAssignmentPools2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignmentPool (networkId,ipRangeStart,ipRangeEnd,ipVersion) VALUES (?,?,?,?)",-1,&_sCreateIpAssignmentPool,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignmentPool WHERE networkId = ?",-1,&_sDeleteIpAssignmentPoolsForNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT ip,ipNetmaskBits,ipVersion FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = 0 ORDER BY ip ASC",-1,&_sGetIpAssignmentsForNode,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT 1 FROM IpAssignment WHERE networkId = ? AND ip = ? AND ipVersion = ? AND \"type\" = ?",-1,&_sCheckIfIpIsAllocated,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO IpAssignment (networkId,nodeId,\"type\",ip,ipNetmaskBits,ipVersion) VALUES (?,?,?,?,?,?)",-1,&_sAllocateIp,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM IpAssignment WHERE networkId = ? AND nodeId = ? AND \"type\" = ?",-1,&_sDeleteIpAllocations,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT rowid,authorized,activeBridge,memberRevision,\"flags\",lastRequestTime,recentHistory FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sGetMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.authorized,m.activeBridge,m.memberRevision,n.identity,m.flags,m.lastRequestTime,m.recentHistory FROM Member AS m LEFT OUTER JOIN Node AS n ON n.id = m.nodeId WHERE m.networkId = ? AND m.nodeId = ?",-1,&_sGetMember2,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT INTO Member (networkId,nodeId,authorized,activeBridge,memberRevision) VALUES (?,?,?,0,(SELECT memberRevisionCounter FROM Network WHERE id = ?))",-1,&_sCreateMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId FROM Member WHERE networkId = ? AND activeBridge > 0 AND authorized > 0",-1,&_sGetActiveBridges,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT m.nodeId,m.memberRevision FROM Member AS m WHERE m.networkId = ? ORDER BY m.nodeId ASC",-1,&_sListNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET authorized = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberAuthorized,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET activeBridge = ?,memberRevision = (SELECT memberRevisionCounter FROM Network WHERE id = ?) WHERE rowid = ?",-1,&_sUpdateMemberActiveBridge,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"UPDATE Member SET \"lastRequestTime\" = ?, \"recentHistory\" = ? WHERE rowid = ?",-1,&_sUpdateMemberHistory,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ? AND nodeId = ?",-1,&_sDeleteMember,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM Member WHERE networkId = ?",-1,&_sDeleteAllNetworkMembers,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT nodeId,recentHistory FROM Member WHERE networkId = ? AND lastRequestTime >= ?",-1,&_sGetActiveNodesOnNetwork,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"INSERT INTO Route (networkId,target,via,targetNetmaskBits,ipVersion,flags,metric) VALUES (?,?,?,?,?,?,?)",-1,&_sCreateRoute,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"SELECT DISTINCT target,via,targetNetmaskBits,ipVersion,flags,metric FROM \"Route\" WHERE networkId = ? ORDER BY ipVersion,target,via",-1,&_sGetRoutes,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"DELETE FROM \"Route\" WHERE networkId = ?",-1,&_sDeleteRoutes,(const char **)0) != SQLITE_OK) - - ||(sqlite3_prepare_v2(_db,"SELECT \"v\" FROM \"Config\" WHERE \"k\" = ?",-1,&_sGetConfig,(const char **)0) != SQLITE_OK) - ||(sqlite3_prepare_v2(_db,"INSERT OR REPLACE INTO \"Config\" (\"k\",\"v\") VALUES (?,?)",-1,&_sSetConfig,(const char **)0) != SQLITE_OK) - - ) { - std::string err(std::string("SqliteNetworkController unable to initialize one or more prepared statements: ") + sqlite3_errmsg(_db)); - sqlite3_close(_db); - throw std::runtime_error(err); - } - - sqlite3_reset(_sGetConfig); - sqlite3_bind_text(_sGetConfig,1,"instanceId",10,SQLITE_STATIC); - if (sqlite3_step(_sGetConfig) != SQLITE_ROW) { - unsigned char sr[32]; - Utils::getSecureRandom(sr,32); - for(unsigned int i=0;i<32;++i) - _instanceId.push_back("0123456789abcdef"[(unsigned int)sr[i] & 0xf]); - - sqlite3_reset(_sSetConfig); - sqlite3_bind_text(_sSetConfig,1,"instanceId",10,SQLITE_STATIC); - sqlite3_bind_text(_sSetConfig,2,_instanceId.c_str(),-1,SQLITE_STATIC); - if (sqlite3_step(_sSetConfig) != SQLITE_DONE) - throw std::runtime_error("SqliteNetworkController unable to read or initialize instanceId"); - } else { - const char *iid = reinterpret_cast(sqlite3_column_text(_sGetConfig,0)); - if (!iid) - throw std::runtime_error("SqliteNetworkController unable to read instanceId (it's NULL)"); - _instanceId = iid; - } - -#ifdef ZT_NETCONF_SQLITE_TRACE - sqlite3_trace(_db,sqliteTraceFunc,(void *)0); -#endif - - _backupThread = Thread::start(this); - */ + OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions } EmbeddedNetworkController::~EmbeddedNetworkController() @@ -653,12 +413,16 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( json network(_readJson(_networkJP(nwid,false))); if (!network.size()) return NetworkController::NETCONF_QUERY_OBJECT_NOT_FOUND; - const std::string memberJP(_memberJP(nwid,identity.address(),false)); + + const std::string memberJP(_memberJP(nwid,identity.address(),true)); json member(_readJson(memberJP)); { std::string haveIdStr(_jS(member["identity"],"")); if (haveIdStr.length() > 0) { + // If we already know this member's identity perform a full compare. This prevents + // a "collision" from being able to auth onto our network in place of an already + // known member. try { if (Identity(haveIdStr.c_str()) != identity) return NetworkController::NETCONF_QUERY_ACCESS_DENIED; @@ -666,6 +430,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( return NetworkController::NETCONF_QUERY_ACCESS_DENIED; } } else { + // If we do not yet know this member's identity, learn it. member["identity"] = identity.toString(false); } } @@ -685,13 +450,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( const char *authorizedBy = (const char *)0; if (!_jB(network["private"],true)) { authorizedBy = "networkIsPublic"; + // If member already has an authorized field, leave it alone. That way its state is + // preserved if the user toggles the network back to private. Otherwise set it to + // true by default for new members of public nets. + if (!member.count("authorized")) member["authorized"] = true; } else if (_jB(member["authorized"],false)) { authorizedBy = "memberIsAuthorized"; } else { char atok[256]; if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) { atok[255] = (char)0; // not necessary but YDIFLO - if (strlen(atok) > 0) { // extra sanity check + if (strlen(atok) > 0) { // extra sanity check since we never want to compare a null token on either side auto authTokens = network["authTokens"]; if (authTokens.is_array()) { for(unsigned long i=0;i now)) && (tok.length() > 0) && (tok == atok) ) { authorizedBy = "token"; + member["authorized"] = true; // tokens actually change member authorization state break; } } @@ -710,10 +480,6 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - // Remember authorization state for public networks and token auth - if (authorizedBy) - member["authorized"] = true; - // Log this request { json rlEntry = json::object(); @@ -748,6 +514,9 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( // If we made it this far, they are authorized. + _NetworkMemberInfo nmi; + _getNetworkMemberInfo(now,nwid,nmi); + nc.networkId = nwid; nc.type = _jB(network["private"],true) ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC; nc.timestamp = now; @@ -758,21 +527,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( Utils::scopy(nc.name,sizeof(nc.name),_jS(network["name"],"").c_str()); nc.multicastLimit = (unsigned int)_jI(network["multicastLimit"],32ULL); - bool amActiveBridge = false; - { - json ab = network["activeBridges"]; - if (ab.is_array()) { - for(unsigned long i=0;i::const_iterator ab(nmi.activeBridges.begin());ab!=nmi.activeBridges.end();++ab) + nc.addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE); auto v4AssignMode = network["v4AssignMode"]; auto v6AssignMode = network["v6AssignMode"]; @@ -780,17 +536,6 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( auto routes = network["routes"]; auto rules = network["rules"]; - if (v6AssignMode.is_object()) { - if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; - } - if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { - nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); - nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; - } - } - if (rules.is_array()) { for(unsigned long i=0;i= ZT_MAX_NETWORK_RULES) @@ -817,6 +562,17 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } + if (v6AssignMode.is_object()) { + if ((_jB(v6AssignMode["rfc4193"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv6rfc4193(nwid,identity.address().toInt()); + nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + } + if ((_jB(v6AssignMode["6plane"],false))&&(nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES)) { + nc.staticIps[nc.staticIpCount++] = InetAddress::makeIpv66plane(nwid,identity.address().toInt()); + nc.flags |= ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION; + } + } + bool haveManagedIpv4AutoAssignment = false; bool haveManagedIpv6AutoAssignment = false; // "special" NDP-emulated address types do not count json ipAssignments = member["ipAssignments"]; @@ -849,11 +605,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( ipAssignments = json::array(); } - std::set allocatedIps; - bool allocatedIpsLoaded = false; - - if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { for(unsigned long p=0;((p 0)&&(!allocatedIps.count(ip6))) { + if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip6))) { ipAssignments.push_back(ip6.toIpString()); member["ipAssignments"] = ipAssignments; ip6.setPort((unsigned int)routedNetmaskBits); @@ -913,8 +665,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } - if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(_jB(v4AssignMode["zt"],false))) && (!haveManagedIpv4AutoAssignment) && (!amActiveBridge) ) { - if (!allocatedIpsLoaded) allocatedIps = _getAlreadyAllocatedIps(nwid); + if ( (ipAssignmentPools.is_array()) && ((v4AssignMode.is_object())&&(_jB(v4AssignMode["zt"],false))) && (!haveManagedIpv4AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { for(unsigned long p=0;((p 0)&&(!allocatedIps.count(ip4))) { + if ((routedNetmaskBits > 0)&&(!nmi.allocatedIps.count(ip4))) { ipAssignments.push_back(ip4.toIpString()); member["ipAssignments"] = ipAssignments; if (nc.staticIpCount < ZT_MAX_ZT_ASSIGNED_ADDRESSES) { @@ -1016,7 +767,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( char addrs[24]; Utils::snprintf(addrs,sizeof(addrs),"%.10llx",address); + // Add non-persisted fields member["clock"] = OSUtils::now(); + responseBody = member.dump(2); responseContentType = "application/json"; @@ -1090,9 +843,11 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( } else { - nlohmann::json o(network); - o["clock"] = OSUtils::now(); - responseBody = o.dump(2); + const uint64_t now = OSUtils::now(); + _NetworkMemberInfo nmi; + _getNetworkMemberInfo(now,nwid,nmi); + _addNetworkNonPersistedFields(network,now,nmi); + responseBody = network.dump(2); responseContentType = "application/json"; return 200; @@ -1146,6 +901,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( } catch ( ... ) { return 400; } + const uint64_t now = OSUtils::now(); if (path[0] == "network") { @@ -1168,6 +924,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( try { if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); + if (b.count("activeBridge")) member["activeBridge"] = _jB(b["activeBridge"],false); if ((b.count("identity"))&&(!member.count("identity"))) member["identity"] = _jS(b["identity"],""); // allow identity to be populated only if not already known if (b.count("ipAssignments")) { @@ -1191,12 +948,13 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (!member.count("authorized")) member["authorized"] = false; if (!member.count("ipAssignments")) member["ipAssignments"] = json::array(); if (!member.count("recentLog")) member["recentLog"] = json::array(); + if (!member.count("activeBridge")) member["activeBridge"] = false; member["id"] = addrs; member["address"] = addrs; // legacy member["nwid"] = nwids; member["objtype"] = "member"; - member["lastModified"] = OSUtils::now(); + member["lastModified"] = now; { auto revj = member["revision"]; const uint64_t rev = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL); @@ -1205,7 +963,9 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( _writeJson(_memberJP(nwid,Address(address),true).c_str(),member); + // Add non-persisted fields member["clock"] = member["lastModified"]; + responseBody = member.dump(2); responseContentType = "application/json"; return 200; @@ -1288,19 +1048,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( if (b.count("allowPassiveBridging")) network["allowPassiveBridging"] = _jB(b["allowPassiveBridging"],false); if (b.count("multicastLimit")) network["multicastLimit"] = _jI(b["multicastLimit"],32ULL); - if (b.count("activeBridges")) { - auto ab = b["activeBridges"]; - if (ab.is_array()) { - json ab2 = json::array(); - for(unsigned long i=0;isecond.jsonResults.append(tmp); } +void EmbeddedNetworkController::_getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi) +{ + Mutex::Lock _mcl(_networkMemberCache_m); + + auto memberCacheEntry = _networkMemberCache[nwid]; + if ((now - memberCacheEntry.second) >= ZT_NETCONF_NETWORK_MEMBER_CACHE_EXPIRE) { + const std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); + std::vector members(OSUtils::listSubdirectories(bp.c_str())); + for(std::vector::iterator m(members.begin());m!=members.end();++m) { + if (m->length() == ZT_ADDRESS_LENGTH_HEX) { + nlohmann::json mj(_readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json")); + if ((mj.is_object())&&(mj.size() > 0)) { + memberCacheEntry.first[Address(*m)] = mj; + } + } + } + memberCacheEntry.second = now; + } + + nmi.totalMemberCount = memberCacheEntry.first.size(); + for(std::map< Address,nlohmann::json >::const_iterator nm(memberCacheEntry.first.begin());nm!=memberCacheEntry.first.end();++nm) { + if (_jB(nm->second["authorized"],false)) { + ++nmi.authorizedMemberCount; + + auto mlog = nm->second["recentLog"]; + if ((mlog.is_array())&&(mlog.size() > 0)) { + auto mlog1 = mlog[0]; + if (mlog1.is_object()) { + if ((now - _jI(mlog1["ts"],0ULL)) < ZT_NETCONF_NODE_ACTIVE_THRESHOLD) + ++nmi.activeMemberCount; + } + } + + if (_jB(nm->second["activeBridge"],false)) { + nmi.activeBridges.insert(nm->first); + } + + auto mips = nm->second["ipAssignments"]; + if (mips.is_array()) { + for(unsigned long i=0;i _getAlreadyAllocatedIps(const uint64_t nwid) + // We cache the members of networks in memory to avoid having to scan the filesystem so much + std::map< uint64_t,std::pair< std::map< Address,nlohmann::json >,uint64_t > > _networkMemberCache; + Mutex _networkMemberCache_m; + + // Gathers a bunch of statistics about members of a network, IP assignments, etc. that we need in various places + // This does lock _networkMemberCache_m + struct _NetworkMemberInfo { - std::set ips; - std::string bp(_networkBP(nwid,false) + ZT_PATH_SEPARATOR_S + "member"); - std::vector members(OSUtils::listSubdirectories(bp.c_str())); - for(std::vector::iterator m(members.begin());m!=members.end();++m) { - if (m->length() == ZT_ADDRESS_LENGTH_HEX) { - nlohmann::json mj = _readJson(bp + ZT_PATH_SEPARATOR_S + *m + ZT_PATH_SEPARATOR_S + "config.json"); - auto ipAssignments = mj["ipAssignments"]; - if (ipAssignments.is_array()) { - for(unsigned long i=0;i activeBridges; + std::set allocatedIps; + unsigned long authorizedMemberCount; + unsigned long activeMemberCount; + unsigned long totalMemberCount; + }; + void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); + + inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const _NetworkMemberInfo &nmi) + { + network["clock"] = now; + network["authorizedMemberCount"] = nmi.authorizedMemberCount; + network["activeMemberCount"] = nmi.activeMemberCount; + network["totalMemberCount"] = nmi.totalMemberCount; } // These are const after construction diff --git a/controller/README.md b/controller/README.md index 0b57dd25d..339adb31b 100644 --- a/controller/README.md +++ b/controller/README.md @@ -11,8 +11,6 @@ Data is stored in JSON format under `controller.d` in the ZeroTier working direc Controllers can in theory host up to 2^24 networks and serve many millions of devices (or more), but we recommend spreading large numbers of networks across many controllers for load balancing and fault tolerance reasons. -Since this implementation uses a JSON store in the filesystem we recommend running it on SSD-backed hosts. Slow disks will become a speed bottleneck under heavy load. For really huge and busy controllers you could consider linking `controller.d/` to a folder under `/dev/shm` (Linux RAM disk) and then setting up an out-of-band periodic snapshot cron job or background process to persist the data and a script to populate `/dev/shm` on boot before the controller starts. This is beyond the scope of this guide but is not particularly hard. - Since ZeroTier nodes are mobile and do not need static IPs, implementing high availability fail-over for controllers is easy. Just replicate their working directories from master to backup and have something automatically fire up the backup if the master goes down. Many modern orchestration tools have built-in support for this. It would also be possible in theory to run controllers on a replicated or distributed filesystem, but we haven't tested this yet. ### Dockerizing Controllers @@ -67,15 +65,15 @@ When POSTing new networks take care that their IDs are not in use, otherwise you | name | string | A short name for this network | YES | | private | boolean | Is access control enabled? | YES | | enableBroadcast | boolean | Ethernet ff:ff:ff:ff:ff:ff allowed? | YES | -| activeBridges | array[string] | Array of ZeroTier addresses of active bridges | YES | | allowPassiveBridging | boolean | Allow any member to bridge (very experimental) | YES | | v4AssignMode | object | IPv4 management and assign options (see below) | YES | | v6AssignMode | object | IPv6 management and assign options (see below) | YES | | multicastLimit | integer | Maximum recipients for a multicast packet | YES | | creationTime | integer | Time network was first created | no | | revision | integer | Network config revision counter | no | -| memberRevisionCounter | integer | Network member revision counter | no | | authorizedMemberCount | integer | Number of authorized members (for private nets) | no | +| activeMemberCount | integer | Number of members that appear to be online | no | +| totalMemberCount | integer | Total known members of this network | no | | routes | array[object] | Managed IPv4 and IPv6 routes; see below | YES | | ipAssignmentPools | array[object] | IP auto-assign ranges; see below | YES | | rules | array[object] | Traffic rules; see below | YES | @@ -84,7 +82,6 @@ Recent changes: * The `ipLocalRoutes` field appeared in older versions but is no longer present. Routes will now show up in `routes`. * The `relays` field is gone since network preferred relays are gone. This capability is replaced by VL1 level federation ("federated roots"). - * Active bridges are now set at the network level, not in individual member configs. Other important points: From 1cadbfb4d1c26f5765253f052b218f7c74a42021 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 13:47:02 -0700 Subject: [PATCH 2/3] Little fixes. --- controller/EmbeddedNetworkController.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index c861e4d87..82a242844 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -60,6 +60,7 @@ using json = nlohmann::json; namespace ZeroTier { +// Get JSON values as unsigned integers, strings, or booleans, doing type conversion if possible static uint64_t _jI(const json &jv,const uint64_t dfl) { if (jv.is_number()) { @@ -97,7 +98,9 @@ static std::string _jS(const json &jv,const char *dfl) if (jv.is_string()) { return jv; } else if (jv.is_number()) { - return jv; + char tmp[64]; + Utils::snprintf(tmp,sizeof(tmp),"%llu",(uint64_t)jv); + return tmp; } else if (jv.is_boolean()) { return ((bool)jv ? std::string("1") : std::string("0")); } @@ -603,6 +606,7 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } else { ipAssignments = json::array(); + member["ipAssignments"] = ipAssignments; } if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { @@ -785,10 +789,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET( responseBody.append((responseBody.length() == 1) ? "\"" : ",\""); responseBody.append(*i); responseBody.append("\":"); - auto rev = member["revision"]; - if (rev.is_number()) - responseBody.append(rev); - else responseBody.push_back('0'); + responseBody.append(_jS(member["revision"],"0")); } } } From 212a5af9a549270abc1230b93a80caf4c0cd4b3c Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 18 Aug 2016 14:37:56 -0700 Subject: [PATCH 3/3] Capabilities and tags in POST JSON. --- controller/EmbeddedNetworkController.cpp | 102 +++++++++++++++++------ controller/EmbeddedNetworkController.hpp | 32 +++++++ 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/controller/EmbeddedNetworkController.cpp b/controller/EmbeddedNetworkController.cpp index 82a242844..d7d5c92c8 100644 --- a/controller/EmbeddedNetworkController.cpp +++ b/controller/EmbeddedNetworkController.cpp @@ -438,6 +438,8 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } + _initMember(member); + // Make sure these are always present no matter what, and increment member revision since we will always at least log something member["id"] = identity.address().toString(); member["address"] = member["id"]; @@ -606,7 +608,6 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest( } } else { ipAssignments = json::array(); - member["ipAssignments"] = ipAssignments; } if ( (ipAssignmentPools.is_array()) && ((v6AssignMode.is_object())&&(_jB(v6AssignMode["zt"],false))) && (!haveManagedIpv6AutoAssignment) && (!_jB(member["activeBridge"],false)) ) { @@ -922,6 +923,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( Utils::snprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address); json member(_readJson(_memberJP(nwid,Address(address),true))); + _initMember(member); try { if (b.count("authorized")) member["authorized"] = _jB(b["authorized"],false); @@ -942,19 +944,46 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST( member["ipAssignments"] = mipa; } } + + if (b.count("tags")) { + auto tags = b["tags"]; + if (tags.is_array()) { + std::map mtags; + for(unsigned long i=0;i::iterator t(mtags.begin());t!=mtags.end();++t) { + json ta = json::array(); + ta.push_back(t->first); + ta.push_back(t->second); + mtagsa.push_back(ta); + } + member["tags"] = mtagsa; + } + } + + if (b.count("capabilities")) { + auto capabilities = b["capabilities"]; + if (capabilities.is_array()) { + json mcaps = json::array(); + for(unsigned long i=0;i ncaps; + for(unsigned long i=0;i::iterator c(ncaps.begin());c!=ncaps.end();++c) + ncapsa.push_back(c->second); + network["capabilities"] = ncapsa; + } + } } catch ( ... ) { return 400; } - if (!network.count("private")) network["private"] = true; - if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); - if (!network.count("name")) network["name"] = ""; - if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; - if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; - if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; - if (!network.count("activeBridges")) network["activeBridges"] = json::array(); - if (!network.count("authTokens")) network["authTokens"] = json::array(); - - if (!network.count("rules")) { - // If unspecified, rules are set to allow anything and behave like a flat L2 segment - network["rules"] = { - { "not",false }, - { "type","ACTION_ACCEPT" } - }; - } - network["id"] = nwids; network["nwid"] = nwids; // legacy auto rev = network["revision"]; network["revision"] = (rev.is_number() ? ((uint64_t)rev + 1ULL) : 1ULL); - network["objtype"] = "network"; network["lastModified"] = now; _writeJson(_networkJP(nwid,true),network); diff --git a/controller/EmbeddedNetworkController.hpp b/controller/EmbeddedNetworkController.hpp index 16a1adbea..e6b4bb59f 100644 --- a/controller/EmbeddedNetworkController.hpp +++ b/controller/EmbeddedNetworkController.hpp @@ -150,6 +150,38 @@ private: }; void _getNetworkMemberInfo(uint64_t now,uint64_t nwid,_NetworkMemberInfo &nmi); + // These init objects with default and static/informational fields + inline void _initMember(nlohmann::json &member) + { + if (!member.count("authorized")) member["authorized"] = false; + if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array(); + if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array(); + if (!member.count("activeBridge")) member["activeBridge"] = false; + if (!member.count("tags")) member["tags"] = nlohmann::json::array(); + if (!member.count("capabilities")) member["capabilities"] = nlohmann::json::array(); + if (!member.count("creationTime")) member["creationTime"] = OSUtils::now(); + member["objtype"] = "member"; + } + inline void _initNetwork(nlohmann::json &network) + { + if (!network.count("private")) network["private"] = true; + if (!network.count("creationTime")) network["creationTime"] = OSUtils::now(); + if (!network.count("name")) network["name"] = ""; + if (!network.count("multicastLimit")) network["multicastLimit"] = (uint64_t)32; + if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}}; + if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}}; + if (!network.count("activeBridges")) network["activeBridges"] = nlohmann::json::array(); + if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array(); + if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array(); + if (!network.count("rules")) { + // If unspecified, rules are set to allow anything and behave like a flat L2 segment + network["rules"] = { + { "not",false }, + { "type","ACTION_ACCEPT" } + }; + } + network["objtype"] = "network"; + } inline void _addNetworkNonPersistedFields(nlohmann::json &network,uint64_t now,const _NetworkMemberInfo &nmi) { network["clock"] = now;