HTTP backend support for JSONDB

controller/EmbeddedNetworkController.cpp

@@ -434,8 +434,6 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa
 	_db(dbPath),
 	_node(node)
 {
-	OSUtils::mkdir(dbPath);
-	OSUtils::lockDownFile(dbPath,true); // networks might contain auth tokens, etc., so restrict directory permissions
 }
 
 EmbeddedNetworkController::~EmbeddedNetworkController()

controller/JSONDB.cpp

@@ -18,43 +18,67 @@
 
 #include "JSONDB.hpp"
 
+#define ZT_JSONDB_HTTP_TIMEOUT 60000
+
 namespace ZeroTier {
 
 static const nlohmann::json _EMPTY_JSON(nlohmann::json::object());
+static const std::map<std::string,std::string> _ZT_JSONDB_GET_HEADERS;
+
+JSONDB::JSONDB(const std::string &basePath) :
+	_basePath(basePath)
+{
+	if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) {
+		// TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported.
+		// Typically it's used with 127.0.0.1 anyway.
+		std::string hn = _basePath.substr(7);
+		std::size_t hnend = hn.find_first_of('/');
+		if (hnend != std::string::npos)
+			hn = hn.substr(0,hnend);
+		std::size_t hnsep = hn.find_last_of(':');
+		if (hnsep != std::string::npos)
+			hn[hnsep] = '/';
+		_httpAddr.fromString(hn);
+		if (hnend != std::string::npos)
+			_basePath = _basePath.substr(7 + hnend);
+		if (_basePath.length() == 0)
+			_basePath = "/";
+		if (_basePath[0] != '/')
+			_basePath = std::string("/") + _basePath;
+	} else {
+		OSUtils::mkdir(_basePath.c_str());
+		OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions
+	}
+	_reload(_basePath,std::string());
+}
 
 bool JSONDB::writeRaw(const std::string &n,const std::string &obj)
 {
 	if (!_isValidObjectName(n))
 		return false;
-
+	if (_httpAddr) {
-	const std::string path(_genPath(n,true));
+		std::map<std::string,std::string> headers;
-	if (!path.length())
+		std::string body;
-		return false;
+		std::map<std::string,std::string> reqHeaders;
-
+		char tmp[64];
-	const std::string buf(obj);
+		Utils::snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length());
-	if (!OSUtils::writeFile(path.c_str(),buf))
+		reqHeaders["Content-Length"] = tmp;
-		return false;
+		reqHeaders["Content-Type"] = "application/json";
-
+		const unsigned int sc = Http::PUT(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),obj.length(),headers,body);
-	return true;
+		return (sc == 200);
+	} else {
+		const std::string path(_genPath(n,true));
+		if (!path.length())
+			return false;
+		return OSUtils::writeFile(path.c_str(),obj);
+	}
 }
 
 bool JSONDB::put(const std::string &n,const nlohmann::json &obj)
 {
-	if (!_isValidObjectName(n))
+	const bool r = writeRaw(n,OSUtils::jsonDump(obj));
-		return false;
+	_db[n].obj = obj;
-
+	return r;
-	const std::string path(_genPath(n,true));
-	if (!path.length())
-		return false;
-
-	const std::string buf(OSUtils::jsonDump(obj));
-	if (!OSUtils::writeFile(path.c_str(),buf))
-		return false;
-
-	_E &e = _db[n];
-	e.obj = obj;
-
-	return true;
 }
 
 const nlohmann::json &JSONDB::get(const std::string &n)
@@ -66,22 +90,28 @@ const nlohmann::json &JSONDB::get(const std::string &n)
 	if (e != _db.end())
 		return e->second.obj;
 
-	const std::string path(_genPath(n,false));
-	if (!path.length())
-		return _EMPTY_JSON;
 	std::string buf;
-	if (!OSUtils::readFile(path.c_str(),buf))
+	if (_httpAddr) {
-		return _EMPTY_JSON;
+		std::map<std::string,std::string> headers;
-
+		const unsigned int sc = Http::GET(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,buf);
-	_E &e2 = _db[n];
+		if (sc != 200)
-	try {
+			return _EMPTY_JSON;
-		e2.obj = OSUtils::jsonParse(buf);
+	} else {
-	} catch ( ... ) {
+		const std::string path(_genPath(n,false));
-		e2.obj = _EMPTY_JSON;
+		if (!path.length())
-		buf = "{}";
+			return _EMPTY_JSON;
+		if (!OSUtils::readFile(path.c_str(),buf))
+			return _EMPTY_JSON;
 	}
 
-	return e2.obj;
+	try {
+		_E &e2 = _db[n];
+		e2.obj = OSUtils::jsonParse(buf);
+		return e2.obj;
+	} catch ( ... ) {
+		_db.erase(n);
+		return _EMPTY_JSON;
+	}
 }
 
 void JSONDB::erase(const std::string &n)
@@ -89,23 +119,50 @@ void JSONDB::erase(const std::string &n)
 	if (!_isValidObjectName(n))
 		return;
 
-	std::string path(_genPath(n,true));
+	if (_httpAddr) {
-	if (!path.length())
+		std::string body;
-		return;
+		std::map<std::string,std::string> headers;
+		Http::DEL(1048576,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body);
+	} else {
+		std::string path(_genPath(n,true));
+		if (!path.length())
+			return;
+		OSUtils::rm(path.c_str());
+	}
 
-	OSUtils::rm(path.c_str());
 	_db.erase(n);
 }
 
 void JSONDB::_reload(const std::string &p,const std::string &b)
 {
-	std::vector<std::string> dl(OSUtils::listDirectory(p.c_str(),true));
+	if (_httpAddr) {
-	for(std::vector<std::string>::const_iterator di(dl.begin());di!=dl.end();++di) {
+		std::string body;
-		printf("%s\n",di->c_str());
+		std::map<std::string,std::string> headers;
-		if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) {
+		const unsigned int sc = Http::GET(2147483647,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body);
-			this->get(b + di->substr(0,di->length() - 5));
+		if (sc == 200) {
-		} else {
+			try {
-			this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR));
+				nlohmann::json dbImg(OSUtils::jsonParse(body));
+				std::string tmp;
+				if (dbImg.is_object()) {
+					for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) {
+						if (i.value().is_object()) {
+							tmp = i.key();
+							_db[tmp].obj = i.value();
+						}
+					}
+				}
+			} catch ( ... ) {
+				// TODO: report error?
+			}
+		}
+	} else {
+		std::vector<std::string> dl(OSUtils::listDirectory(p.c_str(),true));
+		for(std::vector<std::string>::const_iterator di(dl.begin());di!=dl.end();++di) {
+			if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) {
+				this->get(b + di->substr(0,di->length() - 5));
+			} else {
+				this->_reload((p + ZT_PATH_SEPARATOR + *di),(b + *di + ZT_PATH_SEPARATOR));
+			}
 		}
 	}
 }
@@ -130,15 +187,23 @@ std::string JSONDB::_genPath(const std::string &n,bool create)
 	if (pt.size() == 0)
 		return std::string();
 
+	char sep;
+	if (_httpAddr) {
+		sep = '/';
+		create = false;
+	} else {
+		sep = ZT_PATH_SEPARATOR;
+	}
+
 	std::string p(_basePath);
 	if (create) OSUtils::mkdir(p.c_str());
 	for(unsigned long i=0,j=(unsigned long)(pt.size()-1);i<j;++i) {
-		p.push_back(ZT_PATH_SEPARATOR);
+		p.push_back(sep);
 		p.append(pt[i]);
 		if (create) OSUtils::mkdir(p.c_str());
 	}
 
-	p.push_back(ZT_PATH_SEPARATOR);
+	p.push_back(sep);
 	p.append(pt[pt.size()-1]);
 	p.append(".json");
 

controller/JSONDB.hpp

@@ -31,22 +31,21 @@
 
 #include "../node/Constants.hpp"
 #include "../node/Utils.hpp"
+#include "../node/InetAddress.hpp"
+#include "../node/Mutex.hpp"
 #include "../ext/json/json.hpp"
 #include "../osdep/OSUtils.hpp"
+#include "../osdep/Http.hpp"
 
 namespace ZeroTier {
 
 /**
- * Hierarchical JSON store that persists into the filesystem
+ * Hierarchical JSON store that persists into the filesystem or via HTTP
  */
 class JSONDB
 {
 public:
-	JSONDB(const std::string &basePath) :
+	JSONDB(const std::string &basePath);
-		_basePath(basePath)
-	{
-		_reload(_basePath,std::string());
-	}
 
 	inline void reload()
 	{
@@ -106,6 +105,7 @@ private:
 		inline bool operator!=(const _E &e) const { return (obj != e.obj); }
 	};
 
+	InetAddress _httpAddr;
 	std::string _basePath;
 	std::map<std::string,_E> _db;
 };

osdep/Http.hpp

@@ -135,6 +135,39 @@ public:
 			responseBody);
 	}
 
+	/**
+	 * Make HTTP PUT request
+	 *
+	 * It is the responsibility of the caller to set all headers. With PUT, the
+	 * Content-Length and Content-Type headers must be set or the PUT will not
+	 * work.
+	 *
+	 * @return HTTP status code or 0 on error (responseBody will contain error message)
+	 */
+	static inline unsigned int PUT(
+		unsigned long maxResponseSize,
+		unsigned long timeout,
+		const struct sockaddr *remoteAddress,
+		const char *path,
+		const std::map<std::string,std::string> &requestHeaders,
+		const void *postData,
+		unsigned long postDataLength,
+		std::map<std::string,std::string> &responseHeaders,
+		std::string &responseBody)
+	{
+		return _do(
+			"PUT",
+			maxResponseSize,
+			timeout,
+			remoteAddress,
+			path,
+			requestHeaders,
+			postData,
+			postDataLength,
+			responseHeaders,
+			responseBody);
+	}
+
 private:
 	static unsigned int _do(
 		const char *method,

service/OneService.cpp

@@ -381,6 +381,7 @@ public:
 
 	const std::string _homePath;
 	std::string _authToken;
+	std::string _controllerDbPath;
 	EmbeddedNetworkController *_controller;
 	Phy<OneServiceImpl *> _phy;
 	Node *_node;
@@ -482,6 +483,7 @@ public:
 
 	OneServiceImpl(const char *hp,unsigned int port) :
 		_homePath((hp) ? hp : ".")
+		,_controllerDbPath(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH)
 		,_controller((EmbeddedNetworkController *)0)
 		,_phy(this,false,true)
 		,_node((Node *)0)
@@ -747,7 +749,7 @@ public:
 			for(int i=0;i<3;++i)
 				_portsBE[i] = Utils::hton((uint16_t)_ports[i]);
 
-			_controller = new EmbeddedNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S ZT_CONTROLLER_DB_PATH).c_str());
+			_controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str());
 			_node->setNetconfMaster((void *)_controller);
 
 #ifdef ZT_ENABLE_CLUSTER
@@ -1522,6 +1524,26 @@ public:
 					_allowManagementFrom.push_back(nw);
 			}
 		}
+
+		json &controllerDbHttpHost = settings["controllerDbHttpHost"];
+		json &controllerDbHttpPort = settings["controllerDbHttpPort"];
+		json &controllerDbHttpPath = settings["controllerDbHttpPath"];
+		if ((controllerDbHttpHost.is_string())&&(controllerDbHttpPort.is_number())) {
+			_controllerDbPath = "http://";
+			_controllerDbPath.append(controllerDbHttpHost);
+			char dbp[128];
+			Utils::snprintf(dbp,sizeof(dbp),"%d",(int)controllerDbHttpPort);
+			_controllerDbPath.push_back(':');
+			_controllerDbPath.append(dbp);
+			if (controllerDbHttpPath.is_string()) {
+				std::string p = controllerDbHttpPath;
+				if ((p.length() == 0)||(p[0] != '/'))
+					_controllerDbPath.push_back('/');
+				_controllerDbPath.append(p);
+			} else {
+				_controllerDbPath.push_back('/');
+			}
+		}
 	}
 
 	// Checks if a managed IP or route target is allowed