Netconf testing and fixes.

netconf-service/index.js

@@ -32,13 +32,14 @@ var ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP = "ts";
 var ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO = "id";
 var ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_PREFIX_BITS = "mpb";
 var ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_DEPTH = "md";
+var ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_RATES = "mr";
 var ZT_NETWORKCONFIG_DICT_KEY_PRIVATE = "p";
 var ZT_NETWORKCONFIG_DICT_KEY_NAME = "n";
 var ZT_NETWORKCONFIG_DICT_KEY_DESC = "d";
 var ZT_NETWORKCONFIG_DICT_KEY_IPV4_STATIC = "v4s";
 var ZT_NETWORKCONFIG_DICT_KEY_IPV6_STATIC = "v6s";
-var ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_RATES = "mr";
 var ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATE_OF_MEMBERSHIP = "com";
+var ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST = "eb";
 
 // Path to zerotier-idtool binary, invoked to enerate certificates of membership
 var ZEROTIER_IDTOOL = '/usr/local/bin/zerotier-idtool';
@@ -48,6 +49,7 @@ var ZT_NETWORK_AUTOCONF_DELAY = 60000;
 var ZT_NETWORK_CERTIFICATE_TTL_WINDOW = (ZT_NETWORK_AUTOCONF_DELAY * 4);
 
 // Connect to redis, assuming database 0 and no auth (for now)
+var async = require('async');
 var redis = require('redis');
 var DB = redis.createClient();
 DB.on("error",function(err) { console.error('redis query error: '+err); });
@@ -67,7 +69,7 @@ function ztDbTrue(v) { return ((v === '1')||(v === 'true')||(v > 0)); }
 
 function Dictionary(fromStr)
 {
-	var thiz = this;
+	var self = this;
 
 	this.data = {};
 
@@ -115,12 +117,12 @@ function Dictionary(fromStr)
 	this.toString = function() {
 		var str = '';
 
-		for(var key in thiz.data) {
-			str += thiz._esc(key);
+		for(var key in self.data) {
+			str += self._esc(key);
 			str += '=';
-			var value = thiz.data[key];
+			var value = self.data[key];
 			if (value)
-				str += thiz._esc(value.toString());
+				str += self._esc(value.toString());
 			str += '\n';
 		}
 
@@ -128,9 +130,9 @@ function Dictionary(fromStr)
 	};
 
 	this.fromString = function(str) {
-		thiz.data = {};
+		self.data = {};
 		if (typeof str !== 'string')
-			return thiz;
+			return self;
 
 		var lines = str.split('\n');
 		for(var l=0;l<lines.length;++l) {
@@ -146,17 +148,17 @@ function Dictionary(fromStr)
 					break;
 			}
 
-			var k = thiz._unesc(lines[l].substr(0,eqAt));
+			var k = self._unesc(lines[l].substr(0,eqAt));
 			++eqAt;
 			if ((k)&&(k.length > 0))
-				thiz.data[k] = thiz._unesc((eqAt < lines[l].length) ? lines[l].substr(eqAt) : '');
+				self.data[k] = self._unesc((eqAt < lines[l].length) ? lines[l].substr(eqAt) : '');
 		}
 
-		return thiz;
+		return self;
 	};
 
 	if ((typeof fromStr === 'string')&&(fromStr.length > 0))
-		thiz.fromString(fromStr);
+		self.fromString(fromStr);
 };
 
 //
@@ -165,46 +167,45 @@ function Dictionary(fromStr)
 
 function Identity(idstr)
 {
-	var thiz = this;
+	var self = this;
 
 	this.str = '';
 	this.fields = [];
 
 	this.toString = function() {
-		return thiz.str;
+		return self.str;
 	};
 
 	this.address = function() {
-		return ((thiz.fields.length > 0) ? thiz.fields[0] : '0000000000');
+		return ((self.fields.length > 0) ? self.fields[0] : '0000000000');
 	};
 
 	this.fromString = function(str) {
-		thiz.str = '';
-		thiz.fields = [];
+		self.str = '';
+		self.fields = [];
 		if (typeof str !== 'string')
 			return;
 		for(var i=0;i<str.length;++i) {
-			if ("0123456789abcdef:ABCDEF".indexOf(str.charAt(i)) < 0)
+			if ("0123456789abcdef:".indexOf(str.charAt(i)) < 0)
 				return; // invalid character in identity
 		}
 		var fields = str.split(':');
 		if ((fields.length < 3)||(fields[0].length !== 10)||(fields[1] !== '0'))
 			return;
-		thiz.fields = fields;
+		self.str = str;
+		self.fields = fields;
 	};
 
 	this.isValid = function() {
-		if ((thiz.fields.length < 3)||(thiz.fields[0].length !== 10)||(thiz.fields[1] !== '0'))
-			return true;
-		return false;
+		return (! ((self.fields.length < 3)||(self.fields[0].length !== 10)||(self.fields[1] !== '0')) );
 	};
 
 	this.hasPrivate = function() {
-		return ((thiz.isValid())&&(thiz.fields.length >= 4));
+		return ((self.isValid())&&(self.fields.length >= 4));
 	};
 
 	if (typeof idstr === 'string')
-		thiz.fromString(idstr);
+		self.fromString(idstr);
 };
 
 //
@@ -247,12 +248,12 @@ function doNetconfInit(message)
 	if (!netconfSigningIdentity.hasPrivate()) {
 		netconfSigningIdentity = null;
 		console.error('got invalid netconf signing identity in netconf-init');
-	}
+	} // else console.error('got netconf-init, running! id: '+netconfSigningIdentity.address());
 }
 
 function doNetconfRequest(message)
 {
-	if ((!netconfSigningIdentity)||(!netconfSigningIdentity.hasPrivate())) {
+	if ((netconfSigningIdentity === null)||(!netconfSigningIdentity.hasPrivate())) {
 		console.error('got netconf-request before netconf-init, ignored');
 		return;
 	}
@@ -266,6 +267,7 @@ function doNetconfRequest(message)
 		return;
 	}
 
+	var networkKey = 'zt1:network:'+nwid+':~';
 	var memberKey = 'zt1:network:'+nwid+':member:'+peerId.address()+':~';
 	var ipAssignmentsKey = 'zt1:network:'+nwid+':ipAssignments';
 
@@ -283,15 +285,15 @@ function doNetconfRequest(message)
 	async.series([function(next) {
 
 		// network lookup
-		DB.hgetall('zt1:network:'+nwid+':~',function(err,obj) {
+		DB.hgetall(networkKey,function(err,obj) {
 			network = obj;
-			return next(err);
+			return next(null);
 		});
 
 	},function(next) {
 
-		// member record lookup, unless public network
-		if ((!network)||(!('nwid' in network))||(network['nwid'] !== nwid))
+		// member lookup
+		if ((!network)||(!('id' in network))||(network['id'] !== nwid))
 			return next(null);
 
 		DB.hgetall(memberKey,function(err,obj) {
@@ -301,7 +303,7 @@ function doNetconfRequest(message)
 			if (obj) {
 				// Update existing member record with new last seen time, etc.
 				member = obj;
-				authorized = (ztDbTrue(network['private']) || ztDbTrue(member['authorized']));
+				authorized = ((!ztDbTrue(network['private'])) || ztDbTrue(member['authorized']));
 				DB.hmset(memberKey,{
 					'lastSeen': Date.now(),
 					'lastAt': fromIpAndPort,
@@ -448,7 +450,7 @@ function doNetconfRequest(message)
 	}],function(err) {
 
 		if (err) {
-			console.log('error composing response for '+peerId.address()+': '+err);
+			console.error('error answering netconf-request for '+peerId.address()+': '+err);
 			return;
 		}
 
@@ -500,6 +502,7 @@ function doNetconfRequest(message)
 						netconf.data[ZT_NETWORKCONFIG_DICT_KEY_IPV6_STATIC] = v6Assignments.join(',');
 					if (certificateOfMembership !== null)
 						netconf.data[ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATE_OF_MEMBERSHIP] = certificateOfMembership;
+					netconf.data[ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST] = ztDbTrue(network['enableBroadcast']) ? '1' : '0';
 					response.data['netconf'] = netconf.toString();
 				}
 
@@ -538,24 +541,29 @@ function handleMessage(dictStr)
 //
 
 var stdinReadBuffer = '';
+
 process.stdin.on('readable',function() {
 	var chunk = process.stdin.read();
 	if (chunk)
 		stdinReadBuffer += chunk;
-	if ((stdinReadBuffer.length >= 2)&&(stdinReadBuffer.substr(stdinReadBuffer.length - 2) === '\n\n')) {
-		handleMessage(stdinReadBuffer);
-		stdinReadBuffer = '';
+	for(;;) {
+		var boundary = stdinReadBuffer.indexOf('\n\n');
+		if (boundary >= 0) {
+			handleMessage(stdinReadBuffer.substr(0,boundary + 1));
+			stdinReadBuffer = stdinReadBuffer.substr(boundary + 2);
+		} else break;
 	}
 });
 process.stdin.on('end',function() {
-	process.exit(0);
+	//process.exit(0);
 });
 process.stdin.on('close',function() {
-	process.exit(0);
+	//process.exit(0);
 });
 process.stdin.on('error',function() {
-	process.exit(0);
+	//process.exit(0);
 });
 
 // Tell ZeroTier One that the service is running, solicit netconf-init
 process.stdout.write('type=ready\n\n');
+

netconf-service/package.json

@@ -9,6 +9,7 @@
   "author": "ZeroTier Networks LLC",
   "license": "GPL",
   "dependencies": {
-    "redis": "~0.10.1"
+    "redis": "~0.10.2",
+    "async": "~0.9.0"
   }
 }

netconf-service/redis-schema.md

@@ -2,19 +2,23 @@
 
 ## Notes
 
-- Top-level key format is zt1:top-level-record-type:... using :'s as a separator as per de-facto Redis standard.
-- Each top-level record type has a :~ child containing a hash. This is its root "document" and stores anything not requiring a special Redis data structure.
-- Right now there are no SET/ZSET fields to optimize searches for sub-keys such as :\<nwid\>:member:*. The code uses the Redis KEYS command for this, which is not efficient for (very) large databases. This will need to be refactored to use sets if ZT1 is wildly successful since key-globbing will slow things down with hundreds of thousands of keys. At current scales it's trivial so make it work first then make it fast.
-- ! before hash field names denotes required fields for a record to be valid.
-- M before hash field names denotes fields that are user-editable via the web API. (M for malleable/mutable.)
-- R before hash field names denotes read-only fields that are computed or can only be changed by special operations.
-- H before hash field names indicates fields that are not returned by web API queries. These are also read-only and are for internal use only.
-- @ before hash field names denotes fields that are presented by the web API as if they're part of the regular hash but are actually stored in sub-keys as SETs or other Redis data types.
-- Booleans: any value other than "1" or "true" is false, including a missing field.
+- : is used as the key namespace separator as per de-facto Redis standard.
+- A top-level record may have a :~ child containing a hash. This is the root hash and contains any simple key=value properties of the record.
+- Booleans: any value other than "1" or "true" is false.
 - Timestamps are in milliseconds since the epoch and are stored as base-10 integers.
-- IPv6 addresses must be stored *without* shortening, e.g. \:0000\: must be used instead of \:\:. It must be possible to strip colons and convert directly from hex to a 128-bit address.
-- All hexadecimal values must use lower-case letters a-f.
-- 16-digit hex network IDs and 10-digit hex addresses are zero-padded to 16 and 10 digits respectively (as they are everywhere else in the ZT1 universe).
+- IPv4 addresees: stored in standard dot notation e.g. 1.2.3.4
+- IPv6 addresses: :'s are optional and addresses must be stored *without* shortening, e.g. with :0000: instead of ::. It must be possible to strip :'s from the address and get 128 bits of straight hex.
+- Hexadecimal: all hex values must be lower case
+- 16-digit network IDs and 10-digit addresses are left zero-padded to 16 and 10 digits respectively, as they are everywhere else in the ZT1 universe.
+
+Note: right now KEYS globbing is used in a few places in the code to search for stuff. In the future we'll want to add SET/ZSET index fields for these to optimize, but that won't become an issue until there are at least millions of records. (Millions of users will give us lots of "good problems to have." :)
+
+## Field attribute flags used in this documentation (not in database)
+
+- ! required
+- M mutable (user-editable via API)
+- R read-only (at least from API/user perspective)
+- H hidden (not returned by API queries)
 
 ## Base Configuration
 
@@ -39,6 +43,7 @@ Note: users are referred to elsewhere in the database by their compound key \<au
 - R creationTime: :: timestamp of account creation
 - M displayName :: usually First Last, defaults to e-mail address for 'local' auth and whatever the OpenID API says for third party auth such as Google.
 - M defaultCard :: ID of default credit card (actual card objects are stored by Stripe, not in this database)
+- M ui :: arbitrary field that can be used by the UI to store stuff
 - R stripeCustomerId :: customer ID for Stripe credit card service if the user has cards on file (we don't store cards, we let Stripe do that)
 
 ## Networks
@@ -65,7 +70,7 @@ Each network has a network record indexed by its 64-bit network ID in lower-case
 - M v6AssignMode :: 'none' (or null/empty/etc.), 'zt', 'v6native', 'dhcp6'
 - M v6AssignPool :: network/bits from which to assign IPs
 - M subscriptions :: comma-delimited list of subscriptions for this network
-- M ui :: string-serialized JSON blob for use by the user interface, ignored by netconf-master
+- M ui :: arbitrary field that can be used by the UI to store stuff
 
 ### zt1:network:\<nwid\>:member:\<address\>:~
 

netconf-service/test-input.txt

@@ -0,0 +1,15 @@
+type=netconf-init
+netconfId=8a89399f23:0:28067b5c4b9d2a43a25ae6a01babcd943fa60c35b4fd591d39ae5e16d69fe47c196dc90fa2fdaf4e2a4a57b505612c96722e0a0eafbdcf99efbc8031f1d7acd2:b000d2a0e6048d14fa5bb30bea26eb1b0c2c0e725a622f0ca611b2dfe001678c5c1a7326750d7080834e085c6c06621b91c2e00448a8dec5c11d89bd358ac294
+
+type=netconf-request
+peerId=ce9d545ed5:0:6eaf30f54f10bfad20df64a3eb129c08efb40901b35a0d1cb1ab2732ca8ce118f0375dd67db62c30adc5cb9ed9ca03d1090b68b9b59bf1cee163dc5fdbe9bcd5
+from=127.0.0.1:1234
+nwid=8a89399f234dcaf3
+requestId=1
+
+type=netconf-request
+peerId=ce9d545ed5:0:6eaf30f54f10bfad20df64a3eb129c08efb40901b35a0d1cb1ab2732ca8ce118f0375dd67db62c30adc5cb9ed9ca03d1090b68b9b59bf1cee163dc5fdbe9bcd5
+from=127.0.0.1:1234
+nwid=8a89399f234dcaf4
+requestId=2
+