From 6cf1da166f5bab0c306b8504541e70ba1fb4b1a3 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 15:12:28 -0700
Subject: [PATCH 01/73] Add the whole new World, though with test identities at
 this point.

---
 node/Topology.cpp               |   8 ++++++--
 world/alice-test/alice-test.bin | Bin 257 -> 510 bytes
 world/alice-test/alice-test.out |   8 ++++----
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/node/Topology.cpp b/node/Topology.cpp
index e56d1f47b..2114e5a73 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -35,8 +35,12 @@
 
 namespace ZeroTier {
 
-#define ZT_DEFAULT_WORLD_LENGTH 494
-static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x4f,0xdf,0xbf,0xfc,0xbb,0x6c,0x7e,0x15,0x67,0x85,0x1b,0xb4,0x65,0x04,0x01,0xaf,0x56,0xbf,0xe7,0x63,0x9d,0x77,0xef,0xa4,0x1e,0x61,0x53,0x88,0xcb,0x8d,0x78,0xe5,0x47,0x38,0x98,0x5a,0x6c,0x8a,0xdd,0xe6,0x9c,0x65,0xdf,0x1a,0x80,0x63,0xce,0x2e,0x4d,0x48,0x24,0x3d,0x68,0x87,0x96,0x13,0x89,0xba,0x25,0x6f,0xc9,0xb0,0x9f,0x20,0xc5,0x4c,0x51,0x7b,0x30,0xb7,0x5f,0xba,0xca,0xa4,0xc5,0x48,0xa3,0x15,0xab,0x2f,0x1d,0x64,0xe8,0x04,0x42,0xb3,0x1c,0x51,0x8b,0x2a,0x04,0x01,0xf8,0xe1,0x81,0xaf,0x60,0x2f,0x70,0x3e,0xcd,0x0b,0x21,0x38,0x19,0x62,0x02,0xbd,0x0e,0x33,0x1d,0x0a,0x7b,0xf1,0xec,0xad,0xef,0x54,0xb3,0x7b,0x17,0x84,0xaa,0xda,0x0a,0x85,0x5d,0x0b,0x1c,0x05,0x83,0xb9,0x0e,0x3e,0xe3,0xb4,0xd1,0x8b,0x5b,0x64,0xf7,0xcf,0xe1,0xff,0x5d,0xc2,0x2a,0xcf,0x60,0x7b,0x09,0xb4,0xa3,0x86,0x3c,0x5a,0x7e,0x31,0xa0,0xc7,0xb4,0x86,0xe3,0x41,0x33,0x04,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x01,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x01,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09};
+// Old root servers
+//#define ZT_DEFAULT_WORLD_LENGTH 494
+//static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x4f,0xdf,0xbf,0xfc,0xbb,0x6c,0x7e,0x15,0x67,0x85,0x1b,0xb4,0x65,0x04,0x01,0xaf,0x56,0xbf,0xe7,0x63,0x9d,0x77,0xef,0xa4,0x1e,0x61,0x53,0x88,0xcb,0x8d,0x78,0xe5,0x47,0x38,0x98,0x5a,0x6c,0x8a,0xdd,0xe6,0x9c,0x65,0xdf,0x1a,0x80,0x63,0xce,0x2e,0x4d,0x48,0x24,0x3d,0x68,0x87,0x96,0x13,0x89,0xba,0x25,0x6f,0xc9,0xb0,0x9f,0x20,0xc5,0x4c,0x51,0x7b,0x30,0xb7,0x5f,0xba,0xca,0xa4,0xc5,0x48,0xa3,0x15,0xab,0x2f,0x1d,0x64,0xe8,0x04,0x42,0xb3,0x1c,0x51,0x8b,0x2a,0x04,0x01,0xf8,0xe1,0x81,0xaf,0x60,0x2f,0x70,0x3e,0xcd,0x0b,0x21,0x38,0x19,0x62,0x02,0xbd,0x0e,0x33,0x1d,0x0a,0x7b,0xf1,0xec,0xad,0xef,0x54,0xb3,0x7b,0x17,0x84,0xaa,0xda,0x0a,0x85,0x5d,0x0b,0x1c,0x05,0x83,0xb9,0x0e,0x3e,0xe3,0xb4,0xd1,0x8b,0x5b,0x64,0xf7,0xcf,0xe1,0xff,0x5d,0xc2,0x2a,0xcf,0x60,0x7b,0x09,0xb4,0xa3,0x86,0x3c,0x5a,0x7e,0x31,0xa0,0xc7,0xb4,0x86,0xe3,0x41,0x33,0x04,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x01,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x01,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09};
+
+#define ZT_DEFAULT_WORLD_LENGTH 510
+static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0xa6,0x29,0x29,0xcf,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0xe9,0x3d,0x15,0xbb,0x20,0x3e,0xd4,0x08,0xc2,0xec,0xd4,0xfd,0xe3,0x49,0x9c,0x3a,0xe2,0x33,0xd4,0x7d,0x62,0xc1,0xec,0x1b,0x74,0x97,0xb7,0x7d,0x2a,0x3e,0xff,0xe5,0x94,0x70,0xe6,0x51,0x6b,0xc2,0x90,0x80,0x50,0x42,0x7e,0x83,0xde,0xe3,0x3a,0xa4,0xc8,0xfb,0xce,0x2b,0x06,0x5b,0x0a,0xf4,0xca,0x1c,0x41,0x3c,0x2a,0x65,0x1c,0x01,0xe3,0x13,0x79,0x9f,0xd1,0xff,0x13,0xa5,0x51,0xf9,0x3e,0x03,0xf6,0x71,0x84,0xed,0x78,0x42,0x82,0x81,0xe0,0x90,0x30,0x7a,0x78,0xd8,0x1b,0x53,0x15,0x49,0x29,0x62,0x01,0x16,0xeb,0xbd,0x6c,0x5d,0x00,0x47,0xd3,0x9b,0xca,0x9d,0x0a,0x5c,0xf7,0x01,0x48,0xe3,0x9f,0x6c,0x45,0x19,0x9e,0x17,0xe0,0xe3,0x2e,0x4e,0x46,0xca,0xc0,0x1a,0xe5,0xbc,0xb2,0x12,0x24,0x13,0x7b,0x09,0x7f,0x40,0xbd,0xd9,0x82,0xa9,0x21,0xc3,0xaa,0xbd,0xcb,0x9a,0xda,0x8b,0x4f,0x2b,0xb0,0x59,0x37,0x53,0xbf,0xdb,0x21,0xcf,0x12,0xea,0xc2,0x8c,0x8d,0x90,0x42,0x00,0x14,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x68,0xee,0xb6,0x53,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0xac,0x00,0x08,0x09,0x54,0x00,0x00,0xff,0xfe,0x15,0xf3,0xf4,0x27,0x09,0x04,0x80,0xc7,0xb6,0x09,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x1b,0x10,0x01,0x27,0x09,0x04,0x2d,0x21,0x04,0x43,0x27,0x09,0x06,0x26,0x00,0x3c,0x00,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0xb7,0x04,0x27,0x09,0x04,0x8b,0xa2,0x9d,0xf3,0x27,0x09,0x06,0x2a,0x01,0x7e,0x01,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0x3f,0xfd,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09};
 
 Topology::Topology(const RuntimeEnvironment *renv) :
 	RR(renv),
diff --git a/world/alice-test/alice-test.bin b/world/alice-test/alice-test.bin
index 2fdaf7b3066231ea0663e5da6f566610b3e43ad3..eebe6aaf357ed2f6fac641724b206ec05f4aefa2 100644
GIT binary patch
delta 458
zcmZo<`o}ED$N&T!uTF9?Ffaxz)6_gaQP83OrLE|01-mO8hu&QI``B}i)g$98wMhrx
zNS93CUaMvI|LK&1XMx#=CNu;%)ivLHY_;UX?{nI0(Oh3n$vE0*rOGfq7OtFs@xSoW
zz@K)^-wIpaRyZ{^KA2!oRdGW)SkzN9iBatJ-kewl_sg?S&E<;u&gk)YevYf;Jn;vQ
z_59pU9jKRjx@VJ+if}b&y~Ey{O)C`-uiATh)~#-T?G2IU!TWD3o)>y`sHb;=6N3oL
zo@H?x)j8R;m^U0?U}Crc@(n{R1EV@8%ly-ctAR3VEGrr~7#J3UWkM1_GAk|nGe9!z
zA1=r;GK7O=7#Ns<G8yl-1*>z`vneo2epti6!5PB9@c*Ca=Py91hU42fftpkp5*vUT
zAQnjrfQ-{sWN`)=$6y0C>4VM0|NqjrvjAnf7tQ?)l+j|W0|zri#{MrzkHWXjKpBud
u5e(fwAogsY1(x|?5Dt|pV3>UpB6Z{|P-^~ZCZGzik9gKXouF0#bOHd)zn_c%

delta 203
zcmV;+05t#p1Azh&0RR9100`>I3IG5BP=P8`Z;=r|e`AZYwsJ+Ofq~5Ge=pft&d5;`
z!{bDDRPE=DGz(o2JyHal;j%VSbrdjD=u700n(diZqm5n^|7O@v_dFxM0=pizeh2z|
zeo*K@`8RMHb<Y{*gW+5t9UfcdYQE>I>;cx@%4)R$PIeLp*t8%E<WEPAJ)ua!V3`pP
z=Nn{LF>TrrjzA)-V`wblH`<G+7yN&7sxf2o<1>>!DU`%(PeB<ld+3E)asUAYsX32m
FCkfE@Sdst$

diff --git a/world/alice-test/alice-test.out b/world/alice-test/alice-test.out
index 20077bf2e..61d089af5 100644
--- a/world/alice-test/alice-test.out
+++ b/world/alice-test/alice-test.out
@@ -1,5 +1,5 @@
-INFO: generating and signing id==149604618 ts==1445276046447
-INFO: wrote 257 bytes to stdout
+INFO: generating and signing id==149604618 ts==1445896726991
+INFO: wrote 510 bytes to stdout
 
-#define ZT_DEFAULT_WORLD_LENGTH 257
-static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0x81,0x2a,0x54,0x6f,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0x63,0x8b,0xb3,0xb6,0x72,0x45,0xa9,0x81,0x81,0xcc,0xea,0x7f,0x2f,0xd9,0x59,0xce,0xc8,0x51,0x12,0xc3,0xe3,0x44,0x76,0x54,0xed,0xe7,0x8d,0x34,0x0b,0x5d,0x10,0x3d,0x52,0x04,0x9b,0xe1,0xb2,0x36,0x51,0x75,0x14,0x30,0x53,0xe8,0x4b,0xe4,0x91,0x9a,0xed,0x99,0x56,0xa3,0x8d,0x5e,0x14,0xff,0x66,0xd8,0x4f,0xf7,0x3c,0x23,0xbe,0x02,0xbb,0x1e,0xb6,0x7e,0x07,0xfa,0x7c,0x7e,0x50,0xe8,0x40,0xf9,0x37,0x70,0x1a,0x75,0xcf,0x19,0xe6,0x83,0xe1,0x5c,0x20,0x1d,0x1e,0x5b,0xe5,0x6a,0xbe,0xe7,0xab,0xec,0x01,0xd6,0xdd,0xca,0x6a,0xb5,0x00,0x4e,0x76,0x12,0x07,0xd8,0xb4,0x20,0x0b,0xe4,0x4f,0x47,0x8e,0x3d,0xa1,0x48,0xc1,0x60,0x99,0x11,0x0e,0xe7,0x1b,0x64,0x58,0x6d,0xda,0x11,0x8e,0x40,0x22,0xab,0x63,0x68,0x2c,0xe1,0x37,0xda,0x8b,0xa8,0x17,0xfc,0x7f,0x73,0xaa,0x31,0x63,0xf2,0xe3,0x33,0x93,0x3e,0x29,0x94,0xc4,0x6b,0x4f,0x41,0x19,0x30,0x7b,0xe8,0x85,0x5a,0x72,0x00,0x01,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09};
+#define ZT_DEFAULT_WORLD_LENGTH 510
+static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0xa6,0x29,0x29,0xcf,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0xe9,0x3d,0x15,0xbb,0x20,0x3e,0xd4,0x08,0xc2,0xec,0xd4,0xfd,0xe3,0x49,0x9c,0x3a,0xe2,0x33,0xd4,0x7d,0x62,0xc1,0xec,0x1b,0x74,0x97,0xb7,0x7d,0x2a,0x3e,0xff,0xe5,0x94,0x70,0xe6,0x51,0x6b,0xc2,0x90,0x80,0x50,0x42,0x7e,0x83,0xde,0xe3,0x3a,0xa4,0xc8,0xfb,0xce,0x2b,0x06,0x5b,0x0a,0xf4,0xca,0x1c,0x41,0x3c,0x2a,0x65,0x1c,0x01,0xe3,0x13,0x79,0x9f,0xd1,0xff,0x13,0xa5,0x51,0xf9,0x3e,0x03,0xf6,0x71,0x84,0xed,0x78,0x42,0x82,0x81,0xe0,0x90,0x30,0x7a,0x78,0xd8,0x1b,0x53,0x15,0x49,0x29,0x62,0x01,0x16,0xeb,0xbd,0x6c,0x5d,0x00,0x47,0xd3,0x9b,0xca,0x9d,0x0a,0x5c,0xf7,0x01,0x48,0xe3,0x9f,0x6c,0x45,0x19,0x9e,0x17,0xe0,0xe3,0x2e,0x4e,0x46,0xca,0xc0,0x1a,0xe5,0xbc,0xb2,0x12,0x24,0x13,0x7b,0x09,0x7f,0x40,0xbd,0xd9,0x82,0xa9,0x21,0xc3,0xaa,0xbd,0xcb,0x9a,0xda,0x8b,0x4f,0x2b,0xb0,0x59,0x37,0x53,0xbf,0xdb,0x21,0xcf,0x12,0xea,0xc2,0x8c,0x8d,0x90,0x42,0x00,0x14,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x68,0xee,0xb6,0x53,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0xac,0x00,0x08,0x09,0x54,0x00,0x00,0xff,0xfe,0x15,0xf3,0xf4,0x27,0x09,0x04,0x80,0xc7,0xb6,0x09,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x1b,0x10,0x01,0x27,0x09,0x04,0x2d,0x21,0x04,0x43,0x27,0x09,0x06,0x26,0x00,0x3c,0x00,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0xb7,0x04,0x27,0x09,0x04,0x8b,0xa2,0x9d,0xf3,0x27,0x09,0x06,0x2a,0x01,0x7e,0x01,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0x3f,0xfd,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09};

From de761c5a82b7928decf94e93aff4af7adc309e7a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 15:47:32 -0700
Subject: [PATCH 02/73] Fix test world def.

---
 node/Switch.cpp                 |   7 +++++++
 node/Topology.cpp               |   4 ++--
 world/alice-test/alice-test.bin | Bin 510 -> 582 bytes
 world/alice-test/alice-test.out |   8 ++++----
 world/alice-test/mkworld.cpp    |   1 +
 5 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/node/Switch.cpp b/node/Switch.cpp
index 0edaa96d2..709ed802a 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -295,6 +295,8 @@ void Switch::send(const Packet &packet,bool encrypt,uint64_t nwid)
 		return;
 	}
 
+	//TRACE(">> %s to %s (%u bytes, encrypt==%d, nwid==%.16llx)",Packet::verbString(packet.verb()),packet.destination().toString().c_str(),packet.size(),(int)encrypt,nwid);
+
 	if (!_trySend(packet,encrypt,nwid)) {
 		Mutex::Lock _l(_txQueue_m);
 		_txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt,nwid));
@@ -637,6 +639,11 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 	Address source(packet->source());
 	Address destination(packet->destination());
 
+	// Catch this and toss it -- it would never work, but it could happen if we somehow
+	// mistakenly guessed an address we're bound to as a destination for another peer.
+	if (source == RR->identity.address())
+		return;
+
 	//TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size());
 
 	if (destination != RR->identity.address()) {
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 2114e5a73..ff4b7225e 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -39,8 +39,8 @@ namespace ZeroTier {
 //#define ZT_DEFAULT_WORLD_LENGTH 494
 //static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x4f,0xdf,0xbf,0xfc,0xbb,0x6c,0x7e,0x15,0x67,0x85,0x1b,0xb4,0x65,0x04,0x01,0xaf,0x56,0xbf,0xe7,0x63,0x9d,0x77,0xef,0xa4,0x1e,0x61,0x53,0x88,0xcb,0x8d,0x78,0xe5,0x47,0x38,0x98,0x5a,0x6c,0x8a,0xdd,0xe6,0x9c,0x65,0xdf,0x1a,0x80,0x63,0xce,0x2e,0x4d,0x48,0x24,0x3d,0x68,0x87,0x96,0x13,0x89,0xba,0x25,0x6f,0xc9,0xb0,0x9f,0x20,0xc5,0x4c,0x51,0x7b,0x30,0xb7,0x5f,0xba,0xca,0xa4,0xc5,0x48,0xa3,0x15,0xab,0x2f,0x1d,0x64,0xe8,0x04,0x42,0xb3,0x1c,0x51,0x8b,0x2a,0x04,0x01,0xf8,0xe1,0x81,0xaf,0x60,0x2f,0x70,0x3e,0xcd,0x0b,0x21,0x38,0x19,0x62,0x02,0xbd,0x0e,0x33,0x1d,0x0a,0x7b,0xf1,0xec,0xad,0xef,0x54,0xb3,0x7b,0x17,0x84,0xaa,0xda,0x0a,0x85,0x5d,0x0b,0x1c,0x05,0x83,0xb9,0x0e,0x3e,0xe3,0xb4,0xd1,0x8b,0x5b,0x64,0xf7,0xcf,0xe1,0xff,0x5d,0xc2,0x2a,0xcf,0x60,0x7b,0x09,0xb4,0xa3,0x86,0x3c,0x5a,0x7e,0x31,0xa0,0xc7,0xb4,0x86,0xe3,0x41,0x33,0x04,0x7e,0x19,0x87,0x6a,0xba,0x00,0x2a,0x6e,0x2b,0x23,0x18,0x93,0x0f,0x60,0xeb,0x09,0x7f,0x70,0xd0,0xf4,0xb0,0x28,0xb2,0xcd,0x6d,0x3d,0x0c,0x63,0xc0,0x14,0xb9,0x03,0x9f,0xf3,0x53,0x90,0xe4,0x11,0x81,0xf2,0x16,0xfb,0x2e,0x6f,0xa8,0xd9,0x5c,0x1e,0xe9,0x66,0x71,0x56,0x41,0x19,0x05,0xc3,0xdc,0xcf,0xea,0x78,0xd8,0xc6,0xdf,0xaf,0xba,0x68,0x81,0x70,0xb3,0xfa,0x00,0x01,0x04,0xc6,0xc7,0x61,0xdc,0x27,0x09,0x88,0x41,0x40,0x8a,0x2e,0x00,0xbb,0x1d,0x31,0xf2,0xc3,0x23,0xe2,0x64,0xe9,0xe6,0x41,0x72,0xc1,0xa7,0x4f,0x77,0x89,0x95,0x55,0xed,0x10,0x75,0x1c,0xd5,0x6e,0x86,0x40,0x5c,0xde,0x11,0x8d,0x02,0xdf,0xfe,0x55,0x5d,0x46,0x2c,0xcf,0x6a,0x85,0xb5,0x63,0x1c,0x12,0x35,0x0c,0x8d,0x5d,0xc4,0x09,0xba,0x10,0xb9,0x02,0x5d,0x0f,0x44,0x5c,0xf4,0x49,0xd9,0x2b,0x1c,0x00,0x01,0x04,0x6b,0xbf,0x2e,0xd2,0x27,0x09,0x8a,0xcf,0x05,0x9f,0xe3,0x00,0x48,0x2f,0x6e,0xe5,0xdf,0xe9,0x02,0x31,0x9b,0x41,0x9d,0xe5,0xbd,0xc7,0x65,0x20,0x9c,0x0e,0xcd,0xa3,0x8c,0x4d,0x6e,0x4f,0xcf,0x0d,0x33,0x65,0x83,0x98,0xb4,0x52,0x7d,0xcd,0x22,0xf9,0x31,0x12,0xfb,0x9b,0xef,0xd0,0x2f,0xd7,0x8b,0xf7,0x26,0x1b,0x33,0x3f,0xc1,0x05,0xd1,0x92,0xa6,0x23,0xca,0x9e,0x50,0xfc,0x60,0xb3,0x74,0xa5,0x00,0x01,0x04,0xa2,0xf3,0x4d,0x6f,0x27,0x09,0x9d,0x21,0x90,0x39,0xf3,0x00,0x01,0xf0,0x92,0x2a,0x98,0xe3,0xb3,0x4e,0xbc,0xbf,0xf3,0x33,0x26,0x9d,0xc2,0x65,0xd7,0xa0,0x20,0xaa,0xb6,0x9d,0x72,0xbe,0x4d,0x4a,0xcc,0x9c,0x8c,0x92,0x94,0x78,0x57,0x71,0x25,0x6c,0xd1,0xd9,0x42,0xa9,0x0d,0x1b,0xd1,0xd2,0xdc,0xa3,0xea,0x84,0xef,0x7d,0x85,0xaf,0xe6,0x61,0x1f,0xb4,0x3f,0xf0,0xb7,0x41,0x26,0xd9,0x0a,0x6e,0x00,0x01,0x04,0x80,0xc7,0xc5,0xd9,0x27,0x09};
 
-#define ZT_DEFAULT_WORLD_LENGTH 510
-static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0xa6,0x29,0x29,0xcf,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0xe9,0x3d,0x15,0xbb,0x20,0x3e,0xd4,0x08,0xc2,0xec,0xd4,0xfd,0xe3,0x49,0x9c,0x3a,0xe2,0x33,0xd4,0x7d,0x62,0xc1,0xec,0x1b,0x74,0x97,0xb7,0x7d,0x2a,0x3e,0xff,0xe5,0x94,0x70,0xe6,0x51,0x6b,0xc2,0x90,0x80,0x50,0x42,0x7e,0x83,0xde,0xe3,0x3a,0xa4,0xc8,0xfb,0xce,0x2b,0x06,0x5b,0x0a,0xf4,0xca,0x1c,0x41,0x3c,0x2a,0x65,0x1c,0x01,0xe3,0x13,0x79,0x9f,0xd1,0xff,0x13,0xa5,0x51,0xf9,0x3e,0x03,0xf6,0x71,0x84,0xed,0x78,0x42,0x82,0x81,0xe0,0x90,0x30,0x7a,0x78,0xd8,0x1b,0x53,0x15,0x49,0x29,0x62,0x01,0x16,0xeb,0xbd,0x6c,0x5d,0x00,0x47,0xd3,0x9b,0xca,0x9d,0x0a,0x5c,0xf7,0x01,0x48,0xe3,0x9f,0x6c,0x45,0x19,0x9e,0x17,0xe0,0xe3,0x2e,0x4e,0x46,0xca,0xc0,0x1a,0xe5,0xbc,0xb2,0x12,0x24,0x13,0x7b,0x09,0x7f,0x40,0xbd,0xd9,0x82,0xa9,0x21,0xc3,0xaa,0xbd,0xcb,0x9a,0xda,0x8b,0x4f,0x2b,0xb0,0x59,0x37,0x53,0xbf,0xdb,0x21,0xcf,0x12,0xea,0xc2,0x8c,0x8d,0x90,0x42,0x00,0x14,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x68,0xee,0xb6,0x53,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0xac,0x00,0x08,0x09,0x54,0x00,0x00,0xff,0xfe,0x15,0xf3,0xf4,0x27,0x09,0x04,0x80,0xc7,0xb6,0x09,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x1b,0x10,0x01,0x27,0x09,0x04,0x2d,0x21,0x04,0x43,0x27,0x09,0x06,0x26,0x00,0x3c,0x00,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0xb7,0x04,0x27,0x09,0x04,0x8b,0xa2,0x9d,0xf3,0x27,0x09,0x06,0x2a,0x01,0x7e,0x01,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0x3f,0xfd,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09};
+#define ZT_DEFAULT_WORLD_LENGTH 582
+static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0xa6,0x54,0xe4,0x8e,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0xe4,0xe9,0x51,0xc1,0xe1,0x39,0x50,0xcd,0x17,0x82,0x9d,0x74,0xf1,0xa9,0x5b,0x03,0x14,0x2c,0xa7,0xc0,0x7f,0x21,0x8b,0xad,0xdd,0xa5,0x04,0x26,0x35,0xa6,0xab,0xc1,0x49,0x64,0x2c,0xda,0x65,0x52,0x77,0xf3,0xf0,0x70,0x00,0xcd,0xc3,0xff,0x3b,0x19,0x77,0x4c,0xab,0xb6,0x35,0xbb,0x77,0xcf,0x54,0xe5,0x6d,0x01,0x9d,0x43,0x92,0x0a,0x6d,0x00,0x23,0x8e,0x0a,0x3d,0xba,0x36,0xc3,0xa1,0xa4,0xad,0x13,0x8f,0x46,0xff,0xcc,0x8f,0x9e,0xc2,0x3c,0x06,0xf8,0x3b,0xf3,0xa2,0x5f,0x71,0xcc,0x07,0x35,0x7f,0x02,0xd6,0xdd,0xca,0x6a,0xb5,0x00,0x4e,0x76,0x12,0x07,0xd8,0xb4,0x20,0x0b,0xe4,0x4f,0x47,0x8e,0x3d,0xa1,0x48,0xc1,0x60,0x99,0x11,0x0e,0xe7,0x1b,0x64,0x58,0x6d,0xda,0x11,0x8e,0x40,0x22,0xab,0x63,0x68,0x2c,0xe1,0x37,0xda,0x8b,0xa8,0x17,0xfc,0x7f,0x73,0xaa,0x31,0x63,0xf2,0xe3,0x33,0x93,0x3e,0x29,0x94,0xc4,0x6b,0x4f,0x41,0x19,0x30,0x7b,0xe8,0x85,0x5a,0x72,0x00,0x0a,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x68,0xee,0xb6,0x53,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0xac,0x00,0x08,0x09,0x54,0x00,0x00,0xff,0xfe,0x15,0xf3,0xf4,0x27,0x09,0x04,0x80,0xc7,0xb6,0x09,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x1b,0x10,0x01,0x27,0x09,0x16,0xeb,0xbd,0x6c,0x5d,0x00,0x47,0xd3,0x9b,0xca,0x9d,0x0a,0x5c,0xf7,0x01,0x48,0xe3,0x9f,0x6c,0x45,0x19,0x9e,0x17,0xe0,0xe3,0x2e,0x4e,0x46,0xca,0xc0,0x1a,0xe5,0xbc,0xb2,0x12,0x24,0x13,0x7b,0x09,0x7f,0x40,0xbd,0xd9,0x82,0xa9,0x21,0xc3,0xaa,0xbd,0xcb,0x9a,0xda,0x8b,0x4f,0x2b,0xb0,0x59,0x37,0x53,0xbf,0xdb,0x21,0xcf,0x12,0xea,0xc2,0x8c,0x8d,0x90,0x42,0x00,0x0a,0x04,0x2d,0x21,0x04,0x43,0x27,0x09,0x06,0x26,0x00,0x3c,0x00,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0xb7,0x04,0x27,0x09,0x04,0x8b,0xa2,0x9d,0xf3,0x27,0x09,0x06,0x2a,0x01,0x7e,0x01,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0x3f,0xfd,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09};
 
 Topology::Topology(const RuntimeEnvironment *renv) :
 	RR(renv),
diff --git a/world/alice-test/alice-test.bin b/world/alice-test/alice-test.bin
index eebe6aaf357ed2f6fac641724b206ec05f4aefa2..910f036f3c6fd88853fc0d41615ae51dc92edd29 100644
GIT binary patch
delta 208
zcmV;>05AXk1I7dq0RR9100`>I3IG5BP^MJmj*$^Sf8^;=!QnYj%@=~5bn&TM0~9Q$
zz<(i&t=**rCN-w3!AWE++GSFA^YCy0&BOmY8Fx&pwl%wV&s61Y0i8pV3T*%*jtV`x
zHp8K$trL$%|ICk`!aN4}JM*Gnam)ube*)Is%4)R$PIeLp*t8%E<WEPAJ)ua!V3`pP
z=Nn{LDs9>kjzA)-V`wblH`<G+7yN&7sxf2o<1>>!DU`%(PeB<ld+3E)asUdE`ht<N
KM*#|xd;x<0w_A?@

delta 136
zcmV;30C)e!1pWgN0RR9100`>I3IG5BP^Kv<&yf*8Vd*^;yC6Q)2*T{t{o_fTI^r|b
zePY4v8+4bqeJVcx<&<#dQES4GfKWnygWlshq{#cuD+XH%^vWDTJSt@z0pk;SpV9vl
qrBV4l1NL!*?RY|hf#8rZdU)6yQx!=mVgZwU0Y?E8k@|v@!~uf113YN}

diff --git a/world/alice-test/alice-test.out b/world/alice-test/alice-test.out
index 61d089af5..6b9313ce5 100644
--- a/world/alice-test/alice-test.out
+++ b/world/alice-test/alice-test.out
@@ -1,5 +1,5 @@
-INFO: generating and signing id==149604618 ts==1445896726991
-INFO: wrote 510 bytes to stdout
+INFO: generating and signing id==149604618 ts==1445899592846
+INFO: wrote 582 bytes to stdout
 
-#define ZT_DEFAULT_WORLD_LENGTH 510
-static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0xa6,0x29,0x29,0xcf,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0xe9,0x3d,0x15,0xbb,0x20,0x3e,0xd4,0x08,0xc2,0xec,0xd4,0xfd,0xe3,0x49,0x9c,0x3a,0xe2,0x33,0xd4,0x7d,0x62,0xc1,0xec,0x1b,0x74,0x97,0xb7,0x7d,0x2a,0x3e,0xff,0xe5,0x94,0x70,0xe6,0x51,0x6b,0xc2,0x90,0x80,0x50,0x42,0x7e,0x83,0xde,0xe3,0x3a,0xa4,0xc8,0xfb,0xce,0x2b,0x06,0x5b,0x0a,0xf4,0xca,0x1c,0x41,0x3c,0x2a,0x65,0x1c,0x01,0xe3,0x13,0x79,0x9f,0xd1,0xff,0x13,0xa5,0x51,0xf9,0x3e,0x03,0xf6,0x71,0x84,0xed,0x78,0x42,0x82,0x81,0xe0,0x90,0x30,0x7a,0x78,0xd8,0x1b,0x53,0x15,0x49,0x29,0x62,0x01,0x16,0xeb,0xbd,0x6c,0x5d,0x00,0x47,0xd3,0x9b,0xca,0x9d,0x0a,0x5c,0xf7,0x01,0x48,0xe3,0x9f,0x6c,0x45,0x19,0x9e,0x17,0xe0,0xe3,0x2e,0x4e,0x46,0xca,0xc0,0x1a,0xe5,0xbc,0xb2,0x12,0x24,0x13,0x7b,0x09,0x7f,0x40,0xbd,0xd9,0x82,0xa9,0x21,0xc3,0xaa,0xbd,0xcb,0x9a,0xda,0x8b,0x4f,0x2b,0xb0,0x59,0x37,0x53,0xbf,0xdb,0x21,0xcf,0x12,0xea,0xc2,0x8c,0x8d,0x90,0x42,0x00,0x14,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x68,0xee,0xb6,0x53,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0xac,0x00,0x08,0x09,0x54,0x00,0x00,0xff,0xfe,0x15,0xf3,0xf4,0x27,0x09,0x04,0x80,0xc7,0xb6,0x09,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x1b,0x10,0x01,0x27,0x09,0x04,0x2d,0x21,0x04,0x43,0x27,0x09,0x06,0x26,0x00,0x3c,0x00,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0xb7,0x04,0x27,0x09,0x04,0x8b,0xa2,0x9d,0xf3,0x27,0x09,0x06,0x2a,0x01,0x7e,0x01,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0x3f,0xfd,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09};
+#define ZT_DEFAULT_WORLD_LENGTH 582
+static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x00,0x00,0x00,0x00,0x08,0xea,0xc9,0x0a,0x00,0x00,0x01,0x50,0xa6,0x54,0xe4,0x8e,0x72,0xb0,0x3b,0xbe,0x73,0xda,0xbd,0xfb,0x85,0x77,0x9f,0xc9,0x2e,0x17,0xc8,0x11,0x6e,0xda,0x61,0x80,0xd1,0x41,0xcb,0x7c,0x2d,0x2b,0xa4,0x34,0x75,0x19,0x64,0x20,0x80,0x0a,0x22,0x32,0xf2,0x01,0x6c,0xfe,0x79,0xa6,0x7d,0xec,0x10,0x7e,0x03,0xf3,0xa2,0xa0,0x19,0xc8,0x7c,0xfd,0x6c,0x56,0x52,0xa8,0xfb,0xdc,0xfb,0x93,0x81,0x3e,0xe4,0xe9,0x51,0xc1,0xe1,0x39,0x50,0xcd,0x17,0x82,0x9d,0x74,0xf1,0xa9,0x5b,0x03,0x14,0x2c,0xa7,0xc0,0x7f,0x21,0x8b,0xad,0xdd,0xa5,0x04,0x26,0x35,0xa6,0xab,0xc1,0x49,0x64,0x2c,0xda,0x65,0x52,0x77,0xf3,0xf0,0x70,0x00,0xcd,0xc3,0xff,0x3b,0x19,0x77,0x4c,0xab,0xb6,0x35,0xbb,0x77,0xcf,0x54,0xe5,0x6d,0x01,0x9d,0x43,0x92,0x0a,0x6d,0x00,0x23,0x8e,0x0a,0x3d,0xba,0x36,0xc3,0xa1,0xa4,0xad,0x13,0x8f,0x46,0xff,0xcc,0x8f,0x9e,0xc2,0x3c,0x06,0xf8,0x3b,0xf3,0xa2,0x5f,0x71,0xcc,0x07,0x35,0x7f,0x02,0xd6,0xdd,0xca,0x6a,0xb5,0x00,0x4e,0x76,0x12,0x07,0xd8,0xb4,0x20,0x0b,0xe4,0x4f,0x47,0x8e,0x3d,0xa1,0x48,0xc1,0x60,0x99,0x11,0x0e,0xe7,0x1b,0x64,0x58,0x6d,0xda,0x11,0x8e,0x40,0x22,0xab,0x63,0x68,0x2c,0xe1,0x37,0xda,0x8b,0xa8,0x17,0xfc,0x7f,0x73,0xaa,0x31,0x63,0xf2,0xe3,0x33,0x93,0x3e,0x29,0x94,0xc4,0x6b,0x4f,0x41,0x19,0x30,0x7b,0xe8,0x85,0x5a,0x72,0x00,0x0a,0x04,0xbc,0xa6,0x5e,0xb1,0x27,0x09,0x06,0x2a,0x03,0xb0,0xc0,0x00,0x02,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x7d,0x00,0x01,0x27,0x09,0x04,0x9f,0xcb,0x61,0xab,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x08,0x00,0x00,0xa1,0x00,0x00,0x00,0x00,0x00,0x54,0x60,0x01,0x27,0x09,0x04,0xa9,0x39,0x8f,0x68,0x27,0x09,0x06,0x26,0x07,0xf0,0xd0,0x1d,0x01,0x00,0x57,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x02,0x27,0x09,0x04,0x68,0xee,0xb6,0x53,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0xac,0x00,0x08,0x09,0x54,0x00,0x00,0xff,0xfe,0x15,0xf3,0xf4,0x27,0x09,0x04,0x80,0xc7,0xb6,0x09,0x27,0x09,0x06,0x24,0x00,0x61,0x80,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x1b,0x10,0x01,0x27,0x09,0x16,0xeb,0xbd,0x6c,0x5d,0x00,0x47,0xd3,0x9b,0xca,0x9d,0x0a,0x5c,0xf7,0x01,0x48,0xe3,0x9f,0x6c,0x45,0x19,0x9e,0x17,0xe0,0xe3,0x2e,0x4e,0x46,0xca,0xc0,0x1a,0xe5,0xbc,0xb2,0x12,0x24,0x13,0x7b,0x09,0x7f,0x40,0xbd,0xd9,0x82,0xa9,0x21,0xc3,0xaa,0xbd,0xcb,0x9a,0xda,0x8b,0x4f,0x2b,0xb0,0x59,0x37,0x53,0xbf,0xdb,0x21,0xcf,0x12,0xea,0xc2,0x8c,0x8d,0x90,0x42,0x00,0x0a,0x04,0x2d,0x21,0x04,0x43,0x27,0x09,0x06,0x26,0x00,0x3c,0x00,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0xb7,0x04,0x27,0x09,0x04,0x8b,0xa2,0x9d,0xf3,0x27,0x09,0x06,0x2a,0x01,0x7e,0x01,0x00,0x00,0x00,0x00,0xf0,0x3c,0x91,0xff,0xfe,0x67,0x3f,0xfd,0x27,0x09,0x04,0x2d,0x20,0xf6,0xb3,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x58,0x00,0x8b,0xf8,0x54,0x00,0x00,0xff,0xfe,0x15,0xb3,0x9a,0x27,0x09,0x04,0x2d,0x20,0xf8,0x57,0x27,0x09,0x06,0x20,0x01,0x19,0xf0,0x70,0x00,0x9b,0xc9,0x54,0x00,0x00,0xff,0xfe,0x15,0xc4,0xf5,0x27,0x09,0x04,0x9f,0xcb,0x02,0x9a,0x27,0x09,0x06,0x26,0x04,0xa8,0x80,0x0c,0xad,0x00,0xd0,0x00,0x00,0x00,0x00,0x00,0x26,0x70,0x01,0x27,0x09};
diff --git a/world/alice-test/mkworld.cpp b/world/alice-test/mkworld.cpp
index 8940db2cf..277105c34 100644
--- a/world/alice-test/mkworld.cpp
+++ b/world/alice-test/mkworld.cpp
@@ -151,6 +151,7 @@ int main(int argc,char **argv)
 	roots.back().stableEndpoints.push_back(InetAddress("2400:6180:0:d0::1b:1001/9993")); // Singapore IPv6
 
 	// Bob -- global geo-clustered root #2
+	roots.push_back(World::Root());
 	roots.back().identity = Identity("16ebbd6c5d:0:47d39bca9d0a5cf70148e39f6c45199e17e0e32e4e46cac01ae5bcb21224137b097f40bdd982a921c3aabdcb9ada8b4f2bb0593753bfdb21cf12eac28c8d9042");
 	roots.back().stableEndpoints.push_back(InetAddress("45.33.4.67/9993")); // Dallas IPv4
 	roots.back().stableEndpoints.push_back(InetAddress("2600:3c00::f03c:91ff:fe67:b704/9993")); // Dallas IPv6

From 0b82c9ebad55181b9d668498371be84fb5c81b01 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 16:09:56 -0700
Subject: [PATCH 03/73] Fix infinite loop if there are no live roots (never
 happened before?!? wow!)

---
 node/Topology.cpp        | 16 +++++++---------
 service/ControlPlane.cpp | 12 ++++++------
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/node/Topology.cpp b/node/Topology.cpp
index ff4b7225e..6a72cf8c3 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -202,18 +202,16 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 		 * circumnavigate the globe rather than bouncing between just two. */
 
 		if (_rootAddresses.size() > 1) { // gotta be one other than me for this to work
-			std::vector<Address>::const_iterator sna(std::find(_rootAddresses.begin(),_rootAddresses.end(),RR->identity.address()));
-			if (sna != _rootAddresses.end()) { // sanity check -- _amRoot should've been false in this case
-				for(;;) {
-					if (++sna == _rootAddresses.end())
-						sna = _rootAddresses.begin(); // wrap around at end
-					if (*sna != RR->identity.address()) { // pick one other than us -- starting from me+1 in sorted set order
-						SharedPtr<Peer> *p = _peers.get(*sna);
-						if ((p)&&((*p)->hasActiveDirectPath(now))) {
-							bestRoot = *p;
+			for(unsigned long p=0;p<_rootAddresses.size();++p) {
+				if (_rootAddresses[p] == RR->identity.address()) {
+					for(unsigned long q=1;q<_rootAddresses.size();++q) {
+						SharedPtr<Peer> *nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]);
+						if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) {
+							bestRoot = *nextsn;
 							break;
 						}
 					}
+					break;
 				}
 			}
 		}
diff --git a/service/ControlPlane.cpp b/service/ControlPlane.cpp
index 31eca7b61..4978a91d2 100644
--- a/service/ControlPlane.cpp
+++ b/service/ControlPlane.cpp
@@ -265,7 +265,7 @@ unsigned int ControlPlane::handleRequest(
 	std::string &responseBody,
 	std::string &responseContentType)
 {
-	char json[1024];
+	char json[8194];
 	unsigned int scode = 404;
 	std::vector<std::string> ps(Utils::split(path.c_str(),"/","",""));
 	std::map<std::string,std::string> urlArgs;
@@ -365,11 +365,12 @@ unsigned int ControlPlane::handleRequest(
 					_node->clusterStatus(&cs);
 
 					if (cs.clusterSize >= 1) {
-						char t[4096];
-						Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members: [\n",cs.myId,cs.clusterSize);
+						char t[1024];
+						Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members\": [",cs.myId,cs.clusterSize);
 						clusterJson.append(t);
 						for(unsigned int i=0;i<cs.clusterSize;++i) {
-							Utils::snprintf(t,sizeof(t),"\t\t\t{\n\t\t\t\t\"id\": %u,\n\t\t\t\t\"msSinceLastHeartbeat\": %u,\n\t\t\t\t\"alive\": %s,\n\t\t\t\t\"x\": %d,\n\t\t\t\t\"y\": %d,\n\t\t\t\t\"z\": %d,\n\t\t\t\t\"load\": %llu\n\t\t\t\t\"peers\": %llu\n\t\t\t}%s",
+							Utils::snprintf(t,sizeof(t),"%s\t\t\t{\n\t\t\t\t\"id\": %u,\n\t\t\t\t\"msSinceLastHeartbeat\": %u,\n\t\t\t\t\"alive\": %s,\n\t\t\t\t\"x\": %d,\n\t\t\t\t\"y\": %d,\n\t\t\t\t\"z\": %d,\n\t\t\t\t\"load\": %llu,\n\t\t\t\t\"peers\": %llu\n\t\t\t}",
+								((i == 0) ? "\n" : ",\n"),
 								cs.members[i].id,
 								cs.members[i].msSinceLastHeartbeat,
 								(cs.members[i].alive != 0) ? "true" : "false",
@@ -377,8 +378,7 @@ unsigned int ControlPlane::handleRequest(
 								cs.members[i].y,
 								cs.members[i].z,
 								cs.members[i].load,
-								cs.members[i].peers,
-								(i == (cs.clusterSize - 1)) ? "," : "");
+								cs.members[i].peers);
 							clusterJson.append(t);
 						}
 						clusterJson.append(" ]\n\t\t}");

From 8bfb02ba3cf87db37803a87dff0fa7af137944a9 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 16:55:55 -0700
Subject: [PATCH 04/73] Only send redirects for the same address class, and
 elminiate some TRACE noise.

---
 node/Cluster.cpp | 12 +++++++-----
 node/Peer.cpp    |  6 +++---
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 9d25593aa..729461e86 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -200,10 +200,12 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 								}
 #endif
 							}
-							m.lastReceivedAliveAnnouncement = RR->node->now();
 #ifdef ZT_TRACE
-							TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str());
+							if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) {
+								TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str());
+							}
 #endif
+							m.lastReceivedAliveAnnouncement = RR->node->now();
 						}	break;
 
 						case STATE_MESSAGE_HAVE_PEER: {
@@ -583,7 +585,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
 		unsigned int bestMember = _id;
-		TRACE("%s is at %d,%d,%d -- looking for anyone closer than %d,%d,%d (%fkm)",peerPhysicalAddress.toString().c_str(),px,py,pz,_x,_y,_z,bestDistance);
+		//TRACE("%s is at %d,%d,%d -- looking for anyone closer than %d,%d,%d (%fkm)",peerPhysicalAddress.toString().c_str(),px,py,pz,_x,_y,_z,bestDistance);
 		{
 			Mutex::Lock _l(_memberIds_m);
 			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -610,7 +612,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 			} else { */
 				// Otherwise send VERB_RENDEZVOUS for ourselves, which will trick peers into trying other endpoints for us even if they're too old for PUSH_DIRECT_PATHS
 				for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
-					if ((a->ss_family == AF_INET)||(a->ss_family == AF_INET6)) {
+					if (a->ss_family == peerPhysicalAddress.ss_family) {
 						Packet outp(peerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
 						outp.append((uint8_t)0); // no flags
 						RR->identity.address().appendTo(outp); // HACK: rendezvous with ourselves! with really old peers this will only work if I'm a root server!
@@ -629,7 +631,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 
 			return true;
 		} else {
-			TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peerAddress.toString().c_str(),px,py,pz,currentDistance);
+			//TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peerAddress.toString().c_str(),px,py,pz,currentDistance);
 			return false;
 		}
 	} else {
diff --git a/node/Peer.cpp b/node/Peer.cpp
index fa7b3aa40..e4f0767ec 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -201,16 +201,16 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 
 	if (p) {
 		if ((now - p->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
-			TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
+			//TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
 			attemptToContactAt(RR,p->localAddress(),p->address(),now);
 			p->sent(now);
 		} else if (((now - p->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!p->reliable())) {
-			TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
+			//TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
 			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
 			RR->node->putPacket(p->localAddress(),p->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
 			p->sent(now);
 		} else {
-			TRACE("no PING or NAT keepalive: addr==%s reliable==%d %llums/%llums send/receive inactivity",p->address().toString().c_str(),(int)p->reliable(),now - p->lastSend(),now - p->lastReceived());
+			//TRACE("no PING or NAT keepalive: addr==%s reliable==%d %llums/%llums send/receive inactivity",p->address().toString().c_str(),(int)p->reliable(),now - p->lastSend(),now - p->lastReceived());
 		}
 		return true;
 	}

From 98d856daa2488d3589cba058ec2d74e41dc53287 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 17:58:51 -0700
Subject: [PATCH 05/73] Only send redirects to the sending InetAddress and only
 in response to a set of certain frame types to avoid potential race
 conditions.

---
 node/Cluster.cpp |  16 ++++----
 node/Cluster.hpp |   5 ++-
 node/Peer.cpp    | 100 +++++++++++++++++++++++------------------------
 3 files changed, 62 insertions(+), 59 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 729461e86..d4e81ff8b 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -566,7 +566,7 @@ void Cluster::removeMember(uint16_t memberId)
 	_memberIds = newMemberIds;
 }
 
-bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
+bool Cluster::redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &localAddress,const InetAddress &peerPhysicalAddress,bool offload)
 {
 	if (!peerPhysicalAddress) // sanity check
 		return false;
@@ -575,7 +575,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 		// Pick based on location if it can be determined
 		int px = 0,py = 0,pz = 0;
 		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
-			TRACE("no geolocation available for %s",peerPhysicalAddress.toIpString().c_str());
+			TRACE("NO GEOLOCATION available for %s",peerPhysicalAddress.toIpString().c_str());
 			return false;
 		}
 
@@ -585,7 +585,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
 		unsigned int bestMember = _id;
-		//TRACE("%s is at %d,%d,%d -- looking for anyone closer than %d,%d,%d (%fkm)",peerPhysicalAddress.toString().c_str(),px,py,pz,_x,_y,_z,bestDistance);
+		TRACE("%s is at %d,%d,%d -- looking for anyone closer than %d,%d,%d (%fkm)",peerPhysicalAddress.toString().c_str(),px,py,pz,_x,_y,_z,bestDistance);
 		{
 			Mutex::Lock _l(_memberIds_m);
 			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -605,7 +605,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 		}
 
 		if (best.size() > 0) {
-			TRACE("%s seems closer to %u at %fkm, suggesting redirect...",peerAddress.toString().c_str(),bestMember,bestDistance);
+			TRACE("%s seems closer to %u at %fkm, suggesting redirect...",peer->address().toString().c_str(),bestMember,bestDistance);
 
 			/* if (peer->remoteVersionProtocol() >= 5) {
 				// If it's a newer peer send VERB_PUSH_DIRECT_PATHS which is more idiomatic
@@ -613,7 +613,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 				// Otherwise send VERB_RENDEZVOUS for ourselves, which will trick peers into trying other endpoints for us even if they're too old for PUSH_DIRECT_PATHS
 				for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
 					if (a->ss_family == peerPhysicalAddress.ss_family) {
-						Packet outp(peerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
 						outp.append((uint8_t)0); // no flags
 						RR->identity.address().appendTo(outp); // HACK: rendezvous with ourselves! with really old peers this will only work if I'm a root server!
 						outp.append((uint16_t)a->port());
@@ -624,14 +624,16 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 							outp.append((uint8_t)16);
 							outp.append(a->rawIpData(),16);
 						}
-						RR->sw->send(outp,true,0);
+						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+						RR->node->putPacket(localAddress,peerPhysicalAddress,outp.data(),outp.size());
 					}
 				}
 			//}
 
 			return true;
 		} else {
-			//TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peerAddress.toString().c_str(),px,py,pz,currentDistance);
+			//TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peer->address().toString().c_str(),px,py,pz,currentDistance);
 			return false;
 		}
 	} else {
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index be3466593..b1266f27f 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -247,12 +247,13 @@ public:
 	/**
 	 * Redirect this peer to a better cluster member if needed
 	 *
-	 * @param peerAddress Peer to (possibly) redirect
+	 * @param peer Peer to (possibly) redirect
+	 * @param localAddress Local address for path or NULL for none/any
 	 * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
 	 * @param offload Always redirect if possible -- can be used to offload peers during shutdown
 	 * @return True if peer was redirected
 	 */
-	bool redirectPeer(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
+	bool redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &localAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
 	/**
 	 * Fill out ZT_ClusterStatus structure (from core API)
diff --git a/node/Peer.cpp b/node/Peer.cpp
index e4f0767ec..6cd9baabb 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -80,64 +80,67 @@ void Peer::received(
 	uint64_t inRePacketId,
 	Packet::Verb inReVerb)
 {
+#ifdef ZT_ENABLE_CLUSTER
+	if ((RR->cluster)&&(hops == 0)&&((verb == Packet::VERB_HELLO)||(verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME)||(verb == Packet::VERB_MULTICAST_FRAME))) {
+		if (RR->cluster->redirectPeer(SharedPtr<Peer>(this),localAddr,remoteAddr,false))
+			return;
+	}
+#endif
+
 	const uint64_t now = RR->node->now();
 	bool needMulticastGroupAnnounce = false;
 	bool pathIsConfirmed = false;
 
-	{
+	{	// begin _lock
 		Mutex::Lock _l(_lock);
 
 		_lastReceive = now;
 
-		if (!hops) {
-			/* Learn new paths from direct (hops == 0) packets */
-			{
-				unsigned int np = _numPaths;
-				for(unsigned int p=0;p<np;++p) {
-					if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
-						_paths[p].received(now);
-						pathIsConfirmed = true;
-						break;
-					}
+		if (hops == 0) {
+			unsigned int np = _numPaths;
+			for(unsigned int p=0;p<np;++p) {
+				if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
+					_paths[p].received(now);
+					pathIsConfirmed = true;
+					break;
 				}
+			}
 
-				if (!pathIsConfirmed) {
-					if ((verb == Packet::VERB_OK)&&((inReVerb == Packet::VERB_HELLO)||(inReVerb == Packet::VERB_ECHO))) {
+			if (!pathIsConfirmed) {
+				if (verb == Packet::VERB_OK) {
 
-						// Learn paths if they've been confirmed via a HELLO or an ECHO
-						RemotePath *slot = (RemotePath *)0;
-						if (np < ZT_MAX_PEER_NETWORK_PATHS) {
-							slot = &(_paths[np++]);
-						} else {
-							uint64_t slotLRmin = 0xffffffffffffffffULL;
-							for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-								if (_paths[p].lastReceived() <= slotLRmin) {
-									slotLRmin = _paths[p].lastReceived();
-									slot = &(_paths[p]);
-								}
+					RemotePath *slot = (RemotePath *)0;
+					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
+						slot = &(_paths[np++]);
+					} else {
+						uint64_t slotLRmin = 0xffffffffffffffffULL;
+						for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
+							if (_paths[p].lastReceived() <= slotLRmin) {
+								slotLRmin = _paths[p].lastReceived();
+								slot = &(_paths[p]);
 							}
 						}
-						if (slot) {
-							*slot = RemotePath(localAddr,remoteAddr);
-							slot->received(now);
-							_numPaths = np;
-							pathIsConfirmed = true;
-							_sortPaths(now);
-						}
-
-					} else {
-
-						/* If this path is not known, send a HELLO. We don't learn
-						 * paths without confirming that a bidirectional link is in
-						 * fact present, but any packet that decodes and authenticates
-						 * correctly is considered valid. */
-						if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
-							_lastPathConfirmationSent = now;
-							TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
-							attemptToContactAt(RR,localAddr,remoteAddr,now);
-						}
-
 					}
+					if (slot) {
+						*slot = RemotePath(localAddr,remoteAddr);
+						slot->received(now);
+						_numPaths = np;
+						pathIsConfirmed = true;
+						_sortPaths(now);
+					}
+
+				} else {
+
+					/* If this path is not known, send a HELLO. We don't learn
+					 * paths without confirming that a bidirectional link is in
+					 * fact present, but any packet that decodes and authenticates
+					 * correctly is considered valid. */
+					if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
+						_lastPathConfirmationSent = now;
+						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
+						attemptToContactAt(RR,localAddr,remoteAddr,now);
+					}
+
 				}
 			}
 		}
@@ -151,14 +154,11 @@ void Peer::received(
 			_lastUnicastFrame = now;
 		else if (verb == Packet::VERB_MULTICAST_FRAME)
 			_lastMulticastFrame = now;
-	}
+	}	// end _lock
 
 #ifdef ZT_ENABLE_CLUSTER
-	if ((pathIsConfirmed)&&(RR->cluster)) {
-		// Either shuttle this peer off somewhere else or report to other members that we have it
-		if (!RR->cluster->redirectPeer(_id.address(),remoteAddr,false))
-			RR->cluster->replicateHavePeer(_id);
-	}
+	if ((RR->cluster)&&(pathIsConfirmed))
+		RR->cluster->replicateHavePeer(_id);
 #endif
 
 	if (needMulticastGroupAnnounce) {

From e713f7a54c02915120ac3c32e0f28bd1dd744a80 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 26 Oct 2015 18:20:40 -0700
Subject: [PATCH 06/73] Can redirect in response to a few more verbs, just not
 these.

---
 node/Peer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Peer.cpp b/node/Peer.cpp
index 6cd9baabb..4f2fe9314 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -81,7 +81,7 @@ void Peer::received(
 	Packet::Verb inReVerb)
 {
 #ifdef ZT_ENABLE_CLUSTER
-	if ((RR->cluster)&&(hops == 0)&&((verb == Packet::VERB_HELLO)||(verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME)||(verb == Packet::VERB_MULTICAST_FRAME))) {
+	if ((RR->cluster)&&(hops == 0)&&(verb != VERB_OK)&&(verb != VERB_ERROR)&&(verb != VERB_RENDEZVOUS)&&(verb != VERB_PUSH_DIRECT_PATHS)) {
 		if (RR->cluster->redirectPeer(SharedPtr<Peer>(this),localAddr,remoteAddr,false))
 			return;
 	}

From 69857b4ba8bb6ce341ca6dcdc03759fb901a831a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 09:36:48 -0700
Subject: [PATCH 07/73] Refactor cluster redirects to move code to push peers
 out of the actual Cluster function that checks for redirect, and clean up
 Peer::received() to be a bit more logical.

---
 node/Cluster.cpp | 54 +++++++++++++-----------------------------------
 node/Cluster.hpp | 17 ++++++++++-----
 node/Peer.cpp    | 53 +++++++++++++++++++++++++++++++++++------------
 3 files changed, 66 insertions(+), 58 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index d4e81ff8b..a2a99ecd4 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -44,11 +44,10 @@
 #include "CertificateOfMembership.hpp"
 #include "Salsa20.hpp"
 #include "Poly1305.hpp"
-#include "Packet.hpp"
 #include "Identity.hpp"
-#include "Peer.hpp"
+#include "Topology.hpp"
+#include "Packet.hpp"
 #include "Switch.hpp"
-#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -566,17 +565,17 @@ void Cluster::removeMember(uint16_t memberId)
 	_memberIds = newMemberIds;
 }
 
-bool Cluster::redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &localAddress,const InetAddress &peerPhysicalAddress,bool offload)
+InetAddress Cluster::findBetterEndpoint(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
 {
 	if (!peerPhysicalAddress) // sanity check
-		return false;
+		return InetAddress();
 
 	if (_addressToLocationFunction) {
 		// Pick based on location if it can be determined
 		int px = 0,py = 0,pz = 0;
 		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
-			TRACE("NO GEOLOCATION available for %s",peerPhysicalAddress.toIpString().c_str());
-			return false;
+			TRACE("no geolocation data for %s (geo-lookup is lazy/async so it may work next time)",peerPhysicalAddress.toIpString().c_str());
+			return InetAddress();
 		}
 
 		// Find member closest to this peer
@@ -585,7 +584,6 @@ bool Cluster::redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &localA
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
 		unsigned int bestMember = _id;
-		TRACE("%s is at %d,%d,%d -- looking for anyone closer than %d,%d,%d (%fkm)",peerPhysicalAddress.toString().c_str(),px,py,pz,_x,_y,_z,bestDistance);
 		{
 			Mutex::Lock _l(_memberIds_m);
 			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -604,41 +602,17 @@ bool Cluster::redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &localA
 			}
 		}
 
-		if (best.size() > 0) {
-			TRACE("%s seems closer to %u at %fkm, suggesting redirect...",peer->address().toString().c_str(),bestMember,bestDistance);
-
-			/* if (peer->remoteVersionProtocol() >= 5) {
-				// If it's a newer peer send VERB_PUSH_DIRECT_PATHS which is more idiomatic
-			} else { */
-				// Otherwise send VERB_RENDEZVOUS for ourselves, which will trick peers into trying other endpoints for us even if they're too old for PUSH_DIRECT_PATHS
-				for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
-					if (a->ss_family == peerPhysicalAddress.ss_family) {
-						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
-						outp.append((uint8_t)0); // no flags
-						RR->identity.address().appendTo(outp); // HACK: rendezvous with ourselves! with really old peers this will only work if I'm a root server!
-						outp.append((uint16_t)a->port());
-						if (a->ss_family == AF_INET) {
-							outp.append((uint8_t)4);
-							outp.append(a->rawIpData(),4);
-						} else {
-							outp.append((uint8_t)16);
-							outp.append(a->rawIpData(),16);
-						}
-						outp.armor(peer->key(),true);
-						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-						RR->node->putPacket(localAddress,peerPhysicalAddress,outp.data(),outp.size());
-					}
-				}
-			//}
-
-			return true;
-		} else {
-			//TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peer->address().toString().c_str(),px,py,pz,currentDistance);
-			return false;
+		for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
+			if (a->ss_family == peerPhysicalAddress.ss_family) {
+				TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
+				return *a;
+			}
 		}
+		TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance);
+		return InetAddress();
 	} else {
 		// TODO: pick based on load if no location info?
-		return false;
+		return InetAddress();
 	}
 }
 
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index b1266f27f..080c93102 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -245,15 +245,22 @@ public:
 	void removeMember(uint16_t memberId);
 
 	/**
-	 * Redirect this peer to a better cluster member if needed
+	 * Find a better cluster endpoint for this peer
 	 *
-	 * @param peer Peer to (possibly) redirect
-	 * @param localAddress Local address for path or NULL for none/any
+	 * If this endpoint appears to be the best, a NULL/0 InetAddres is returned.
+	 * Otherwise the InetAddress of a better endpoint is returned and the peer
+	 * can then then be told to contact us there.
+	 *
+	 * Redirection is only done within the same address family, so the returned
+	 * endpoint will always be the same ss_family as the supplied physical
+	 * address.
+	 *
+	 * @param peerAddress Address of peer to (possibly) redirect
 	 * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
 	 * @param offload Always redirect if possible -- can be used to offload peers during shutdown
-	 * @return True if peer was redirected
+	 * @return InetAddress or NULL if there does not seem to be a better endpoint
 	 */
-	bool redirectPeer(const SharedPtr<Peer> &peer,const InetAddress &localAddress,const InetAddress &peerPhysicalAddress,bool offload);
+	InetAddress findBetterEndpoint(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
 	/**
 	 * Fill out ZT_ClusterStatus structure (from core API)
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 4f2fe9314..31a20eeab 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -35,6 +35,7 @@
 #include "AntiRecursion.hpp"
 #include "SelfAwareness.hpp"
 #include "Cluster.hpp"
+#include "Packet.hpp"
 
 #include <algorithm>
 
@@ -81,9 +82,28 @@ void Peer::received(
 	Packet::Verb inReVerb)
 {
 #ifdef ZT_ENABLE_CLUSTER
-	if ((RR->cluster)&&(hops == 0)&&(verb != VERB_OK)&&(verb != VERB_ERROR)&&(verb != VERB_RENDEZVOUS)&&(verb != VERB_PUSH_DIRECT_PATHS)) {
-		if (RR->cluster->redirectPeer(SharedPtr<Peer>(this),localAddr,remoteAddr,false))
-			return;
+	bool redirected = false;
+	if ((RR->cluster)&&(hops == 0)&&(verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS)) {
+		InetAddress redirectTo(RR->cluster->findBetterEndpoint(_id.address(),remoteAddr,false));
+		if (redirectTo) {
+			// For older peers we send RENDEZVOUS with ourselves. This will only work if we are
+			// a root server.
+			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+			outp.append((uint8_t)0); // no flags
+			RR->identity.address().appendTo(outp);
+			outp.append((uint16_t)redirectTo.port());
+			if (redirectTo.ss_family == AF_INET) {
+				outp.append((uint8_t)4);
+				outp.append(redirectTo.rawIpData(),4);
+			} else {
+				outp.append((uint8_t)16);
+				outp.append(redirectTo.rawIpData(),16);
+			}
+			outp.armor(_key,true);
+			RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+			RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			redirected = true;
+		}
 	}
 #endif
 
@@ -95,6 +115,23 @@ void Peer::received(
 		Mutex::Lock _l(_lock);
 
 		_lastReceive = now;
+		if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
+			_lastUnicastFrame = now;
+		else if (verb == Packet::VERB_MULTICAST_FRAME)
+			_lastMulticastFrame = now;
+
+#ifdef ZT_ENABLE_CLUSTER
+		// If we're in cluster mode and have sent the peer a better endpoint, stop
+		// here and don't confirm paths, replicate multicast info, etc. The new
+		// endpoint should do that.
+		if (redirected)
+			return;
+#endif
+
+		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
+			_lastAnnouncedTo = now;
+			needMulticastGroupAnnounce = true;
+		}
 
 		if (hops == 0) {
 			unsigned int np = _numPaths;
@@ -144,16 +181,6 @@ void Peer::received(
 				}
 			}
 		}
-
-		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
-			_lastAnnouncedTo = now;
-			needMulticastGroupAnnounce = true;
-		}
-
-		if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
-			_lastUnicastFrame = now;
-		else if (verb == Packet::VERB_MULTICAST_FRAME)
-			_lastMulticastFrame = now;
 	}	// end _lock
 
 #ifdef ZT_ENABLE_CLUSTER

From fb3b7a3baaf1fc1a6d922ea7aaa24a37c7a14952 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 09:41:12 -0700
Subject: [PATCH 08/73] Take -DZT_ENABLE_CLUSTER out of Mac defaults.

---
 make-mac.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/make-mac.mk b/make-mac.mk
index e53212c0e..174216fb1 100644
--- a/make-mac.mk
+++ b/make-mac.mk
@@ -6,7 +6,7 @@ ifeq ($(origin CXX),default)
 endif
 
 INCLUDES=
-DEFS=-DZT_ENABLE_CLUSTER
+DEFS=
 LIBS=
 ARCH_FLAGS=-arch x86_64
 

From 9617208e4027bec8a72e72a1fd89aa19ea9367d8 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 09:53:43 -0700
Subject: [PATCH 09/73] Some cleanup, and use VERB_PUSH_DIRECT_PATHS to
 redirect newer peers.

---
 node/IncomingPacket.cpp |  2 +-
 node/Packet.hpp         |  3 ---
 node/Peer.cpp           | 50 ++++++++++++++++++++++++++++-------------
 3 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index c8526dfba..d8f458e86 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -280,8 +280,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 		// VALID -- if we made it here, packet passed identity and authenticity checks!
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
 
 		if (destAddr)
 			RR->sa->iam(id.address(),_remoteAddress,destAddr,RR->topology->isRoot(id),RR->node->now());
diff --git a/node/Packet.hpp b/node/Packet.hpp
index 0e8426b68..985d25d0e 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -924,9 +924,6 @@ public:
 		 *   0x04 - Disable encryption (trust: privacy)
 		 *   0x08 - Disable encryption and authentication (trust: ultimate)
 		 *
-		 * Address types and addresses are of the same format as the destination
-		 * address type and address in HELLO.
-		 *
 		 * The receiver may, upon receiving a push, attempt to establish a
 		 * direct link to one or more of the indicated addresses. It is the
 		 * responsibility of the sender to limit which peers it pushes direct
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 31a20eeab..f16f14212 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -85,23 +85,43 @@ void Peer::received(
 	bool redirected = false;
 	if ((RR->cluster)&&(hops == 0)&&(verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS)) {
 		InetAddress redirectTo(RR->cluster->findBetterEndpoint(_id.address(),remoteAddr,false));
-		if (redirectTo) {
-			// For older peers we send RENDEZVOUS with ourselves. This will only work if we are
-			// a root server.
-			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
-			outp.append((uint8_t)0); // no flags
-			RR->identity.address().appendTo(outp);
-			outp.append((uint16_t)redirectTo.port());
-			if (redirectTo.ss_family == AF_INET) {
-				outp.append((uint8_t)4);
-				outp.append(redirectTo.rawIpData(),4);
+		if ((redirectTo.ss_family == AF_INET)||(redirectTo.ss_family == AF_INET6)) {
+			if (_vProto >= 5) {
+				// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+				outp.append((uint16_t)1); // count == 1
+				outp.append((uint8_t)0); // no flags
+				outp.append((uint16_t)0); // no extensions
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append((uint8_t)6);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)6);
+					outp.append((uint8_t)18);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.append((uint16_t)redirectTo.port());
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			} else {
-				outp.append((uint8_t)16);
-				outp.append(redirectTo.rawIpData(),16);
+				// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+				outp.append((uint8_t)0); // no flags
+				RR->identity.address().appendTo(outp);
+				outp.append((uint16_t)redirectTo.port());
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)16);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			}
-			outp.armor(_key,true);
-			RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-			RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			redirected = true;
 		}
 	}

From 8a7a0b6b88b2de95ebf98cafc183f7c69ec6c84f Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 10:37:39 -0700
Subject: [PATCH 10/73] Cleanup, including simplification of root server
 picking algorithm since we no longer need all that craziness.

---
 node/Cluster.cpp  |  2 +-
 node/Cluster.hpp  | 15 ++++++++++++++-
 node/Topology.cpp | 36 ++++++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index a2a99ecd4..bd4559335 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -239,7 +239,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							const MAC mac(dmsg.field(ptr,6),6); ptr += 6;
 							const uint32_t adi = dmsg.at<uint32_t>(ptr); ptr += 4;
 							RR->mc->add(RR->node->now(),nwid,MulticastGroup(mac,adi),address);
-							TRACE("[%u] %s likes %s/%u on %.16llu",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
+							TRACE("[%u] %s likes %s/%.8x on %.16llu",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
 						}	break;
 
 						case STATE_MESSAGE_COM: {
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 080c93102..7d0c6a089 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -158,7 +158,20 @@ public:
 		 * while PROXY_SEND is used to implement proxy sending (which right
 		 * now is only used to send RENDEZVOUS).
 		 */
-		STATE_MESSAGE_PROXY_SEND = 6
+		STATE_MESSAGE_PROXY_SEND = 6,
+
+		/**
+		 * Replicate a network config for a network we belong to:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of network config>
+		 *   <[...] serialized network config>
+		 *
+		 * This is used by clusters to avoid every member having to query
+		 * for the same netconf for networks all members belong to.
+		 *
+		 * TODO: not implemented yet!
+		 */
+		STATE_MESSAGE_NETWORK_CONFIG = 7
 	};
 
 	/**
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 6a72cf8c3..49854f0e5 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -215,10 +215,45 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 				}
 			}
 		}
+
 	} else {
 		/* If I am not a root server, the best root server is the active one with
 		 * the lowest latency. */
 
+		unsigned int bestLatencyOverall = ~((unsigned int)0);
+		unsigned int bestLatencyNotAvoid = ~((unsigned int)0);
+		const SharedPtr<Peer> *bestOverall = (const SharedPtr<Peer> *)0;
+		const SharedPtr<Peer> *bestNotAvoid = (const SharedPtr<Peer> *)0;
+
+		for(std::vector< SharedPtr<Peer> >::const_iterator r(_rootPeers.begin());r!=_rootPeers.end();++r) {
+			if ((*r)->hasActiveDirectPath(now)) {
+				bool avoiding = false;
+				for(unsigned int i=0;i<avoidCount;++i) {
+					if (avoid[i] == (*r)->address()) {
+						avoiding = true;
+						break;
+					}
+				}
+				unsigned int l = (*r)->latency();
+				if (!l) l = ~l; // zero latency indicates no measurment, so make this 'max'
+				if (l <= bestLatencyOverall) {
+					bestLatencyOverall = l;
+					bestOverall = &(*r);
+				}
+				if ((!avoiding)&&(l <= bestLatencyNotAvoid)) {
+					bestLatencyNotAvoid = l;
+					bestNotAvoid = &(*r);
+				}
+			}
+		}
+
+		if (bestNotAvoid)
+			return *bestNotAvoid;
+		else if ((!strictAvoid)&&(bestOverall))
+			return *bestOverall;
+		return SharedPtr<Peer>();
+
+		/*
 		unsigned int l,bestLatency = 65536;
 		uint64_t lds,ldr;
 
@@ -278,6 +313,7 @@ keep_searching_for_roots:
 				}
 			}
 		}
+		*/
 	}
 
 	if (bestRoot)

From 17e7528e2c1cd0e9f48eec462bef570b9a04164b Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 10:40:31 -0700
Subject: [PATCH 11/73] More root cleanup.

---
 node/Topology.cpp | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/node/Topology.cpp b/node/Topology.cpp
index 49854f0e5..7bb4b4491 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -191,7 +191,6 @@ void Topology::saveIdentity(const Identity &id)
 
 SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCount,bool strictAvoid)
 {
-	SharedPtr<Peer> bestRoot;
 	const uint64_t now = RR->node->now();
 	Mutex::Lock _l(_lock);
 
@@ -207,8 +206,8 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 					for(unsigned long q=1;q<_rootAddresses.size();++q) {
 						SharedPtr<Peer> *nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]);
 						if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) {
-							bestRoot = *nextsn;
-							break;
+							(*nextsn)->use(now);
+							return *nextsn;
 						}
 					}
 					break;
@@ -247,11 +246,13 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 			}
 		}
 
-		if (bestNotAvoid)
+		if (bestNotAvoid) {
+			(*bestNotAvoid)->use(now);
 			return *bestNotAvoid;
-		else if ((!strictAvoid)&&(bestOverall))
+		} else if ((!strictAvoid)&&(bestOverall)) {
+			(*bestOverall)->use(now);
 			return *bestOverall;
-		return SharedPtr<Peer>();
+		}
 
 		/*
 		unsigned int l,bestLatency = 65536;
@@ -315,10 +316,7 @@ keep_searching_for_roots:
 		}
 		*/
 	}
-
-	if (bestRoot)
-		bestRoot->use(now);
-	return bestRoot;
+	return SharedPtr<Peer>();
 }
 
 bool Topology::isUpstream(const Identity &id) const

From 700c3166b7dcdec61b0dc47a4649776b9ba046e8 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 10:51:11 -0700
Subject: [PATCH 12/73] Fix inverted sense bug.

---
 node/IncomingPacket.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index d8f458e86..5ade8517b 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -888,7 +888,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
-		if ((now - peer->lastDirectPathPushReceived()) >= ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL) {
+		if ((now - peer->lastDirectPathPushReceived()) < ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL) {
 			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): too frequent!",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
 		}

From f32e9d07dd493bfb1c2fcb8e3638d5c634e40030 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 10:58:01 -0700
Subject: [PATCH 13/73] Don't include COM if not necessary (fix).

---
 node/Multicaster.cpp | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp
index 6a8d63793..6e6cd628a 100644
--- a/node/Multicaster.cpp
+++ b/node/Multicaster.cpp
@@ -233,22 +233,21 @@ void Multicaster::send(
 
 		if ((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY) {
 			gs.lastExplicitGather = now;
-			SharedPtr<Peer> sn(RR->topology->getBestRoot());
-			if (sn) {
+			SharedPtr<Peer> r(RR->topology->getBestRoot());
+			if (r) {
 				TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
 
 				const CertificateOfMembership *com = (CertificateOfMembership *)0;
-				SharedPtr<NetworkConfig> nconf;
-				if (sn->needsOurNetworkMembershipCertificate(nwid,now,true)) {
+				{
 					SharedPtr<Network> nw(RR->node->network(nwid));
 					if (nw) {
-						nconf = nw->config2();
-						if (nconf)
+						SharedPtr<NetworkConfig> nconf(nw->config2());
+						if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
 							com = &(nconf->com());
 					}
 				}
 
-				Packet outp(sn->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
+				Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
 				outp.append(nwid);
 				outp.append((uint8_t)(com ? 0x01 : 0x00));
 				mg.mac().appendTo(outp);
@@ -256,8 +255,8 @@ void Multicaster::send(
 				outp.append((uint32_t)gatherLimit);
 				if (com)
 					com->serialize(outp);
-				outp.armor(sn->key(),true);
-				sn->send(RR,outp.data(),outp.size(),now);
+				outp.armor(r->key(),true);
+				r->send(RR,outp.data(),outp.size(),now);
 			}
 			gatherLimit = 0;
 		}

From 62db18b6dd0002520d4828b0091995a40b4eb2e9 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 11:01:58 -0700
Subject: [PATCH 14/73] Lessen this limit just a bit to make cluster settle
 faster.

---
 node/Constants.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Constants.hpp b/node/Constants.hpp
index e45602f72..3ad07e4c0 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -327,7 +327,7 @@
 /**
  * Minimum interval between direct path pushes from a given peer or we will ignore them
  */
-#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2500
+#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2000
 
 /**
  * How long (max) to remember network certificates of membership?

From 54a99d8e32e3ee0aed222069e961d44ddd748399 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 11:14:07 -0700
Subject: [PATCH 15/73] Well that was broken.

---
 node/IncomingPacket.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 5ade8517b..51e883643 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -622,7 +622,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 
 		// Iterate through 18-byte network,MAC,ADI tuples
 		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18) {
-			const uint32_t nwid(at<uint64_t>(ptr));
+			const uint64_t nwid(at<uint64_t>(ptr));
 			const MulticastGroup group(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14));
 			RR->mc->add(now,nwid,group,peer->address());
 #ifdef ZT_ENABLE_CLUSTER

From a1a0ee4edb0933c9b82abad8715def6a63049658 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 12:01:00 -0700
Subject: [PATCH 16/73] Fix infinite loop in Cluster, clean up some stuff
 elsewhere, and back out rate limiting in PUSH_DIRECT_PATHS for now (but we
 will do something else to mitigate amplification attacks)

---
 node/Cluster.cpp        |  5 ++--
 node/Constants.hpp      |  7 +----
 node/IncomingPacket.cpp | 12 ++------
 node/Peer.cpp           |  1 -
 node/Peer.hpp           | 17 ++---------
 node/SelfAwareness.cpp  | 12 ++++----
 node/Topology.cpp       | 62 +----------------------------------------
 7 files changed, 16 insertions(+), 100 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index bd4559335..eef02bc79 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -239,7 +239,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							const MAC mac(dmsg.field(ptr,6),6); ptr += 6;
 							const uint32_t adi = dmsg.at<uint32_t>(ptr); ptr += 4;
 							RR->mc->add(RR->node->now(),nwid,MulticastGroup(mac,adi),address);
-							TRACE("[%u] %s likes %s/%.8x on %.16llu",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
+							TRACE("[%u] %s likes %s/%.8x on %.16llx",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
 						}	break;
 
 						case STATE_MESSAGE_COM: {
@@ -376,11 +376,12 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 		Mutex::Lock _l2(_peerAffinities_m);
 		std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),_PeerAffinity(toPeerAddress,0,0))); // O(log(n))
 		while ((i != _peerAffinities.end())&&(i->address() == toPeerAddress)) {
-			uint16_t mid = i->clusterMemberId();
+			const uint16_t mid = i->clusterMemberId();
 			if ((mid != _id)&&(i->timestamp > mostRecentTimestamp)) {
 				mostRecentTimestamp = i->timestamp;
 				canHasPeer = mid;
 			}
+			++i;
 		}
 	}
 
diff --git a/node/Constants.hpp b/node/Constants.hpp
index 3ad07e4c0..bef1183a8 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -324,11 +324,6 @@
  */
 #define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
 
-/**
- * Minimum interval between direct path pushes from a given peer or we will ignore them
- */
-#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2000
-
 /**
  * How long (max) to remember network certificates of membership?
  *
@@ -355,7 +350,7 @@
 /**
  * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235)
  */
-#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 8
+#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 2
 
 /**
  * A test pseudo-network-ID that can be joined
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 51e883643..2514cd64b 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -622,7 +622,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 
 		// Iterate through 18-byte network,MAC,ADI tuples
 		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18) {
-			const uint64_t nwid(at<uint64_t>(ptr));
+			const uint64_t nwid = at<uint64_t>(ptr);
 			const MulticastGroup group(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14));
 			RR->mc->add(now,nwid,group,peer->address());
 #ifdef ZT_ENABLE_CLUSTER
@@ -888,12 +888,6 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
-		if ((now - peer->lastDirectPathPushReceived()) < ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL) {
-			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): too frequent!",source().toString().c_str(),_remoteAddress.toString().c_str());
-			return true;
-		}
-		peer->setLastDirectPathPushReceived(now);
-
 		const RemotePath *currentBest = peer->getBestPath(now);
 
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
@@ -916,7 +910,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 						if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
 							if ((!currentBest)||(currentBest->address() != a))
-								peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+								peer->attemptToContactAt(RR,_localAddress,a,now);
 						}
 					}
 				}	break;
@@ -926,7 +920,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 						if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
 							if ((!currentBest)||(currentBest->address() != a))
-								peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+								peer->attemptToContactAt(RR,_localAddress,a,now);
 						}
 					}
 				}	break;
diff --git a/node/Peer.cpp b/node/Peer.cpp
index f16f14212..d5367b17c 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -55,7 +55,6 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
 	_lastDirectPathPushSent(0),
-	_lastDirectPathPushReceived(0),
 	_lastPathSort(0),
 	_vProto(0),
 	_vMajor(0),
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 0aca70b4f..04b541af0 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -407,16 +407,6 @@ public:
 	 */
 	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
 
-	/**
-	 * @return Time last direct path push was received
-	 */
-	inline uint64_t lastDirectPathPushReceived() const throw() { return _lastDirectPathPushReceived; }
-
-	/**
-	 * @param t New time of last direct path push received
-	 */
-	inline void setLastDirectPathPushReceived(uint64_t t) throw() { _lastDirectPathPushReceived = t; }
-
 	/**
 	 * Perform periodic cleaning operations
 	 */
@@ -450,7 +440,7 @@ public:
 		const unsigned int recSizePos = b.size();
 		b.addSize(4); // space for uint32_t field length
 
-		b.append((uint16_t)1); // version of serialized Peer data
+		b.append((uint16_t)0); // version of serialized Peer data
 
 		_id.serialize(b,false);
 
@@ -461,7 +451,6 @@ public:
 		b.append((uint64_t)_lastAnnouncedTo);
 		b.append((uint64_t)_lastPathConfirmationSent);
 		b.append((uint64_t)_lastDirectPathPushSent);
-		b.append((uint64_t)_lastDirectPathPushReceived);
 		b.append((uint64_t)_lastPathSort);
 		b.append((uint16_t)_vProto);
 		b.append((uint16_t)_vMajor);
@@ -513,7 +502,7 @@ public:
 		const unsigned int recSize = b.template at<uint32_t>(p); p += 4;
 		if ((p + recSize) > b.size())
 			return SharedPtr<Peer>(); // size invalid
-		if (b.template at<uint16_t>(p) != 1)
+		if (b.template at<uint16_t>(p) != 0)
 			return SharedPtr<Peer>(); // version mismatch
 		p += 2;
 
@@ -531,7 +520,6 @@ public:
 		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
 		np->_lastPathConfirmationSent = b.template at<uint64_t>(p); p += 8;
 		np->_lastDirectPathPushSent = b.template at<uint64_t>(p); p += 8;
-		np->_lastDirectPathPushReceived = b.template at<uint64_t>(p); p += 8;
 		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
 		np->_vProto = b.template at<uint16_t>(p); p += 2;
 		np->_vMajor = b.template at<uint16_t>(p); p += 2;
@@ -581,7 +569,6 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
 	uint64_t _lastDirectPathPushSent;
-	uint64_t _lastDirectPathPushReceived;
 	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index 1b70f17cb..81d193694 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -123,16 +123,16 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 
 		// For all peers for whom we forgot an address, send a packet indirectly if
 		// they are still considered alive so that we will re-establish direct links.
-		SharedPtr<Peer> sn(RR->topology->getBestRoot());
-		if (sn) {
-			RemotePath *snp = sn->getBestPath(now);
-			if (snp) {
+		SharedPtr<Peer> r(RR->topology->getBestRoot());
+		if (r) {
+			RemotePath *rp = r->getBestPath(now);
+			if (rp) {
 				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
 					if ((*p)->alive(now)) {
-						TRACE("sending indirect NOP to %s via %s(%s) to re-establish link",(*p)->address().toString().c_str(),sn->address().toString().c_str(),snp->address().toString().c_str());
+						TRACE("sending indirect NOP to %s via %s to re-establish link",(*p)->address().toString().c_str(),r->address().toString().c_str());
 						Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
 						outp.armor((*p)->key(),true);
-						snp->send(RR,outp.data(),outp.size(),now);
+						rp->send(RR,outp.data(),outp.size(),now);
 					}
 				}
 			}
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 7bb4b4491..09668ef55 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -254,68 +254,8 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 			return *bestOverall;
 		}
 
-		/*
-		unsigned int l,bestLatency = 65536;
-		uint64_t lds,ldr;
-
-		// First look for a best root by comparing latencies, but exclude
-		// root servers that have not responded to direct messages in order to
-		// try to exclude any that are dead or unreachable.
-		for(std::vector< SharedPtr<Peer> >::const_iterator sn(_rootPeers.begin());sn!=_rootPeers.end();) {
-			// Skip explicitly avoided relays
-			for(unsigned int i=0;i<avoidCount;++i) {
-				if (avoid[i] == (*sn)->address())
-					goto keep_searching_for_roots;
-			}
-
-			// Skip possibly comatose or unreachable relays
-			lds = (*sn)->lastDirectSend();
-			ldr = (*sn)->lastDirectReceive();
-			if ((lds)&&(lds > ldr)&&((lds - ldr) > ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD))
-				goto keep_searching_for_roots;
-
-			if ((*sn)->hasActiveDirectPath(now)) {
-				l = (*sn)->latency();
-				if (bestRoot) {
-					if ((l)&&(l < bestLatency)) {
-						bestLatency = l;
-						bestRoot = *sn;
-					}
-				} else {
-					if (l)
-						bestLatency = l;
-					bestRoot = *sn;
-				}
-			}
-
-keep_searching_for_roots:
-			++sn;
-		}
-
-		if (bestRoot) {
-			bestRoot->use(now);
-			return bestRoot;
-		} else if (strictAvoid)
-			return SharedPtr<Peer>();
-
-		// If we have nothing from above, just pick one without avoidance criteria.
-		for(std::vector< SharedPtr<Peer> >::const_iterator sn=_rootPeers.begin();sn!=_rootPeers.end();++sn) {
-			if ((*sn)->hasActiveDirectPath(now)) {
-				unsigned int l = (*sn)->latency();
-				if (bestRoot) {
-					if ((l)&&(l < bestLatency)) {
-						bestLatency = l;
-						bestRoot = *sn;
-					}
-				} else {
-					if (l)
-						bestLatency = l;
-					bestRoot = *sn;
-				}
-			}
-		}
-		*/
 	}
+
 	return SharedPtr<Peer>();
 }
 

From 0ffbd05c0e14088ddb7eaecba7f19541651e859b Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 12:21:57 -0700
Subject: [PATCH 17/73] --wtf; prevent roots from TCP fallback

---
 node/Node.cpp          | 4 ++--
 service/OneService.cpp | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/node/Node.cpp b/node/Node.cpp
index 2b2989039..87871e20d 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -321,8 +321,8 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
 			// Update online status, post status change as event
-			bool oldOnline = _online;
-			_online = ((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT);
+			const bool oldOnline = _online;
+			_online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot()));
 			if (oldOnline != _online)
 				postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE);
 		} catch ( ... ) {
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 729812edb..7a473b67e 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -867,6 +867,7 @@ public:
 	{
 #ifdef ZT_ENABLE_CLUSTER
 		if (sock == _clusterMessageSocket) {
+			_lastDirectReceiveFromGlobal = OSUtils::now();
 			_node->clusterHandleIncomingMessage(data,len);
 			return;
 		}
@@ -1030,7 +1031,7 @@ public:
 							if (from) {
 								ZT_ResultCode rc = _node->processWirePacket(
 									OSUtils::now(),
-									0,
+									&ZT_SOCKADDR_NULL,
 									reinterpret_cast<struct sockaddr_storage *>(&from),
 									data,
 									plen,

From cfe166ef359c0d92b1521e6c127e2b92238c0731 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 12:29:01 -0700
Subject: [PATCH 18/73] Tweak some size limits.

---
 include/ZeroTierOne.h | 2 +-
 node/Cluster.hpp      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 7af4f7601..7371b9f0a 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -141,7 +141,7 @@ extern "C" {
 /**
  * Maximum allowed cluster message length in bytes
  */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 6)
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 4)
 
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 7d0c6a089..bb7d3b397 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
 
 namespace ZeroTier {
 

From 7295fcfa86aafdca76ac0210ca59ad9a70a2d67a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 12:50:23 -0700
Subject: [PATCH 19/73] Merge Phy<> from netcon.

---
 osdep/Http.cpp         |   8 +++
 osdep/Phy.hpp          | 123 ++++++++++++++++++-----------------------
 selftest.cpp           |   4 +-
 service/OneService.cpp |   4 +-
 4 files changed, 65 insertions(+), 74 deletions(-)

diff --git a/osdep/Http.cpp b/osdep/Http.cpp
index 0eb7c4c6d..6d812a142 100644
--- a/osdep/Http.cpp
+++ b/osdep/Http.cpp
@@ -100,6 +100,14 @@ struct HttpPhyHandler
 			phy->setNotifyWritable(sock,false);
 	}
 
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
+#ifdef __UNIX_LIKE__
+	inline void phyOnUnixAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN) {}
+	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
+	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
+	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
+#endif // __UNIX_LIKE__
+
 	http_parser parser;
 	std::string currentHeaderField;
 	std::string currentHeaderValue;
diff --git a/osdep/Phy.hpp b/osdep/Phy.hpp
index 6737034e3..94130fafa 100644
--- a/osdep/Phy.hpp
+++ b/osdep/Phy.hpp
@@ -78,11 +78,6 @@
 #define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS
 #define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage
 
-#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-#define ZT_PHY_HAVE_EVENTFD 1
-#include <sys/eventfd.h>
-#endif
-
 #endif // Windows or not
 
 namespace ZeroTier {
@@ -109,6 +104,7 @@ typedef void PhySocket;
  * phyOnTcpClose(PhySocket *sock,void **uptr)
  * phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len)
  * phyOnTcpWritable(PhySocket *sock,void **uptr)
+ * phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable)
  *
  * On Linux/OSX/Unix only (not required/used on Windows or elsewhere):
  *
@@ -116,9 +112,6 @@ typedef void PhySocket;
  * phyOnUnixClose(PhySocket *sock,void **uptr)
  * phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len)
  * phyOnUnixWritable(PhySocket *sock,void **uptr)
- * phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr)
- * phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len)
- * phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr)
  *
  * These templates typically refer to function objects. Templates are used to
  * avoid the call overhead of indirection, which is surprisingly high for high
@@ -154,11 +147,10 @@ private:
 		ZT_PHY_SOCKET_TCP_OUT_CONNECTED = 0x02,
 		ZT_PHY_SOCKET_TCP_IN = 0x03,
 		ZT_PHY_SOCKET_TCP_LISTEN = 0x04,
-		ZT_PHY_SOCKET_RAW = 0x05,
-		ZT_PHY_SOCKET_UDP = 0x06,
+		ZT_PHY_SOCKET_UDP = 0x05,
+		ZT_PHY_SOCKET_FD = 0x06,
 		ZT_PHY_SOCKET_UNIX_IN = 0x07,
-		ZT_PHY_SOCKET_UNIX_LISTEN = 0x08,
-		ZT_PHY_SOCKET_PAIR_ENDPOINT = 0x09
+		ZT_PHY_SOCKET_UNIX_LISTEN = 0x08
 	};
 
 	struct PhySocketImpl
@@ -277,57 +269,47 @@ public:
 	 */
 	inline unsigned long maxCount() const throw() { return ZT_PHY_MAX_SOCKETS; }
 
-#ifdef __UNIX_LIKE__
 	/**
-	 * Create a two-way socket pair
+	 * Wrap a raw file descriptor in a PhySocket structure
 	 *
-	 * This uses socketpair() to create a local domain pair. The returned
-	 * PhySocket holds the local side of the socket pair, while the
-	 * supplied fd variable is set to the descriptor for the remote side.
+	 * This can be used to select/poll on a raw file descriptor as part of this
+	 * class's I/O loop. By default the fd is set for read notification but
+	 * this can be controlled with setNotifyReadable(). When any detected
+	 * condition is present, the phyOnFileDescriptorActivity() callback is
+	 * called with one or both of its arguments 'true'.
 	 *
-	 * The local side is set to O_NONBLOCK to work with our poll loop, but
-	 * the remote descriptor is left untouched. It's up to the caller to
-	 * set any required fcntl(), ioctl(), or setsockopt() settings there.
-	 * It's also up to the caller to close the remote descriptor when
-	 * done, if necessary.
+	 * The Phy<>::close() method *must* be called when you're done with this
+	 * file descriptor to remove it from the select/poll set, but unlike other
+	 * types of sockets Phy<> does not actually close the underlying fd or
+	 * otherwise manage its life cycle. There is also no close notification
+	 * callback for this fd, since Phy<> doesn't actually perform reading or
+	 * writing or detect error conditions. This is only useful for adding a
+	 * file descriptor to Phy<> to select/poll on it.
 	 *
-	 * @param remoteSocketDescriptor Result parameter set to remote end of socket pair's socket FD
-	 * @param uptr Pointer to associate with local side of socket pair
-	 * @return PhySocket for local side of socket pair
+	 * @param fd Raw file descriptor
+	 * @param uptr User pointer to supply to callbacks
+	 * @return PhySocket wrapping fd or NULL on failure (out of memory or too many sockets)
 	 */
-	inline PhySocket *createSocketPair(ZT_PHY_SOCKFD_TYPE &remoteSocketDescriptor,void *uptr = (void *)0)
+	inline PhySocket *wrapSocket(ZT_PHY_SOCKFD_TYPE fd,void *uptr = (void *)0)
 	{
 		if (_socks.size() >= ZT_PHY_MAX_SOCKETS)
 			return (PhySocket *)0;
-
-		int fd[2]; fd[0] = -1; fd[1] = -1;
-		if ((::socketpair(PF_LOCAL,SOCK_STREAM,0,fd) != 0)||(fd[0] <= 0)||(fd[1] <= 0))
-			return (PhySocket *)0;
-		fcntl(fd[0],F_SETFL,O_NONBLOCK);
-
 		try {
 			_socks.push_back(PhySocketImpl());
 		} catch ( ... ) {
-			ZT_PHY_CLOSE_SOCKET(fd[0]);
-			ZT_PHY_CLOSE_SOCKET(fd[1]);
 			return (PhySocket *)0;
 		}
 		PhySocketImpl &sws = _socks.back();
-
-		if ((long)fd[0] > _nfds)
-			_nfds = (long)fd[0];
-		FD_SET(fd[0],&_readfds);
-		sws.type = ZT_PHY_SOCKET_PAIR_ENDPOINT;
-		sws.sock = fd[0];
+		if ((long)fd > _nfds)
+			_nfds = (long)fd;
+		FD_SET(fd,&_readfds);
+		sws.type = ZT_PHY_SOCKET_FD;
+		sws.sock = fd;
 		sws.uptr = uptr;
 		memset(&(sws.saddr),0,sizeof(struct sockaddr_storage));
 		// no sockaddr for this socket type, leave saddr null
-
-		remoteSocketDescriptor = fd[1];
-
 		return (PhySocket *)&sws;
 	}
-#endif // __UNIX_LIKE__
 
 	/**
 	 * Bind a UDP socket
@@ -787,6 +769,26 @@ public:
 		}
 	}
 
+	/**
+	 * Set whether we want to be notified that a socket is readable
+	 *
+	 * This is primarily for raw sockets added with wrapSocket(). It could be
+	 * used with others, but doing so would essentially lock them and prevent
+	 * data from being read from them until this is set to 'true' again.
+	 *
+	 * @param sock Socket to modify
+	 * @param notifyReadable True if socket should be monitored for readability
+	 */
+	inline const void setNotifyReadable(PhySocket *sock,bool notifyReadable)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+		if (notifyReadable) {
+			FD_SET(sws.sock,&_readfds);
+		} else {
+			FD_CLR(sws.sock,&_readfds);
+		}
+	}
+
 	/**
 	 * Wait for activity and handle one or more events
 	 *
@@ -936,7 +938,7 @@ public:
 					}
 					if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) {
 						try {
-							_handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr));
+							//_handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr));
 						} catch ( ... ) {}
 					}
 #endif // __UNIX_LIKE__
@@ -971,25 +973,15 @@ public:
 #endif // __UNIX_LIKE__
 					break;
 
-				case ZT_PHY_SOCKET_PAIR_ENDPOINT: {
-#ifdef __UNIX_LIKE__
-					ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable
-					if (FD_ISSET(sock,&rfds)) {
-						long n = (long)::read(sock,buf,sizeof(buf));
-						if (n <= 0) {
-							this->close((PhySocket *)&(*s),true);
-						} else {
-							try {
-								_handler->phyOnSocketPairEndpointData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n);
-							} catch ( ... ) {}
-						}
-					}
-					if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) {
+				case ZT_PHY_SOCKET_FD: {
+					ZT_PHY_SOCKFD_TYPE sock = s->sock;
+					const bool readable = ((FD_ISSET(sock,&rfds))&&(FD_ISSET(sock,&_readfds)));
+					const bool writable = ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds)));
+					if ((readable)||(writable)) {
 						try {
-							_handler->phyOnSocketPairEndpointWritable((PhySocket *)&(*s),&(s->uptr));
+							_handler->phyOnFileDescriptorActivity((PhySocket *)&(*s),&(s->uptr),readable,writable);
 						} catch ( ... ) {}
 					}
-#endif // __UNIX_LIKE__
 				}	break;
 
 				default:
@@ -1021,7 +1013,8 @@ public:
 		FD_CLR(sws.sock,&_exceptfds);
 #endif
 
-		ZT_PHY_CLOSE_SOCKET(sws.sock);
+		if (sws.type != ZT_PHY_SOCKET_FD)
+			ZT_PHY_CLOSE_SOCKET(sws.sock);
 
 #ifdef __UNIX_LIKE__
 		if (sws.type == ZT_PHY_SOCKET_UNIX_LISTEN)
@@ -1048,12 +1041,6 @@ public:
 					} catch ( ... ) {}
 #endif // __UNIX_LIKE__
 					break;
-				case ZT_PHY_SOCKET_PAIR_ENDPOINT:
-#ifdef __UNIX_LIKE__
-					try {
-						_handler->phyOnSocketPairEndpointClose(sock,&(sws.uptr));
-					} catch ( ... ) {}
-#endif // __UNIX_LIKE__
 				default:
 					break;
 			}
diff --git a/selftest.cpp b/selftest.cpp
index 9c357dc4c..fa5eec0fb 100644
--- a/selftest.cpp
+++ b/selftest.cpp
@@ -837,9 +837,7 @@ struct TestPhyHandlers
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
-	inline void phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr) {}
-  inline void phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-  inline void phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 #endif // __UNIX_LIKE__
 };
 static int testPhy()
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 7a473b67e..1765b5c44 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -1085,9 +1085,7 @@ public:
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
-	inline void phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr) {}
-  inline void phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-  inline void phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 
 	inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwc)
 	{

From 40e0a34a5c22276e5546dc7835eec87282ac9484 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 13:04:08 -0700
Subject: [PATCH 20/73] Add set buffer sizes code to Phy<>

---
 osdep/Phy.hpp | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/osdep/Phy.hpp b/osdep/Phy.hpp
index 94130fafa..8dde279ce 100644
--- a/osdep/Phy.hpp
+++ b/osdep/Phy.hpp
@@ -655,6 +655,36 @@ public:
 		return (PhySocket *)&sws;
 	}
 
+	/**
+	 * Try to set buffer sizes as close to the given value as possible
+	 *
+	 * This will try the specified value and then lower values in 16K increments
+	 * until one works.
+	 *
+	 * @param sock Socket
+	 * @param bufferSize Desired buffer sizes
+	 */
+	inline void setBufferSizes(const PhySocket *sock,int bufferSize)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+		if (bufferSize > 0) {
+			int bs = bufferSize;
+			while (bs >= 65536) {
+				int tmpbs = bs;
+				if (::setsockopt(sws.sock,SOL_SOCKET,SO_RCVBUF,(const char *)&tmpbs,sizeof(tmpbs)) == 0)
+					break;
+				bs -= 16384;
+			}
+			bs = bufferSize;
+			while (bs >= 65536) {
+				int tmpbs = bs;
+				if (::setsockopt(sws.sock,SOL_SOCKET,SO_SNDBUF,(const char *)&tmpbs,sizeof(tmpbs)) == 0)
+					break;
+				bs -= 16384;
+			}
+		}
+	}
+
 	/**
 	 * Attempt to send data to a stream socket (non-blocking)
 	 *

From f692cec763d67caae54a4f47446657c390563319 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 14:04:12 -0700
Subject: [PATCH 21/73] Change how cluster relays packets -- just PROXY_UNITE
 and then send packet via normal ZeroTier front plane -- more efficient and
 eliminates fragmentation issues.

---
 include/ZeroTierOne.h |   2 +-
 node/Cluster.cpp      | 189 +++++++++++++++++++++---------------------
 node/Cluster.hpp      |  22 +++--
 node/Switch.cpp       |  37 +++++----
 node/Switch.hpp       |   8 +-
 5 files changed, 130 insertions(+), 128 deletions(-)

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 7371b9f0a..3457634bb 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -141,7 +141,7 @@ extern "C" {
 /**
  * Maximum allowed cluster message length in bytes
  */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 4)
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48)
 
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index eef02bc79..c18663bc7 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -250,102 +250,87 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							}
 						}	break;
 
-						case STATE_MESSAGE_RELAY: {
+						case STATE_MESSAGE_PROXY_UNITE: {
+							const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 							const unsigned int numRemotePeerPaths = dmsg[ptr++];
 							InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max
 							for(unsigned int i=0;i<numRemotePeerPaths;++i)
 								ptr += remotePeerPaths[i].deserialize(dmsg,ptr);
-							const unsigned int packetLen = dmsg.at<uint16_t>(ptr); ptr += 2;
-							const void *packet = (const void *)dmsg.field(ptr,packetLen); ptr += packetLen;
 
-							if (packetLen >= ZT_PROTO_MIN_FRAGMENT_LENGTH) { // ignore anything too short to contain a dest address
-								const Address destinationAddress(reinterpret_cast<const char *>(packet) + 8,ZT_ADDRESS_LENGTH);
-								TRACE("[%u] relay %u bytes to %s (%u remote paths included)",(unsigned int)fromMemberId,packetLen,destinationAddress.toString().c_str(),numRemotePeerPaths);
+							TRACE("[%u] requested proxy unite between local peer %s and remote peer %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
 
-								SharedPtr<Peer> destinationPeer(RR->topology->getPeer(destinationAddress));
-								if (destinationPeer) {
-									if (
-									    (destinationPeer->send(RR,packet,packetLen,RR->node->now()))&&
-									    (numRemotePeerPaths > 0)&&
-									    (packetLen >= 18)&&
-									    (reinterpret_cast<const unsigned char *>(packet)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR)
-									   ) {
-										// If remote peer paths were sent with this relayed packet, we do
-										// RENDEZVOUS. It's handled here for cluster-relayed packets since
-										// we don't have both Peer records so this is a different path.
+							SharedPtr<Peer> localPeer(RR->topology->getPeer(localPeerAddress));
+							if ((localPeer)&&(numRemotePeerPaths > 0)) {
+								InetAddress bestLocalV4,bestLocalV6;
+								localPeer->getBestActiveAddresses(RR->node->now(),bestLocalV4,bestLocalV6);
 
-										const Address remotePeerAddress(reinterpret_cast<const char *>(packet) + 13,ZT_ADDRESS_LENGTH);
-
-										InetAddress bestDestV4,bestDestV6;
-										destinationPeer->getBestActiveAddresses(RR->node->now(),bestDestV4,bestDestV6);
-										InetAddress bestRemoteV4,bestRemoteV6;
-										for(unsigned int i=0;i<numRemotePeerPaths;++i) {
-											if ((bestRemoteV4)&&(bestRemoteV6))
-												break;
-											switch(remotePeerPaths[i].ss_family) {
-												case AF_INET:
-													if (!bestRemoteV4)
-														bestRemoteV4 = remotePeerPaths[i];
-													break;
-												case AF_INET6:
-													if (!bestRemoteV6)
-														bestRemoteV6 = remotePeerPaths[i];
-													break;
-											}
-										}
-
-										Packet rendezvousForDest(destinationAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
-										rendezvousForDest.append((uint8_t)0);
-										remotePeerAddress.appendTo(rendezvousForDest);
-
-										Buffer<2048> rendezvousForOtherEnd;
-										remotePeerAddress.appendTo(rendezvousForOtherEnd);
-										rendezvousForOtherEnd.append((uint8_t)Packet::VERB_RENDEZVOUS);
-										const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForOtherEnd.size();
-										rendezvousForOtherEnd.addSize(2); // space for actual packet payload length
-										rendezvousForOtherEnd.append((uint8_t)0); // flags == 0
-										destinationAddress.appendTo(rendezvousForOtherEnd);
-
-										bool haveMatch = false;
-										if ((bestDestV6)&&(bestRemoteV6)) {
-											haveMatch = true;
-
-											rendezvousForDest.append((uint16_t)bestRemoteV6.port());
-											rendezvousForDest.append((uint8_t)16);
-											rendezvousForDest.append(bestRemoteV6.rawIpData(),16);
-
-											rendezvousForOtherEnd.append((uint16_t)bestDestV6.port());
-											rendezvousForOtherEnd.append((uint8_t)16);
-											rendezvousForOtherEnd.append(bestDestV6.rawIpData(),16);
-											rendezvousForOtherEnd.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
-										} else if ((bestDestV4)&&(bestRemoteV4)) {
-											haveMatch = true;
-
-											rendezvousForDest.append((uint16_t)bestRemoteV4.port());
-											rendezvousForDest.append((uint8_t)4);
-											rendezvousForDest.append(bestRemoteV4.rawIpData(),4);
-
-											rendezvousForOtherEnd.append((uint16_t)bestDestV4.port());
-											rendezvousForOtherEnd.append((uint8_t)4);
-											rendezvousForOtherEnd.append(bestDestV4.rawIpData(),4);
-											rendezvousForOtherEnd.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
-										}
-
-										if (haveMatch) {
-											_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForOtherEnd.data(),rendezvousForOtherEnd.size());
-											RR->sw->send(rendezvousForDest,true,0);
-										}
+								InetAddress bestRemoteV4,bestRemoteV6;
+								for(unsigned int i=0;i<numRemotePeerPaths;++i) {
+									if ((bestRemoteV4)&&(bestRemoteV6))
+										break;
+									switch(remotePeerPaths[i].ss_family) {
+										case AF_INET:
+											if (!bestRemoteV4)
+												bestRemoteV4 = remotePeerPaths[i];
+											break;
+										case AF_INET6:
+											if (!bestRemoteV6)
+												bestRemoteV6 = remotePeerPaths[i];
+											break;
 									}
 								}
+
+								Packet rendezvousForLocal(localPeerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+								rendezvousForLocal.append((uint8_t)0);
+								remotePeerAddress.appendTo(rendezvousForLocal);
+
+								Buffer<2048> rendezvousForRemote;
+								remotePeerAddress.appendTo(rendezvousForRemote);
+								rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS);
+								const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForRemote.size();
+								rendezvousForRemote.addSize(2); // space for actual packet payload length
+								rendezvousForRemote.append((uint8_t)0); // flags == 0
+								localPeerAddress.appendTo(rendezvousForRemote);
+
+								bool haveMatch = false;
+								if ((bestLocalV6)&&(bestRemoteV6)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV6.port());
+									rendezvousForLocal.append((uint8_t)16);
+									rendezvousForLocal.append(bestRemoteV6.rawIpData(),16);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV6.port());
+									rendezvousForRemote.append((uint8_t)16);
+									rendezvousForRemote.append(bestLocalV6.rawIpData(),16);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
+								} else if ((bestLocalV4)&&(bestRemoteV4)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV4.port());
+									rendezvousForLocal.append((uint8_t)4);
+									rendezvousForLocal.append(bestRemoteV4.rawIpData(),4);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV4.port());
+									rendezvousForRemote.append((uint8_t)4);
+									rendezvousForRemote.append(bestLocalV4.rawIpData(),4);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
+								}
+
+								if (haveMatch) {
+									_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
+									RR->sw->send(rendezvousForLocal,true,0);
+								}
 							}
 						}	break;
 
 						case STATE_MESSAGE_PROXY_SEND: {
-							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 							const Packet::Verb verb = (Packet::Verb)dmsg[ptr++];
 							const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
 							Packet outp(rcpt,RR->identity.address(),verb);
-							outp.append(dmsg.field(ptr,len),len);
+							outp.append(dmsg.field(ptr,len),len); ptr += len;
 							RR->sw->send(outp,true,0);
 							TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
 						}	break;
@@ -364,13 +349,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 	}
 }
 
-bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len)
+bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
 {
 	if (len > 16384) // sanity check
 		return false;
 
 	uint64_t mostRecentTimestamp = 0;
-	uint16_t canHasPeer = 0;
+	unsigned int canHasPeer = 0;
 
 	{	// Anyone got this peer?
 		Mutex::Lock _l2(_peerAffinities_m);
@@ -387,25 +372,37 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 
 	const uint64_t now = RR->node->now();
 	if ((now - mostRecentTimestamp) < ZT_PEER_ACTIVITY_TIMEOUT) {
-		Buffer<16384> buf;
+		Buffer<2048> buf;
 
-		InetAddress v4,v6;
-		if (fromPeerAddress) {
-			SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
-			if (fromPeer)
-				fromPeer->getBestActiveAddresses(now,v4,v6);
+		if (unite) {
+			InetAddress v4,v6;
+			if (fromPeerAddress) {
+				SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
+				if (fromPeer)
+					fromPeer->getBestActiveAddresses(now,v4,v6);
+			}
+			uint8_t addrCount = 0;
+			if (v4)
+				++addrCount;
+			if (v6)
+				++addrCount;
+			if (addrCount) {
+				toPeerAddress.appendTo(buf);
+				fromPeerAddress.appendTo(buf);
+				buf.append(addrCount);
+				if (v4)
+					v4.serialize(buf);
+				if (v6)
+					v6.serialize(buf);
+			}
 		}
-		buf.append((uint8_t)( (v4) ? ((v6) ? 2 : 1) : ((v6) ? 1 : 0) ));
-		if (v4)
-			v4.serialize(buf);
-		if (v6)
-			v6.serialize(buf);
-		buf.append((uint16_t)len);
-		buf.append(data,len);
 
 		{
 			Mutex::Lock _l2(_members[canHasPeer].lock);
-			_send(canHasPeer,STATE_MESSAGE_RELAY,buf.data(),buf.size());
+			if (buf.size() > 0)
+				_send(canHasPeer,STATE_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+			if (_members[canHasPeer].zeroTierPhysicalEndpoints.size() > 0)
+				RR->node->putPacket(InetAddress(),_members[canHasPeer].zeroTierPhysicalEndpoints.front(),data,len);
 		}
 
 		TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index bb7d3b397..282d81200 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 50
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
 
 namespace ZeroTier {
 
@@ -136,13 +136,18 @@ public:
 		STATE_MESSAGE_COM = 4,
 
 		/**
-		 * Relay a packet to a peer:
-		 *   <[1] 8-bit number of sending peer active path addresses>
-		 *   <[...] series of serialized InetAddresses of sending peer's paths>
-		 *   <[2] 16-bit packet length>
-		 *   <[...] packet or packet fragment>
+		 * Request that VERB_RENDEZVOUS be sent to a peer that we have:
+		 *   <[5] ZeroTier address of peer on recipient's side>
+		 *   <[5] ZeroTier address of peer on sender's side>
+		 *   <[1] 8-bit number of sender's peer's active path addresses>
+		 *   <[...] series of serialized InetAddresses of sender's peer's paths>
+		 *
+		 * This requests that we perform NAT-t introduction between a peer that
+		 * we have and one on the sender's side. The sender furnishes contact
+		 * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours
+		 * directly and with PROXY_SEND to theirs.
 		 */
-		STATE_MESSAGE_RELAY = 5,
+		STATE_MESSAGE_PROXY_UNITE = 5,
 
 		/**
 		 * Request that a cluster member send a packet to a locally-known peer:
@@ -211,9 +216,10 @@ public:
 	 * @param toPeerAddress Destination peer address
 	 * @param data Packet or packet fragment data
 	 * @param len Length of packet or fragment
+	 * @param unite If true, also request proxy unite across cluster
 	 * @return True if this data was sent via another cluster member, false if none have this peer
 	 */
-	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len);
+	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
 
 	/**
 	 * Advertise to the cluster that we have this peer
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 709ed802a..772eaf023 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -303,11 +303,10 @@ void Switch::send(const Packet &packet,bool encrypt,uint64_t nwid)
 	}
 }
 
-bool Switch::unite(const Address &p1,const Address &p2,bool force)
+bool Switch::unite(const Address &p1,const Address &p2)
 {
 	if ((p1 == RR->identity.address())||(p2 == RR->identity.address()))
 		return false;
-
 	SharedPtr<Peer> p1p = RR->topology->getPeer(p1);
 	if (!p1p)
 		return false;
@@ -317,14 +316,6 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 
 	const uint64_t now = RR->node->now();
 
-	{
-		Mutex::Lock _l(_lastUniteAttempt_m);
-		uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
-		if (((now - luts) < ZT_MIN_UNITE_INTERVAL)&&(!force))
-			return false;
-		luts = now;
-	}
-
 	std::pair<InetAddress,InetAddress> cg(Peer::findCommonGround(*p1p,*p2p,now));
 	if ((!(cg.first))||(cg.first.ipScope() != cg.second.ipScope()))
 		return false;
@@ -571,7 +562,7 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
 			if ((!relayTo)||(!relayTo->send(RR,fragment.data(),fragment.size(),RR->node->now()))) {
 #ifdef ZT_ENABLE_CLUSTER
-				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size())))
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false)))
 					return; // sent by way of another member of this cluster
 #endif
 
@@ -634,7 +625,8 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 
 void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,RR->node->now()));
+	const uint64_t now = RR->node->now();
+	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,now));
 
 	Address source(packet->source());
 	Address destination(packet->destination());
@@ -652,17 +644,18 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 			packet->incrementHops();
 
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),RR->node->now())))) {
-				unite(source,destination,false);
+			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),now)))) {
+				if (_shouldTryUnite(now,source,destination))
+					unite(source,destination);
 			} else {
 #ifdef ZT_ENABLE_CLUSTER
-				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size())))
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size(),_shouldTryUnite(now,source,destination))))
 					return; // sent by way of another member of this cluster
 #endif
 
 				relayTo = RR->topology->getBestRoot(&source,1,true);
 				if (relayTo)
-					relayTo->send(RR,packet->data(),packet->size(),RR->node->now());
+					relayTo->send(RR,packet->data(),packet->size(),now);
 			}
 		} else {
 			TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet->source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str());
@@ -677,7 +670,7 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 		if (!dq.creationTime) {
 			// If we have no other fragments yet, create an entry and save the head
 
-			dq.creationTime = RR->node->now();
+			dq.creationTime = now;
 			dq.frag0 = packet;
 			dq.totalFragments = 0; // 0 == unknown, waiting for Packet::Fragment
 			dq.haveFragments = 1; // head is first bit (left to right)
@@ -805,4 +798,14 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 	return false;
 }
 
+bool Switch::_shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2)
+{
+	Mutex::Lock _l(_lastUniteAttempt_m);
+	uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
+	if ((now - luts) < ZT_MIN_UNITE_INTERVAL)
+		return false;
+	luts = now;
+	return true;
+}
+
 } // namespace ZeroTier
diff --git a/node/Switch.hpp b/node/Switch.hpp
index 3bdc0c47c..42e87ca56 100644
--- a/node/Switch.hpp
+++ b/node/Switch.hpp
@@ -127,15 +127,10 @@ public:
 	 * This only works if both peers are known, with known working direct
 	 * links to this peer. The best link for each peer is sent to the other.
 	 *
-	 * A rate limiter is in effect via the _lastUniteAttempt map. If force
-	 * is true, a unite attempt is made even if one has been made less than
-	 * ZT_MIN_UNITE_INTERVAL milliseconds ago.
-	 *
 	 * @param p1 One of two peers (order doesn't matter)
 	 * @param p2 Second of pair
-	 * @param force If true, send now regardless of interval
 	 */
-	bool unite(const Address &p1,const Address &p2,bool force);
+	bool unite(const Address &p1,const Address &p2);
 
 	/**
 	 * Attempt NAT traversal to peer at a given physical address
@@ -185,6 +180,7 @@ private:
 	void _handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid);
+	bool _shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2);
 
 	const RuntimeEnvironment *const RR;
 	uint64_t _lastBeaconResponse;

From 40976c02a42b8e9078519f92a7c7412b8464e9bc Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 14:37:38 -0700
Subject: [PATCH 22/73] Forget paths to peers if we are handing them off.

---
 node/Cluster.cpp | 14 ++++----
 node/Cluster.hpp | 15 +++------
 node/Peer.cpp    | 88 +++++++++++++++++++++++++-----------------------
 3 files changed, 56 insertions(+), 61 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index c18663bc7..73ff58463 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -563,17 +563,14 @@ void Cluster::removeMember(uint16_t memberId)
 	_memberIds = newMemberIds;
 }
 
-InetAddress Cluster::findBetterEndpoint(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
+bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
 {
-	if (!peerPhysicalAddress) // sanity check
-		return InetAddress();
-
 	if (_addressToLocationFunction) {
 		// Pick based on location if it can be determined
 		int px = 0,py = 0,pz = 0;
 		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
 			TRACE("no geolocation data for %s (geo-lookup is lazy/async so it may work next time)",peerPhysicalAddress.toIpString().c_str());
-			return InetAddress();
+			return false;
 		}
 
 		// Find member closest to this peer
@@ -603,14 +600,15 @@ InetAddress Cluster::findBetterEndpoint(const Address &peerAddress,const InetAdd
 		for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
 			if (a->ss_family == peerPhysicalAddress.ss_family) {
 				TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
-				return *a;
+				redirectTo = *a;
+				return true;
 			}
 		}
 		TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance);
-		return InetAddress();
+		return false;
 	} else {
 		// TODO: pick based on load if no location info?
-		return InetAddress();
+		return false;
 	}
 }
 
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 282d81200..45395b0f0 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -264,22 +264,15 @@ public:
 	void removeMember(uint16_t memberId);
 
 	/**
-	 * Find a better cluster endpoint for this peer
-	 *
-	 * If this endpoint appears to be the best, a NULL/0 InetAddres is returned.
-	 * Otherwise the InetAddress of a better endpoint is returned and the peer
-	 * can then then be told to contact us there.
-	 *
-	 * Redirection is only done within the same address family, so the returned
-	 * endpoint will always be the same ss_family as the supplied physical
-	 * address.
+	 * Find a better cluster endpoint for this peer (if any)
 	 *
+	 * @param redirectTo InetAddress to be set to a better endpoint (if there is one)
 	 * @param peerAddress Address of peer to (possibly) redirect
 	 * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
 	 * @param offload Always redirect if possible -- can be used to offload peers during shutdown
-	 * @return InetAddress or NULL if there does not seem to be a better endpoint
+	 * @return True if redirectTo was set to a new address, false if redirectTo was not modified
 	 */
-	InetAddress findBetterEndpoint(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
+	bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
 	/**
 	 * Fill out ZT_ClusterStatus structure (from core API)
diff --git a/node/Peer.cpp b/node/Peer.cpp
index d5367b17c..009e2be58 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -81,47 +81,49 @@ void Peer::received(
 	Packet::Verb inReVerb)
 {
 #ifdef ZT_ENABLE_CLUSTER
-	bool redirected = false;
-	if ((RR->cluster)&&(hops == 0)&&(verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS)) {
-		InetAddress redirectTo(RR->cluster->findBetterEndpoint(_id.address(),remoteAddr,false));
-		if ((redirectTo.ss_family == AF_INET)||(redirectTo.ss_family == AF_INET6)) {
-			if (_vProto >= 5) {
-				// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
-				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
-				outp.append((uint16_t)1); // count == 1
-				outp.append((uint8_t)0); // no flags
-				outp.append((uint16_t)0); // no extensions
-				if (redirectTo.ss_family == AF_INET) {
-					outp.append((uint8_t)4);
-					outp.append((uint8_t)6);
-					outp.append(redirectTo.rawIpData(),4);
+	InetAddress redirectTo;
+	if ((RR->cluster)&&(hops == 0)) {
+		// Note: findBetterEndpoint() is first since we still want to check
+		// for a better endpoint even if we don't actually send a redirect.
+		if ( (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),remoteAddr,false)) && (verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS) ) {
+			if ((redirectTo.ss_family == AF_INET)||(redirectTo.ss_family == AF_INET6)) {
+				if (_vProto >= 5) {
+					// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
+					Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+					outp.append((uint16_t)1); // count == 1
+					outp.append((uint8_t)0); // no flags
+					outp.append((uint16_t)0); // no extensions
+					if (redirectTo.ss_family == AF_INET) {
+						outp.append((uint8_t)4);
+						outp.append((uint8_t)6);
+						outp.append(redirectTo.rawIpData(),4);
+					} else {
+						outp.append((uint8_t)6);
+						outp.append((uint8_t)18);
+						outp.append(redirectTo.rawIpData(),16);
+					}
+					outp.append((uint16_t)redirectTo.port());
+					outp.armor(_key,true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+					RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 				} else {
-					outp.append((uint8_t)6);
-					outp.append((uint8_t)18);
-					outp.append(redirectTo.rawIpData(),16);
+					// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
+					Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+					outp.append((uint8_t)0); // no flags
+					RR->identity.address().appendTo(outp);
+					outp.append((uint16_t)redirectTo.port());
+					if (redirectTo.ss_family == AF_INET) {
+						outp.append((uint8_t)4);
+						outp.append(redirectTo.rawIpData(),4);
+					} else {
+						outp.append((uint8_t)16);
+						outp.append(redirectTo.rawIpData(),16);
+					}
+					outp.armor(_key,true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+					RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 				}
-				outp.append((uint16_t)redirectTo.port());
-				outp.armor(_key,true);
-				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
-			} else {
-				// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
-				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
-				outp.append((uint8_t)0); // no flags
-				RR->identity.address().appendTo(outp);
-				outp.append((uint16_t)redirectTo.port());
-				if (redirectTo.ss_family == AF_INET) {
-					outp.append((uint8_t)4);
-					outp.append(redirectTo.rawIpData(),4);
-				} else {
-					outp.append((uint8_t)16);
-					outp.append(redirectTo.rawIpData(),16);
-				}
-				outp.armor(_key,true);
-				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			}
-			redirected = true;
 		}
 	}
 #endif
@@ -140,11 +142,13 @@ void Peer::received(
 			_lastMulticastFrame = now;
 
 #ifdef ZT_ENABLE_CLUSTER
-		// If we're in cluster mode and have sent the peer a better endpoint, stop
-		// here and don't confirm paths, replicate multicast info, etc. The new
-		// endpoint should do that.
-		if (redirected)
+		// If we're in cluster mode and there's a better endpoint, stop here and don't
+		// learn or confirm paths. Also reset any existing paths, since they should
+		// go there and no longer talk to us here.
+		if (redirectTo) {
+			_numPaths = 0;
 			return;
+		}
 #endif
 
 		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {

From 16bc3e03982286232e1df2da17d8b2fc3c5a5c55 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 15:00:16 -0700
Subject: [PATCH 23/73] Factor out RemotePath subclass of Path -- no longer
 needed, just cruft.

---
 include/ZeroTierOne.h   |   3 +-
 node/Cluster.cpp        |   1 +
 node/IncomingPacket.cpp |   5 +-
 node/Multicaster.cpp    |   1 +
 node/Network.cpp        |   1 +
 node/Node.cpp           |  14 ++--
 node/Node.hpp           |   6 +-
 node/Path.cpp           |  45 +++++++++++
 node/Path.hpp           | 117 +++++++++++++++++++++++++++--
 node/Peer.cpp           |  32 ++++----
 node/Peer.hpp           |  24 +++---
 node/RemotePath.hpp     | 161 ----------------------------------------
 node/SelfAwareness.cpp  |   2 +-
 node/Switch.cpp         |   2 +-
 objects.mk              |   1 +
 service/OneService.cpp  |   4 +-
 16 files changed, 208 insertions(+), 211 deletions(-)
 create mode 100644 node/Path.cpp
 delete mode 100644 node/RemotePath.hpp

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 3457634bb..01c8bcdeb 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -1356,11 +1356,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * reject bad, empty, and unusable addresses.
  *
  * @param addr Local interface address
- * @param metric Local interface metric
  * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust);
 
 /**
  * Clear local interface addresses
diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 73ff58463..07ca0ba13 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -48,6 +48,7 @@
 #include "Topology.hpp"
 #include "Packet.hpp"
 #include "Switch.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 2514cd64b..7015535a4 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -44,6 +44,7 @@
 #include "SHA512.hpp"
 #include "World.hpp"
 #include "Cluster.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -888,7 +889,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
-		const RemotePath *currentBest = peer->getBestPath(now);
+		const Path *currentBest = peer->getBestPath(now);
 
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2;
@@ -1036,7 +1037,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 				remainingHopsPtr += ZT_ADDRESS_LENGTH;
 				SharedPtr<Peer> nhp(RR->topology->getPeer(nextHop[h]));
 				if (nhp) {
-					RemotePath *const rp = nhp->getBestPath(now);
+					Path *const rp = nhp->getBestPath(now);
 					if (rp)
 						nextHopBestPathAddress[h] = rp->address();
 				}
diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp
index 6e6cd628a..e43d7d886 100644
--- a/node/Multicaster.cpp
+++ b/node/Multicaster.cpp
@@ -37,6 +37,7 @@
 #include "Peer.hpp"
 #include "C25519.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
diff --git a/node/Network.cpp b/node/Network.cpp
index cd30e386d..afbe10740 100644
--- a/node/Network.cpp
+++ b/node/Network.cpp
@@ -37,6 +37,7 @@
 #include "Packet.hpp"
 #include "Buffer.hpp"
 #include "NetworkController.hpp"
+#include "Node.hpp"
 
 #include "../version.h"
 
diff --git a/node/Node.cpp b/node/Node.cpp
index 87871e20d..82cda66d8 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -448,10 +448,10 @@ ZT_PeerList *Node::peers() const
 		p->latency = pi->second->latency();
 		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
-		std::vector<RemotePath> paths(pi->second->paths());
-		RemotePath *bestPath = pi->second->getBestPath(_now);
+		std::vector<Path> paths(pi->second->paths());
+		Path *bestPath = pi->second->getBestPath(_now);
 		p->pathCount = 0;
-		for(std::vector<RemotePath>::iterator path(paths.begin());path!=paths.end();++path) {
+		for(std::vector<Path>::iterator path(paths.begin());path!=paths.end();++path) {
 			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
 			p->paths[p->pathCount].lastSend = path->lastSend();
 			p->paths[p->pathCount].lastReceive = path->lastReceived();
@@ -499,11 +499,11 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
-		_directPaths.push_back(Path(*(reinterpret_cast<const InetAddress *>(addr)),metric,(Path::Trust)trust));
+		_directPaths.push_back(*(reinterpret_cast<const InetAddress *>(addr)));
 		std::sort(_directPaths.begin(),_directPaths.end());
 		_directPaths.erase(std::unique(_directPaths.begin(),_directPaths.end()),_directPaths.end());
 		return 1;
@@ -900,10 +900,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust)
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,trust);
 	} catch ( ... ) {
 		return 0;
 	}
diff --git a/node/Node.hpp b/node/Node.hpp
index 4094a79e9..48c5ead8f 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -105,7 +105,7 @@ public:
 	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
 	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
@@ -207,7 +207,7 @@ public:
 	/**
 	 * @return Potential direct paths to me a.k.a. local interface addresses
 	 */
-	inline std::vector<Path> directPaths() const
+	inline std::vector<InetAddress> directPaths() const
 	{
 		Mutex::Lock _l(_directPaths_m);
 		return _directPaths;
@@ -285,7 +285,7 @@ private:
 	std::vector< ZT_CircuitTest * > _circuitTests;
 	Mutex _circuitTests_m;
 
-	std::vector<Path> _directPaths;
+	std::vector<InetAddress> _directPaths;
 	Mutex _directPaths_m;
 
 	Mutex _backgroundTasksLock;
diff --git a/node/Path.cpp b/node/Path.cpp
new file mode 100644
index 000000000..e2475751c
--- /dev/null
+++ b/node/Path.cpp
@@ -0,0 +1,45 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include "Path.hpp"
+#include "AntiRecursion.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Node.hpp"
+
+namespace ZeroTier {
+
+bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+{
+	if (RR->node->putPacket(_localAddress,address(),data,len)) {
+		sent(now);
+		RR->antiRec->logOutgoingZT(data,len);
+		return true;
+	}
+	return false;
+}
+
+} // namespace ZeroTier
diff --git a/node/Path.hpp b/node/Path.hpp
index 6fa3c52e6..99f6590b2 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -28,12 +28,19 @@
 #ifndef ZT_PATH_HPP
 #define ZT_PATH_HPP
 
+#include <stdint.h>
+#include <string.h>
+
+#include <stdexcept>
+#include <algorithm>
+
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "Utils.hpp"
 
 namespace ZeroTier {
 
+class RuntimeEnvironment;
+
 /**
  * Base class for paths
  *
@@ -67,19 +74,87 @@ public:
 	};
 
 	Path() :
+		_lastSend(0),
+		_lastReceived(0),
 		_addr(),
+		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_trust(TRUST_NORMAL)
+		_trust(TRUST_NORMAL),
+		_flags(0)
 	{
 	}
 
-	Path(const InetAddress &addr,int metric,Trust trust) :
+	Path(const InetAddress &localAddress,const InetAddress &addr,Trust trust) :
+		_lastSend(0),
+		_lastReceived(0),
 		_addr(addr),
+		_localAddress(localAddress),
 		_ipScope(addr.ipScope()),
-		_trust(trust)
+		_trust(trust),
+		_flags(0)
 	{
 	}
 
+	/**
+	 * Called when a packet is sent to this remote path
+	 *
+	 * This is called automatically by Path::send().
+	 *
+	 * @param t Time of send
+	 */
+	inline void sent(uint64_t t)
+		throw()
+	{
+		_lastSend = t;
+	}
+
+	/**
+	 * Called when a packet is received from this remote path
+	 *
+	 * @param t Time of receive
+	 */
+	inline void received(uint64_t t)
+		throw()
+	{
+		_lastReceived = t;
+	}
+
+	/**
+	 * @param now Current time
+	 * @return True if this path appears active
+	 */
+	inline bool active(uint64_t now) const
+		throw()
+	{
+		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
+	}
+
+	/**
+	 * Send a packet via this path
+	 *
+	 * @param RR Runtime environment
+	 * @param data Packet data
+	 * @param len Packet length
+	 * @param now Current time
+	 * @return True if transport reported success
+	 */
+	bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now);
+
+	/**
+	 * @return Address of local side of this path or NULL if unspecified
+	 */
+	inline const InetAddress &localAddress() const throw() { return _localAddress; }
+
+	/**
+	 * @return Time of last send to this path
+	 */
+	inline uint64_t lastSend() const throw() { return _lastSend; }
+
+	/**
+	 * @return Time of last receive from this path
+	 */
+	inline uint64_t lastReceived() const throw() { return _lastReceived; }
+
 	/**
 	 * @return Physical address
 	 */
@@ -157,10 +232,42 @@ public:
 		return false;
 	}
 
-protected:
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)0); // version
+		b.append((uint64_t)_lastSend);
+		b.append((uint64_t)_lastReceived);
+		_addr.serialize(b);
+		_localAddress.serialize(b);
+		b.append((uint8_t)_trust);
+		b.append((uint16_t)_flags);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		if (b[p++] != 0)
+			throw std::invalid_argument("invalid serialized Path");
+		_lastSend = b.template at<uint64_t>(p); p += 8;
+		_lastReceived = b.template at<uint64_t>(p); p += 8;
+		p += _addr.deserialize(b,p);
+		p += _localAddress.deserialize(b,p);
+		_ipScope = _addr.ipScope();
+		_trust = (Path::Trust)b[p++];
+		_flags = b.template at<uint16_t>(p); p += 2;
+		return (p - startAt);
+	}
+
+private:
+	uint64_t _lastSend;
+	uint64_t _lastReceived;
 	InetAddress _addr;
+	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	Trust _trust;
+	uint16_t _flags;
 };
 
 } // namespace ZeroTier
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 009e2be58..ebdb60264 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -169,7 +169,7 @@ void Peer::received(
 			if (!pathIsConfirmed) {
 				if (verb == Packet::VERB_OK) {
 
-					RemotePath *slot = (RemotePath *)0;
+					Path *slot = (Path *)0;
 					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
 						slot = &(_paths[np++]);
 					} else {
@@ -182,7 +182,7 @@ void Peer::received(
 						}
 					}
 					if (slot) {
-						*slot = RemotePath(localAddr,remoteAddr);
+						*slot = Path(localAddr,remoteAddr,Path::TRUST_NORMAL);
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
@@ -240,7 +240,7 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo
 
 bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily)
 {
-	RemotePath *p = (RemotePath *)0;
+	Path *p = (Path *)0;
 
 	Mutex::Lock _l(_lock);
 	if (inetAddressFamily != 0) {
@@ -268,7 +268,7 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 	return false;
 }
 
-void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
+void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force)
 {
 #ifdef ZT_ENABLE_CLUSTER
 	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
@@ -281,7 +281,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	if (((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
 		_lastDirectPathPushSent = now;
 
-		std::vector<Path> dps(RR->node->directPaths());
+		std::vector<InetAddress> dps(RR->node->directPaths());
 		if (dps.empty())
 			return;
 
@@ -291,13 +291,13 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 			for(std::vector<Path>::const_iterator p(dps.begin());p!=dps.end();++p) {
 				if (ps.length() > 0)
 					ps.push_back(',');
-				ps.append(p->address().toString());
+				ps.append(p->toString());
 			}
 			TRACE("pushing %u direct paths to %s: %s",(unsigned int)dps.size(),_id.address().toString().c_str(),ps.c_str());
 		}
 #endif
 
-		std::vector<Path>::const_iterator p(dps.begin());
+		std::vector<InetAddress>::const_iterator p(dps.begin());
 		while (p != dps.end()) {
 			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
 			outp.addSize(2); // leave room for count
@@ -305,7 +305,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 			unsigned int count = 0;
 			while ((p != dps.end())&&((outp.size() + 24) < ZT_PROTO_MAX_PACKET_LENGTH)) {
 				uint8_t addressType = 4;
-				switch(p->address().ss_family) {
+				switch(p->ss_family) {
 					case AF_INET:
 						break;
 					case AF_INET6:
@@ -317,6 +317,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 				}
 
 				uint8_t flags = 0;
+				/* TODO: path trust is not implemented yet
 				switch(p->trust()) {
 					default:
 						break;
@@ -327,13 +328,14 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 						flags |= (0x04 | 0x08); // no encryption, no authentication (redundant but go ahead and set both)
 						break;
 				}
+				*/
 
 				outp.append(flags);
 				outp.append((uint16_t)0); // no extensions
 				outp.append(addressType);
 				outp.append((uint8_t)((addressType == 4) ? 6 : 18));
-				outp.append(p->address().rawIpData(),((addressType == 4) ? 4 : 16));
-				outp.append((uint16_t)p->address().port());
+				outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+				outp.append((uint16_t)p->port());
 
 				++count;
 				++p;
@@ -506,7 +508,7 @@ struct _SortPathsByQuality
 {
 	uint64_t _now;
 	_SortPathsByQuality(const uint64_t now) : _now(now) {}
-	inline bool operator()(const RemotePath &a,const RemotePath &b) const
+	inline bool operator()(const Path &a,const Path &b) const
 	{
 		const uint64_t qa = (
 			((uint64_t)a.active(_now) << 63) |
@@ -526,7 +528,7 @@ void Peer::_sortPaths(const uint64_t now)
 	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
 }
 
-RemotePath *Peer::_getBestPath(const uint64_t now)
+Path *Peer::_getBestPath(const uint64_t now)
 {
 	// assumes _lock is locked
 	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
@@ -538,10 +540,10 @@ RemotePath *Peer::_getBestPath(const uint64_t now)
 		if (_paths[0].active(now))
 			return &(_paths[0]);
 	}
-	return (RemotePath *)0;
+	return (Path *)0;
 }
 
-RemotePath *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
+Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 {
 	// assumes _lock is locked
 	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
@@ -553,7 +555,7 @@ RemotePath *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 		}
 		_sortPaths(now);
 	}
-	return (RemotePath *)0;
+	return (Path *)0;
 }
 
 } // namespace ZeroTier
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 04b541af0..aa75b3f48 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -41,7 +41,7 @@
 
 #include "RuntimeEnvironment.hpp"
 #include "CertificateOfMembership.hpp"
-#include "RemotePath.hpp"
+#include "Path.hpp"
 #include "Address.hpp"
 #include "Utils.hpp"
 #include "Identity.hpp"
@@ -135,7 +135,7 @@ public:
 	 * @param now Current time
 	 * @return Best path or NULL if there are no active direct paths
 	 */
-	inline RemotePath *getBestPath(uint64_t now)
+	inline Path *getBestPath(uint64_t now)
 	{
 		Mutex::Lock _l(_lock);
 		return _getBestPath(now);
@@ -150,14 +150,14 @@ public:
 	 * @param now Current time
 	 * @return Path used on success or NULL on failure
 	 */
-	inline RemotePath *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+	inline Path *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 	{
-		RemotePath *bestPath = getBestPath(now);
+		Path *bestPath = getBestPath(now);
 		if (bestPath) {
 			if (bestPath->send(RR,data,len,now))
 				return bestPath;
 		}
-		return (RemotePath *)0;
+		return (Path *)0;
 	}
 
 	/**
@@ -191,14 +191,14 @@ public:
 	 * @param now Current time
 	 * @param force If true, push regardless of rate limit
 	 */
-	void pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force);
+	void pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force);
 
 	/**
 	 * @return All known direct paths to this peer
 	 */
-	inline std::vector<RemotePath> paths() const
+	inline std::vector<Path> paths() const
 	{
-		std::vector<RemotePath> pp;
+		std::vector<Path> pp;
 		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			pp.push_back(_paths[p]);
@@ -533,7 +533,7 @@ public:
 				p += np->_paths[np->_numPaths++].deserialize(b,p);
 			} else {
 				// Skip any paths beyond max, but still read stream
-				RemotePath foo;
+				Path foo;
 				p += foo.deserialize(b,p);
 			}
 		}
@@ -557,8 +557,8 @@ public:
 
 private:
 	void _sortPaths(const uint64_t now);
-	RemotePath *_getBestPath(const uint64_t now);
-	RemotePath *_getBestPath(const uint64_t now,int inetAddressFamily);
+	Path *_getBestPath(const uint64_t now);
+	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized
 
@@ -575,7 +575,7 @@ private:
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 	Identity _id;
-	RemotePath _paths[ZT_MAX_PEER_NETWORK_PATHS];
+	Path _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	unsigned int _numPaths;
 	unsigned int _latency;
 
diff --git a/node/RemotePath.hpp b/node/RemotePath.hpp
deleted file mode 100644
index 8b37621a8..000000000
--- a/node/RemotePath.hpp
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_REMOTEPATH_HPP
-#define ZT_REMOTEPATH_HPP
-
-#include <stdint.h>
-#include <string.h>
-
-#include <stdexcept>
-#include <algorithm>
-
-#include "Path.hpp"
-#include "Node.hpp"
-#include "AntiRecursion.hpp"
-#include "RuntimeEnvironment.hpp"
-
-namespace ZeroTier {
-
-/**
- * Path to a remote peer
- *
- * This extends Path to include status information about path activity.
- */
-class RemotePath : public Path
-{
-public:
-	RemotePath() :
-		Path(),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(),
-		_flags(0) {}
-
-	RemotePath(const InetAddress &localAddress,const InetAddress &addr) :
-		Path(addr,0,TRUST_NORMAL),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(localAddress),
-		_flags(0) {}
-
-	inline const InetAddress &localAddress() const throw() { return _localAddress; }
-
-	inline uint64_t lastSend() const throw() { return _lastSend; }
-	inline uint64_t lastReceived() const throw() { return _lastReceived; }
-
-	/**
-	 * Called when a packet is sent to this remote path
-	 *
-	 * This is called automatically by RemotePath::send().
-	 *
-	 * @param t Time of send
-	 */
-	inline void sent(uint64_t t)
-		throw()
-	{
-		_lastSend = t;
-	}
-
-	/**
-	 * Called when a packet is received from this remote path
-	 *
-	 * @param t Time of receive
-	 */
-	inline void received(uint64_t t)
-		throw()
-	{
-		_lastReceived = t;
-	}
-
-	/**
-	 * @param now Current time
-	 * @return True if this path appears active
-	 */
-	inline bool active(uint64_t now) const
-		throw()
-	{
-		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
-	}
-
-	/**
-	 * Send a packet via this path
-	 *
-	 * @param RR Runtime environment
-	 * @param data Packet data
-	 * @param len Packet length
-	 * @param now Current time
-	 * @return True if transport reported success
-	 */
-	inline bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
-	{
-		if (RR->node->putPacket(_localAddress,address(),data,len)) {
-			sent(now);
-			RR->antiRec->logOutgoingZT(data,len);
-			return true;
-		}
-		return false;
-	}
-
-	template<unsigned int C>
-	inline void serialize(Buffer<C> &b) const
-	{
-		b.append((uint8_t)1); // version
-		_addr.serialize(b);
-		b.append((uint8_t)_trust);
-		b.append((uint64_t)_lastSend);
-		b.append((uint64_t)_lastReceived);
-		_localAddress.serialize(b);
-		b.append((uint16_t)_flags);
-	}
-
-	template<unsigned int C>
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-	{
-		unsigned int p = startAt;
-		if (b[p++] != 1)
-			throw std::invalid_argument("invalid serialized RemotePath");
-		p += _addr.deserialize(b,p);
-		_ipScope = _addr.ipScope();
-		_trust = (Path::Trust)b[p++];
-		_lastSend = b.template at<uint64_t>(p); p += 8;
-		_lastReceived = b.template at<uint64_t>(p); p += 8;
-		p += _localAddress.deserialize(b,p);
-		_flags = b.template at<uint16_t>(p); p += 2;
-		return (p - startAt);
-	}
-
-protected:
-	uint64_t _lastSend;
-	uint64_t _lastReceived;
-	InetAddress _localAddress;
-	uint16_t _flags;
-};
-
-} // namespace ZeroTier
-
-#endif
diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index 81d193694..b4841544b 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -125,7 +125,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 		// they are still considered alive so that we will re-establish direct links.
 		SharedPtr<Peer> r(RR->topology->getBestRoot());
 		if (r) {
-			RemotePath *rp = r->getBestPath(now);
+			Path *rp = r->getBestPath(now);
 			if (rp) {
 				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
 					if ((*p)->alive(now)) {
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 772eaf023..2f72f57af 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -736,7 +736,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // sanity check: unconfigured network? why are we trying to talk to it?
 		}
 
-		RemotePath *viaPath = peer->getBestPath(now);
+		Path *viaPath = peer->getBestPath(now);
 		SharedPtr<Peer> relay;
 		if (!viaPath) {
 			// See if this network has a preferred relay (if packet has an associated network)
diff --git a/objects.mk b/objects.mk
index 6dd5ea30e..540072d5d 100644
--- a/objects.mk
+++ b/objects.mk
@@ -15,6 +15,7 @@ OBJS=\
 	node/Node.o \
 	node/OutboundMulticast.o \
 	node/Packet.o \
+	node/Path.o \
 	node/Peer.o \
 	node/Poly1305.o \
 	node/Salsa20.o \
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 1765b5c44..4e3f24c79 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -731,7 +731,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -749,7 +749,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 								}
 							}
 							ifa = ifa->ifa_next;

From 218ef07d8e7fcc361aa0875f755eea7b5171f02c Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 15:01:11 -0700
Subject: [PATCH 24/73] Build fix in TRACE mode.

---
 node/Peer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Peer.cpp b/node/Peer.cpp
index ebdb60264..e56c1ecaa 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -288,7 +288,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,
 #ifdef ZT_TRACE
 		{
 			std::string ps;
-			for(std::vector<Path>::const_iterator p(dps.begin());p!=dps.end();++p) {
+			for(std::vector<InetAddress>::const_iterator p(dps.begin());p!=dps.end();++p) {
 				if (ps.length() > 0)
 					ps.push_back(',');
 				ps.append(p->toString());

From 6399f6f0940b6f20819d021a0dc3dcf0d289f002 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 15:02:15 -0700
Subject: [PATCH 25/73] This no longer has to be quite so fast.

---
 node/Cluster.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 45395b0f0..bab56785a 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
 
 namespace ZeroTier {
 

From cc6080fe3898ddd1419050ee3a2c45cc87dd140b Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 15:57:26 -0700
Subject: [PATCH 26/73] (1) No need to confirm if we are a root (small
 optimization), (2) Refactor peer affinity tracking.

---
 node/Cluster.cpp | 132 ++++++++++++++++++++---------------------------
 node/Cluster.hpp |  29 ++++-------
 node/Peer.cpp    |  70 ++++++++++++-------------
 3 files changed, 99 insertions(+), 132 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 07ca0ba13..b2f3d5854 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -82,7 +82,8 @@ Cluster::Cluster(
 	_z(z),
 	_id(id),
 	_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
-	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS])
+	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
+	_peerAffinities(65536)
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -214,19 +215,12 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 								ptr += id.deserialize(dmsg,ptr);
 								if (id) {
 									RR->topology->saveIdentity(id);
-
-									{	// Add or update peer affinity entry
-										_PeerAffinity pa(id.address(),fromMemberId,RR->node->now());
+									{
 										Mutex::Lock _l2(_peerAffinities_m);
-										std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),pa)); // O(log(n))
-										if ((i != _peerAffinities.end())&&(i->key == pa.key)) {
-											i->timestamp = pa.timestamp;
-										} else {
-											_peerAffinities.push_back(pa);
-											std::sort(_peerAffinities.begin(),_peerAffinities.end()); // probably a more efficient way to insert but okay for now
-										}
+										_PA &pa = _peerAffinities[id.address()];
+										pa.ts = RR->node->now();
+										pa.mid = fromMemberId;
 			 						}
-
 			 						TRACE("[%u] has %s",(unsigned int)fromMemberId,id.address().toString().c_str());
 			 					}
 							} catch ( ... ) {
@@ -355,83 +349,66 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 	if (len > 16384) // sanity check
 		return false;
 
-	uint64_t mostRecentTimestamp = 0;
+	const uint64_t now = RR->node->now();
 	unsigned int canHasPeer = 0;
 
 	{	// Anyone got this peer?
 		Mutex::Lock _l2(_peerAffinities_m);
-		std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),_PeerAffinity(toPeerAddress,0,0))); // O(log(n))
-		while ((i != _peerAffinities.end())&&(i->address() == toPeerAddress)) {
-			const uint16_t mid = i->clusterMemberId();
-			if ((mid != _id)&&(i->timestamp > mostRecentTimestamp)) {
-				mostRecentTimestamp = i->timestamp;
-				canHasPeer = mid;
-			}
-			++i;
-		}
+		_PA *pa = _peerAffinities.get(toPeerAddress);
+		if ((pa)&&(pa->mid != _id)&&((now - pa->ts) < ZT_PEER_ACTIVITY_TIMEOUT))
+			canHasPeer = pa->mid;
+		else return false;
 	}
 
-	const uint64_t now = RR->node->now();
-	if ((now - mostRecentTimestamp) < ZT_PEER_ACTIVITY_TIMEOUT) {
-		Buffer<2048> buf;
-
-		if (unite) {
-			InetAddress v4,v6;
-			if (fromPeerAddress) {
-				SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
-				if (fromPeer)
-					fromPeer->getBestActiveAddresses(now,v4,v6);
-			}
-			uint8_t addrCount = 0;
+	Buffer<2048> buf;
+	if (unite) {
+		InetAddress v4,v6;
+		if (fromPeerAddress) {
+			SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
+			if (fromPeer)
+				fromPeer->getBestActiveAddresses(now,v4,v6);
+		}
+		uint8_t addrCount = 0;
+		if (v4)
+			++addrCount;
+		if (v6)
+			++addrCount;
+		if (addrCount) {
+			toPeerAddress.appendTo(buf);
+			fromPeerAddress.appendTo(buf);
+			buf.append(addrCount);
 			if (v4)
-				++addrCount;
+				v4.serialize(buf);
 			if (v6)
-				++addrCount;
-			if (addrCount) {
-				toPeerAddress.appendTo(buf);
-				fromPeerAddress.appendTo(buf);
-				buf.append(addrCount);
-				if (v4)
-					v4.serialize(buf);
-				if (v6)
-					v6.serialize(buf);
-			}
+				v6.serialize(buf);
 		}
-
-		{
-			Mutex::Lock _l2(_members[canHasPeer].lock);
-			if (buf.size() > 0)
-				_send(canHasPeer,STATE_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
-			if (_members[canHasPeer].zeroTierPhysicalEndpoints.size() > 0)
-				RR->node->putPacket(InetAddress(),_members[canHasPeer].zeroTierPhysicalEndpoints.front(),data,len);
-		}
-
-		TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
-		return true;
-	} else {
-		TRACE("sendViaCluster(): unable to relay %u bytes from %s to %s since no cluster members seem to have it!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
-		return false;
 	}
+	{
+		Mutex::Lock _l2(_members[canHasPeer].lock);
+		if (buf.size() > 0)
+			_send(canHasPeer,STATE_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+		if (_members[canHasPeer].zeroTierPhysicalEndpoints.size() > 0)
+			RR->node->putPacket(InetAddress(),_members[canHasPeer].zeroTierPhysicalEndpoints.front(),data,len);
+	}
+
+	TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
+
+	return true;
 }
 
 void Cluster::replicateHavePeer(const Identity &peerId)
 {
+	const uint64_t now = RR->node->now();
 	{	// Use peer affinity table to track our own last announce time for peers
-		_PeerAffinity pa(peerId.address(),_id,RR->node->now());
 		Mutex::Lock _l2(_peerAffinities_m);
-		std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),pa)); // O(log(n))
-		if ((i != _peerAffinities.end())&&(i->key == pa.key)) {
-			if ((pa.timestamp - i->timestamp) >= ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
-				i->timestamp = pa.timestamp;
-				// continue to announcement
-			} else {
-				// we've already announced this peer recently, so skip
-				return;
-			}
+		_PA &pa = _peerAffinities[peerId.address()];
+		if (pa.mid != _id) {
+			pa.ts = now;
+			pa.mid = _id;
+		} else if ((now - pa.ts) >= ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
+			return;
 		} else {
-			_peerAffinities.push_back(pa);
-			std::sort(_peerAffinities.begin(),_peerAffinities.end()); // probably a more efficient way to insert but okay for now
-			// continue to announcement
+			pa.ts = now;
 		}
 	}
 
@@ -598,6 +575,7 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 			}
 		}
 
+		// Suggestion redirection if a closer member was found
 		for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
 			if (a->ss_family == peerPhysicalAddress.ss_family) {
 				TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
@@ -661,10 +639,12 @@ void Cluster::status(ZT_ClusterStatus &status) const
 
 	{
 		Mutex::Lock _l2(_peerAffinities_m);
-		for(std::vector<_PeerAffinity>::const_iterator pi(_peerAffinities.begin());pi!=_peerAffinities.end();++pi) {
-			unsigned int mid = pi->clusterMemberId();
-			if ((ms[mid])&&(mid != _id)&&((now - pi->timestamp) < ZT_PEER_ACTIVITY_TIMEOUT))
-				++ms[mid]->peers;
+		Address *k = (Address *)0;
+		_PA *v = (_PA *)0;
+		Hashtable< Address,_PA >::Iterator i(const_cast<Cluster *>(this)->_peerAffinities);
+		while (i.next(k,v)) {
+			if ( (ms[v->mid]) && (v->mid != _id) && ((now - v->ts) < ZT_PEER_ACTIVITY_TIMEOUT) )
+				++ms[v->mid]->peers;
 		}
 	}
 }
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index bab56785a..42a26c7ff 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -52,7 +52,7 @@
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD 60000
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD 30000
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
@@ -285,7 +285,7 @@ private:
 	void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len);
 	void _flush(uint16_t memberId);
 
-	// These are initialized in the constructor and remain static
+	// These are initialized in the constructor and remain immutable
 	uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	const RuntimeEnvironment *RR;
@@ -298,6 +298,7 @@ private:
 	const int32_t _z;
 	const uint16_t _id;
 	const std::vector<InetAddress> _zeroTierPhysicalEndpoints;
+	// end immutable fields
 
 	struct _Member
 	{
@@ -330,30 +331,18 @@ private:
 		_Member() { this->clear(); }
 		~_Member() { Utils::burn(key,sizeof(key)); }
 	};
-
-	_Member *const _members; // cluster IDs can be from 0 to 65535 (16-bit)
+	_Member *const _members;
 
 	std::vector<uint16_t> _memberIds;
 	Mutex _memberIds_m;
 
-	// Record tracking which members have which peers and how recently they claimed this -- also used to track our last claimed time
-	struct _PeerAffinity
+	struct _PA
 	{
-		_PeerAffinity(const Address &a,uint16_t mid,uint64_t ts) :
-			key((a.toInt() << 16) | (uint64_t)mid),
-			timestamp(ts) {}
-
-		uint64_t key;
-		uint64_t timestamp;
-
-		inline Address address() const throw() { return Address(key >> 16); }
-		inline uint16_t clusterMemberId() const throw() { return (uint16_t)(key & 0xffff); }
-
-		inline bool operator<(const _PeerAffinity &pi) const throw() { return (key < pi.key); }
+		_PA() : ts(0),mid(0xffff) {}
+		uint64_t ts;
+		uint16_t mid;
 	};
-
-	// A memory-efficient packed map of _PeerAffinity records searchable with std::binary_search() and std::lower_bound()
-	std::vector<_PeerAffinity> _peerAffinities;
+	Hashtable< Address,_PA > _peerAffinities;
 	Mutex _peerAffinities_m;
 };
 
diff --git a/node/Peer.cpp b/node/Peer.cpp
index e56c1ecaa..99e2156e5 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -86,43 +86,41 @@ void Peer::received(
 		// Note: findBetterEndpoint() is first since we still want to check
 		// for a better endpoint even if we don't actually send a redirect.
 		if ( (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),remoteAddr,false)) && (verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS) ) {
-			if ((redirectTo.ss_family == AF_INET)||(redirectTo.ss_family == AF_INET6)) {
-				if (_vProto >= 5) {
-					// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
-					Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
-					outp.append((uint16_t)1); // count == 1
-					outp.append((uint8_t)0); // no flags
-					outp.append((uint16_t)0); // no extensions
-					if (redirectTo.ss_family == AF_INET) {
-						outp.append((uint8_t)4);
-						outp.append((uint8_t)6);
-						outp.append(redirectTo.rawIpData(),4);
-					} else {
-						outp.append((uint8_t)6);
-						outp.append((uint8_t)18);
-						outp.append(redirectTo.rawIpData(),16);
-					}
-					outp.append((uint16_t)redirectTo.port());
-					outp.armor(_key,true);
-					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-					RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			if (_vProto >= 5) {
+				// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+				outp.append((uint16_t)1); // count == 1
+				outp.append((uint8_t)0); // no flags
+				outp.append((uint16_t)0); // no extensions
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append((uint8_t)6);
+					outp.append(redirectTo.rawIpData(),4);
 				} else {
-					// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
-					Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
-					outp.append((uint8_t)0); // no flags
-					RR->identity.address().appendTo(outp);
-					outp.append((uint16_t)redirectTo.port());
-					if (redirectTo.ss_family == AF_INET) {
-						outp.append((uint8_t)4);
-						outp.append(redirectTo.rawIpData(),4);
-					} else {
-						outp.append((uint8_t)16);
-						outp.append(redirectTo.rawIpData(),16);
-					}
-					outp.armor(_key,true);
-					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-					RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+					outp.append((uint8_t)6);
+					outp.append((uint8_t)18);
+					outp.append(redirectTo.rawIpData(),16);
 				}
+				outp.append((uint16_t)redirectTo.port());
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			} else {
+				// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+				outp.append((uint8_t)0); // no flags
+				RR->identity.address().appendTo(outp);
+				outp.append((uint16_t)redirectTo.port());
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)16);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			}
 		}
 	}
@@ -167,7 +165,7 @@ void Peer::received(
 			}
 
 			if (!pathIsConfirmed) {
-				if (verb == Packet::VERB_OK) {
+				if ((verb == Packet::VERB_OK)||(RR->topology->amRoot())) {
 
 					Path *slot = (Path *)0;
 					if (np < ZT_MAX_PEER_NETWORK_PATHS) {

From cc1b275ad97bf186f21b487aa57d7893bee3c956 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 16:47:13 -0700
Subject: [PATCH 27/73] Replicate peer endpoints and forget paths if we have
 them -- this allows two clusters to talk to each other, whereas forgetting
 all paths does not.

---
 node/Cluster.cpp   | 45 +++++++++++++++++++++++++++------------------
 node/Cluster.hpp   |  7 ++++++-
 node/Constants.hpp |  4 ++--
 node/Peer.cpp      | 11 ++++-------
 node/Peer.hpp      | 19 +++++++++++++++++++
 5 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index b2f3d5854..0797d83d0 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -210,22 +210,30 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						}	break;
 
 						case STATE_MESSAGE_HAVE_PEER: {
-							try {
-								Identity id;
-								ptr += id.deserialize(dmsg,ptr);
-								if (id) {
-									RR->topology->saveIdentity(id);
-									{
-										Mutex::Lock _l2(_peerAffinities_m);
-										_PA &pa = _peerAffinities[id.address()];
-										pa.ts = RR->node->now();
-										pa.mid = fromMemberId;
-			 						}
-			 						TRACE("[%u] has %s",(unsigned int)fromMemberId,id.address().toString().c_str());
-			 					}
-							} catch ( ... ) {
-								// ignore invalid identities
-							}
+							Identity id;
+							InetAddress physicalAddress;
+							ptr += id.deserialize(dmsg,ptr);
+							ptr += physicalAddress.deserialize(dmsg,ptr);
+							if (id) {
+								// Forget any paths that we have to this peer at its address
+								if (physicalAddress) {
+									SharedPtr<Peer> myPeerRecord(RR->topology->getPeer(id.address()));
+									if (myPeerRecord)
+										myPeerRecord->removePathByAddress(physicalAddress);
+								}
+
+								// Always save identity to update file time
+								RR->topology->saveIdentity(id);
+
+								// Set peer affinity to its new home
+								{
+									Mutex::Lock _l2(_peerAffinities_m);
+									_PA &pa = _peerAffinities[id.address()];
+									pa.ts = RR->node->now();
+									pa.mid = fromMemberId;
+		 						}
+		 						TRACE("[%u] has %s @ %s",(unsigned int)fromMemberId,id.address().toString().c_str(),physicalAddress.toString().c_str());
+		 					}
 						}	break;
 
 						case STATE_MESSAGE_MULTICAST_LIKE: {
@@ -396,7 +404,7 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 	return true;
 }
 
-void Cluster::replicateHavePeer(const Identity &peerId)
+void Cluster::replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress)
 {
 	const uint64_t now = RR->node->now();
 	{	// Use peer affinity table to track our own last announce time for peers
@@ -405,7 +413,7 @@ void Cluster::replicateHavePeer(const Identity &peerId)
 		if (pa.mid != _id) {
 			pa.ts = now;
 			pa.mid = _id;
-		} else if ((now - pa.ts) >= ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
+		} else if ((now - pa.ts) < ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
 			return;
 		} else {
 			pa.ts = now;
@@ -415,6 +423,7 @@ void Cluster::replicateHavePeer(const Identity &peerId)
 	// announcement
 	Buffer<4096> buf;
 	peerId.serialize(buf,false);
+	physicalAddress.serialize(buf);
 	{
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 42a26c7ff..c3367d574 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -117,6 +117,10 @@ public:
 		/**
 		 * Cluster member has this peer:
 		 *   <[...] binary serialized peer identity>
+		 *   <[...] binary serialized peer remote physical address>
+		 *
+		 * Clusters send this message when they learn a path to a peer. The
+		 * replicated physical address is the one learned.
 		 */
 		STATE_MESSAGE_HAVE_PEER = 2,
 
@@ -225,8 +229,9 @@ public:
 	 * Advertise to the cluster that we have this peer
 	 *
 	 * @param peerId Identity of peer that we have
+	 * @param physicalAddress Physical address of peer (from our POV)
 	 */
-	void replicateHavePeer(const Identity &peerId);
+	void replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress);
 
 	/**
 	 * Advertise a multicast LIKE to the cluster
diff --git a/node/Constants.hpp b/node/Constants.hpp
index bef1183a8..4b06db449 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -317,7 +317,7 @@
 /**
  * Minimum delay between attempts to confirm new paths to peers (to avoid HELLO flooding)
  */
-#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 5000
+#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 1000
 
 /**
  * Interval between direct path pushes in milliseconds
@@ -350,7 +350,7 @@
 /**
  * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235)
  */
-#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 2
+#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 4
 
 /**
  * A test pseudo-network-ID that can be joined
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 99e2156e5..99eb32c71 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -140,13 +140,10 @@ void Peer::received(
 			_lastMulticastFrame = now;
 
 #ifdef ZT_ENABLE_CLUSTER
-		// If we're in cluster mode and there's a better endpoint, stop here and don't
-		// learn or confirm paths. Also reset any existing paths, since they should
-		// go there and no longer talk to us here.
-		if (redirectTo) {
-			_numPaths = 0;
+		// If we think this peer belongs elsewhere, don't learn this path or
+		// do other connection init stuff.
+		if (redirectTo)
 			return;
-		}
 #endif
 
 		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
@@ -206,7 +203,7 @@ void Peer::received(
 
 #ifdef ZT_ENABLE_CLUSTER
 	if ((RR->cluster)&&(pathIsConfirmed))
-		RR->cluster->replicateHavePeer(_id);
+		RR->cluster->replicateHavePeer(_id,remoteAddr);
 #endif
 
 	if (needMulticastGroupAnnounce) {
diff --git a/node/Peer.hpp b/node/Peer.hpp
index aa75b3f48..69343f20c 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -412,6 +412,25 @@ public:
 	 */
 	void clean(const RuntimeEnvironment *RR,uint64_t now);
 
+	/**
+	 * Remove all paths with this remote address
+	 *
+	 * @param addr Remote address to remove
+	 */
+	inline void removePathByAddress(const InetAddress &addr)
+	{
+		Mutex::Lock _l(_lock);
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].address() != addr)
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
+
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *

From 4221552c0b3283d106a7c3a44959a02fefd31af6 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 16:52:44 -0700
Subject: [PATCH 28/73] Use getPeerNoCache() in Cluster to avoid keeping all
 peers cached everywhere.

---
 node/Cluster.cpp  |  5 +++--
 node/Topology.hpp | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 0797d83d0..0535f9ee3 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -210,6 +210,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						}	break;
 
 						case STATE_MESSAGE_HAVE_PEER: {
+							const uint64_t now = RR->node->now();
 							Identity id;
 							InetAddress physicalAddress;
 							ptr += id.deserialize(dmsg,ptr);
@@ -217,7 +218,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							if (id) {
 								// Forget any paths that we have to this peer at its address
 								if (physicalAddress) {
-									SharedPtr<Peer> myPeerRecord(RR->topology->getPeer(id.address()));
+									SharedPtr<Peer> myPeerRecord(RR->topology->getPeerNoCache(id.address(),now));
 									if (myPeerRecord)
 										myPeerRecord->removePathByAddress(physicalAddress);
 								}
@@ -229,7 +230,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 								{
 									Mutex::Lock _l2(_peerAffinities_m);
 									_PA &pa = _peerAffinities[id.address()];
-									pa.ts = RR->node->now();
+									pa.ts = now;
 									pa.mid = fromMemberId;
 		 						}
 		 						TRACE("[%u] has %s @ %s",(unsigned int)fromMemberId,id.address().toString().c_str(),physicalAddress.toString().c_str());
diff --git a/node/Topology.hpp b/node/Topology.hpp
index ee9827b95..16836e075 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -78,6 +78,23 @@ public:
 	 */
 	SharedPtr<Peer> getPeer(const Address &zta);
 
+	/**
+	 * Get a peer only if it is presently in memory (no disk cache)
+	 *
+	 * @param zta ZeroTier address
+	 * @param now Current time
+	 */
+	inline SharedPtr<Peer> getPeerNoCache(const Address &zta,const uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		const SharedPtr<Peer> *ap = _peers.get(zta);
+		if (ap) {
+			(*ap)->use(now);
+			return *ap;
+		}
+		return SharedPtr<Peer>();
+	}
+
 	/**
 	 * Get the identity of a peer
 	 *

From 51fcc753549e4f7c18efb889a841c4dd4fb9e6cf Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 17:36:47 -0700
Subject: [PATCH 29/73] Some cleanup, and use getPeerNoCache() exclusively in
 Cluster.

---
 node/Cluster.cpp | 42 ++++++++++++++++++++++++++++++------------
 node/Cluster.hpp |  3 ++-
 node/Path.hpp    |  8 ++++++++
 3 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 0535f9ee3..9b0348221 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -83,7 +83,8 @@ Cluster::Cluster(
 	_id(id),
 	_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
 	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
-	_peerAffinities(65536)
+	_peerAffinities(65536),
+	_lastCleanedPeerAffinities(0)
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -247,11 +248,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 						}	break;
 
 						case STATE_MESSAGE_COM: {
+							/* not currently used so not decoded yet
 							CertificateOfMembership com;
 							ptr += com.deserialize(dmsg,ptr);
 							if (com) {
 								TRACE("[%u] COM for %s on %.16llu rev %llu",(unsigned int)fromMemberId,com.issuedTo().toString().c_str(),com.networkId(),com.revision());
 							}
+							*/
 						}	break;
 
 						case STATE_MESSAGE_PROXY_UNITE: {
@@ -262,12 +265,13 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							for(unsigned int i=0;i<numRemotePeerPaths;++i)
 								ptr += remotePeerPaths[i].deserialize(dmsg,ptr);
 
-							TRACE("[%u] requested proxy unite between local peer %s and remote peer %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
+							TRACE("[%u] requested that we unite local %s with remote %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
 
-							SharedPtr<Peer> localPeer(RR->topology->getPeer(localPeerAddress));
+							const uint64_t now = RR->node->now();
+							SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress,now));
 							if ((localPeer)&&(numRemotePeerPaths > 0)) {
 								InetAddress bestLocalV4,bestLocalV6;
-								localPeer->getBestActiveAddresses(RR->node->now(),bestLocalV4,bestLocalV6);
+								localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6);
 
 								InetAddress bestRemoteV4,bestRemoteV6;
 								for(unsigned int i=0;i<numRemotePeerPaths;++i) {
@@ -369,11 +373,11 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 		else return false;
 	}
 
-	Buffer<2048> buf;
+	Buffer<1024> buf;
 	if (unite) {
 		InetAddress v4,v6;
 		if (fromPeerAddress) {
-			SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
+			SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress,now));
 			if (fromPeer)
 				fromPeer->getBestActiveAddresses(now,v4,v6);
 		}
@@ -408,7 +412,7 @@ bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPee
 void Cluster::replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress)
 {
 	const uint64_t now = RR->node->now();
-	{	// Use peer affinity table to track our own last announce time for peers
+	{
 		Mutex::Lock _l2(_peerAffinities_m);
 		_PA &pa = _peerAffinities[peerId.address()];
 		if (pa.mid != _id) {
@@ -436,7 +440,7 @@ void Cluster::replicateHavePeer(const Identity &peerId,const InetAddress &physic
 
 void Cluster::replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group)
 {
-	Buffer<2048> buf;
+	Buffer<1024> buf;
 	buf.append((uint64_t)nwid);
 	peerAddress.appendTo(buf);
 	group.mac().appendTo(buf);
@@ -453,7 +457,7 @@ void Cluster::replicateMulticastLike(uint64_t nwid,const Address &peerAddress,co
 
 void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembership &com)
 {
-	Buffer<2048> buf;
+	Buffer<4096> buf;
 	com.serialize(buf);
 	TRACE("replicating %s COM for %.16llx to all members",com.issuedTo().toString().c_str(),com.networkId());
 	{
@@ -502,6 +506,20 @@ void Cluster::doPeriodicTasks()
 			_flush(*mid); // does nothing if nothing to flush
 		}
 	}
+
+	{
+		if ((now - _lastCleanedPeerAffinities) >= (ZT_PEER_ACTIVITY_TIMEOUT * 10)) {
+			_lastCleanedPeerAffinities = now;
+			Address *k = (Address *)0;
+			_PA *v = (_PA *)0;
+			Mutex::Lock _l(_peerAffinities_m);
+			Hashtable< Address,_PA >::Iterator i(_peerAffinities);
+			while (i.next(k,v)) {
+				if ((now - v->ts) >= (ZT_PEER_ACTIVITY_TIMEOUT * 10))
+					_peerAffinities.erase(*k);
+			}
+		}
+	}
 }
 
 void Cluster::addMember(uint16_t memberId)
@@ -563,7 +581,7 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 
 		// Find member closest to this peer
 		const uint64_t now = RR->node->now();
-		std::vector<InetAddress> best; // initial "best" is for peer to stay put
+		std::vector<InetAddress> best;
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
 		unsigned int bestMember = _id;
@@ -575,7 +593,7 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 
 				// Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to
 				if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) {
-					double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
+					const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
 					if (mdist < bestDistance) {
 						bestDistance = mdist;
 						bestMember = *mid;
@@ -585,7 +603,7 @@ bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddr
 			}
 		}
 
-		// Suggestion redirection if a closer member was found
+		// Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family
 		for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
 			if (a->ss_family == peerPhysicalAddress.ss_family) {
 				TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index c3367d574..cc9edd1d1 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -47,7 +47,7 @@
 /**
  * Timeout for cluster members being considered "alive"
  */
-#define ZT_CLUSTER_TIMEOUT 10000
+#define ZT_CLUSTER_TIMEOUT 20000
 
 /**
  * How often should we announce that we have a peer?
@@ -349,6 +349,7 @@ private:
 	};
 	Hashtable< Address,_PA > _peerAffinities;
 	Mutex _peerAffinities_m;
+	uint64_t _lastCleanedPeerAffinities;
 };
 
 } // namespace ZeroTier
diff --git a/node/Path.hpp b/node/Path.hpp
index 99f6590b2..2b05b8126 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -95,6 +95,14 @@ public:
 	{
 	}
 
+	inline Path &operator=(const Path &p)
+		throw()
+	{
+		if (this != &p)
+			memcpy(this,&p,sizeof(Path));
+		return *this;
+	}
+
 	/**
 	 * Called when a packet is sent to this remote path
 	 *

From 88b100e5d0db5df16622fa48899cf652e09b3e91 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 17:59:17 -0700
Subject: [PATCH 30/73] More cleanup.

---
 node/IncomingPacket.cpp | 49 +++++++++++++++++++++++++++--------------
 node/InetAddress.hpp    | 11 ++++++---
 2 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index 7015535a4..b1a9af3b0 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -45,6 +45,7 @@
 #include "World.hpp"
 #include "Cluster.hpp"
 #include "Node.hpp"
+#include "AntiRecursion.hpp"
 
 namespace ZeroTier {
 
@@ -122,7 +123,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 			case Packet::ERROR_OBJ_NOT_FOUND:
 				if (inReVerb == Packet::VERB_WHOIS) {
-					if (RR->topology->isRoot(peer->identity()))
+					if (RR->topology->isUpstream(peer->identity()))
 						RR->sw->cancelWhoisRequest(Address(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH));
 				} else if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
 					SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
@@ -155,6 +156,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
 						nconf->com().serialize(outp);
 						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					}
 				}
@@ -202,13 +204,13 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		const uint64_t timestamp = at<uint64_t>(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP);
 
 		Identity id;
-		InetAddress destAddr;
+		InetAddress externalSurfaceAddress;
 		uint64_t worldId = ZT_WORLD_ID_NULL;
 		uint64_t worldTimestamp = 0;
 		{
 			unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
 			if (ptr < size()) // ZeroTier One < 1.0.3 did not include physical destination address info
-				ptr += destAddr.deserialize(*this,ptr);
+				ptr += externalSurfaceAddress.deserialize(*this,ptr);
 			if ((ptr + 16) <= size()) { // older versions also did not include World IDs or timestamps
 				worldId = at<uint64_t>(ptr); ptr += 8;
 				worldTimestamp = at<uint64_t>(ptr);
@@ -281,11 +283,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 		// VALID -- if we made it here, packet passed identity and authenticity checks!
 
-		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
-		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
-
-		if (destAddr)
-			RR->sa->iam(id.address(),_remoteAddress,destAddr,RR->topology->isRoot(id),RR->node->now());
+		if (externalSurfaceAddress)
+			RR->sa->iam(id.address(),_remoteAddress,externalSurfaceAddress,RR->topology->isRoot(id),RR->node->now());
 
 		Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_HELLO);
@@ -308,7 +307,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		}
 
 		outp.armor(peer->key(),true);
+		RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+
+		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -332,12 +335,17 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION];
 				const unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION);
 
+				if (vProto < ZT_PROTO_VERSION_MIN) {
+					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
+					return true;
+				}
+
 				const bool trusted = RR->topology->isRoot(peer->identity());
 
-				InetAddress destAddr;
+				InetAddress externalSurfaceAddress;
 				unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2;
 				if (ptr < size()) // ZeroTier One < 1.0.3 did not include this field
-					ptr += destAddr.deserialize(*this,ptr);
+					ptr += externalSurfaceAddress.deserialize(*this,ptr);
 				if ((trusted)&&((ptr + 2) <= size())) { // older versions also did not include this field, and right now we only use if from a root
 					World worldUpdate;
 					const unsigned int worldLen = at<uint16_t>(ptr); ptr += 2;
@@ -348,18 +356,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					}
 				}
 
-				if (vProto < ZT_PROTO_VERSION_MIN) {
-					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
-					return true;
-				}
-
 				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((destAddr) ? destAddr.toString().c_str() : "(none)"));
 
 				peer->addDirectLatencyMeasurment(latency);
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
-				if (destAddr)
-					RR->sa->iam(peer->address(),_remoteAddress,destAddr,trusted,RR->node->now());
+				if (externalSurfaceAddress)
+					RR->sa->iam(peer->address(),_remoteAddress,externalSurfaceAddress,trusted,RR->node->now());
 			}	break;
 
 			case Packet::VERB_WHOIS: {
@@ -443,6 +446,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append(packetId());
 				queried.serialize(outp,false);
 				outp.armor(peer->key(),true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			} else {
 				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
@@ -451,6 +455,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 				outp.append(payload(),ZT_ADDRESS_LENGTH);
 				outp.armor(peer->key(),true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 		} else {
@@ -608,6 +613,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		outp.append((unsigned char)Packet::VERB_ECHO);
 		outp.append((uint64_t)pid);
 		outp.append(field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD),size() - ZT_PACKET_IDX_PAYLOAD);
+		RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP);
 	} catch ( ... ) {
@@ -693,6 +699,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 						if (outp.size() > ZT_PROTO_MAX_PACKET_LENGTH) { // sanity check
 							TRACE("NETWORK_CONFIG_REQUEST failed: internal error: netconf size %u is too large",(unsigned int)netconfStr.length());
 						} else {
+							RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 							RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 						}
 					}
@@ -705,6 +712,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				}	break;
 
@@ -715,6 +723,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				} break;
 
@@ -737,6 +746,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
 			outp.armor(peer->key(),true);
+			RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 		}
 	} catch ( ... ) {
@@ -781,6 +791,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 			outp.append((uint32_t)mg.adi());
 			if (RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit)) {
 				outp.armor(peer->key(),true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 		}
@@ -873,6 +884,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 				outp.append((unsigned char)0x02); // flag 0x02 = contains gather results
 				if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
 					outp.armor(peer->key(),true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				}
 			}
@@ -1173,6 +1185,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 						outp.append((uint16_t)sizeof(result));
 						outp.append(result,sizeof(result));
 						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					} else {
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
@@ -1180,6 +1193,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 						outp.append(pid);
 						outp.append((unsigned char)Packet::ERROR_INVALID_REQUEST);
 						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					}
 				}	break;
@@ -1285,6 +1299,7 @@ void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,cons
 	outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
 	outp.append(nwid);
 	outp.armor(peer->key(),true);
+	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 	RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 }
 
diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp
index ecafcf51f..fcbed4b15 100644
--- a/node/InetAddress.hpp
+++ b/node/InetAddress.hpp
@@ -346,14 +346,19 @@ struct InetAddress : public sockaddr_storage
 	}
 
 	/**
+	 * Performs an IP-only comparison or, if that is impossible, a memcmp()
+	 *
 	 * @param a InetAddress to compare again
 	 * @return True if only IP portions are equal (false for non-IP or null addresses)
 	 */
 	inline bool ipsEqual(const InetAddress &a) const
 	{
-		switch(ss_family) {
-			case AF_INET: return (reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr);
-			case AF_INET6: return (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0);
+		if (ss_family == a.ss_family) {
+			if (ss_family == AF_INET)
+				return (reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr);
+			if (ss_family == AF_INET6)
+				return (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0);
+			return (memcmp(this,&a,sizeof(InetAddress)) == 0);
 		}
 		return false;
 	}

From cdc99bfee10ac58a8dab1aabcb85e69f3862b7ad Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Tue, 27 Oct 2015 18:18:26 -0700
Subject: [PATCH 31/73] Add a circuit breaker for VERB_PUSH_DIRECT_PATHS.

---
 node/Constants.hpp      | 26 ++++++++++++++++++++------
 node/IncomingPacket.cpp |  5 +++++
 node/Peer.cpp           |  2 ++
 node/Peer.hpp           | 31 +++++++++++++++++++++++++++++--
 4 files changed, 56 insertions(+), 8 deletions(-)

diff --git a/node/Constants.hpp b/node/Constants.hpp
index 4b06db449..53cc64c7e 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -319,11 +319,6 @@
  */
 #define ZT_MIN_PATH_CONFIRMATION_INTERVAL 1000
 
-/**
- * Interval between direct path pushes in milliseconds
- */
-#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
-
 /**
  * How long (max) to remember network certificates of membership?
  *
@@ -347,10 +342,29 @@
  */
 #define ZT_MAX_BRIDGE_SPAM 16
 
+/**
+ * Interval between direct path pushes in milliseconds
+ */
+#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
+
 /**
  * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235)
  */
-#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 4
+#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 5
+
+/**
+ * Time horizon for push direct paths cutoff
+ */
+#define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000
+
+/**
+ * Maximum number of direct path pushes within cutoff time
+ *
+ * This limits response to PUSH_DIRECT_PATHS to CUTOFF_LIMIT responses
+ * per CUTOFF_TIME milliseconds per peer to prevent this from being
+ * useful for DOS amplification attacks.
+ */
+#define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5
 
 /**
  * A test pseudo-network-ID that can be joined
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index b1a9af3b0..e985b34c4 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -901,6 +901,11 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
+		if (!peer->shouldRespondToDirectPathPush(now)) {
+			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str());
+			return true;
+		}
+
 		const Path *currentBest = peer->getBestPath(now);
 
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 99eb32c71..976c7c449 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -55,6 +55,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
 	_lastDirectPathPushSent(0),
+	_lastDirectPathPushReceive(0),
 	_lastPathSort(0),
 	_vProto(0),
 	_vMajor(0),
@@ -63,6 +64,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_id(peerIdentity),
 	_numPaths(0),
 	_latency(0),
+	_directPathPushCutoffCount(0),
 	_networkComs(4),
 	_lastPushedComs(4)
 {
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 69343f20c..d9ef5fcba 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -431,6 +431,27 @@ public:
 		_numPaths = y;
 	}
 
+	/**
+	 * Update direct path push stats and return true if we should respond
+	 *
+	 * This is a circuit breaker to make VERB_PUSH_DIRECT_PATHS not particularly
+	 * useful as a DDOS amplification attack vector. Otherwise a malicious peer
+	 * could send loads of these and cause others to bombard arbitrary IPs with
+	 * traffic.
+	 *
+	 * @param now Current time
+	 * @return True if we should respond
+	 */
+	inline bool shouldRespondToDirectPathPush(const uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME)
+			++_directPathPushCutoffCount;
+		else _directPathPushCutoffCount = 0;
+		_lastDirectPathPushReceive = now;
+		return (_directPathPushCutoffCount >= ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT);
+	}
+
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *
@@ -459,7 +480,7 @@ public:
 		const unsigned int recSizePos = b.size();
 		b.addSize(4); // space for uint32_t field length
 
-		b.append((uint16_t)0); // version of serialized Peer data
+		b.append((uint16_t)1); // version of serialized Peer data
 
 		_id.serialize(b,false);
 
@@ -470,12 +491,14 @@ public:
 		b.append((uint64_t)_lastAnnouncedTo);
 		b.append((uint64_t)_lastPathConfirmationSent);
 		b.append((uint64_t)_lastDirectPathPushSent);
+		b.append((uint64_t)_lastDirectPathPushReceive);
 		b.append((uint64_t)_lastPathSort);
 		b.append((uint16_t)_vProto);
 		b.append((uint16_t)_vMajor);
 		b.append((uint16_t)_vMinor);
 		b.append((uint16_t)_vRevision);
 		b.append((uint32_t)_latency);
+		b.append((uint16_t)_directPathPushCutoffCount);
 
 		b.append((uint16_t)_numPaths);
 		for(unsigned int i=0;i<_numPaths;++i)
@@ -521,7 +544,7 @@ public:
 		const unsigned int recSize = b.template at<uint32_t>(p); p += 4;
 		if ((p + recSize) > b.size())
 			return SharedPtr<Peer>(); // size invalid
-		if (b.template at<uint16_t>(p) != 0)
+		if (b.template at<uint16_t>(p) != 1)
 			return SharedPtr<Peer>(); // version mismatch
 		p += 2;
 
@@ -539,12 +562,14 @@ public:
 		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
 		np->_lastPathConfirmationSent = b.template at<uint64_t>(p); p += 8;
 		np->_lastDirectPathPushSent = b.template at<uint64_t>(p); p += 8;
+		np->_lastDirectPathPushReceive = b.template at<uint64_t>(p); p += 8;
 		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
 		np->_vProto = b.template at<uint16_t>(p); p += 2;
 		np->_vMajor = b.template at<uint16_t>(p); p += 2;
 		np->_vMinor = b.template at<uint16_t>(p); p += 2;
 		np->_vRevision = b.template at<uint16_t>(p); p += 2;
 		np->_latency = b.template at<uint32_t>(p); p += 4;
+		np->_directPathPushCutoffCount = b.template at<uint16_t>(p); p += 2;
 
 		const unsigned int numPaths = b.template at<uint16_t>(p); p += 2;
 		for(unsigned int i=0;i<numPaths;++i) {
@@ -588,6 +613,7 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
 	uint64_t _lastDirectPathPushSent;
+	uint64_t _lastDirectPathPushReceive;
 	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
@@ -597,6 +623,7 @@ private:
 	Path _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	unsigned int _numPaths;
 	unsigned int _latency;
+	unsigned int _directPathPushCutoffCount;
 
 	struct _NetworkCom
 	{

From da9371284625d2481ed496921505c8afd2dae1f0 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 09:11:30 -0700
Subject: [PATCH 32/73] Clean up PUSH_DIRECT_PATH limits a bit more and make
 them a bit smarter.

---
 node/Constants.hpp      | 10 +++++-----
 node/IncomingPacket.cpp | 25 +++++++++++++++----------
 node/InetAddress.hpp    |  8 +++++++-
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/node/Constants.hpp b/node/Constants.hpp
index 53cc64c7e..5a4d4a4de 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -347,11 +347,6 @@
  */
 #define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
 
-/**
- * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235)
- */
-#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 5
-
 /**
  * Time horizon for push direct paths cutoff
  */
@@ -366,6 +361,11 @@
  */
 #define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5
 
+/**
+ * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6)
+ */
+#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 1
+
 /**
  * A test pseudo-network-ID that can be joined
  *
diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index e985b34c4..f06eb30c3 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -901,16 +901,19 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
+
+		// First, subject this to a rate limit
 		if (!peer->shouldRespondToDirectPathPush(now)) {
 			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
 		}
 
-		const Path *currentBest = peer->getBestPath(now);
+		// Second, limit addresses by scope and type
+		uint8_t countPerScope[ZT_INETADDRESS_MAX_SCOPE+1][2]; // [][0] is v4, [][1] is v6
+		memset(countPerScope,0,sizeof(countPerScope));
 
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2;
-		unsigned int v4Count = 0,v6Count = 0;
 
 		while (count--) { // if ptr overflows Buffer will throw
 			// TODO: some flags are not yet implemented
@@ -925,20 +928,22 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 				case 4: {
 					InetAddress a(field(ptr,4),4,at<uint16_t>(ptr + 4));
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) {
-						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
-							if ((!currentBest)||(currentBest->address() != a))
-								peer->attemptToContactAt(RR,_localAddress,a,now);
+						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
+							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
+							peer->attemptToContactAt(RR,_localAddress,a,now);
+						} else {
+							TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str());
 						}
 					}
 				}	break;
 				case 6: {
 					InetAddress a(field(ptr,16),16,at<uint16_t>(ptr + 16));
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) {
-						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
-							if ((!currentBest)||(currentBest->address() != a))
-								peer->attemptToContactAt(RR,_localAddress,a,now);
+						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
+							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
+							peer->attemptToContactAt(RR,_localAddress,a,now);
+						} else {
+							TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str());
 						}
 					}
 				}	break;
diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp
index fcbed4b15..5e5eb06ee 100644
--- a/node/InetAddress.hpp
+++ b/node/InetAddress.hpp
@@ -42,6 +42,11 @@
 
 namespace ZeroTier {
 
+/**
+ * Maximum integer value of enum IpScope
+ */
+#define ZT_INETADDRESS_MAX_SCOPE 7
+
 /**
  * Extends sockaddr_storage with friendly C++ methods
  *
@@ -66,7 +71,8 @@ struct InetAddress : public sockaddr_storage
 	 * IP address scope
 	 *
 	 * Note that these values are in ascending order of path preference and
-	 * MUST remain that way or Path must be changed to reflect.
+	 * MUST remain that way or Path must be changed to reflect. Also be sure
+	 * to change ZT_INETADDRESS_MAX_SCOPE if the max changes.
 	 */
 	enum IpScope
 	{

From c1b0329969d3601fe80ef3298837edac5bdbbed2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 09:32:07 -0700
Subject: [PATCH 33/73] Only check IP equality to detect external surface
 changes (should prevent some spurious resets under symmetric NATs), and
 simplify some logic.

---
 node/SelfAwareness.cpp | 50 ++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index b4841544b..856892ffa 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -36,6 +36,7 @@
 #include "Topology.hpp"
 #include "Packet.hpp"
 #include "Peer.hpp"
+#include "Switch.hpp"
 
 // Entry timeout -- make it fairly long since this is just to prevent stale buildup
 #define ZT_SELFAWARENESS_ENTRY_TIMEOUT 3600000
@@ -65,7 +66,8 @@ private:
 };
 
 SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) :
-	RR(renv)
+	RR(renv),
+	_phy(32)
 {
 }
 
@@ -77,35 +79,35 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 {
 	const InetAddress::IpScope scope = myPhysicalAddress.ipScope();
 
+	// This would be weird, e.g. a public IP talking to 10.0.0.1, so just ignore it.
+	// If your network is this weird it's probably not reliable information.
+	if (scope != reporterPhysicalAddress.ipScope())
+		return;
+
+	// Some scopes we ignore, and global scope IPs are only used for this
+	// mechanism if they come from someone we trust (e.g. a root).
 	switch(scope) {
 		case InetAddress::IP_SCOPE_NONE:
 		case InetAddress::IP_SCOPE_LOOPBACK:
 		case InetAddress::IP_SCOPE_MULTICAST:
 			return;
 		case InetAddress::IP_SCOPE_GLOBAL:
-			if ((!trusted)||(scope != reporterPhysicalAddress.ipScope()))
+			if (!trusted)
 				return;
 			break;
 		default:
-			if (scope != reporterPhysicalAddress.ipScope())
-				return;
 			break;
 	}
 
 	Mutex::Lock _l(_phy_m);
-
 	PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,reporterPhysicalAddress,scope)];
 
-	if ((now - entry.ts) >= ZT_SELFAWARENESS_ENTRY_TIMEOUT) {
+	if ( ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) {
 		entry.mySurface = myPhysicalAddress;
 		entry.ts = now;
-		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced <null>)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str());
-	} else if (entry.mySurface != myPhysicalAddress) {
-		entry.mySurface = myPhysicalAddress;
-		entry.ts = now;
-		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced %s, resetting all in scope)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
+		TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
 
-		// Erase all entries in this scope that were not reported by this remote address to prevent 'thrashing'
+		// Erase all entries in this scope that were not reported from this remote address to prevent 'thrashing'
 		// due to multiple reports of endpoint change.
 		// Don't use 'entry' after this since hash table gets modified.
 		{
@@ -118,26 +120,22 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 			}
 		}
 
+		// Reset all paths within this scope
 		_ResetWithinScope rset(RR,now,(InetAddress::IpScope)scope);
 		RR->topology->eachPeer<_ResetWithinScope &>(rset);
 
-		// For all peers for whom we forgot an address, send a packet indirectly if
-		// they are still considered alive so that we will re-establish direct links.
-		SharedPtr<Peer> r(RR->topology->getBestRoot());
-		if (r) {
-			Path *rp = r->getBestPath(now);
-			if (rp) {
-				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
-					if ((*p)->alive(now)) {
-						TRACE("sending indirect NOP to %s via %s to re-establish link",(*p)->address().toString().c_str(),r->address().toString().c_str());
-						Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
-						outp.armor((*p)->key(),true);
-						rp->send(RR,outp.data(),outp.size(),now);
-					}
-				}
+		// Send a NOP to all peers for whom we forgot a path. This will cause direct
+		// links to be re-established if possible, possibly using a root server or some
+		// other relay.
+		for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
+			if ((*p)->alive(now)) {
+				TRACE("sending indirect NOP to %s via %s to re-establish link",(*p)->address().toString().c_str(),r->address().toString().c_str());
+				Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
+				RR->sw->send(outp,true,0);
 			}
 		}
 	} else {
+		entry.mySurface = myPhysicalAddress;
 		entry.ts = now;
 	}
 }

From fdc3e103ccc3207c4a00b8476d8635772c6c6dc4 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 09:38:33 -0700
Subject: [PATCH 34/73] Cleanup and docs.

---
 node/InetAddress.hpp | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/node/InetAddress.hpp b/node/InetAddress.hpp
index 5e5eb06ee..c4d5cfda0 100644
--- a/node/InetAddress.hpp
+++ b/node/InetAddress.hpp
@@ -52,8 +52,8 @@ namespace ZeroTier {
  *
  * This is basically a "mixin" for sockaddr_storage. It adds methods and
  * operators, but does not modify the structure. This can be cast to/from
- * sockaddr_storage and used interchangeably. Don't change this as it's
- * used in a few places.
+ * sockaddr_storage and used interchangeably. DO NOT change this by e.g.
+ * adding non-static fields, since much code depends on this identity.
  */
 struct InetAddress : public sockaddr_storage
 {
@@ -326,7 +326,7 @@ struct InetAddress : public sockaddr_storage
 	inline bool isV6() const throw() { return (ss_family == AF_INET6); }
 
 	/**
-	 * @return pointer to raw IP address bytes
+	 * @return pointer to raw address bytes or NULL if not available
 	 */
 	inline const void *rawIpData() const
 		throw()
@@ -338,19 +338,6 @@ struct InetAddress : public sockaddr_storage
 		}
 	}
 
-	/**
-	 * @return pointer to raw IP address bytes
-	 */
-	inline void *rawIpData()
-		throw()
-	{
-		switch(ss_family) {
-			case AF_INET: return (void *)&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr);
-			case AF_INET6: return (void *)(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr);
-			default: return 0;
-		}
-	}
-
 	/**
 	 * Performs an IP-only comparison or, if that is impossible, a memcmp()
 	 *

From 938d0a970b84c6a50465bb6ebb64798e4ffec202 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 10:01:32 -0700
Subject: [PATCH 35/73] TRACE build fixes.

---
 node/IncomingPacket.cpp | 2 +-
 node/SelfAwareness.cpp  | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index f06eb30c3..b0d65159b 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -356,7 +356,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					}
 				}
 
-				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((destAddr) ? destAddr.toString().c_str() : "(none)"));
+				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)"));
 
 				peer->addDirectLatencyMeasurment(latency);
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index 856892ffa..d8eca0718 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -129,7 +129,6 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 		// other relay.
 		for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
 			if ((*p)->alive(now)) {
-				TRACE("sending indirect NOP to %s via %s to re-establish link",(*p)->address().toString().c_str(),r->address().toString().c_str());
 				Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
 				RR->sw->send(outp,true,0);
 			}

From 0fd15d9cf3a7bf6e85247533e83edade69cd01d0 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 10:38:37 -0700
Subject: [PATCH 36/73] Fix inverted sense bug.

---
 node/Peer.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Peer.hpp b/node/Peer.hpp
index d9ef5fcba..39acffd97 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -449,7 +449,7 @@ public:
 			++_directPathPushCutoffCount;
 		else _directPathPushCutoffCount = 0;
 		_lastDirectPathPushReceive = now;
-		return (_directPathPushCutoffCount >= ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT);
+		return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT);
 	}
 
 	/**

From 0034efafe4f141d06d07464e7df2e151f4304294 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 11:08:15 -0700
Subject: [PATCH 37/73] On semi-undocumented test net, assign a RFC4193 IPv6
 address too. Will be useful for our at-scale tests.

---
 node/NetworkConfig.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/node/NetworkConfig.cpp b/node/NetworkConfig.cpp
index cd32600f1..35e238377 100644
--- a/node/NetworkConfig.cpp
+++ b/node/NetworkConfig.cpp
@@ -55,6 +55,9 @@ SharedPtr<NetworkConfig> NetworkConfig::createTestNetworkConfig(const Address &s
 	if ((ip & 0x000000ff) == 0x00000000) ip ^= 0x00000001; // or .0
 	nc->_staticIps.push_back(InetAddress(Utils::hton(ip),8));
 
+	// Assign an RFC4193-compliant IPv6 address -- will never collide
+	nc->_staticIps.push_back(InetAddress::makeIpv6rfc4193(ZT_TEST_NETWORK_ID,self.toInt()));
+
 	return nc;
 }
 

From c6a918d9962dcf2354483b709b8bf0fffbbc3983 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 12:50:48 -0700
Subject: [PATCH 38/73] HTTP test code.

---
 tests/http/README.md    |   5 +
 tests/http/agent.js     | 224 ++++++++++++++++++++++++++++++++++++++++
 tests/http/package.json |  16 +++
 tests/http/server.js    |  48 +++++++++
 4 files changed, 293 insertions(+)
 create mode 100644 tests/http/README.md
 create mode 100644 tests/http/agent.js
 create mode 100644 tests/http/package.json
 create mode 100644 tests/http/server.js

diff --git a/tests/http/README.md b/tests/http/README.md
new file mode 100644
index 000000000..ae7f08f10
--- /dev/null
+++ b/tests/http/README.md
@@ -0,0 +1,5 @@
+HTTP one-to-all test
+======
+
+This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+
diff --git a/tests/http/agent.js b/tests/http/agent.js
new file mode 100644
index 000000000..14964d873
--- /dev/null
+++ b/tests/http/agent.js
@@ -0,0 +1,224 @@
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+// How frequently in ms to run tests
+//var RUN_TEST_EVERY = (60 * 5 * 1000);
+var RUN_TEST_EVERY = 1000;
+
+// Maximum test duration in milliseconds (must be less than RUN_TEST_EVERY)
+var TEST_DURATION = (60 * 1000);
+
+// Where should I contact to register and query a list of other nodes?
+var SERVER_HOST = '127.0.0.1';
+var SERVER_PORT = 18080;
+
+// Which port should agents use for their HTTP?
+var AGENT_PORT = 18888;
+
+// Payload size in bytes
+var PAYLOAD_SIZE = 4096;
+
+// ---------------------------------------------------------------------------
+
+var ipaddr = require('ipaddr.js');
+var os = require('os');
+var http = require('http');
+var async = require('async');
+
+var express = require('express');
+var app = express();
+
+// Find our ZeroTier-assigned RFC4193 IPv6 address
+var thisAgentId = null;
+var interfaces = os.networkInterfaces();
+if (!interfaces) {
+	console.error('FATAL: os.networkInterfaces() failed.');
+	process.exit(1);
+}
+for(var ifname in interfaces) {
+	var ifaddrs = interfaces[ifname];
+	if (Array.isArray(ifaddrs)) {
+		for(var i=0;i<ifaddrs.length;++i) {
+			if (ifaddrs[i].family == 'IPv6') {
+				try {
+					var ipbytes = ipaddr.parse(ifaddrs[i].address).toByteArray();
+					if ((ipbytes.length === 16)&&(ipbytes[0] == 0xfd)&&(ipbytes[9] == 0x99)&&(ipbytes[10] == 0x93)) {
+						thisAgentId = '';
+						for(var j=0;j<16;++j) {
+							var tmp = ipbytes[j].toString(16);
+							if (tmp.length === 1)
+								thisAgentId += '0';
+							thisAgentId += tmp;
+						}
+					}
+				} catch (e) {
+					console.error(e);
+				}
+			}
+		}
+	}
+}
+if (thisAgentId === null) {
+	console.error('FATAL: no ZeroTier-assigned RFC4193 IPv6 addresses found on any local interface!');
+	process.exit(1);
+}
+
+//console.log(thisAgentId);
+
+// Create a random (and therefore not very compressable) payload
+var payload = '';
+while (payload.length < PAYLOAD_SIZE) {
+	payload += String.fromCharCode(Math.round(Math.random() * 255.0));
+}
+
+// Incremented for each test
+var testCounter = 0;
+
+function registerAndGetPeers(callback)
+{
+	http.get({
+		host: SERVER_HOST,
+		port: SERVER_PORT,
+		path: '/'+thisAgentId
+	},function(res) {
+		var body = '';
+		res.on('data',function(chunk) { body += chunk.toString(); });
+		res.on('end',function() {
+			try {
+				var peers = JSON.parse(body);
+				if (Array.isArray(peers))
+					return callback(null,peers);
+				else return callback(new Error('invalid JSON response from server'),null);
+			} catch (e) {
+				return callback(new Error('invalid JSON response from server'),null);
+			}
+		});
+	}).on('error',function(e) {
+		return callback(e,null);
+	});
+};
+
+function performTestOnAllPeers(peers,callback)
+{
+	var allResults = {};
+	var timedOut = false;
+	var endOfTestTimer = setTimeout(function() {
+		timedOut = true;
+		return callback(allResults);
+	},TEST_DURATION);
+	var testStartTime = Date.now();
+
+	async.each(peers,function(peer,next) {
+		if (timedOut)
+			return next(null);
+		if (peer.length !== 32)
+			return next(null);
+
+		var connectionStartTime = Date.now();
+		allResults[peer] = {
+			testStart: testStartTime,
+			start: connectionStartTime,
+			end: null,
+			error: null,
+			bytes: 0,
+			test: testCounter
+		};
+
+		var peerHost = '';
+		peerHost += peer.substr(0,4);
+		peerHost += ':';
+		peerHost += peer.substr(4,4);
+		peerHost += ':';
+		peerHost += peer.substr(8,4);
+		peerHost += ':';
+		peerHost += peer.substr(12,4);
+		peerHost += ':';
+		peerHost += peer.substr(16,4);
+		peerHost += ':';
+		peerHost += peer.substr(20,4);
+		peerHost += ':';
+		peerHost += peer.substr(24,4);
+		peerHost += ':';
+		peerHost += peer.substr(28,4);
+
+		http.get({
+			host: peerHost,
+			port: AGENT_PORT,
+			path: '/'
+		},function(res) {
+			var bytes = 0;
+			res.on('data',function(chunk) { bytes += chunk.length; });
+			res.on('end',function() {
+				if (timedOut)
+					return next(null);
+				allResults[peer] = {
+					testStart: testStartTime,
+					start: connectionStartTime,
+					end: Date.now(),
+					error: null,
+					bytes: bytes,
+					test: testCounter
+				};
+				return next(null);
+			});
+		}).on('error',function(e) {
+			if (timedOut)
+				return next(null);
+			allResults[peer] = {
+				testStart: testStartTime,
+				start: connectionStartTime,
+				end: Date.now(),
+				error: e.toString(),
+				bytes: 0,
+				test: testCounter
+			};
+			return next(null);
+		});
+	},function(err) {
+		if (!timedOut) {
+			clearTimeout(endOfTestTimer);
+			return callback(allResults);
+		}
+	});
+};
+
+// Agents just serve up a test payload
+app.get('/',function(req,res) {
+	return res.status(200).send(payload);
+});
+
+var expressServer = app.listen(AGENT_PORT,function () {
+	registerAndGetPeers(function(err,peers) {
+		if (err) {
+			console.error('FATAL: unable to contact or query server: '+err.toString());
+			process.exit(1);
+		}
+
+		setInterval(function() {
+			++testCounter;
+
+			registerAndGetPeers(function(err,peers) {
+				if (err) {
+					console.error('WARNING: unable to contact or query server, test aborted: '+err.toString());
+					return;
+				}
+
+				performTestOnAllPeers(peers,function(results) {
+					console.log(results);
+
+					var submit = http.request({
+						host: SERVER_HOST,
+						port: SERVER_PORT,
+						path: '/'+thisAgentId,
+						method: 'POST'
+					},function(res) {
+					}).on('error',function(e) {
+						console.error('WARNING: unable to submit results to server: '+err.toString());
+					});
+					submit.write(JSON.stringify(results));
+					submit.end();
+				});
+			});
+		},RUN_TEST_EVERY);
+	});
+});
diff --git a/tests/http/package.json b/tests/http/package.json
new file mode 100644
index 000000000..173a6f99b
--- /dev/null
+++ b/tests/http/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "zerotier-test-http",
+  "version": "1.0.0",
+  "description": "ZeroTier in-network HTTP test",
+  "main": "agent.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "ZeroTier, Inc.",
+  "license": "GPL-3.0",
+  "dependencies": {
+    "async": "^1.5.0",
+    "express": "^4.13.3",
+    "ipaddr.js": "^1.0.3"
+  }
+}
diff --git a/tests/http/server.js b/tests/http/server.js
new file mode 100644
index 000000000..221dcda9e
--- /dev/null
+++ b/tests/http/server.js
@@ -0,0 +1,48 @@
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+var SERVER_PORT = 18080;
+
+// ---------------------------------------------------------------------------
+
+var express = require('express');
+var app = express();
+
+app.use(function(req,res,next) {
+	req.rawBody = '';
+	req.on('data', function(chunk) { req.rawBody += chunk.toString(); });
+	req.on('end', function() { return next(); });
+});
+
+var knownAgents = {};
+
+app.get('/:agentId',function(req,res) {
+	var agentId = req.params.agentId;
+	if ((!agentId)||(agentId.length !== 32))
+		return res.status(404).send('');
+	knownAgents[agentId] = Date.now();
+	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
+});
+
+app.post('/:agentId',function(req,res) {
+	var agentId = req.params.agentId;
+	if ((!agentId)||(agentId.length !== 32))
+		return res.status(404).send('');
+	var resultData = null;
+	try {
+		resultData = JSON.parse(req.rawBody);
+	} catch (e) {
+		resultData = req.rawBody;
+	}
+	result = {
+		agentId: agentId,
+		result: resultData
+	};
+	console.log(result);
+	return res.status(200).send('');
+});
+
+var expressServer = app.listen(SERVER_PORT,function () {
+	console.log('LISTENING ON '+SERVER_PORT);
+	console.log('');
+});

From c03550de3598e3c55ea6c181148286b8673b6df1 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 13:14:53 -0700
Subject: [PATCH 39/73] HTTP test works!

---
 tests/http/agent.js  | 16 +++++++++-------
 tests/http/server.js | 14 +++++++++++++-
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 14964d873..348374111 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -9,14 +9,14 @@ var RUN_TEST_EVERY = 1000;
 var TEST_DURATION = (60 * 1000);
 
 // Where should I contact to register and query a list of other nodes?
-var SERVER_HOST = '127.0.0.1';
+var SERVER_HOST = '174.136.102.178';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
 var AGENT_PORT = 18888;
 
 // Payload size in bytes
-var PAYLOAD_SIZE = 4096;
+var PAYLOAD_SIZE = 100000;
 
 // ---------------------------------------------------------------------------
 
@@ -66,9 +66,9 @@ if (thisAgentId === null) {
 //console.log(thisAgentId);
 
 // Create a random (and therefore not very compressable) payload
-var payload = '';
-while (payload.length < PAYLOAD_SIZE) {
-	payload += String.fromCharCode(Math.round(Math.random() * 255.0));
+var payload = new Buffer(PAYLOAD_SIZE);
+for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
+	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
 // Incremented for each test
@@ -147,7 +147,9 @@ function performTestOnAllPeers(peers,callback)
 			path: '/'
 		},function(res) {
 			var bytes = 0;
-			res.on('data',function(chunk) { bytes += chunk.length; });
+			res.on('data',function(chunk) {
+				bytes += chunk.length;
+			});
 			res.on('end',function() {
 				if (timedOut)
 					return next(null);
@@ -204,7 +206,7 @@ var expressServer = app.listen(AGENT_PORT,function () {
 				}
 
 				performTestOnAllPeers(peers,function(results) {
-					console.log(results);
+					//console.log(results);
 
 					var submit = http.request({
 						host: SERVER_HOST,
diff --git a/tests/http/server.js b/tests/http/server.js
index 221dcda9e..a58756bc1 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -5,6 +5,8 @@ var SERVER_PORT = 18080;
 
 // ---------------------------------------------------------------------------
 
+var fs = require('fs');
+
 var express = require('express');
 var app = express();
 
@@ -28,6 +30,8 @@ app.post('/:agentId',function(req,res) {
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
+
+	var receiveTime = Date.now();
 	var resultData = null;
 	try {
 		resultData = JSON.parse(req.rawBody);
@@ -36,9 +40,17 @@ app.post('/:agentId',function(req,res) {
 	}
 	result = {
 		agentId: agentId,
+		receiveTime: receiveTime,
 		result: resultData
 	};
-	console.log(result);
+
+	var nows = receiveTime.toString(16);
+	while (nows.length < 16)
+		nows = '0' + nows;
+	fs.writeFile('result_'+agentId+'_'+nows,JSON.stringify(result),function(err) {
+		console.log(result);
+	});
+
 	return res.status(200).send('');
 });
 

From 1cae7327aeabeb4546a302c00985fb9413b63443 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 13:35:52 -0700
Subject: [PATCH 40/73] Basic Dockerfile for building test agents.

---
 .gitignore                |  5 ++---
 tests/http/Dockerfile     | 23 +++++++++++++++++++++++
 tests/http/docker-main.sh |  6 ++++++
 3 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 tests/http/Dockerfile
 create mode 100644 tests/http/docker-main.sh

diff --git a/.gitignore b/.gitignore
index 06b06b7d3..89ab049fe 100755
--- a/.gitignore
+++ b/.gitignore
@@ -34,7 +34,7 @@ Thumbs.db
 /world/mkworld
 /world/*.c25519
 
-# Miscellaneous file types that we don't want to check in
+# Miscellaneous temporaries, build files, etc.
 *.log
 *.opensdf
 *.user
@@ -50,10 +50,9 @@ Thumbs.db
 *.autosave
 *.tmp
 node_modules
-
-# cluster-geo stuff
 cluster-geo/cluster-geo/config.js
 cluster-geo/cluster-geo/cache.*
+tests/http/zerotier-one
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*
diff --git a/tests/http/Dockerfile b/tests/http/Dockerfile
new file mode 100644
index 000000000..02578cd53
--- /dev/null
+++ b/tests/http/Dockerfile
@@ -0,0 +1,23 @@
+FROM centos:latest
+
+MAINTAINER https://www.zerotier.com/
+
+EXPOSE 9993/udp
+
+RUN yum -y update && yum -y install epel-release && yum -y install nodejs npm && yum clean all
+
+RUN mkdir -p /var/lib/zerotier-one
+RUN mkdir -p /var/lib/zerotier-one/networks.d
+RUN touch /var/lib/zerotier-one/networks.d/ffffffffffffffff.conf
+
+ADD package.json /
+RUN npm install
+
+ADD zerotier-one /
+RUN chmod a+x /zerotier-one
+
+ADD agent.js /
+ADD main.sh /
+RUN chmod a+x /docker-main.sh
+
+CMD ["./docker-main.sh"]
diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
new file mode 100644
index 000000000..947ccf47a
--- /dev/null
+++ b/tests/http/docker-main.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
+
+/zerotier-one -d
+exec node --harmony /agent.js

From 07c1b4ddee1b6ba1a4399dc62c8e3c6f5367afa0 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:16:58 -0700
Subject: [PATCH 41/73] test stuff

---
 tests/http/Dockerfile     |   2 +-
 tests/http/agent.js       | 148 +++++++++++++++++++-------------------
 tests/http/docker-main.sh |   7 +-
 tests/http/server.js      |   6 +-
 4 files changed, 83 insertions(+), 80 deletions(-)
 mode change 100644 => 100755 tests/http/docker-main.sh

diff --git a/tests/http/Dockerfile b/tests/http/Dockerfile
index 02578cd53..7bba2fc07 100644
--- a/tests/http/Dockerfile
+++ b/tests/http/Dockerfile
@@ -17,7 +17,7 @@ ADD zerotier-one /
 RUN chmod a+x /zerotier-one
 
 ADD agent.js /
-ADD main.sh /
+ADD docker-main.sh /
 RUN chmod a+x /docker-main.sh
 
 CMD ["./docker-main.sh"]
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 348374111..465a2e28e 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -1,14 +1,13 @@
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
-// How frequently in ms to run tests
-//var RUN_TEST_EVERY = (60 * 5 * 1000);
-var RUN_TEST_EVERY = 1000;
+// Maximum test duration in milliseconds
+var TEST_DURATION = (30 * 1000);
 
-// Maximum test duration in milliseconds (must be less than RUN_TEST_EVERY)
-var TEST_DURATION = (60 * 1000);
+// Interval between tests (should be several times longer than TEST_DURATION)
+var TEST_INTERVAL = (60 * 2 * 1000);
 
-// Where should I contact to register and query a list of other nodes?
+// Where should I contact to register and query a list of other test agents?
 var SERVER_HOST = '174.136.102.178';
 var SERVER_PORT = 18080;
 
@@ -71,8 +70,29 @@ for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
 	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
-// Incremented for each test
-var testCounter = 0;
+// Incremented with each test
+var testNumber = 0;
+
+function agentIdToIp(agentId)
+{
+	var ip = '';
+	ip += agentId.substr(0,4);
+	ip += ':';
+	ip += agentId.substr(4,4);
+	ip += ':';
+	ip += agentId.substr(8,4);
+	ip += ':';
+	ip += agentId.substr(12,4);
+	ip += ':';
+	ip += agentId.substr(16,4);
+	ip += ':';
+	ip += agentId.substr(20,4);
+	ip += ':';
+	ip += agentId.substr(24,4);
+	ip += ':';
+	ip += agentId.substr(28,4);
+	return ip;
+};
 
 function registerAndGetPeers(callback)
 {
@@ -84,13 +104,15 @@ function registerAndGetPeers(callback)
 		var body = '';
 		res.on('data',function(chunk) { body += chunk.toString(); });
 		res.on('end',function() {
+			if (!body)
+				return callback(null,[]);
 			try {
 				var peers = JSON.parse(body);
 				if (Array.isArray(peers))
 					return callback(null,peers);
 				else return callback(new Error('invalid JSON response from server'),null);
 			} catch (e) {
-				return callback(new Error('invalid JSON response from server'),null);
+				return callback(new Error('invalid JSON response from server: '+e.toString()),null);
 			}
 		});
 	}).on('error',function(e) {
@@ -101,12 +123,13 @@ function registerAndGetPeers(callback)
 function performTestOnAllPeers(peers,callback)
 {
 	var allResults = {};
+	var allRequests = [];
 	var timedOut = false;
 	var endOfTestTimer = setTimeout(function() {
 		timedOut = true;
-		return callback(allResults);
+		for(var x=0;x<allRequests.length;++x)
+			allRequests[x].abort();
 	},TEST_DURATION);
-	var testStartTime = Date.now();
 
 	async.each(peers,function(peer,next) {
 		if (timedOut)
@@ -116,33 +139,15 @@ function performTestOnAllPeers(peers,callback)
 
 		var connectionStartTime = Date.now();
 		allResults[peer] = {
-			testStart: testStartTime,
 			start: connectionStartTime,
-			end: null,
+			end: 0,
 			error: null,
-			bytes: 0,
-			test: testCounter
+			timedOut: false,
+			bytes: 0
 		};
 
-		var peerHost = '';
-		peerHost += peer.substr(0,4);
-		peerHost += ':';
-		peerHost += peer.substr(4,4);
-		peerHost += ':';
-		peerHost += peer.substr(8,4);
-		peerHost += ':';
-		peerHost += peer.substr(12,4);
-		peerHost += ':';
-		peerHost += peer.substr(16,4);
-		peerHost += ':';
-		peerHost += peer.substr(20,4);
-		peerHost += ':';
-		peerHost += peer.substr(24,4);
-		peerHost += ':';
-		peerHost += peer.substr(28,4);
-
-		http.get({
-			host: peerHost,
+		allRequests.push(http.get({
+			host: agentIdToIp(peer),
 			port: AGENT_PORT,
 			path: '/'
 		},function(res) {
@@ -151,76 +156,67 @@ function performTestOnAllPeers(peers,callback)
 				bytes += chunk.length;
 			});
 			res.on('end',function() {
-				if (timedOut)
-					return next(null);
 				allResults[peer] = {
-					testStart: testStartTime,
 					start: connectionStartTime,
 					end: Date.now(),
 					error: null,
-					bytes: bytes,
-					test: testCounter
+					timedOut: timedOut,
+					bytes: bytes
 				};
 				return next(null);
 			});
 		}).on('error',function(e) {
-			if (timedOut)
-				return next(null);
 			allResults[peer] = {
-				testStart: testStartTime,
 				start: connectionStartTime,
 				end: Date.now(),
 				error: e.toString(),
-				bytes: 0,
-				test: testCounter
+				timedOut: timedOut,
+				bytes: 0
 			};
 			return next(null);
-		});
+		}));
 	},function(err) {
-		if (!timedOut) {
+		if (!timedOut)
 			clearTimeout(endOfTestTimer);
-			return callback(allResults);
-		}
+		return callback(allResults);
 	});
 };
 
+function doTestsAndReport()
+{
+	registerAndGetPeers(function(err,peers) {
+		if (err) {
+			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
+		} else {
+			performTestOnAllPeers(peers,function(results) {
+				++testNumber;
+				var submit = http.request({
+					host: SERVER_HOST,
+					port: SERVER_PORT,
+					path: '/'+testNumber+'/'+thisAgentId,
+					method: 'POST'
+				},function(res) {
+				}).on('error',function(e) {
+					console.error('WARNING: unable to submit results to server: '+err.toString());
+				});
+				submit.write(JSON.stringify(results));
+				submit.end();
+			});
+		}
+	});
+}
+
 // Agents just serve up a test payload
 app.get('/',function(req,res) {
 	return res.status(200).send(payload);
 });
 
-var expressServer = app.listen(AGENT_PORT,function () {
+var expressServer = app.listen(AGENT_PORT,agentIdToIp(thisAgentId),function () {
 	registerAndGetPeers(function(err,peers) {
 		if (err) {
 			console.error('FATAL: unable to contact or query server: '+err.toString());
 			process.exit(1);
 		}
-
-		setInterval(function() {
-			++testCounter;
-
-			registerAndGetPeers(function(err,peers) {
-				if (err) {
-					console.error('WARNING: unable to contact or query server, test aborted: '+err.toString());
-					return;
-				}
-
-				performTestOnAllPeers(peers,function(results) {
-					//console.log(results);
-
-					var submit = http.request({
-						host: SERVER_HOST,
-						port: SERVER_PORT,
-						path: '/'+thisAgentId,
-						method: 'POST'
-					},function(res) {
-					}).on('error',function(e) {
-						console.error('WARNING: unable to submit results to server: '+err.toString());
-					});
-					submit.write(JSON.stringify(results));
-					submit.end();
-				});
-			});
-		},RUN_TEST_EVERY);
+		setInterval(doTestsAndReport,TEST_INTERVAL);
 	});
 });
diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
old mode 100644
new mode 100755
index 947ccf47a..ad80af0ce
--- a/tests/http/docker-main.sh
+++ b/tests/http/docker-main.sh
@@ -3,4 +3,9 @@
 export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
 
 /zerotier-one -d
-exec node --harmony /agent.js
+
+while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
+	sleep 0.25
+done
+
+exec node --harmony /agent.js >>agent.out 2>&1
diff --git a/tests/http/server.js b/tests/http/server.js
index a58756bc1..6211b4eea 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -26,7 +26,8 @@ app.get('/:agentId',function(req,res) {
 	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
 });
 
-app.post('/:agentId',function(req,res) {
+app.post('/:testNumber/:agentId',function(req,res) {
+	var testNumber = req.params.testNumber;
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
@@ -40,8 +41,9 @@ app.post('/:agentId',function(req,res) {
 	}
 	result = {
 		agentId: agentId,
+		testNumber: testNumber,
 		receiveTime: receiveTime,
-		result: resultData
+		results: resultData
 	};
 
 	var nows = receiveTime.toString(16);

From 9653531242dd5f66e331fc716c4aacd1aece30c5 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:18:58 -0700
Subject: [PATCH 42/73] .

---
 tests/http/agent.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 465a2e28e..53b2e9f9c 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -204,7 +204,7 @@ function doTestsAndReport()
 			});
 		}
 	});
-}
+};
 
 // Agents just serve up a test payload
 app.get('/',function(req,res) {

From 4c24e0cfb0bb9b61a4e19ac81b89dc1cbce6ea99 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:24:54 -0700
Subject: [PATCH 43/73] More tweaks to tests... just about ready to run at
 scale.

---
 .gitignore           |  1 +
 tests/http/agent.js  |  1 +
 tests/http/server.js | 10 +++++-----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 89ab049fe..87e387a6d 100755
--- a/.gitignore
+++ b/.gitignore
@@ -53,6 +53,7 @@ node_modules
 cluster-geo/cluster-geo/config.js
 cluster-geo/cluster-geo/cache.*
 tests/http/zerotier-one
+tests/http/result_*
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 53b2e9f9c..061c7ba70 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -217,6 +217,7 @@ var expressServer = app.listen(AGENT_PORT,agentIdToIp(thisAgentId),function () {
 			console.error('FATAL: unable to contact or query server: '+err.toString());
 			process.exit(1);
 		}
+		doTestsAndReport();
 		setInterval(doTestsAndReport,TEST_INTERVAL);
 	});
 });
diff --git a/tests/http/server.js b/tests/http/server.js
index 6211b4eea..69ccf527e 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -41,15 +41,15 @@ app.post('/:testNumber/:agentId',function(req,res) {
 	}
 	result = {
 		agentId: agentId,
-		testNumber: testNumber,
+		testNumber: parseInt(testNumber),
 		receiveTime: receiveTime,
 		results: resultData
 	};
 
-	var nows = receiveTime.toString(16);
-	while (nows.length < 16)
-		nows = '0' + nows;
-	fs.writeFile('result_'+agentId+'_'+nows,JSON.stringify(result),function(err) {
+	testNumber = testNumber.toString();
+	while (testNumber.length < 10)
+		testNumber = '0' + testNumber;
+	fs.writeFile('result_'+testNumber+'_'+agentId,JSON.stringify(result),function(err) {
 		console.log(result);
 	});
 

From 68d6d3c4ff4d3921408f7d9bab3e31e0e028a871 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 14:29:08 -0700
Subject: [PATCH 44/73] Fix bug in peer count.

---
 node/Cluster.cpp  | 2 +-
 node/Topology.cpp | 4 ++--
 node/Topology.hpp | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 9b0348221..e95f6acca 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -634,7 +634,7 @@ void Cluster::status(ZT_ClusterStatus &status) const
 	ms[_id]->x = _x;
 	ms[_id]->y = _y;
 	ms[_id]->z = _z;
-	ms[_id]->peers = RR->topology->countAlive();
+	ms[_id]->peers = RR->topology->countActive();
 	for(std::vector<InetAddress>::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) {
 		if (ms[_id]->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
 			break;
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 09668ef55..9027eff16 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -308,7 +308,7 @@ void Topology::clean(uint64_t now)
 	}
 }
 
-unsigned long Topology::countAlive() const
+unsigned long Topology::countActive() const
 {
 	const uint64_t now = RR->node->now();
 	unsigned long cnt = 0;
@@ -317,7 +317,7 @@ unsigned long Topology::countAlive() const
 	Address *a = (Address *)0;
 	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 	while (i.next(a,p)) {
-		if ((*p)->alive(now))
+		if ((*p)->hasActiveDirectPath(now))
 			++cnt;
 	}
 	return cnt;
diff --git a/node/Topology.hpp b/node/Topology.hpp
index 16836e075..b9f063c85 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -210,9 +210,9 @@ public:
 	void clean(uint64_t now);
 
 	/**
-	 * @return Number of 'alive' peers
+	 * @return Number of peers with active direct paths
 	 */
-	unsigned long countAlive() const;
+	unsigned long countActive() const;
 
 	/**
 	 * Apply a function or function object to all peers

From 1f5ef968cff51721d97587cb1c5402c988c92d5e Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 15:08:00 -0700
Subject: [PATCH 45/73] Test need a more recent version of NodeJS so update
 Dockerfile.

---
 tests/http/Dockerfile         | 3 ++-
 tests/http/agent.js           | 2 +-
 tests/http/docker-main.sh     | 3 +++
 tests/http/nodesource-el.repo | 6 ++++++
 4 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 tests/http/nodesource-el.repo

diff --git a/tests/http/Dockerfile b/tests/http/Dockerfile
index 7bba2fc07..e19b3feef 100644
--- a/tests/http/Dockerfile
+++ b/tests/http/Dockerfile
@@ -4,7 +4,8 @@ MAINTAINER https://www.zerotier.com/
 
 EXPOSE 9993/udp
 
-RUN yum -y update && yum -y install epel-release && yum -y install nodejs npm && yum clean all
+ADD nodesource-el.repo /etc/yum.repos.d/nodesource-el.repo
+RUN yum -y update && yum install -y nodejs && yum clean all
 
 RUN mkdir -p /var/lib/zerotier-one
 RUN mkdir -p /var/lib/zerotier-one/networks.d
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 061c7ba70..8b1cf5121 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -211,7 +211,7 @@ app.get('/',function(req,res) {
 	return res.status(200).send(payload);
 });
 
-var expressServer = app.listen(AGENT_PORT,agentIdToIp(thisAgentId),function () {
+var expressServer = app.listen(AGENT_PORT,function () {
 	registerAndGetPeers(function(err,peers) {
 		if (err) {
 			console.error('FATAL: unable to contact or query server: '+err.toString());
diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
index ad80af0ce..720236687 100755
--- a/tests/http/docker-main.sh
+++ b/tests/http/docker-main.sh
@@ -8,4 +8,7 @@ while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
 	sleep 0.25
 done
 
+sleep 2
+
 exec node --harmony /agent.js >>agent.out 2>&1
+#exec node --harmony /agent.js
diff --git a/tests/http/nodesource-el.repo b/tests/http/nodesource-el.repo
new file mode 100644
index 000000000..b785d3d07
--- /dev/null
+++ b/tests/http/nodesource-el.repo
@@ -0,0 +1,6 @@
+[nodesource]
+name=Node.js Packages for Enterprise Linux 7 - $basearch
+baseurl=https://rpm.nodesource.com/pub_4.x/el/7/$basearch
+failovermethod=priority
+enabled=1
+gpgcheck=0

From cabb8752cb9d9c85dc2aa2cac22a4ad101614577 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 15:28:05 -0700
Subject: [PATCH 46/73] docs

---
 tests/http/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/http/README.md b/tests/http/README.md
index ae7f08f10..7852ac167 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -3,3 +3,6 @@ HTTP one-to-all test
 
 This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
 
+A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container.
+
+Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently.

From e3d811b04b7fb04981d65a85d9042e2bd31798b7 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Wed, 28 Oct 2015 15:55:40 -0700
Subject: [PATCH 47/73] docs

---
 tests/http/README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/http/README.md b/tests/http/README.md
index 7852ac167..ab4827ecc 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -3,6 +3,8 @@ HTTP one-to-all test
 
 This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
 
-A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container.
-
 Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently.
+
+A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. You can run a bunch with a command like:
+
+    for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done

From 883c84bdb95b0374e4f4ea2238b2288787547897 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 29 Oct 2015 09:39:36 -0700
Subject: [PATCH 48/73] Tweak some timings, and remove some dead code.

---
 node/Cluster.hpp |  2 +-
 node/Peer.hpp    | 31 -------------------------------
 2 files changed, 1 insertion(+), 32 deletions(-)

diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index cc9edd1d1..0d8c0f155 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -57,7 +57,7 @@
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
 
 namespace ZeroTier {
 
diff --git a/node/Peer.hpp b/node/Peer.hpp
index 39acffd97..e5db3bde5 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -205,32 +205,6 @@ public:
 		return pp;
 	}
 
-	/**
-	 * @return Time of last direct packet receive for any path
-	 */
-	inline uint64_t lastDirectReceive() const
-		throw()
-	{
-		Mutex::Lock _l(_lock);
-		uint64_t x = 0;
-		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			x = std::max(x,_paths[p].lastReceived());
-		return x;
-	}
-
-	/**
-	 * @return Time of last direct packet send for any path
-	 */
-	inline uint64_t lastDirectSend() const
-		throw()
-	{
-		Mutex::Lock _l(_lock);
-		uint64_t x = 0;
-		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			x = std::max(x,_paths[p].lastSend());
-		return x;
-	}
-
 	/**
 	 * @return Time of last receive of anything, whether direct or relayed
 	 */
@@ -285,11 +259,6 @@ public:
 		else _latency = std::min(l,(unsigned int)65535);
 	}
 
-	/**
-	 * @return True if this peer has at least one direct IP address path
-	 */
-	inline bool hasDirectPath() const throw() { return (_numPaths != 0); }
-
 	/**
 	 * @param now Current time
 	 * @return True if this peer has at least one active direct path

From a994573a436ec2835781d13ecd2307e18c67855b Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 29 Oct 2015 09:42:15 -0700
Subject: [PATCH 49/73] Eliminate some more dead code. We may do path trust,
 but not like that.

---
 include/ZeroTierOne.h  | 17 +----------------
 node/Node.cpp          |  6 +++---
 node/Node.hpp          |  2 +-
 node/Path.hpp          | 16 +++-------------
 node/Peer.cpp          |  2 +-
 service/OneService.cpp |  4 ++--
 6 files changed, 11 insertions(+), 36 deletions(-)

diff --git a/include/ZeroTierOne.h b/include/ZeroTierOne.h
index 01c8bcdeb..e9b38c524 100644
--- a/include/ZeroTierOne.h
+++ b/include/ZeroTierOne.h
@@ -424,15 +424,6 @@ enum ZT_VirtualNetworkConfigOperation
 	ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4
 };
 
-/**
- * Local interface trust levels
- */
-enum ZT_LocalInterfaceAddressTrust {
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
-};
-
 /**
  * What trust hierarchy role does this peer have?
  */
@@ -1337,11 +1328,6 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
 /**
  * Add a local interface address
  *
- * Local interface addresses may be added if you want remote peers
- * with whom you have a trust relatinship (e.g. common network membership)
- * to receive information about these endpoints as potential endpoints for
- * direct communication.
- *
  * Take care that these are never ZeroTier interface addresses, otherwise
  * strange things might happen or they simply won't work.
  *
@@ -1356,10 +1342,9 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * reject bad, empty, and unusable addresses.
  *
  * @param addr Local interface address
- * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr);
 
 /**
  * Clear local interface addresses
diff --git a/node/Node.cpp b/node/Node.cpp
index 82cda66d8..42180e990 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -499,7 +499,7 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
@@ -900,10 +900,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,enum ZT_LocalInterfaceAddressTrust trust)
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr);
 	} catch ( ... ) {
 		return 0;
 	}
diff --git a/node/Node.hpp b/node/Node.hpp
index 48c5ead8f..9b85b8326 100644
--- a/node/Node.hpp
+++ b/node/Node.hpp
@@ -105,7 +105,7 @@ public:
 	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
 	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,ZT_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
diff --git a/node/Path.hpp b/node/Path.hpp
index 2b05b8126..c01829903 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -79,18 +79,16 @@ public:
 		_addr(),
 		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_trust(TRUST_NORMAL),
 		_flags(0)
 	{
 	}
 
-	Path(const InetAddress &localAddress,const InetAddress &addr,Trust trust) :
+	Path(const InetAddress &localAddress,const InetAddress &addr) :
 		_lastSend(0),
 		_lastReceived(0),
 		_addr(addr),
 		_localAddress(localAddress),
 		_ipScope(addr.ipScope()),
-		_trust(trust),
 		_flags(0)
 	{
 	}
@@ -187,11 +185,6 @@ public:
 		return ( ((int)_ipScope * 2) + ((_addr.ss_family == AF_INET6) ? 1 : 0) );
 	}
 
-	/**
-	 * @return Path trust level
-	 */
-	inline Trust trust() const throw() { return _trust; }
-
 	/**
 	 * @return True if path is considered reliable (no NAT keepalives etc. are needed)
 	 */
@@ -243,12 +236,11 @@ public:
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
 	{
-		b.append((uint8_t)0); // version
+		b.append((uint8_t)1); // version
 		b.append((uint64_t)_lastSend);
 		b.append((uint64_t)_lastReceived);
 		_addr.serialize(b);
 		_localAddress.serialize(b);
-		b.append((uint8_t)_trust);
 		b.append((uint16_t)_flags);
 	}
 
@@ -256,14 +248,13 @@ public:
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		unsigned int p = startAt;
-		if (b[p++] != 0)
+		if (b[p++] != 1)
 			throw std::invalid_argument("invalid serialized Path");
 		_lastSend = b.template at<uint64_t>(p); p += 8;
 		_lastReceived = b.template at<uint64_t>(p); p += 8;
 		p += _addr.deserialize(b,p);
 		p += _localAddress.deserialize(b,p);
 		_ipScope = _addr.ipScope();
-		_trust = (Path::Trust)b[p++];
 		_flags = b.template at<uint16_t>(p); p += 2;
 		return (p - startAt);
 	}
@@ -274,7 +265,6 @@ private:
 	InetAddress _addr;
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
-	Trust _trust;
 	uint16_t _flags;
 };
 
diff --git a/node/Peer.cpp b/node/Peer.cpp
index 976c7c449..9d0d78e5a 100644
--- a/node/Peer.cpp
+++ b/node/Peer.cpp
@@ -179,7 +179,7 @@ void Peer::received(
 						}
 					}
 					if (slot) {
-						*slot = Path(localAddr,remoteAddr,Path::TRUST_NORMAL);
+						*slot = Path(localAddr,remoteAddr);
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
diff --git a/service/OneService.cpp b/service/OneService.cpp
index 4e3f24c79..8c8ff1ed1 100644
--- a/service/OneService.cpp
+++ b/service/OneService.cpp
@@ -731,7 +731,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)));
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -749,7 +749,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip));
 								}
 							}
 							ifa = ifa->ifa_next;

From 9f0f0197fe5a79ed04647b110485a530ed06ed8e Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 29 Oct 2015 09:44:25 -0700
Subject: [PATCH 50/73] More dead code removal.

---
 node/Path.hpp | 33 ++-------------------------------
 1 file changed, 2 insertions(+), 31 deletions(-)

diff --git a/node/Path.hpp b/node/Path.hpp
index c01829903..39a18c430 100644
--- a/node/Path.hpp
+++ b/node/Path.hpp
@@ -49,37 +49,12 @@ class RuntimeEnvironment;
 class Path
 {
 public:
-	/**
-	 * Path trust category
-	 *
-	 * Note that this is NOT peer trust and has nothing to do with root server
-	 * designations or other trust metrics. This indicates how much we trust
-	 * this path to be secure and/or private. A trust level of normal means
-	 * encrypt and authenticate all traffic. Privacy trust means we can send
-	 * traffic in the clear. Ultimate trust means we don't even need
-	 * authentication. Generally a private path would be a hard-wired local
-	 * LAN, while an ultimate trust path would be a physically isolated private
-	 * server backplane.
-	 *
-	 * Nearly all paths will be normal trust. The other levels are for high
-	 * performance local SDN use only.
-	 *
-	 * These values MUST match ZT_LocalInterfaceAddressTrust in ZeroTierOne.h
-	 */
-	enum Trust // NOTE: max 255
-	{
-		TRUST_NORMAL = 0,
-		TRUST_PRIVACY = 10,
-		TRUST_ULTIMATE = 20
-	};
-
 	Path() :
 		_lastSend(0),
 		_lastReceived(0),
 		_addr(),
 		_localAddress(),
-		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_flags(0)
+		_ipScope(InetAddress::IP_SCOPE_NONE)
 	{
 	}
 
@@ -88,8 +63,7 @@ public:
 		_lastReceived(0),
 		_addr(addr),
 		_localAddress(localAddress),
-		_ipScope(addr.ipScope()),
-		_flags(0)
+		_ipScope(addr.ipScope())
 	{
 	}
 
@@ -241,7 +215,6 @@ public:
 		b.append((uint64_t)_lastReceived);
 		_addr.serialize(b);
 		_localAddress.serialize(b);
-		b.append((uint16_t)_flags);
 	}
 
 	template<unsigned int C>
@@ -255,7 +228,6 @@ public:
 		p += _addr.deserialize(b,p);
 		p += _localAddress.deserialize(b,p);
 		_ipScope = _addr.ipScope();
-		_flags = b.template at<uint16_t>(p); p += 2;
 		return (p - startAt);
 	}
 
@@ -265,7 +237,6 @@ private:
 	InetAddress _addr;
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
-	uint16_t _flags;
 };
 
 } // namespace ZeroTier

From d6c0d176ee3bc71f25105503e807a7fe0d45674a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 29 Oct 2015 10:10:09 -0700
Subject: [PATCH 51/73] Periodically re-announce peers that we have.

---
 node/Cluster.cpp | 53 ++++++++++++++++++++++++++++++++++--------------
 node/Cluster.hpp | 11 +++++++---
 2 files changed, 46 insertions(+), 18 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index e95f6acca..93b69a08a 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -84,7 +84,8 @@ Cluster::Cluster(
 	_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
 	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
 	_peerAffinities(65536),
-	_lastCleanedPeerAffinities(0)
+	_lastCleanedPeerAffinities(0),
+	_lastCheckedPeersForAnnounce(0)
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -328,6 +329,7 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 
 								if (haveMatch) {
 									_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
+									_flush(fromMemberId); // we want this to go ASAP, since with port restricted cone NATs success can be timing-sensitive
 									RR->sw->send(rendezvousForLocal,true,0);
 								}
 							}
@@ -469,10 +471,45 @@ void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembers
 	}
 }
 
+struct _ClusterAnnouncePeers
+{
+	_ClusterAnnouncePeers(const uint64_t now_,Cluster *parent_) : now(now_),parent(parent_) {}
+	const uint64_t now;
+	Cluster *const parent;
+	inline void operator()(const Topology &t,const SharedPtr<Peer> &peer)
+	{
+		Path *p = peer->getBestPath(now);
+		if (p)
+			parent->replicateHavePeer(peer->identity(),p->address());
+	}
+};
 void Cluster::doPeriodicTasks()
 {
 	const uint64_t now = RR->node->now();
 
+	// Erase old peer affinity entries just to control table size
+	if ((now - _lastCleanedPeerAffinities) >= (ZT_PEER_ACTIVITY_TIMEOUT * 5)) {
+		_lastCleanedPeerAffinities = now;
+		Address *k = (Address *)0;
+		_PA *v = (_PA *)0;
+		Mutex::Lock _l(_peerAffinities_m);
+		Hashtable< Address,_PA >::Iterator i(_peerAffinities);
+		while (i.next(k,v)) {
+			if ((now - v->ts) >= (ZT_PEER_ACTIVITY_TIMEOUT * 5))
+				_peerAffinities.erase(*k);
+		}
+	}
+
+	// Announce peers that we have active direct paths to -- note that we forget paths
+	// that other cluster members claim they have, which prevents us from fighting
+	// with other cluster members (route flapping) over specific paths.
+	if ((now - _lastCheckedPeersForAnnounce) >= (ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD / 4)) {
+		_lastCheckedPeersForAnnounce = now;
+		_ClusterAnnouncePeers func(now,this);
+		RR->topology->eachPeer<_ClusterAnnouncePeers &>(func);
+	}
+
+	// Flush outgoing packet send queue every doPeriodicTasks()
 	{
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -506,20 +543,6 @@ void Cluster::doPeriodicTasks()
 			_flush(*mid); // does nothing if nothing to flush
 		}
 	}
-
-	{
-		if ((now - _lastCleanedPeerAffinities) >= (ZT_PEER_ACTIVITY_TIMEOUT * 10)) {
-			_lastCleanedPeerAffinities = now;
-			Address *k = (Address *)0;
-			_PA *v = (_PA *)0;
-			Mutex::Lock _l(_peerAffinities_m);
-			Hashtable< Address,_PA >::Iterator i(_peerAffinities);
-			while (i.next(k,v)) {
-				if ((now - v->ts) >= (ZT_PEER_ACTIVITY_TIMEOUT * 10))
-					_peerAffinities.erase(*k);
-			}
-		}
-	}
 }
 
 void Cluster::addMember(uint16_t memberId)
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 0d8c0f155..7d7a1ced4 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -46,18 +46,21 @@
 
 /**
  * Timeout for cluster members being considered "alive"
+ *
+ * A cluster member is considered dead and will no longer have peers
+ * redirected to it if we have not heard a heartbeat in this long.
  */
-#define ZT_CLUSTER_TIMEOUT 20000
+#define ZT_CLUSTER_TIMEOUT 10000
 
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD 30000
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ((ZT_PEER_ACTIVITY_TIMEOUT / 2) - 1000)
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
 
 namespace ZeroTier {
 
@@ -349,7 +352,9 @@ private:
 	};
 	Hashtable< Address,_PA > _peerAffinities;
 	Mutex _peerAffinities_m;
+
 	uint64_t _lastCleanedPeerAffinities;
+	uint64_t _lastCheckedPeersForAnnounce;
 };
 
 } // namespace ZeroTier

From e2fc20876d1f171b1d3f1bfabb9fc23fb8455a5a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Thu, 29 Oct 2015 18:23:41 -0700
Subject: [PATCH 52/73] docs

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 248065047..00efed368 100644
--- a/README.md
+++ b/README.md
@@ -166,6 +166,10 @@ If you're interested, there's a [technical deep dive about NAT traversal on our
 
 If a firewall between you and the Internet blocks ZeroTier's UDP traffic, you will fall back to last-resort TCP tunneling to rootservers over port 443 (https impersonation). This will work almost anywhere but is *very slow* compared to UDP or direct peer to peer connectivity.
 
+### Contributing
+
+There are three main branches: **edge**, **test**, and **master**. Other branches may be for specific features, tests, or use cases. In general **edge** is "bleeding" and may or may not work, while **test** should be relatively stable and **master** is the latest tagged release. Pull requests should generally be done against **test** or **edge**, since pull requests against **master** may be working against a branch that is somewhat out of date.
+
 ### License
 
 The ZeroTier source code is open source and is licensed under the GNU GPL v3 (not LGPL). If you'd like to embed it in a closed-source commercial product or appliance, please e-mail [contact@zerotier.com](mailto:contact@zerotier.com) to discuss commercial licensing. Otherwise it can be used for free.

From 80e62ad29180800fd1669aa68515876f0e8add54 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 10:55:05 -0700
Subject: [PATCH 53/73] docs

---
 tests/http/README.md | 4 ++--
 tests/http/agent.js  | 2 ++
 tests/http/server.js | 2 ++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/http/README.md b/tests/http/README.md
index ab4827ecc..58d4a9892 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -3,8 +3,8 @@ HTTP one-to-all test
 
 This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
 
-Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently.
+Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently. You will also want to edit the Dockerfile if you want to build containers and change the network ID to the network you want to run tests over.
 
-A Dockerfile is also included which will build a simple Docker image that runs the agent. The image must be launched with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. You can run a bunch with a command like:
+The Dockerfile builds an image that launches the agent. The image must be "docker run" with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. (Unfortunately CAP_NET_ADMIN may not work due to a bug in Docker and/or Linux.) You can run a bunch with a command like:
 
     for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done
diff --git a/tests/http/agent.js b/tests/http/agent.js
index 8b1cf5121..8a8b785e5 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -1,3 +1,5 @@
+// ZeroTier distributed HTTP test agent
+
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
diff --git a/tests/http/server.js b/tests/http/server.js
index 69ccf527e..30d8339a3 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -1,3 +1,5 @@
+// ZeroTier distributed HTTP test coordinator and result-reporting server
+
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 

From 5bfa29ddedaa918fb0d5912537391f1ca4b0ba07 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 11:09:40 -0700
Subject: [PATCH 54/73] Make antirec tail len slightly shorter, better
 performance and still plenty long enough.

---
 node/AntiRecursion.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/AntiRecursion.hpp b/node/AntiRecursion.hpp
index c5aa92d80..4bb24bf3b 100644
--- a/node/AntiRecursion.hpp
+++ b/node/AntiRecursion.hpp
@@ -35,7 +35,7 @@
 
 namespace ZeroTier {
 
-#define ZT_ANTIRECURSION_TAIL_LEN 256
+#define ZT_ANTIRECURSION_TAIL_LEN 128
 
 /**
  * Filter to prevent recursion (ZeroTier-over-ZeroTier)

From b6725c44150af306130d38a2875ab31a640607c8 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 11:48:33 -0700
Subject: [PATCH 55/73] Optimize AntiRecursion.

---
 node/AntiRecursion.hpp | 66 +++++++++++++++++++++++++++---------------
 node/Constants.hpp     |  5 ----
 2 files changed, 42 insertions(+), 29 deletions(-)

diff --git a/node/AntiRecursion.hpp b/node/AntiRecursion.hpp
index 4bb24bf3b..8629d19a9 100644
--- a/node/AntiRecursion.hpp
+++ b/node/AntiRecursion.hpp
@@ -35,28 +35,28 @@
 
 namespace ZeroTier {
 
-#define ZT_ANTIRECURSION_TAIL_LEN 128
+/**
+ * Size of anti-recursion history
+ */
+#define ZT_ANTIRECURSION_HISTORY_SIZE 16
 
 /**
  * Filter to prevent recursion (ZeroTier-over-ZeroTier)
  *
  * This works by logging ZeroTier packets that we send. It's then invoked
- * again against packets read from local Ethernet taps. If the last N
+ * again against packets read from local Ethernet taps. If the last 32
  * bytes representing the ZeroTier packet match in the tap frame, then
  * the frame is a re-injection of a frame that we sent and is rejected.
  *
  * This means that ZeroTier packets simply will not traverse ZeroTier
  * networks, which would cause all sorts of weird problems.
  *
- * NOTE: this is applied to low-level packets before they are sent to
- * SocketManager and/or sockets, not to fully assembled packets before
- * (possible) fragmentation.
+ * This is highly optimized code since it's checked for every packet.
  */
 class AntiRecursion
 {
 public:
 	AntiRecursion()
-		throw()
 	{
 		memset(_history,0,sizeof(_history));
 		_ptr = 0;
@@ -68,13 +68,20 @@ public:
 	 * @param data ZT packet data
 	 * @param len Length of packet
 	 */
-	inline void logOutgoingZT(const void *data,unsigned int len)
-		throw()
+	inline void logOutgoingZT(const void *const data,const unsigned int len)
 	{
-		ArItem *i = &(_history[_ptr++ % ZT_ANTIRECURSION_HISTORY_SIZE]);
-		const unsigned int tl = (len > ZT_ANTIRECURSION_TAIL_LEN) ? ZT_ANTIRECURSION_TAIL_LEN : len;
-		memcpy(i->tail,((const unsigned char *)data) + (len - tl),tl);
-		i->len = tl;
+		if (len < 32)
+			return;
+#ifdef ZT_NO_TYPE_PUNNING
+		memcpy(_history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail,reinterpret_cast<const uint8_t *>(data) + (len - 32),32);
+#else
+		uint64_t *t = _history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail;
+		const uint64_t *p = reinterpret_cast<const uint64_t *>(reinterpret_cast<const uint8_t *>(data) + (len - 32));
+		*(t++) = *(p++);
+		*(t++) = *(p++);
+		*(t++) = *(p++);
+		*t = *p;
+#endif
 	}
 
 	/**
@@ -84,25 +91,36 @@ public:
 	 * @param len Length of frame
 	 * @return True if frame is OK to be passed, false if it's a ZT frame that we sent
 	 */
-	inline bool checkEthernetFrame(const void *data,unsigned int len)
-		throw()
+	inline bool checkEthernetFrame(const void *const data,const unsigned int len) const
 	{
-		for(unsigned int h=0;h<ZT_ANTIRECURSION_HISTORY_SIZE;++h) {
-			ArItem *i = &(_history[h]);
-			if ((i->len > 0)&&(len >= i->len)&&(!memcmp(((const unsigned char *)data) + (len - i->len),i->tail,i->len)))
+		if (len < 32)
+			return true;
+		const uint8_t *const pp = reinterpret_cast<const uint8_t *>(data) + (len - 32);
+		const _ArItem *i = _history;
+		const _ArItem *const end = i + ZT_ANTIRECURSION_HISTORY_SIZE;
+		while (i != end) {
+#ifdef ZT_NO_TYPE_PUNNING
+			if (!memcmp(pp,i->tail,32))
 				return false;
+#else
+			const uint64_t *t = i->tail;
+			const uint64_t *p = reinterpret_cast<const uint64_t *>(pp);
+			uint64_t bits = *(t++) ^ *(p++);
+			bits |= *(t++) ^ *(p++);
+			bits |= *(t++) ^ *(p++);
+			bits |= *t ^ *p;
+			if (!bits)
+				return false;
+#endif
+			++i;
 		}
 		return true;
 	}
 
 private:
-	struct ArItem
-	{
-		unsigned char tail[ZT_ANTIRECURSION_TAIL_LEN];
-		unsigned int len;
-	};
-	ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE];
-	volatile unsigned int _ptr;
+	struct _ArItem { uint64_t tail[4]; };
+	_ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE];
+	volatile unsigned long _ptr;
 };
 
 } // namespace ZeroTier
diff --git a/node/Constants.hpp b/node/Constants.hpp
index 5a4d4a4de..1d5fa6f42 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -309,11 +309,6 @@
  */
 #define ZT_NAT_T_TACTICAL_ESCALATION_DELAY 1000
 
-/**
- * Size of anti-recursion history (see AntiRecursion.hpp)
- */
-#define ZT_ANTIRECURSION_HISTORY_SIZE 16
-
 /**
  * Minimum delay between attempts to confirm new paths to peers (to avoid HELLO flooding)
  */

From b845dd1b88b966c7931524721d8369e6db240ed7 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 12:38:12 -0700
Subject: [PATCH 56/73] Set contact IP for real test.

---
 tests/http/agent.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 8a8b785e5..d0c33917b 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -10,7 +10,7 @@ var TEST_DURATION = (30 * 1000);
 var TEST_INTERVAL = (60 * 2 * 1000);
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '174.136.102.178';
+var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?

From f808138a942893188537bf82a064932a2523d34d Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 13:05:34 -0700
Subject: [PATCH 57/73] docs and stuff

---
 tests/http/README.md         |  4 +++-
 tests/http/run-a-big-test.sh | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100755 tests/http/run-a-big-test.sh

diff --git a/tests/http/README.md b/tests/http/README.md
index 58d4a9892..23a956054 100644
--- a/tests/http/README.md
+++ b/tests/http/README.md
@@ -1,10 +1,12 @@
 HTTP one-to-all test
 ======
 
-This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+*This is really internal use code. You're free to test it out but expect to do some editing/tweaking to make it work. We used this to run some massive scale tests of our new geo-cluster-based root server infrastructure prior to taking it live.*
 
 Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently. You will also want to edit the Dockerfile if you want to build containers and change the network ID to the network you want to run tests over.
 
+This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+
 The Dockerfile builds an image that launches the agent. The image must be "docker run" with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. (Unfortunately CAP_NET_ADMIN may not work due to a bug in Docker and/or Linux.) You can run a bunch with a command like:
 
     for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done
diff --git a/tests/http/run-a-big-test.sh b/tests/http/run-a-big-test.sh
new file mode 100755
index 000000000..1c125345d
--- /dev/null
+++ b/tests/http/run-a-big-test.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+docker ps -q | xargs -n 1 docker kill
+docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+for ((n=0;n<$NUM_CONTAINERS;n++)); do
+	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+done
+
+exit 0

From f974517f64aac6b527fefa8f3b30088b804d2ae2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 13:06:30 -0700
Subject: [PATCH 58/73] Save zerotier output in containers.

---
 tests/http/docker-main.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/http/docker-main.sh b/tests/http/docker-main.sh
index 720236687..f9e11de57 100755
--- a/tests/http/docker-main.sh
+++ b/tests/http/docker-main.sh
@@ -2,7 +2,7 @@
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
 
-/zerotier-one -d
+/zerotier-one -d >>zerotier-one.out 2>&1
 
 while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
 	sleep 0.25

From 377ccff600859da2d0e7ecd65a38953bd471d04d Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 13:39:28 -0700
Subject: [PATCH 59/73] getPeer() had a small potential to be unsafe.

---
 node/Topology.cpp | 38 +++++++++++++++++++++-----------------
 node/Topology.hpp |  2 +-
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/node/Topology.cpp b/node/Topology.cpp
index 9027eff16..031c0b1b6 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -146,26 +146,30 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 		return SharedPtr<Peer>();
 	}
 
-	Mutex::Lock _l(_lock);
-
-	SharedPtr<Peer> &ap = _peers[zta];
-
-	if (ap) {
-		ap->use(RR->node->now());
-		return ap;
+	{
+		Mutex::Lock _l(_lock);
+		const SharedPtr<Peer> *const ap = _peers.get(zta);
+		if (ap) {
+			(*ap)->use(RR->node->now());
+			return *ap;
+		}
 	}
 
-	Identity id(_getIdentity(zta));
-	if (id) {
-		try {
-			ap = SharedPtr<Peer>(new Peer(RR->identity,id));
-			ap->use(RR->node->now());
-			return ap;
-		} catch ( ... ) {} // invalid identity?
-	}
+	try {
+		Identity id(_getIdentity(zta));
+		if (id) {
+			SharedPtr<Peer> np(new Peer(RR->identity,id));
+			{
+				Mutex::Lock _l(_lock);
+				SharedPtr<Peer> &ap = _peers[zta];
+				if (!ap)
+					ap.swap(np);
+				ap->use(RR->node->now());
+				return ap;
+			}
+		}
+	} catch ( ... ) {} // invalid identity on disk?
 
-	// If we get here it means we read an invalid cache identity or had some other error
-	_peers.erase(zta);
 	return SharedPtr<Peer>();
 }
 
diff --git a/node/Topology.hpp b/node/Topology.hpp
index b9f063c85..d6f453aca 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -87,7 +87,7 @@ public:
 	inline SharedPtr<Peer> getPeerNoCache(const Address &zta,const uint64_t now)
 	{
 		Mutex::Lock _l(_lock);
-		const SharedPtr<Peer> *ap = _peers.get(zta);
+		const SharedPtr<Peer> *const ap = _peers.get(zta);
 		if (ap) {
 			(*ap)->use(now);
 			return *ap;

From d8dbbf7484f5df16a3e36b35da31383cc9589081 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 14:11:10 -0700
Subject: [PATCH 60/73] Add some debug code in TRACE mode to catch a bug.

---
 node/Topology.cpp | 12 +++++++-----
 node/Topology.hpp | 11 +++++++++--
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/node/Topology.cpp b/node/Topology.cpp
index 031c0b1b6..5e086116b 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -120,10 +120,12 @@ Topology::~Topology()
 
 SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
 {
-	if (peer->address() == RR->identity.address()) {
-		TRACE("BUG: addPeer() caught and ignored attempt to add peer for self");
-		throw std::logic_error("cannot add peer for self");
+#ifdef ZT_TRACE
+	if ((!peer)||(peer->address() == RR->identity.address())) {
+		TRACE("BUG: addPeer() caught and ignored attempt to add peer for self or add a NULL peer");
+		abort();
 	}
+#endif
 
 	SharedPtr<Peer> np;
 	{
@@ -133,6 +135,7 @@ SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
 			hp = peer;
 		np = hp;
 	}
+
 	np->use(RR->node->now());
 	saveIdentity(np->identity());
 
@@ -321,8 +324,7 @@ unsigned long Topology::countActive() const
 	Address *a = (Address *)0;
 	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 	while (i.next(a,p)) {
-		if ((*p)->hasActiveDirectPath(now))
-			++cnt;
+		cnt += (unsigned long)((*p)->hasActiveDirectPath(now));
 	}
 	return cnt;
 }
diff --git a/node/Topology.hpp b/node/Topology.hpp
index d6f453aca..999338666 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -234,8 +234,15 @@ public:
 		Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
-		while (i.next(a,p))
-			f(*this,*p);
+		while (i.next(a,p)) {
+#ifdef ZT_TRACE
+			if (!(*p)) {
+				ZT_TRACE("eachPeer() caught NULL peer for %s",a->toString().c_str());
+				abort();
+			}
+#endif
+			f(*this,*((const SharedPtr<Peer> *)p));
+		}
 	}
 
 	/**

From 2fbb5d0bbf206c6c87c30fc9b7cdf9b7975a5c0a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 14:11:45 -0700
Subject: [PATCH 61/73] .

---
 node/Topology.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Topology.hpp b/node/Topology.hpp
index 999338666..f74b130f5 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -237,7 +237,7 @@ public:
 		while (i.next(a,p)) {
 #ifdef ZT_TRACE
 			if (!(*p)) {
-				ZT_TRACE("eachPeer() caught NULL peer for %s",a->toString().c_str());
+				TRACE("eachPeer() caught NULL peer for %s",a->toString().c_str());
 				abort();
 			}
 #endif

From 641b0dec44d641f6b21c67d5807418d9c89e4033 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 14:12:20 -0700
Subject: [PATCH 62/73] .

---
 node/Topology.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Topology.hpp b/node/Topology.hpp
index f74b130f5..f48a89b2f 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -237,7 +237,7 @@ public:
 		while (i.next(a,p)) {
 #ifdef ZT_TRACE
 			if (!(*p)) {
-				TRACE("eachPeer() caught NULL peer for %s",a->toString().c_str());
+				fprintf(stderr,"eachPeer() caught NULL peer for %s",a->toString().c_str());
 				abort();
 			}
 #endif

From 7382c328daa33b5ff21baf78ee022f23092cda3b Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 14:23:28 -0700
Subject: [PATCH 63/73] Null pointer bug appears fixed... testing again at
 large scale.

---
 node/Topology.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/node/Topology.hpp b/node/Topology.hpp
index f48a89b2f..f7804c293 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -237,7 +237,7 @@ public:
 		while (i.next(a,p)) {
 #ifdef ZT_TRACE
 			if (!(*p)) {
-				fprintf(stderr,"eachPeer() caught NULL peer for %s",a->toString().c_str());
+				fprintf(stderr,"FATAL BUG: eachPeer() caught NULL peer for %s -- peer pointers in Topology should NEVER be NULL",a->toString().c_str());
 				abort();
 			}
 #endif

From 1b4cc4af5c7c0e47f73f3728ca36dc665d0e3224 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Fri, 30 Oct 2015 15:54:40 -0700
Subject: [PATCH 64/73] Fix evil bug, and instrument/assert on some other
 stuff, and a bit of cleanup.

---
 node/Cluster.cpp   |  2 +-
 node/Hashtable.hpp |  2 --
 node/Topology.cpp  | 33 ++++++++++++++++++---------------
 node/Topology.hpp  | 16 ++--------------
 selftest.cpp       | 28 +++++++++++++++++++++++++---
 5 files changed, 46 insertions(+), 35 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index 93b69a08a..d0daae437 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -476,7 +476,7 @@ struct _ClusterAnnouncePeers
 	_ClusterAnnouncePeers(const uint64_t now_,Cluster *parent_) : now(now_),parent(parent_) {}
 	const uint64_t now;
 	Cluster *const parent;
-	inline void operator()(const Topology &t,const SharedPtr<Peer> &peer)
+	inline void operator()(const Topology &t,const SharedPtr<Peer> &peer) const
 	{
 		Path *p = peer->getBestPath(now);
 		if (p)
diff --git a/node/Hashtable.hpp b/node/Hashtable.hpp
index 1d8d9e5d2..e3512fefd 100644
--- a/node/Hashtable.hpp
+++ b/node/Hashtable.hpp
@@ -322,7 +322,6 @@ public:
 		b->next = _t[bidx];
 		_t[bidx] = b;
 		++_s;
-
 		return b->v;
 	}
 
@@ -351,7 +350,6 @@ public:
 		b->next = _t[bidx];
 		_t[bidx] = b;
 		++_s;
-
 		return b->v;
 	}
 
diff --git a/node/Topology.cpp b/node/Topology.cpp
index 5e086116b..b8bb55f29 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -65,7 +65,7 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 			if (!p)
 				break; // stop if invalid records
 			if (p->address() != RR->identity.address())
-				_peers[p->address()] = p;
+				_peers.set(p->address(),p);
 		} catch ( ... ) {
 			break; // stop if invalid records
 		}
@@ -122,7 +122,9 @@ SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
 {
 #ifdef ZT_TRACE
 	if ((!peer)||(peer->address() == RR->identity.address())) {
-		TRACE("BUG: addPeer() caught and ignored attempt to add peer for self or add a NULL peer");
+		if (!peer)
+			fprintf(stderr,"FATAL BUG: addPeer() caught attempt to add NULL peer"ZT_EOL_S);
+		else fprintf(stderr,"FATAL BUG: addPeer() caught attempt to add peer for self"ZT_EOL_S);
 		abort();
 	}
 #endif
@@ -171,7 +173,10 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 				return ap;
 			}
 		}
-	} catch ( ... ) {} // invalid identity on disk?
+	} catch ( ... ) {
+		fprintf(stderr,"EXCEPTION in getPeer() part 2\n");
+		abort();
+	} // invalid identity on disk?
 
 	return SharedPtr<Peer>();
 }
@@ -180,9 +185,9 @@ Identity Topology::getIdentity(const Address &zta)
 {
 	{
 		Mutex::Lock _l(_lock);
-		SharedPtr<Peer> &ap = _peers[zta];
+		const SharedPtr<Peer> *const ap = _peers.get(zta);
 		if (ap)
-			return ap->identity();
+			return (*ap)->identity();
 	}
 	return _getIdentity(zta);
 }
@@ -207,18 +212,16 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 		 * causes packets searching for a route to pretty much literally
 		 * circumnavigate the globe rather than bouncing between just two. */
 
-		if (_rootAddresses.size() > 1) { // gotta be one other than me for this to work
-			for(unsigned long p=0;p<_rootAddresses.size();++p) {
-				if (_rootAddresses[p] == RR->identity.address()) {
-					for(unsigned long q=1;q<_rootAddresses.size();++q) {
-						SharedPtr<Peer> *nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]);
-						if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) {
-							(*nextsn)->use(now);
-							return *nextsn;
-						}
+		for(unsigned long p=0;p<_rootAddresses.size();++p) {
+			if (_rootAddresses[p] == RR->identity.address()) {
+				for(unsigned long q=1;q<_rootAddresses.size();++q) {
+					const SharedPtr<Peer> *const nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]);
+					if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now))) {
+						(*nextsn)->use(now);
+						return *nextsn;
 					}
-					break;
 				}
+				break;
 			}
 		}
 
diff --git a/node/Topology.hpp b/node/Topology.hpp
index f7804c293..4c1a2ab37 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -113,24 +113,12 @@ public:
 	 */
 	void saveIdentity(const Identity &id);
 
-	/**
-	 * @return Vector of peers that are root servers
-	 */
-	inline std::vector< SharedPtr<Peer> > rootPeers() const
-	{
-		Mutex::Lock _l(_lock);
-		return _rootPeers;
-	}
-
 	/**
 	 * Get the current favorite root server
 	 *
 	 * @return Root server with lowest latency or NULL if none
 	 */
-	inline SharedPtr<Peer> getBestRoot()
-	{
-		return getBestRoot((const Address *)0,0,false);
-	}
+	inline SharedPtr<Peer> getBestRoot() { return getBestRoot((const Address *)0,0,false); }
 
 	/**
 	 * Get the best root server, avoiding root servers listed in an array
@@ -237,7 +225,7 @@ public:
 		while (i.next(a,p)) {
 #ifdef ZT_TRACE
 			if (!(*p)) {
-				fprintf(stderr,"FATAL BUG: eachPeer() caught NULL peer for %s -- peer pointers in Topology should NEVER be NULL",a->toString().c_str());
+				fprintf(stderr,"FATAL BUG: eachPeer() caught NULL peer for %s -- peer pointers in Topology should NEVER be NULL"ZT_EOL_S,a->toString().c_str());
 				abort();
 			}
 #endif
diff --git a/selftest.cpp b/selftest.cpp
index fa5eec0fb..0787925f8 100644
--- a/selftest.cpp
+++ b/selftest.cpp
@@ -593,23 +593,44 @@ static int testOther()
 	std::cout << "[other] Testing Hashtable... "; std::cout.flush();
 	{
 		Hashtable<uint64_t,std::string> ht;
-		Hashtable<uint64_t,std::string> ht2;
 		std::map<uint64_t,std::string> ref; // assume std::map works correctly :)
 		for(int x=0;x<2;++x) {
-			for(int i=0;i<25000;++i) {
+			for(int i=0;i<77777;++i) {
 				uint64_t k = rand();
 				while ((k == 0)||(ref.count(k) > 0))
 					++k;
 				std::string v("!");
 				for(int j=0;j<(int)(k % 64);++j)
 					v.push_back("0123456789"[rand() % 10]);
-				ht.set(k,v);
 				ref[k] = v;
+				ht.set(0xffffffffffffffffULL,v);
+				std::string &vref = ht[k];
+				vref = v;
+				ht.erase(0xffffffffffffffffULL);
 			}
 			if (ht.size() != ref.size()) {
 				std::cout << "FAILED! (size mismatch, original)" << std::endl;
 				return -1;
 			}
+			{
+				Hashtable<uint64_t,std::string>::Iterator i(ht);
+				uint64_t *k = (uint64_t *)0;
+				std::string *v = (std::string *)0;
+				while(i.next(k,v)) {
+					if (ref.find(*k)->second != *v) {
+						std::cout << "FAILED! (data mismatch!)" << std::endl;
+						return -1;
+					}
+				}
+			}
+			for(std::map<uint64_t,std::string>::const_iterator i(ref.begin());i!=ref.end();++i) {
+				if (ht[i->first] != i->second) {
+					std::cout << "FAILED! (data mismatch!)" << std::endl;
+					return -1;
+				}
+			}
+
+			Hashtable<uint64_t,std::string> ht2;
 			ht2 = ht;
 			Hashtable<uint64_t,std::string> ht3(ht2);
 			if (ht2.size() != ref.size()) {
@@ -620,6 +641,7 @@ static int testOther()
 				std::cout << "FAILED! (size mismatch, copied)" << std::endl;
 				return -1;
 			}
+
 			for(std::map<uint64_t,std::string>::iterator i(ref.begin());i!=ref.end();++i) {
 				std::string *v = ht.get(i->first);
 				if (!v) {

From f1b6427e63d71b6be79e55bd1abf44ec519e2d11 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 09:32:56 -0800
Subject: [PATCH 65/73] Decided to make this 1.1.0 (semantic versioning
 increment is warranted), and add a legacy hack for older clients working with
 clusters.

---
 node/IncomingPacket.cpp | 31 ++++++++++++++++++++++++++++++-
 node/Packet.hpp         |  7 ++++---
 version.h               |  4 ++--
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp
index b0d65159b..32229ba65 100644
--- a/node/IncomingPacket.cpp
+++ b/node/IncomingPacket.cpp
@@ -294,7 +294,36 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
 		outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
 		outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
-		_remoteAddress.serialize(outp);
+		if (protoVersion >= 5) {
+			_remoteAddress.serialize(outp);
+		} else {
+			/* LEGACY COMPATIBILITY HACK:
+			 *
+			 * For a while now (since 1.0.3), ZeroTier has recognized changes in
+			 * its network environment empirically by examining its external network
+			 * address as reported by trusted peers. In versions prior to 1.1.0
+			 * (protocol version < 5), they did this by saving a snapshot of this
+			 * information (in SelfAwareness.hpp) keyed by reporting device ID and
+			 * address type.
+			 *
+			 * This causes problems when clustering is combined with symmetric NAT.
+			 * Symmetric NAT remaps ports, so different endpoints in a cluster will
+			 * report back different exterior addresses. Since the old code keys
+			 * this by device ID and not sending physical address and compares the
+			 * entire address including port, it constantly thinks its external
+			 * surface is changing and resets connections when talking to a cluster.
+			 *
+			 * In new code we key by sending physical address and device and we also
+			 * take the more conservative position of only interpreting changes in
+			 * IP address (neglecting port) as a change in network topology that
+			 * necessitates a reset. But we can make older clients work here by
+			 * nulling out the port field. Since this info is only used for empirical
+			 * detection of link changes, it doesn't break anything else.
+			 */
+			InetAddress tmpa(_remoteAddress);
+			tmpa.setPort(0);
+			tmpa.serialize(outp);
+		}
 
 		if ((worldId != ZT_WORLD_ID_NULL)&&(RR->topology->worldTimestamp() > worldTimestamp)&&(worldId == RR->topology->worldId())) {
 			World w(RR->topology->world());
diff --git a/node/Packet.hpp b/node/Packet.hpp
index 985d25d0e..63c49ce33 100644
--- a/node/Packet.hpp
+++ b/node/Packet.hpp
@@ -57,10 +57,11 @@
  *   + New crypto completely changes key agreement cipher
  * 4 - 0.6.0 ... 1.0.6
  *   + New identity format based on hashcash design
- * 5 - 1.0.6 ... CURRENT
+ * 5 - 1.1.0 ... CURRENT
  *   + Supports circuit test, proof of work, and echo
- *   + Supports in-band world (root definition) updates
- *   + Otherwise backward compatible with 4
+ *   + Supports in-band world (root server definition) updates
+ *   + Clustering! (Though this will work with protocol v4 clients.)
+ *   + Otherwise backward compatible with protocol v4
  */
 #define ZT_PROTO_VERSION 5
 
diff --git a/version.h b/version.h
index 4c36cb5e9..e27738776 100644
--- a/version.h
+++ b/version.h
@@ -36,11 +36,11 @@
 /**
  * Minor version
  */
-#define ZEROTIER_ONE_VERSION_MINOR 0
+#define ZEROTIER_ONE_VERSION_MINOR 1
 
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 6
+#define ZEROTIER_ONE_VERSION_REVISION 0
 
 #endif

From 29249db5d295b72cd28f73550ff7727b34fd5c9a Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 11:37:32 -0800
Subject: [PATCH 66/73] Big test stuff.

---
 tests/http/big-test-hosts                     |  4 +++
 .../{run-a-big-test.sh => big-test-kill.sh}   | 14 +++++----
 tests/http/big-test-start.sh                  | 30 +++++++++++++++++++
 3 files changed, 42 insertions(+), 6 deletions(-)
 create mode 100644 tests/http/big-test-hosts
 rename tests/http/{run-a-big-test.sh => big-test-kill.sh} (63%)
 create mode 100755 tests/http/big-test-start.sh

diff --git a/tests/http/big-test-hosts b/tests/http/big-test-hosts
new file mode 100644
index 000000000..27c0c6565
--- /dev/null
+++ b/tests/http/big-test-hosts
@@ -0,0 +1,4 @@
+root@104.156.246.48
+root@104.156.252.136
+root@46.101.72.130
+root@188.166.240.16
diff --git a/tests/http/run-a-big-test.sh b/tests/http/big-test-kill.sh
similarity index 63%
rename from tests/http/run-a-big-test.sh
rename to tests/http/big-test-kill.sh
index 1c125345d..fbb34c104 100755
--- a/tests/http/run-a-big-test.sh
+++ b/tests/http/big-test-kill.sh
@@ -14,15 +14,17 @@ CONTAINER_IMAGE=zerotier/http-test
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
 # Kill and clean up old test containers if any -- note that this kills all containers on the system!
-docker ps -q | xargs -n 1 docker kill
-docker ps -aq | xargs -n 1 docker rm
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
 
 # Pull latest if needed -- change this to your image name and/or where to pull it from
-docker pull $CONTAINER_IMAGE
+#docker pull $CONTAINER_IMAGE
 
 # Run NUM_CONTAINERS
-for ((n=0;n<$NUM_CONTAINERS;n++)); do
-	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
-done
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -p 256 "docker ps -q | xargs -r -n 128 docker kill && docker ps -aq | xargs -r -P 16 -n 1 docker rm"
 
 exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
new file mode 100755
index 000000000..79b6f93a3
--- /dev/null
+++ b/tests/http/big-test-start.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -o big-test-out -h big-test-hosts -i -p 256 "docker pull $CONTAINER_IMAGE && for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+
+exit 0

From e53ef9642e0201880672699ca12edd50d103be9e Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 12:31:34 -0800
Subject: [PATCH 67/73] test stuff.

---
 tests/http/agent.js                          | 26 ++++---
 tests/http/big-test-hosts                    |  2 -
 tests/http/big-test-kill.sh                  | 14 +---
 tests/http/big-test-out/root@104.156.246.48  | 67 +++++++++++++++++
 tests/http/big-test-out/root@104.156.252.136 | 75 ++++++++++++++++++++
 tests/http/big-test-out/root@188.166.240.16  |  0
 tests/http/big-test-out/root@46.101.72.130   |  0
 tests/http/big-test-ready.sh                 | 30 ++++++++
 tests/http/big-test-start.sh                 |  4 +-
 9 files changed, 193 insertions(+), 25 deletions(-)
 create mode 100644 tests/http/big-test-out/root@104.156.246.48
 create mode 100644 tests/http/big-test-out/root@104.156.252.136
 create mode 100644 tests/http/big-test-out/root@188.166.240.16
 create mode 100644 tests/http/big-test-out/root@46.101.72.130
 create mode 100755 tests/http/big-test-ready.sh

diff --git a/tests/http/agent.js b/tests/http/agent.js
index d0c33917b..e11fed60c 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -214,12 +214,22 @@ app.get('/',function(req,res) {
 });
 
 var expressServer = app.listen(AGENT_PORT,function () {
-	registerAndGetPeers(function(err,peers) {
-		if (err) {
-			console.error('FATAL: unable to contact or query server: '+err.toString());
-			process.exit(1);
-		}
-		doTestsAndReport();
-		setInterval(doTestsAndReport,TEST_INTERVAL);
-	});
+	var serverUp = false;
+	async.whilst(
+		function() { return (!serverUp); },
+		function(nextTry) {
+			registerAndGetPeers(function(err,peers) {
+				if ((err)||(!peers)) {
+					setTimeout(nextTry,1000);
+				} else {
+					serverUp = true;
+					return nextTry(null);
+				}
+			});
+		},
+		function(err) {
+			console.log('Server up, starting!');
+			doTestsAndReport();
+			setInterval(doTestsAndReport,TEST_INTERVAL);
+		});
 });
diff --git a/tests/http/big-test-hosts b/tests/http/big-test-hosts
index 27c0c6565..93b6f23f3 100644
--- a/tests/http/big-test-hosts
+++ b/tests/http/big-test-hosts
@@ -1,4 +1,2 @@
 root@104.156.246.48
 root@104.156.252.136
-root@46.101.72.130
-root@188.166.240.16
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index fbb34c104..917a7791c 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,18 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-# Kill and clean up old test containers if any -- note that this kills all containers on the system!
-#docker ps -q | xargs -n 1 docker kill
-#docker ps -aq | xargs -n 1 docker rm
-
-# Pull latest if needed -- change this to your image name and/or where to pull it from
-#docker pull $CONTAINER_IMAGE
-
-# Run NUM_CONTAINERS
-#for ((n=0;n<$NUM_CONTAINERS;n++)); do
-#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
-#done
-
-pssh -h big-test-hosts -i -p 256 "docker ps -q | xargs -r -n 128 docker kill && docker ps -aq | xargs -r -P 16 -n 1 docker rm"
+pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -q | xargs -r docker kill && docker ps -aq | xargs -r docker rm"
 
 exit 0
diff --git a/tests/http/big-test-out/root@104.156.246.48 b/tests/http/big-test-out/root@104.156.246.48
new file mode 100644
index 000000000..afcda19f2
--- /dev/null
+++ b/tests/http/big-test-out/root@104.156.246.48
@@ -0,0 +1,67 @@
+5fc1676570b10196c5e261dac6ff28998c4f66c706b131d6423532a38b8b7a15
+894bbfa612f772aa0170d5c5ddb9362a9edb0f5fbe22c81591b50dfa0a0eadeb
+bae9bf7c37806adbbdfe9e01b4e62034953d01ecae55e0cd450a21b1415f5c00
+26e83eccbd6083f2c7a8532786fd1ec5c96d6a51bb3508ec4fd1919a1883630a
+90e95ac0cd3b4ba6ffa33fe946a1f12a2cdba61168770af9fca7b42df45c9530
+88a11bbc89c0b3579de7faf4db4dbe3f0a5a073aa49fd7a5482eff35c93fab4e
+50be80765f09000085d6c0895958e6d3794f672f562ff5bc27634b15c77b7583
+99630786ef067f4c053f7bca96e2869f14c9ceb9f1d07b7635945c143700b950
+8422d4cb09ed44e34488163a96910ece8f1dd39c280daf3d369d17ce2ac70766
+e0a0af014418076adc351827fabbd9ef6da7404f2d6184f20d601736c4685154
+387cd6dbbd8e8b17d9ccf943fd13dcaddf27c178834a9630e357407a42733fb8
+b06b75cb0c5748321ab4501e1055dd5ff8457e81502548954702ead843011388
+59d874f5e36f15b62e8540b86d552950c72f086d90667780b6d82b5595326fce
+69dc1347f8671ba4bbe1da12f26e8f67b0980f1e2ad73bc0b77cc06e5cbf1b06
+68ef4b43d3f5559b5b0c82ba2f396a5b6dcb6001a67efd3a3b3b2a415c2b7b61
+3ecc9f45151f95a194d8274a88f433f83540f5397523de7a86db714cd9155bb2
+3b24b66b1dcd5e8ce1fa24d33cb2eb3cc55f3a157602a09bca4942089e25790f
+ba34465f6cedd2f306022cc9259ef4e43819959f51f980a8cb94ea33a29c7e34
+4e9a701f18a0ac42ee24f3bf2dd6dc442706a0bf4b6288336447b03752640852
+5865df2182165576a0825644d9b7537314c9fd4323cb2023260382acc2f9d7fd
+f1498fda2dd6d1f1fdfc95d539d6ee511ca8baad65b1a1b44d76309e84015550
+7d9232ecda50856096523dc2d0c1fae46481053067d8bfe024e1dfccf8f9c0fc
+173b47f327a3c21187e25d2d02dbde49760182a40e00cf4a64746c00537a3ff6
+9221fdb92693a29f720a5c41f09791c35f2b220459a2e4bdcc24fd5e3af4ad85
+ef8def0260ba3ff21aa292bb482bedf94b17fd9306e529f6feae5cab04e1bdc2
+7392179b9d51cad626b6cf78b00c355b280ddccccd7013e11a8cecbdd2db1c15
+fc31766af9265f2c496ee3847971ee0f5249aca8ad0cd214620643c7347436b2
+7f8bd6d1fde6948a09b132b4b2c3919ca11c932bfdf8a279c4cbe140daf24287
+ce9596c1d8dd42929820e4d3a56e8b1f8eb2d8b67474c525bbed1b888143cc14
+67c9de4f3d88daa89796e8a03306d2c5f2bcc4c409eef89e67c6ffc6a6282060
+705c1973e4a9ef77ffe6319375e585aec1d762a031c8b93ba9883af98e377590
+c7fdcc10eed007a30da7c7bbb8767d0b6a7287fadd67d935651f0bb265a71e1b
+5102e854fe72cdcd91b228d520a9380a36a273df7949de054355e8f99bd14e95
+77c37e503ccaa1350bf2fe16daaef486ec223348bc8f678b6f6d6bb477ae10c1
+fd97792d7bd61449163fa4e953212446e11cf02b27a45a2073a1de5148da63e8
+7d5b84b290f727713ad02f0817856c9b891996bdcc6ad4d0994005608cf9bbb7
+104c41c9cce7934f0e205cfc90b65f5ba89ca696cab41771a12b4384af7f6805
+75e1751a7e290a34faddea54c98e870b4a1cd5bb37c810cafabda8d3ba1faa5f
+2293a17e82ff54e04218c5aafd904079d15d71e47496249dc125f90f0960039f
+3f3563782349fddb61cdc638cf5f54030f726d9759cb104253f5b8b04ae8e2b0
+2328da71617e1abc0e2335e5974a70f45f8abaedba641292ad4d87e2f27a6b83
+00e79b478925c7b866f7d669ee73af1a7b377fb8bf22a04f2bd5356f256f59ba
+2960f757cca294c32abd51f072798f5457a1552de52bd42ca9233cd075da086f
+efbc248c9a3ce7b7a52724b67f14e27b02f5f37da295bc905d4f8fabe847cd00
+a068d735478e065236e840e50697a078d77aad9b82f906555279e5dee074db5d
+c4a366870dd1f3d1fb776b25f009d9079a6f7d0d83b03cd178b237b412d8dce6
+20340c7bd3bd9d32d6ecb7a451c377fb239bfe2d2c976886e2bd59746ed180ee
+8c522ecbabb9580528794d94f82bdef2982f0458b677598f236acc14e819e480
+8dfcc3407b050a39d82af263eec6e332bd69cb848ffa660344644ea10e3c2221
+74e06d9deba29982f9dccdb841163715dd419dee6a54eaf3ac987cd3458a1a2a
+725d990ca2ea34dc3e9acc02480cc8009a9c2016414c9c9cca3c7b135bd384f4
+120d7eac6a5bd761ee6acfb751f48bf7075c0316123326ce6bbb6ce3fe05c3a7
+91aa44d2650ff308877d9c81357619c51c0c0d05dc9f4c899df9ce460673b2aa
+3c203cd73c6be2606397357154789a94fbfe8670271825f75bbfc6c97fa0e048
+f9018c8390a472798c7a39bfe834cd01bb62bb4b0882dfe1108bf43334a3bd0e
+cdf8afae641e0423b2a7a1ff92cca80c7db478ff8deb1d81032808ac84415921
+9a5364df1df5460f6a9ed15e020b8bd283c47464556a7c5896707cee00c01a14
+df9365b8816e9d67f484adf94eef53aab236a92a588a1f3f650fd36b8073f7bb
+bd0172f67fa20716502e2bee82387de7f426e3cdb19d5ac6ce9dcc177f919cde
+90dd259c03b11625c09b8db614f45759e67edf07fe350681d273bfd988b45443
+9f69d376248b6851aa7837a7d09a1b9eda917601049e5942796815069d09a80f
+1a065bee20e8f4c6a91bb92ec9cca6ca16e8eb434798ed433a5248c48d91f596
+fb5a6a9397546a97ccdc4252603d5e774d8430195b06ec74926c48cf372b9906
+51cf0867773bb298140eb09c88d69587aae3a6da3e63275d32f3d32d98a737d7
+70574f3e616413ce90b045e0e9fd92353766e216eabf8139556fa61efea9c3b4
+ccd21257994f9eaf309d06b2fc5652b14ea80796a79face304fcc8fd1da53423
+7d8417be656f17fbfb779a9803b2de045e2f496f75ea5f1ea69e223572bde2a5
diff --git a/tests/http/big-test-out/root@104.156.252.136 b/tests/http/big-test-out/root@104.156.252.136
new file mode 100644
index 000000000..c11311b49
--- /dev/null
+++ b/tests/http/big-test-out/root@104.156.252.136
@@ -0,0 +1,75 @@
+5ea6049a0b92070494f40a5ccecccaf788a5aafccee7c2eada9b9eb8731bc002
+798f8beecd2e3fbb50df49b7ef57cdd1e8e00c0680046b3c2d53a3554f956fda
+dd20da6340210e1b7612d8922aaa4b045e84da32f264add073a65a15f676a9e4
+0479f14d0aa68e835c07dc5ca413febd9da19b6554fd8bdba7e319e5f4661f80
+df6747868f90cfa069f5f9f954626b7392cd99026e43e6d6c83ab7c16d5cbdc6
+c24799f74233b1bc7d7d936d57699b955000c640531f3db38be8196a87eb262c
+46b00a65d527738c0bfad924051bd2117563e0c6ad74b803b662e74720d8d085
+dc14b9428032771388d30c6002bb5cba05131972cab53360f088c51769786c47
+e7dc364aeccf60bafa5a42787cc6de231612782252f57b9f03ebae3a309b2352
+4e8e578a8948ab384525646a17c2e0cb9f2b9ef67fd0c489ad6aa2bffbfabddc
+cc626d978e32dfe14782011218ba265ae4e69886f44335a2c402001dc0c4c3c7
+36a148254d34f954906a810ba4a8644a4433e8847d3cc30e091b1f63723f0590
+1bdd06b691fa06b3af77d2868b78f2a01b91026b4f483fb278cb8872a9150987
+d6d5709039708a515b295519a4007c3b49361ef67465ddca2dfba9a473b9c37c
+f8a3a0b3dd5ce42bd87cdba28df7469071f948fb28151955cfa75abe0455a000
+f7eb37fecb17571091f05f1bfc66f6fba731ff934988a529813f4751451401af
+1c2f2c77c429095b1dad53032684df672f351489ed6b7e00e1097f7dc1c0ba97
+361dd9cc6764facd2aca0b462e64a2400f6b41e124c4d9be71466d801763270a
+3f5f49ce854439bc672c4b0cf4c1e1b9a978e8f2db14709977ddcfd39b7d6bb3
+5221bec1aec2c9f08ddec548a24b0700e3d0c0568d10caa753564c914b35c95f
+ebdcc27c264d326619262f82b5d7dfcbf102d720ad9bab4428b9725118bf627a
+bc01c863316a29c8f119da7cce1c891185c43d385521d46b06186a89cb6cff9d
+47fb8b119e3d098f22cc6f4a7867f2016f244cc8b114aa66630dadbd4bfb2a0c
+4ca2402d762adbd7ce860c3f3d072e948bb33afb7c2830ad51ae9d3fb2c714df
+328e4fda6dd1befcfda64d5c89c458fcc9386d88375218cb5197d479d3d292d0
+099636c67de66af06ee8493ccd55b588cf8bbacf67352877f37a077a2166117a
+624e0c032e4b8a78af1387c0199c5b02da68e0795e9b6397ba8bdd5ebcf7daa4
+265e038f17e4bc3c99737bf4ac98364c98a12d9d22a28b6a9302ce5a83c7d4f8
+de198984870126801aa20f25c459ea8f89bb7a4782614e91659b820c42a33c93
+ad48788c3f91a1292fb28dde84750f94e27ef150b2ebb52b7807f3d0c7986c4b
+34ab2f912eae5e75bbde977b8bb2952e552e36c83f29c2cd64d7ac3165aa9726
+951efa9c57da7eb2a1f6578927f809ab0c9feac2aa4326ea42f182e3ad74e600
+8b8193d9ebd89c36a728a3fb89282854bfb89b27172d93952747126c22ea6a97
+79a78a9f96f2c961b2d06dfd484a495ddc3809228243c608ecd51715a228e528
+4301dfc330137391585d36ddc4b54971999a1da96b1c12aa43221dd92cc32e79
+f956660447a893adbc0fc4382fe67ad2a7bb8591a647a130faa9e17aa380328a
+0cee22f18107559eff4f4eba20320c349d70d82b72197ae380fc514e6241730c
+d74601767fec3cff13062ad1393fad9a88277bbc6710c4fce6d78c21d001bfa4
+c1c46fd958b7806bf0c22c77c997317b7e4dffd7243d0a5919d4922fafd841a2
+a07e0f4e1cac4a84e3ca922b8c59a37ed8049096d9154be76e6d2d9094ba3714
+6861c27e7584e83a792ce710d004f7dc22213cc732b23f0a2025606cc5e9e325
+ee334626da5143ddf49561522483eb689663d031ba6cf9891204b709b279b28b
+3be351d25381f07b85c7f5c2bb08b80b5bbed80c348515187d727ccbf293b13b
+268da41127aa3b7768bbf6075baad5397f0af4f0ff16f5b8dff2cad9c3019750
+74cc47af92d6cf6b2315f62e0261d4461c6297c5ffc19b50a97784cd6271acfc
+8e52beef1bd61be4d223c460b589d32d1a64d5525406ab179d1962fccd734309
+9471871b2f1ec2331c1f9855b408d212ba868965f99aeee91ddd0b7dd76b2985
+16eb9e7e24b61ade0939345118d5d14032ae496de3b5fc702c0c1356662b8a80
+f503549eb9c03c8dc7e54559bf076e1f7eaf7c8599c84722062851597a4c91c1
+d9dce304b504003c97ffdf9d076ff348343d7d0ce50070038d049b71239bc70e
+2ed388e1730860b7605527213d7d61bbc8f29703a2f586d127b37e7ed8eca708
+d20988195a901d19597bf4ed2b136c3d2aed2d169093409a5d3ea8daf1f983af
+a9be22365634b68f0ff5ba9228550eba9b3923319eff07f5cf5785eec85fa11e
+99b512fe14b21569c81d358b161976c9dce45c608cd1a03da6f6063dbabb6c41
+88f76143cb289e8daf8edbc183a4b760c0d86efd1cc9da3933e59603bf92c539
+62054c1a23221461f1c06b94148614754f7da0cd7145eb85290bb11bdf7b9af8
+8dffffd5b63672f8749160dbff55c489f1f1f6c41ce01e086f2d9aa8bc6150ad
+f86a0512be3ec599065dec982545ecb96bdb0d82dda7285c58b3c3b7666988c9
+773786ea8094e7f013702ef721a2dde7657dcb6c44927f41f5acb32161957d83
+ef981b569d0c98e79bfc5bb387cf00457c4254e13f1ec625d0149b06ef92d5e1
+77841624d9fbba73cf722327f531ea36cfa42be59b4709c3b4ca4e7e453ac7fa
+f90ada865c766ef7e0e47d0a677b64183aa1e612f14f3f1f996d411b7197ea3e
+11c278763c1f2ff8d9b2ee5e3100a40538adcc1b74f6ca7d9668fcc8c1ab2f9a
+4a72c59eda21b43136cb5f308298aba39235bbd227668c2c9f3830fbab7b4a34
+f12a3d9308eb7800a1436b3258d899aae3b643f2c78648258f126266f9707032
+ce651585240975fc5c954c526203a048c6e4326e2d16feb083ca3a3ff4ad682a
+43aa6c22cb636983ff1b6e169057b0d7a70ad75754ef1b3a3bc7f49461c84cdb
+7e71c166a6f285c4c501b2125713e698575d1987f1819b76d4fabbbf246eae6c
+d38478eec0109a5b76da1a6e3de982382cf01bbc871b651d3e258330642cbe27
+c4c6ce60a0b1c4f2553ecb0f551de2c7df2e2d2cf2101e80af48b29e03cfefd5
+38905fb0f59281f1d4aa80894654e56df76653f3d3545a883e37c80053e72977
+1a6ee044aa753748035975b12885e73504ffde1bf129a7ff992f012d9cff111b
+f0d37e0a5ccb7871a30b143fad68a4e07624aa2a153e295022868e68e34ff770
+ee9d0ef6b557bd5e7c8fba9e087a428a98ee5350ec86785205db6ea10493b21c
+53ca280c12ba4d5be4ea78144fb2d411ecd9910f5105d04537d4bec362865c40
diff --git a/tests/http/big-test-out/root@188.166.240.16 b/tests/http/big-test-out/root@188.166.240.16
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/http/big-test-out/root@46.101.72.130 b/tests/http/big-test-out/root@46.101.72.130
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/http/big-test-ready.sh b/tests/http/big-test-ready.sh
new file mode 100755
index 000000000..391ca2a1b
--- /dev/null
+++ b/tests/http/big-test-ready.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -t 128 -p 256 "docker pull $CONTAINER_IMAGE"
+
+exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index 79b6f93a3..a4c7e6c12 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=100
+NUM_CONTAINERS=64
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -o big-test-out -h big-test-hosts -i -p 256 "docker pull $CONTAINER_IMAGE && for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+pssh -o big-test-out -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
 
 exit 0

From fd3916a49e7923e95c47c70afb8696f110b79951 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 13:17:11 -0800
Subject: [PATCH 68/73] More test stuff... make it more granular and less batch
 based.

---
 tests/http/agent.js                          | 132 ++++++++++++-------
 tests/http/big-test-kill.sh                  |   2 +-
 tests/http/big-test-out/root@104.156.246.48  |  67 ----------
 tests/http/big-test-out/root@104.156.252.136 |  75 -----------
 tests/http/big-test-out/root@188.166.240.16  |   0
 tests/http/big-test-out/root@46.101.72.130   |   0
 tests/http/big-test-start.sh                 |   4 +-
 tests/http/server.js                         |  42 ++----
 8 files changed, 101 insertions(+), 221 deletions(-)
 delete mode 100644 tests/http/big-test-out/root@104.156.246.48
 delete mode 100644 tests/http/big-test-out/root@104.156.252.136
 delete mode 100644 tests/http/big-test-out/root@188.166.240.16
 delete mode 100644 tests/http/big-test-out/root@46.101.72.130

diff --git a/tests/http/agent.js b/tests/http/agent.js
index e11fed60c..1d4a43200 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -3,21 +3,23 @@
 // ---------------------------------------------------------------------------
 // Customizable parameters:
 
-// Maximum test duration in milliseconds
-var TEST_DURATION = (30 * 1000);
+// Maximum interval between test attempts
+//var TEST_INTERVAL_MAX = (60 * 1 * 1000);
+var TEST_INTERVAL_MAX = 1000;
 
-// Interval between tests (should be several times longer than TEST_DURATION)
-var TEST_INTERVAL = (60 * 2 * 1000);
+// Test timeout in ms
+var TEST_TIMEOUT = 30000;
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '104.238.141.145';
+var SERVER_HOST = '127.0.0.1';
+//var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
 var AGENT_PORT = 18888;
 
 // Payload size in bytes
-var PAYLOAD_SIZE = 100000;
+var PAYLOAD_SIZE = 10000;
 
 // ---------------------------------------------------------------------------
 
@@ -72,9 +74,6 @@ for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
 	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
 }
 
-// Incremented with each test
-var testNumber = 0;
-
 function agentIdToIp(agentId)
 {
 	var ip = '';
@@ -96,32 +95,93 @@ function agentIdToIp(agentId)
 	return ip;
 };
 
-function registerAndGetPeers(callback)
+var lastTestResult = null;
+var allOtherAgents = [];
+
+function doTest()
 {
-	http.get({
+	var submit = http.request({
 		host: SERVER_HOST,
 		port: SERVER_PORT,
-		path: '/'+thisAgentId
+		path: '/'+thisAgentId,
+		method: 'POST'
 	},function(res) {
 		var body = '';
 		res.on('data',function(chunk) { body += chunk.toString(); });
 		res.on('end',function() {
-			if (!body)
-				return callback(null,[]);
-			try {
-				var peers = JSON.parse(body);
-				if (Array.isArray(peers))
-					return callback(null,peers);
-				else return callback(new Error('invalid JSON response from server'),null);
-			} catch (e) {
-				return callback(new Error('invalid JSON response from server: '+e.toString()),null);
+
+			if (body) {
+				try {
+					var peers = JSON.parse(body);
+					if (Array.isArray(peers))
+						allOtherAgents = peers;
+				} catch (e) {}
 			}
+
+			if (allOtherAgents.length > 0) {
+
+				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+
+				var testRequest = null;
+				var timeoutId = null;
+				timeoutId = setTimeout(function() {
+					if (testRequest !== null)
+						testRequest.abort();
+					timeoutId = null;
+				});
+				var startTime = Date.now();
+
+				testRequest = http.get({
+					host: agentIdToIp(target),
+					port: AGENT_PORT,
+					path: '/'
+				},function(res) {
+					var bytes = 0;
+					res.on('data',function(chunk) { bytes += chunk.length; });
+					res.on('end',function() {
+						lastTestResult = {
+							source: thisAgentId,
+							target: target,
+							time: (Date.now() - startTime),
+							bytes: bytes,
+							timedOut: (timeoutId === null),
+							error: null
+						};
+						if (timeoutId !== null)
+							clearTimeout(timeoutId);
+						return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+					});
+				}).on('error',function(e) {
+					lastTestResult = {
+						source: thisAgentId,
+						target: target,
+						time: (Date.now() - startTime),
+						bytes: 0,
+						timedOut: (timeoutId === null),
+						error: e.toString()
+					};
+					if (timeoutId !== null)
+						clearTimeout(timeoutId);
+					return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				});
+
+			} else {
+				return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+			}
+
 		});
 	}).on('error',function(e) {
-		return callback(e,null);
+		console.log('POST failed: '+e.toString());
+		return setTimeout(doTest,1000);
 	});
+	if (lastTestResult !== null) {
+		submit.write(JSON.stringify(lastTestResult));
+		lastTestResult = null;
+	}
+	submit.end();
 };
 
+/*
 function performTestOnAllPeers(peers,callback)
 {
 	var allResults = {};
@@ -191,11 +251,10 @@ function doTestsAndReport()
 			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
 		} else {
 			performTestOnAllPeers(peers,function(results) {
-				++testNumber;
 				var submit = http.request({
 					host: SERVER_HOST,
 					port: SERVER_PORT,
-					path: '/'+testNumber+'/'+thisAgentId,
+					path: '/'+thisAgentId,
 					method: 'POST'
 				},function(res) {
 				}).on('error',function(e) {
@@ -207,29 +266,12 @@ function doTestsAndReport()
 		}
 	});
 };
+*/
 
 // Agents just serve up a test payload
-app.get('/',function(req,res) {
-	return res.status(200).send(payload);
-});
+app.get('/',function(req,res) { return res.status(200).send(payload); });
 
 var expressServer = app.listen(AGENT_PORT,function () {
-	var serverUp = false;
-	async.whilst(
-		function() { return (!serverUp); },
-		function(nextTry) {
-			registerAndGetPeers(function(err,peers) {
-				if ((err)||(!peers)) {
-					setTimeout(nextTry,1000);
-				} else {
-					serverUp = true;
-					return nextTry(null);
-				}
-			});
-		},
-		function(err) {
-			console.log('Server up, starting!');
-			doTestsAndReport();
-			setInterval(doTestsAndReport,TEST_INTERVAL);
-		});
+	// Start timeout-based loop
+	doTest();
 });
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index 917a7791c..4a764d1fb 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,6 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -q | xargs -r docker kill && docker ps -aq | xargs -r docker rm"
+pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -aq | xargs -r docker rm -f"
 
 exit 0
diff --git a/tests/http/big-test-out/root@104.156.246.48 b/tests/http/big-test-out/root@104.156.246.48
deleted file mode 100644
index afcda19f2..000000000
--- a/tests/http/big-test-out/root@104.156.246.48
+++ /dev/null
@@ -1,67 +0,0 @@
-5fc1676570b10196c5e261dac6ff28998c4f66c706b131d6423532a38b8b7a15
-894bbfa612f772aa0170d5c5ddb9362a9edb0f5fbe22c81591b50dfa0a0eadeb
-bae9bf7c37806adbbdfe9e01b4e62034953d01ecae55e0cd450a21b1415f5c00
-26e83eccbd6083f2c7a8532786fd1ec5c96d6a51bb3508ec4fd1919a1883630a
-90e95ac0cd3b4ba6ffa33fe946a1f12a2cdba61168770af9fca7b42df45c9530
-88a11bbc89c0b3579de7faf4db4dbe3f0a5a073aa49fd7a5482eff35c93fab4e
-50be80765f09000085d6c0895958e6d3794f672f562ff5bc27634b15c77b7583
-99630786ef067f4c053f7bca96e2869f14c9ceb9f1d07b7635945c143700b950
-8422d4cb09ed44e34488163a96910ece8f1dd39c280daf3d369d17ce2ac70766
-e0a0af014418076adc351827fabbd9ef6da7404f2d6184f20d601736c4685154
-387cd6dbbd8e8b17d9ccf943fd13dcaddf27c178834a9630e357407a42733fb8
-b06b75cb0c5748321ab4501e1055dd5ff8457e81502548954702ead843011388
-59d874f5e36f15b62e8540b86d552950c72f086d90667780b6d82b5595326fce
-69dc1347f8671ba4bbe1da12f26e8f67b0980f1e2ad73bc0b77cc06e5cbf1b06
-68ef4b43d3f5559b5b0c82ba2f396a5b6dcb6001a67efd3a3b3b2a415c2b7b61
-3ecc9f45151f95a194d8274a88f433f83540f5397523de7a86db714cd9155bb2
-3b24b66b1dcd5e8ce1fa24d33cb2eb3cc55f3a157602a09bca4942089e25790f
-ba34465f6cedd2f306022cc9259ef4e43819959f51f980a8cb94ea33a29c7e34
-4e9a701f18a0ac42ee24f3bf2dd6dc442706a0bf4b6288336447b03752640852
-5865df2182165576a0825644d9b7537314c9fd4323cb2023260382acc2f9d7fd
-f1498fda2dd6d1f1fdfc95d539d6ee511ca8baad65b1a1b44d76309e84015550
-7d9232ecda50856096523dc2d0c1fae46481053067d8bfe024e1dfccf8f9c0fc
-173b47f327a3c21187e25d2d02dbde49760182a40e00cf4a64746c00537a3ff6
-9221fdb92693a29f720a5c41f09791c35f2b220459a2e4bdcc24fd5e3af4ad85
-ef8def0260ba3ff21aa292bb482bedf94b17fd9306e529f6feae5cab04e1bdc2
-7392179b9d51cad626b6cf78b00c355b280ddccccd7013e11a8cecbdd2db1c15
-fc31766af9265f2c496ee3847971ee0f5249aca8ad0cd214620643c7347436b2
-7f8bd6d1fde6948a09b132b4b2c3919ca11c932bfdf8a279c4cbe140daf24287
-ce9596c1d8dd42929820e4d3a56e8b1f8eb2d8b67474c525bbed1b888143cc14
-67c9de4f3d88daa89796e8a03306d2c5f2bcc4c409eef89e67c6ffc6a6282060
-705c1973e4a9ef77ffe6319375e585aec1d762a031c8b93ba9883af98e377590
-c7fdcc10eed007a30da7c7bbb8767d0b6a7287fadd67d935651f0bb265a71e1b
-5102e854fe72cdcd91b228d520a9380a36a273df7949de054355e8f99bd14e95
-77c37e503ccaa1350bf2fe16daaef486ec223348bc8f678b6f6d6bb477ae10c1
-fd97792d7bd61449163fa4e953212446e11cf02b27a45a2073a1de5148da63e8
-7d5b84b290f727713ad02f0817856c9b891996bdcc6ad4d0994005608cf9bbb7
-104c41c9cce7934f0e205cfc90b65f5ba89ca696cab41771a12b4384af7f6805
-75e1751a7e290a34faddea54c98e870b4a1cd5bb37c810cafabda8d3ba1faa5f
-2293a17e82ff54e04218c5aafd904079d15d71e47496249dc125f90f0960039f
-3f3563782349fddb61cdc638cf5f54030f726d9759cb104253f5b8b04ae8e2b0
-2328da71617e1abc0e2335e5974a70f45f8abaedba641292ad4d87e2f27a6b83
-00e79b478925c7b866f7d669ee73af1a7b377fb8bf22a04f2bd5356f256f59ba
-2960f757cca294c32abd51f072798f5457a1552de52bd42ca9233cd075da086f
-efbc248c9a3ce7b7a52724b67f14e27b02f5f37da295bc905d4f8fabe847cd00
-a068d735478e065236e840e50697a078d77aad9b82f906555279e5dee074db5d
-c4a366870dd1f3d1fb776b25f009d9079a6f7d0d83b03cd178b237b412d8dce6
-20340c7bd3bd9d32d6ecb7a451c377fb239bfe2d2c976886e2bd59746ed180ee
-8c522ecbabb9580528794d94f82bdef2982f0458b677598f236acc14e819e480
-8dfcc3407b050a39d82af263eec6e332bd69cb848ffa660344644ea10e3c2221
-74e06d9deba29982f9dccdb841163715dd419dee6a54eaf3ac987cd3458a1a2a
-725d990ca2ea34dc3e9acc02480cc8009a9c2016414c9c9cca3c7b135bd384f4
-120d7eac6a5bd761ee6acfb751f48bf7075c0316123326ce6bbb6ce3fe05c3a7
-91aa44d2650ff308877d9c81357619c51c0c0d05dc9f4c899df9ce460673b2aa
-3c203cd73c6be2606397357154789a94fbfe8670271825f75bbfc6c97fa0e048
-f9018c8390a472798c7a39bfe834cd01bb62bb4b0882dfe1108bf43334a3bd0e
-cdf8afae641e0423b2a7a1ff92cca80c7db478ff8deb1d81032808ac84415921
-9a5364df1df5460f6a9ed15e020b8bd283c47464556a7c5896707cee00c01a14
-df9365b8816e9d67f484adf94eef53aab236a92a588a1f3f650fd36b8073f7bb
-bd0172f67fa20716502e2bee82387de7f426e3cdb19d5ac6ce9dcc177f919cde
-90dd259c03b11625c09b8db614f45759e67edf07fe350681d273bfd988b45443
-9f69d376248b6851aa7837a7d09a1b9eda917601049e5942796815069d09a80f
-1a065bee20e8f4c6a91bb92ec9cca6ca16e8eb434798ed433a5248c48d91f596
-fb5a6a9397546a97ccdc4252603d5e774d8430195b06ec74926c48cf372b9906
-51cf0867773bb298140eb09c88d69587aae3a6da3e63275d32f3d32d98a737d7
-70574f3e616413ce90b045e0e9fd92353766e216eabf8139556fa61efea9c3b4
-ccd21257994f9eaf309d06b2fc5652b14ea80796a79face304fcc8fd1da53423
-7d8417be656f17fbfb779a9803b2de045e2f496f75ea5f1ea69e223572bde2a5
diff --git a/tests/http/big-test-out/root@104.156.252.136 b/tests/http/big-test-out/root@104.156.252.136
deleted file mode 100644
index c11311b49..000000000
--- a/tests/http/big-test-out/root@104.156.252.136
+++ /dev/null
@@ -1,75 +0,0 @@
-5ea6049a0b92070494f40a5ccecccaf788a5aafccee7c2eada9b9eb8731bc002
-798f8beecd2e3fbb50df49b7ef57cdd1e8e00c0680046b3c2d53a3554f956fda
-dd20da6340210e1b7612d8922aaa4b045e84da32f264add073a65a15f676a9e4
-0479f14d0aa68e835c07dc5ca413febd9da19b6554fd8bdba7e319e5f4661f80
-df6747868f90cfa069f5f9f954626b7392cd99026e43e6d6c83ab7c16d5cbdc6
-c24799f74233b1bc7d7d936d57699b955000c640531f3db38be8196a87eb262c
-46b00a65d527738c0bfad924051bd2117563e0c6ad74b803b662e74720d8d085
-dc14b9428032771388d30c6002bb5cba05131972cab53360f088c51769786c47
-e7dc364aeccf60bafa5a42787cc6de231612782252f57b9f03ebae3a309b2352
-4e8e578a8948ab384525646a17c2e0cb9f2b9ef67fd0c489ad6aa2bffbfabddc
-cc626d978e32dfe14782011218ba265ae4e69886f44335a2c402001dc0c4c3c7
-36a148254d34f954906a810ba4a8644a4433e8847d3cc30e091b1f63723f0590
-1bdd06b691fa06b3af77d2868b78f2a01b91026b4f483fb278cb8872a9150987
-d6d5709039708a515b295519a4007c3b49361ef67465ddca2dfba9a473b9c37c
-f8a3a0b3dd5ce42bd87cdba28df7469071f948fb28151955cfa75abe0455a000
-f7eb37fecb17571091f05f1bfc66f6fba731ff934988a529813f4751451401af
-1c2f2c77c429095b1dad53032684df672f351489ed6b7e00e1097f7dc1c0ba97
-361dd9cc6764facd2aca0b462e64a2400f6b41e124c4d9be71466d801763270a
-3f5f49ce854439bc672c4b0cf4c1e1b9a978e8f2db14709977ddcfd39b7d6bb3
-5221bec1aec2c9f08ddec548a24b0700e3d0c0568d10caa753564c914b35c95f
-ebdcc27c264d326619262f82b5d7dfcbf102d720ad9bab4428b9725118bf627a
-bc01c863316a29c8f119da7cce1c891185c43d385521d46b06186a89cb6cff9d
-47fb8b119e3d098f22cc6f4a7867f2016f244cc8b114aa66630dadbd4bfb2a0c
-4ca2402d762adbd7ce860c3f3d072e948bb33afb7c2830ad51ae9d3fb2c714df
-328e4fda6dd1befcfda64d5c89c458fcc9386d88375218cb5197d479d3d292d0
-099636c67de66af06ee8493ccd55b588cf8bbacf67352877f37a077a2166117a
-624e0c032e4b8a78af1387c0199c5b02da68e0795e9b6397ba8bdd5ebcf7daa4
-265e038f17e4bc3c99737bf4ac98364c98a12d9d22a28b6a9302ce5a83c7d4f8
-de198984870126801aa20f25c459ea8f89bb7a4782614e91659b820c42a33c93
-ad48788c3f91a1292fb28dde84750f94e27ef150b2ebb52b7807f3d0c7986c4b
-34ab2f912eae5e75bbde977b8bb2952e552e36c83f29c2cd64d7ac3165aa9726
-951efa9c57da7eb2a1f6578927f809ab0c9feac2aa4326ea42f182e3ad74e600
-8b8193d9ebd89c36a728a3fb89282854bfb89b27172d93952747126c22ea6a97
-79a78a9f96f2c961b2d06dfd484a495ddc3809228243c608ecd51715a228e528
-4301dfc330137391585d36ddc4b54971999a1da96b1c12aa43221dd92cc32e79
-f956660447a893adbc0fc4382fe67ad2a7bb8591a647a130faa9e17aa380328a
-0cee22f18107559eff4f4eba20320c349d70d82b72197ae380fc514e6241730c
-d74601767fec3cff13062ad1393fad9a88277bbc6710c4fce6d78c21d001bfa4
-c1c46fd958b7806bf0c22c77c997317b7e4dffd7243d0a5919d4922fafd841a2
-a07e0f4e1cac4a84e3ca922b8c59a37ed8049096d9154be76e6d2d9094ba3714
-6861c27e7584e83a792ce710d004f7dc22213cc732b23f0a2025606cc5e9e325
-ee334626da5143ddf49561522483eb689663d031ba6cf9891204b709b279b28b
-3be351d25381f07b85c7f5c2bb08b80b5bbed80c348515187d727ccbf293b13b
-268da41127aa3b7768bbf6075baad5397f0af4f0ff16f5b8dff2cad9c3019750
-74cc47af92d6cf6b2315f62e0261d4461c6297c5ffc19b50a97784cd6271acfc
-8e52beef1bd61be4d223c460b589d32d1a64d5525406ab179d1962fccd734309
-9471871b2f1ec2331c1f9855b408d212ba868965f99aeee91ddd0b7dd76b2985
-16eb9e7e24b61ade0939345118d5d14032ae496de3b5fc702c0c1356662b8a80
-f503549eb9c03c8dc7e54559bf076e1f7eaf7c8599c84722062851597a4c91c1
-d9dce304b504003c97ffdf9d076ff348343d7d0ce50070038d049b71239bc70e
-2ed388e1730860b7605527213d7d61bbc8f29703a2f586d127b37e7ed8eca708
-d20988195a901d19597bf4ed2b136c3d2aed2d169093409a5d3ea8daf1f983af
-a9be22365634b68f0ff5ba9228550eba9b3923319eff07f5cf5785eec85fa11e
-99b512fe14b21569c81d358b161976c9dce45c608cd1a03da6f6063dbabb6c41
-88f76143cb289e8daf8edbc183a4b760c0d86efd1cc9da3933e59603bf92c539
-62054c1a23221461f1c06b94148614754f7da0cd7145eb85290bb11bdf7b9af8
-8dffffd5b63672f8749160dbff55c489f1f1f6c41ce01e086f2d9aa8bc6150ad
-f86a0512be3ec599065dec982545ecb96bdb0d82dda7285c58b3c3b7666988c9
-773786ea8094e7f013702ef721a2dde7657dcb6c44927f41f5acb32161957d83
-ef981b569d0c98e79bfc5bb387cf00457c4254e13f1ec625d0149b06ef92d5e1
-77841624d9fbba73cf722327f531ea36cfa42be59b4709c3b4ca4e7e453ac7fa
-f90ada865c766ef7e0e47d0a677b64183aa1e612f14f3f1f996d411b7197ea3e
-11c278763c1f2ff8d9b2ee5e3100a40538adcc1b74f6ca7d9668fcc8c1ab2f9a
-4a72c59eda21b43136cb5f308298aba39235bbd227668c2c9f3830fbab7b4a34
-f12a3d9308eb7800a1436b3258d899aae3b643f2c78648258f126266f9707032
-ce651585240975fc5c954c526203a048c6e4326e2d16feb083ca3a3ff4ad682a
-43aa6c22cb636983ff1b6e169057b0d7a70ad75754ef1b3a3bc7f49461c84cdb
-7e71c166a6f285c4c501b2125713e698575d1987f1819b76d4fabbbf246eae6c
-d38478eec0109a5b76da1a6e3de982382cf01bbc871b651d3e258330642cbe27
-c4c6ce60a0b1c4f2553ecb0f551de2c7df2e2d2cf2101e80af48b29e03cfefd5
-38905fb0f59281f1d4aa80894654e56df76653f3d3545a883e37c80053e72977
-1a6ee044aa753748035975b12885e73504ffde1bf129a7ff992f012d9cff111b
-f0d37e0a5ccb7871a30b143fad68a4e07624aa2a153e295022868e68e34ff770
-ee9d0ef6b557bd5e7c8fba9e087a428a98ee5350ec86785205db6ea10493b21c
-53ca280c12ba4d5be4ea78144fb2d411ecd9910f5105d04537d4bec362865c40
diff --git a/tests/http/big-test-out/root@188.166.240.16 b/tests/http/big-test-out/root@188.166.240.16
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/http/big-test-out/root@46.101.72.130 b/tests/http/big-test-out/root@46.101.72.130
deleted file mode 100644
index e69de29bb..000000000
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index a4c7e6c12..a5e71ef14 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=64
+NUM_CONTAINERS=100
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -o big-test-out -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; done"
+pssh -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0
diff --git a/tests/http/server.js b/tests/http/server.js
index 30d8339a3..57109392f 100644
--- a/tests/http/server.js
+++ b/tests/http/server.js
@@ -20,44 +20,24 @@ app.use(function(req,res,next) {
 
 var knownAgents = {};
 
-app.get('/:agentId',function(req,res) {
+app.post('/:agentId',function(req,res) {
 	var agentId = req.params.agentId;
 	if ((!agentId)||(agentId.length !== 32))
 		return res.status(404).send('');
+
+	if (req.rawBody) {
+		var receiveTime = Date.now();
+		var resultData = null;
+		try {
+			resultData = JSON.parse(req.rawBody);
+			console.log(resultData.source+','+resultData.target+','+resultData.time+','+resultData.bytes+','+resultData.timedOut+',"'+((resultData.error) ? resultData.error : '')+'"');
+		} catch (e) {}
+	}
+
 	knownAgents[agentId] = Date.now();
 	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
 });
 
-app.post('/:testNumber/:agentId',function(req,res) {
-	var testNumber = req.params.testNumber;
-	var agentId = req.params.agentId;
-	if ((!agentId)||(agentId.length !== 32))
-		return res.status(404).send('');
-
-	var receiveTime = Date.now();
-	var resultData = null;
-	try {
-		resultData = JSON.parse(req.rawBody);
-	} catch (e) {
-		resultData = req.rawBody;
-	}
-	result = {
-		agentId: agentId,
-		testNumber: parseInt(testNumber),
-		receiveTime: receiveTime,
-		results: resultData
-	};
-
-	testNumber = testNumber.toString();
-	while (testNumber.length < 10)
-		testNumber = '0' + testNumber;
-	fs.writeFile('result_'+testNumber+'_'+agentId,JSON.stringify(result),function(err) {
-		console.log(result);
-	});
-
-	return res.status(200).send('');
-});
-
 var expressServer = app.listen(SERVER_PORT,function () {
 	console.log('LISTENING ON '+SERVER_PORT);
 	console.log('');

From ab27a91b07278146975087e873577bed43793554 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 13:53:27 -0800
Subject: [PATCH 69/73] .

---
 tests/http/agent.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/http/agent.js b/tests/http/agent.js
index 1d4a43200..bc7c475e6 100644
--- a/tests/http/agent.js
+++ b/tests/http/agent.js
@@ -4,15 +4,13 @@
 // Customizable parameters:
 
 // Maximum interval between test attempts
-//var TEST_INTERVAL_MAX = (60 * 1 * 1000);
-var TEST_INTERVAL_MAX = 1000;
+var TEST_INTERVAL_MAX = 60000;
 
 // Test timeout in ms
 var TEST_TIMEOUT = 30000;
 
 // Where should I contact to register and query a list of other test agents?
-var SERVER_HOST = '127.0.0.1';
-//var SERVER_HOST = '104.238.141.145';
+var SERVER_HOST = '104.238.141.145';
 var SERVER_PORT = 18080;
 
 // Which port should agents use for their HTTP?
@@ -118,9 +116,11 @@ function doTest()
 				} catch (e) {}
 			}
 
-			if (allOtherAgents.length > 0) {
+			if (allOtherAgents.length > 1) {
 
 				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+				while (target === thisAgentId)
+					target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
 
 				var testRequest = null;
 				var timeoutId = null;
@@ -128,7 +128,7 @@ function doTest()
 					if (testRequest !== null)
 						testRequest.abort();
 					timeoutId = null;
-				});
+				},TEST_TIMEOUT);
 				var startTime = Date.now();
 
 				testRequest = http.get({
@@ -166,7 +166,7 @@ function doTest()
 				});
 
 			} else {
-				return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				return setTimeout(doTest,1000);
 			}
 
 		});

From 60ce886605c0298fc22dbce48beb106a96bd35e2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 15:15:20 -0800
Subject: [PATCH 70/73] Tweak some timings for better reliability.

---
 node/Cluster.cpp             |   6 +-
 node/Cluster.hpp             |   8 +-
 node/Constants.hpp           |  24 +---
 node/Multicaster.cpp         | 216 ++++++++++++++++++-----------------
 node/Node.cpp                |  13 +--
 tests/http/big-test-kill.sh  |   2 +-
 tests/http/big-test-ready.sh |   2 +-
 tests/http/big-test-start.sh |   4 +-
 8 files changed, 129 insertions(+), 146 deletions(-)

diff --git a/node/Cluster.cpp b/node/Cluster.cpp
index d0daae437..e9e31edeb 100644
--- a/node/Cluster.cpp
+++ b/node/Cluster.cpp
@@ -85,7 +85,8 @@ Cluster::Cluster(
 	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
 	_peerAffinities(65536),
 	_lastCleanedPeerAffinities(0),
-	_lastCheckedPeersForAnnounce(0)
+	_lastCheckedPeersForAnnounce(0),
+	_lastFlushed(0)
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -510,7 +511,8 @@ void Cluster::doPeriodicTasks()
 	}
 
 	// Flush outgoing packet send queue every doPeriodicTasks()
-	{
+	if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) {
+		_lastFlushed = now;
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
 			Mutex::Lock _l2(_members[*mid].lock);
diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index 7d7a1ced4..f1caa436d 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -55,13 +55,18 @@
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ((ZT_PEER_ACTIVITY_TIMEOUT / 2) - 1000)
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2)
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
 #define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
 
+/**
+ * How often to flush outgoing message queues (maximum interval)
+ */
+#define ZT_CLUSTER_FLUSH_PERIOD 500
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -355,6 +360,7 @@ private:
 
 	uint64_t _lastCleanedPeerAffinities;
 	uint64_t _lastCheckedPeersForAnnounce;
+	uint64_t _lastFlushed;
 };
 
 } // namespace ZeroTier
diff --git a/node/Constants.hpp b/node/Constants.hpp
index 1d5fa6f42..bb62484d0 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -173,13 +173,8 @@
 
 /**
  * Timeout for receipt of fragmented packets in ms
- *
- * Since there's no retransmits, this is just a really bad case scenario for
- * transit time. It's short enough that a DOS attack from exhausing buffers is
- * very unlikely, as the transfer rate would have to be fast enough to fill
- * system memory in this time.
  */
-#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000
+#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 500
 
 /**
  * Length of secret key in bytes -- 256-bit -- do not change
@@ -194,7 +189,7 @@
 /**
  * Overriding granularity for timer tasks to prevent CPU-intensive thrashing on every packet
  */
-#define ZT_CORE_TIMER_TASK_GRANULARITY 1000
+#define ZT_CORE_TIMER_TASK_GRANULARITY 500
 
 /**
  * How long to remember peer records in RAM if they haven't been used
@@ -269,7 +264,7 @@
 /**
  * Delay between ordinary case pings of direct links
  */
-#define ZT_PEER_DIRECT_PING_DELAY 120000
+#define ZT_PEER_DIRECT_PING_DELAY 60000
 
 /**
  * Delay between requests for updated network autoconf information
@@ -279,18 +274,7 @@
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
-#define ZT_PEER_ACTIVITY_TIMEOUT (ZT_PEER_DIRECT_PING_DELAY + (ZT_PING_CHECK_INVERVAL * 3))
-
-/**
- * Stop relaying via peers that have not responded to direct sends
- *
- * When we send something (including frames), we generally expect a response.
- * Switching relays if no response in a short period of time causes more
- * rapid failover if a root server goes down or becomes unreachable. In the
- * mistaken case, little harm is done as it'll pick the next-fastest
- * root server and will switch back eventually.
- */
-#define ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD 10000
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 3) + (ZT_PING_CHECK_INVERVAL * 2))
 
 /**
  * Minimum interval between attempts by relays to unite peers
diff --git a/node/Multicaster.cpp b/node/Multicaster.cpp
index e43d7d886..01e6b7999 100644
--- a/node/Multicaster.cpp
+++ b/node/Multicaster.cpp
@@ -175,128 +175,130 @@ void Multicaster::send(
 	unsigned long idxbuf[8194];
 	unsigned long *indexes = idxbuf;
 
-	Mutex::Lock _l(_groups_m);
-	MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
+	try {
+		Mutex::Lock _l(_groups_m);
+		MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
 
-	if (!gs.members.empty()) {
-		// Allocate a memory buffer if group is monstrous
-		if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
-			indexes = new unsigned long[gs.members.size()];
+		if (!gs.members.empty()) {
+			// Allocate a memory buffer if group is monstrous
+			if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
+				indexes = new unsigned long[gs.members.size()];
 
-		// Generate a random permutation of member indexes
-		for(unsigned long i=0;i<gs.members.size();++i)
-			indexes[i] = i;
-		for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
-			unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
-			unsigned long tmp = indexes[j];
-			indexes[j] = indexes[i];
-			indexes[i] = tmp;
-		}
-	}
-
-	if (gs.members.size() >= limit) {
-		// Skip queue if we already have enough members to complete the send operation
-		OutboundMulticast out;
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			1, // we'll still gather a little from peers to keep multicast list fresh
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendOnly(RR,*ast);
-				if (++count >= limit)
-					break;
+			// Generate a random permutation of member indexes
+			for(unsigned long i=0;i<gs.members.size();++i)
+				indexes[i] = i;
+			for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
+				unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
+				unsigned long tmp = indexes[j];
+				indexes[j] = indexes[i];
+				indexes[i] = tmp;
 			}
 		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendOnly(RR,ma);
-				++count;
-			}
-		}
-	} else {
-		unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
+		if (gs.members.size() >= limit) {
+			// Skip queue if we already have enough members to complete the send operation
+			OutboundMulticast out;
 
-		if ((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY) {
-			gs.lastExplicitGather = now;
-			SharedPtr<Peer> r(RR->topology->getBestRoot());
-			if (r) {
-				TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				1, // we'll still gather a little from peers to keep multicast list fresh
+				src,
+				mg,
+				etherType,
+				data,
+				len);
 
-				const CertificateOfMembership *com = (CertificateOfMembership *)0;
-				{
-					SharedPtr<Network> nw(RR->node->network(nwid));
-					if (nw) {
-						SharedPtr<NetworkConfig> nconf(nw->config2());
-						if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
-							com = &(nconf->com());
-					}
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send
+					if (++count >= limit)
+						break;
 				}
-
-				Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
-				outp.append(nwid);
-				outp.append((uint8_t)(com ? 0x01 : 0x00));
-				mg.mac().appendTo(outp);
-				outp.append((uint32_t)mg.adi());
-				outp.append((uint32_t)gatherLimit);
-				if (com)
-					com->serialize(outp);
-				outp.armor(r->key(),true);
-				r->send(RR,outp.data(),outp.size(),now);
 			}
-			gatherLimit = 0;
-		}
 
-		gs.txQueue.push_back(OutboundMulticast());
-		OutboundMulticast &out = gs.txQueue.back();
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send
+					++count;
+				}
+			}
+		} else {
+			unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
 
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			gatherLimit,
-			src,
-			mg,
-			etherType,
-			data,
-			len);
+			if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) {
+				gs.lastExplicitGather = now;
+				SharedPtr<Peer> r(RR->topology->getBestRoot());
+				if (r) {
+					TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
 
-		unsigned int count = 0;
+					const CertificateOfMembership *com = (CertificateOfMembership *)0;
+					{
+						SharedPtr<Network> nw(RR->node->network(nwid));
+						if (nw) {
+							SharedPtr<NetworkConfig> nconf(nw->config2());
+							if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
+								com = &(nconf->com());
+						}
+					}
 
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendAndLog(RR,*ast);
-				if (++count >= limit)
-					break;
+					Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
+					outp.append(nwid);
+					outp.append((uint8_t)(com ? 0x01 : 0x00));
+					mg.mac().appendTo(outp);
+					outp.append((uint32_t)mg.adi());
+					outp.append((uint32_t)gatherLimit);
+					if (com)
+						com->serialize(outp);
+					outp.armor(r->key(),true);
+					r->send(RR,outp.data(),outp.size(),now);
+				}
+				gatherLimit = 0;
+			}
+
+			gs.txQueue.push_back(OutboundMulticast());
+			OutboundMulticast &out = gs.txQueue.back();
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				gatherLimit,
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendAndLog(RR,*ast);
+					if (++count >= limit)
+						break;
+				}
+			}
+
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendAndLog(RR,ma);
+					++count;
+				}
 			}
 		}
-
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendAndLog(RR,ma);
-				++count;
-			}
-		}
-	}
+	} catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted
 
 	// Free allocated memory buffer if any
 	if (indexes != idxbuf)
diff --git a/node/Node.cpp b/node/Node.cpp
index 42180e990..74acc869b 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -305,18 +305,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Attempt to contact network preferred relays that we don't have direct links to
-			std::sort(networkRelays.begin(),networkRelays.end());
-			networkRelays.erase(std::unique(networkRelays.begin(),networkRelays.end()),networkRelays.end());
-			for(std::vector< std::pair<Address,InetAddress> >::const_iterator nr(networkRelays.begin());nr!=networkRelays.end();++nr) {
-				if (nr->second) {
-					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
-					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,InetAddress(),nr->second,now);
-				}
-			}
-
-			// Ping living or root server/relay peers
+			// Do pings and keepalives
 			_PingPeersThatNeedPing pfunc(RR,now,networkRelays);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
diff --git a/tests/http/big-test-kill.sh b/tests/http/big-test-kill.sh
index 4a764d1fb..59f367884 100755
--- a/tests/http/big-test-kill.sh
+++ b/tests/http/big-test-kill.sh
@@ -13,6 +13,6 @@ CONTAINER_IMAGE=zerotier/http-test
 
 export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker ps -aq | xargs -r docker rm -f"
+pssh -h big-test-hosts -i -t 0 -p 256 "docker ps -aq | xargs -r docker rm -f"
 
 exit 0
diff --git a/tests/http/big-test-ready.sh b/tests/http/big-test-ready.sh
index 391ca2a1b..aa540bba2 100755
--- a/tests/http/big-test-ready.sh
+++ b/tests/http/big-test-ready.sh
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -h big-test-hosts -i -t 128 -p 256 "docker pull $CONTAINER_IMAGE"
+pssh -h big-test-hosts -i -t 0 -p 256 "docker pull $CONTAINER_IMAGE"
 
 exit 0
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index a5e71ef14..43166c6eb 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=100
+NUM_CONTAINERS=25
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -h big-test-hosts -i -t 128 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
+pssh -h big-test-hosts -i -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0

From 7fbe2f7adf3575f3a21fc1ab3a5a2a036e18e6e2 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 15:38:53 -0800
Subject: [PATCH 71/73] Tweak some more timings for better reliability.

---
 node/Cluster.hpp             |  2 +-
 node/Constants.hpp           | 12 ++++++------
 node/Node.cpp                |  2 +-
 node/Peer.hpp                |  4 ++--
 node/SelfAwareness.cpp       |  2 +-
 node/Switch.cpp              |  6 +++---
 node/Topology.hpp            |  9 ++++++---
 tests/http/big-test-start.sh |  4 ++--
 8 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/node/Cluster.hpp b/node/Cluster.hpp
index f1caa436d..ee2209998 100644
--- a/node/Cluster.hpp
+++ b/node/Cluster.hpp
@@ -55,7 +55,7 @@
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD (ZT_PEER_DIRECT_PING_DELAY / 2)
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ZT_PEER_DIRECT_PING_DELAY
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
diff --git a/node/Constants.hpp b/node/Constants.hpp
index bb62484d0..552688a62 100644
--- a/node/Constants.hpp
+++ b/node/Constants.hpp
@@ -266,16 +266,16 @@
  */
 #define ZT_PEER_DIRECT_PING_DELAY 60000
 
+/**
+ * Timeout for overall peer activity (measured from last receive)
+ */
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 4) + ZT_PING_CHECK_INVERVAL)
+
 /**
  * Delay between requests for updated network autoconf information
  */
 #define ZT_NETWORK_AUTOCONF_DELAY 60000
 
-/**
- * Timeout for overall peer activity (measured from last receive)
- */
-#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 3) + (ZT_PING_CHECK_INVERVAL * 2))
-
 /**
  * Minimum interval between attempts by relays to unite peers
  *
@@ -283,7 +283,7 @@
  * a RENDEZVOUS message no more than this often. This instructs the peers
  * to attempt NAT-t and gives each the other's corresponding IP:port pair.
  */
-#define ZT_MIN_UNITE_INTERVAL 60000
+#define ZT_MIN_UNITE_INTERVAL 30000
 
 /**
  * Delay between initial direct NAT-t packet and more aggressive techniques
diff --git a/node/Node.cpp b/node/Node.cpp
index 74acc869b..82cb7ddbc 100644
--- a/node/Node.cpp
+++ b/node/Node.cpp
@@ -263,7 +263,7 @@ public:
 			}
 
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
-		} else if (p->alive(_now)) {
+		} else if (p->activelyTransferringFrames(_now)) {
 			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
 			p->doPingAndKeepalive(RR,_now,0);
 		}
diff --git a/node/Peer.hpp b/node/Peer.hpp
index e5db3bde5..ad4c67463 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -231,9 +231,9 @@ public:
 	inline uint64_t lastAnnouncedTo() const throw() { return _lastAnnouncedTo; }
 
 	/**
-	 * @return True if peer has received an actual data frame within ZT_PEER_ACTIVITY_TIMEOUT milliseconds
+	 * @return True if this peer is actively sending real network frames
 	 */
-	inline uint64_t alive(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
+	inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
 
 	/**
 	 * @return Current latency or 0 if unknown (max: 65535)
diff --git a/node/SelfAwareness.cpp b/node/SelfAwareness.cpp
index d8eca0718..ce75eb03e 100644
--- a/node/SelfAwareness.cpp
+++ b/node/SelfAwareness.cpp
@@ -128,7 +128,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 		// links to be re-established if possible, possibly using a root server or some
 		// other relay.
 		for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
-			if ((*p)->alive(now)) {
+			if ((*p)->activelyTransferringFrames(now)) {
 				Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
 				RR->sw->send(outp,true,0);
 			}
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 2f72f57af..120ce7a4d 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -442,8 +442,8 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		Mutex::Lock _l(_contactQueue_m);
 		for(std::list<ContactQueueEntry>::iterator qi(_contactQueue.begin());qi!=_contactQueue.end();) {
 			if (now >= qi->fireAtTime) {
-				if ((!qi->peer->alive(now))||(qi->peer->hasActiveDirectPath(now))) {
-					// Cancel attempt if we've already connected or peer is no longer "alive"
+				if (qi->peer->hasActiveDirectPath(now)) {
+					// Cancel if connection has succeeded
 					_contactQueue.erase(qi++);
 					continue;
 				} else {
@@ -539,7 +539,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		_LastUniteKey *k = (_LastUniteKey *)0;
 		uint64_t *v = (uint64_t *)0;
 		while (i.next(k,v)) {
-			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 16))
+			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 8))
 				_lastUniteAttempt.erase(*k);
 		}
 	}
diff --git a/node/Topology.hpp b/node/Topology.hpp
index 4c1a2ab37..a0c28b0fe 100644
--- a/node/Topology.hpp
+++ b/node/Topology.hpp
@@ -81,6 +81,11 @@ public:
 	/**
 	 * Get a peer only if it is presently in memory (no disk cache)
 	 *
+	 * This also does not update the lastUsed() time for peers, which means
+	 * that it won't prevent them from falling out of RAM. This is currently
+	 * used in the Cluster code to update peer info without forcing all peers
+	 * across the entire cluster to remain in memory cache.
+	 *
 	 * @param zta ZeroTier address
 	 * @param now Current time
 	 */
@@ -88,10 +93,8 @@ public:
 	{
 		Mutex::Lock _l(_lock);
 		const SharedPtr<Peer> *const ap = _peers.get(zta);
-		if (ap) {
-			(*ap)->use(now);
+		if (ap)
 			return *ap;
-		}
 		return SharedPtr<Peer>();
 	}
 
diff --git a/tests/http/big-test-start.sh b/tests/http/big-test-start.sh
index 43166c6eb..f300ac612 100755
--- a/tests/http/big-test-start.sh
+++ b/tests/http/big-test-start.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
-NUM_CONTAINERS=25
+NUM_CONTAINERS=50
 CONTAINER_IMAGE=zerotier/http-test
 
 #
@@ -25,6 +25,6 @@ export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
 #	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
 #done
 
-pssh -h big-test-hosts -i -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
+pssh -h big-test-hosts -o big-test-out -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
 
 exit 0

From 00dcb0f22c6c4ee9e983510cff783c678afd43fa Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 15:39:09 -0800
Subject: [PATCH 72/73] .

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 87e387a6d..28d7ec5d6 100755
--- a/.gitignore
+++ b/.gitignore
@@ -54,6 +54,7 @@ cluster-geo/cluster-geo/config.js
 cluster-geo/cluster-geo/cache.*
 tests/http/zerotier-one
 tests/http/result_*
+tests/http/big-test-out
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*

From 4e9d4304761f93a1764d3ec2d2b0c38140decad8 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@gmail.com>
Date: Mon, 2 Nov 2015 16:03:28 -0800
Subject: [PATCH 73/73] Make root and relay selection somewhat more robust.

---
 node/Peer.hpp     | 26 ++++++++++++++++++++++----
 node/Switch.cpp   |  7 +++++--
 node/Topology.cpp | 37 +++++++++++++++++--------------------
 3 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/node/Peer.hpp b/node/Peer.hpp
index ad4c67463..a70d98688 100644
--- a/node/Peer.hpp
+++ b/node/Peer.hpp
@@ -236,15 +236,33 @@ public:
 	inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
 
 	/**
-	 * @return Current latency or 0 if unknown (max: 65535)
+	 * @return Latency in milliseconds or 0 if unknown
 	 */
-	inline unsigned int latency() const
-		throw()
+	inline unsigned int latency() const { return _latency; }
+
+	/**
+	 * This computes a quality score for relays and root servers
+	 *
+	 * If we haven't heard anything from these in ZT_PEER_ACTIVITY_TIMEOUT, they
+	 * receive the worst possible quality (max unsigned int). Otherwise the
+	 * quality is a product of latency and the number of potential missed
+	 * pings. This causes roots and relays to switch over a bit faster if they
+	 * fail.
+	 *
+	 * @return Relay quality score computed from latency and other factors, lower is better
+	 */
+	inline unsigned int relayQuality(const uint64_t now) const
 	{
+		const uint64_t tsr = now - _lastReceive;
+		if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT)
+			return (~(unsigned int)0);
 		unsigned int l = _latency;
-		return std::min(l,(unsigned int)65535);
+		if (!l)
+			l = 0xffff;
+		return (l * (((unsigned int)tsr / (ZT_PEER_DIRECT_PING_DELAY + 1000)) + 1));
 	}
 
+
 	/**
 	 * Update latency with a new direct measurment
 	 *
diff --git a/node/Switch.cpp b/node/Switch.cpp
index 120ce7a4d..b7a9c5227 100644
--- a/node/Switch.cpp
+++ b/node/Switch.cpp
@@ -741,12 +741,15 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 		if (!viaPath) {
 			// See if this network has a preferred relay (if packet has an associated network)
 			if (nconf) {
-				unsigned int latency = ~((unsigned int)0);
+				unsigned int bestq = ~((unsigned int)0);
 				for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(nconf->relays().begin());r!=nconf->relays().end();++r) {
 					if (r->first != peer->address()) {
 						SharedPtr<Peer> rp(RR->topology->getPeer(r->first));
-						if ((rp)&&(rp->hasActiveDirectPath(now))&&(rp->latency() <= latency))
+						const unsigned int q = rp->relayQuality(now);
+						if ((rp)&&(q < bestq)) { // SUBTILE: < == don't use these if they are nil quality (unsigned int max), instead use a root
+							bestq = q;
 							rp.swap(relay);
+						}
 					}
 				}
 			}
diff --git a/node/Topology.cpp b/node/Topology.cpp
index b8bb55f29..bea97ab9f 100644
--- a/node/Topology.cpp
+++ b/node/Topology.cpp
@@ -227,33 +227,30 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 
 	} else {
 		/* If I am not a root server, the best root server is the active one with
-		 * the lowest latency. */
+		 * the lowest quality score. (lower == better) */
 
-		unsigned int bestLatencyOverall = ~((unsigned int)0);
-		unsigned int bestLatencyNotAvoid = ~((unsigned int)0);
+		unsigned int bestQualityOverall = ~((unsigned int)0);
+		unsigned int bestQualityNotAvoid = ~((unsigned int)0);
 		const SharedPtr<Peer> *bestOverall = (const SharedPtr<Peer> *)0;
 		const SharedPtr<Peer> *bestNotAvoid = (const SharedPtr<Peer> *)0;
 
 		for(std::vector< SharedPtr<Peer> >::const_iterator r(_rootPeers.begin());r!=_rootPeers.end();++r) {
-			if ((*r)->hasActiveDirectPath(now)) {
-				bool avoiding = false;
-				for(unsigned int i=0;i<avoidCount;++i) {
-					if (avoid[i] == (*r)->address()) {
-						avoiding = true;
-						break;
-					}
-				}
-				unsigned int l = (*r)->latency();
-				if (!l) l = ~l; // zero latency indicates no measurment, so make this 'max'
-				if (l <= bestLatencyOverall) {
-					bestLatencyOverall = l;
-					bestOverall = &(*r);
-				}
-				if ((!avoiding)&&(l <= bestLatencyNotAvoid)) {
-					bestLatencyNotAvoid = l;
-					bestNotAvoid = &(*r);
+			bool avoiding = false;
+			for(unsigned int i=0;i<avoidCount;++i) {
+				if (avoid[i] == (*r)->address()) {
+					avoiding = true;
+					break;
 				}
 			}
+			const unsigned int q = (*r)->relayQuality(now);
+			if (q <= bestQualityOverall) {
+				bestQualityOverall = q;
+				bestOverall = &(*r);
+			}
+			if ((!avoiding)&&(q <= bestQualityNotAvoid)) {
+				bestQualityNotAvoid = q;
+				bestNotAvoid = &(*r);
+			}
 		}
 
 		if (bestNotAvoid) {