diff --git a/ext/lz4/lz4hc.c b/ext/lz4/lz4hc.c deleted file mode 100644 index 608674902..000000000 --- a/ext/lz4/lz4hc.c +++ /dev/null @@ -1,892 +0,0 @@ -/* - LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ - - - -/************************************** - Tuning Parameter -**************************************/ -#define LZ4HC_DEFAULT_COMPRESSIONLEVEL 8 - - -/************************************** - Memory routines -**************************************/ -#include /* calloc, free */ -#define ALLOCATOR(s) calloc(1,s) -#define FREEMEM free -#include /* memset, memcpy */ -#define MEM_INIT memset - - -/************************************** - CPU Feature Detection -**************************************/ -/* 32 or 64 bits ? */ -#if (defined(__x86_64__) || defined(_M_X64) || defined(_WIN64) \ - || defined(__powerpc64__) || defined(__powerpc64le__) \ - || defined(__ppc64__) || defined(__ppc64le__) \ - || defined(__PPC64__) || defined(__PPC64LE__) \ - || defined(__ia64) || defined(__itanium__) || defined(_M_IA64) ) /* Detects 64 bits mode */ -# define LZ4_ARCH64 1 -#else -# define LZ4_ARCH64 0 -#endif - -/* - * Little Endian or Big Endian ? - * Overwrite the #define below if you know your architecture endianess - */ -#include /* Apparently required to detect endianess */ -#if defined (__GLIBC__) -# include -# if (__BYTE_ORDER == __BIG_ENDIAN) -# define LZ4_BIG_ENDIAN 1 -# endif -#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) -# define LZ4_BIG_ENDIAN 1 -#elif defined(__sparc) || defined(__sparc__) \ - || defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) \ - || defined(__hpux) || defined(__hppa) \ - || defined(_MIPSEB) || defined(__s390__) -# define LZ4_BIG_ENDIAN 1 -#else -/* Little Endian assumed. PDP Endian and other very rare endian format are unsupported. */ -#endif - -/* - * Unaligned memory access is automatically enabled for "common" CPU, such as x86. - * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected - * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance - */ -#if defined(__ARM_FEATURE_UNALIGNED) -# define LZ4_FORCE_UNALIGNED_ACCESS 1 -#endif - -/* Define this parameter if your target system or compiler does not support hardware bit count */ -#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -/************************************** - Compiler Options -**************************************/ -#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -/* "restrict" is a known keyword */ -#else -# define restrict /* Disable restrict */ -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# if LZ4_ARCH64 /* 64-bits */ -# pragma intrinsic(_BitScanForward64) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse64) /* For Visual 2005 */ -# else /* 32-bits */ -# pragma intrinsic(_BitScanForward) /* For Visual 2005 */ -# pragma intrinsic(_BitScanReverse) /* For Visual 2005 */ -# endif -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable used */ -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - -#ifdef _MSC_VER /* Visual Studio */ -# define lz4_bswap16(x) _byteswap_ushort(x) -#else -# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) -#endif - - -/************************************** - Includes -**************************************/ -#include "lz4hc.h" -#include "lz4.h" - - -/************************************** - Basic Types -**************************************/ -#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(LZ4_FORCE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U16_S { U16 v; } _PACKED U16_S; -typedef struct _U32_S { U32 v; } _PACKED U32_S; -typedef struct _U64_S { U64 v; } _PACKED U64_S; - -#if !defined(LZ4_FORCE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A64(x) (((U64_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A16(x) (((U16_S *)(x))->v) - - -/************************************** - Constants -**************************************/ -#define MINMATCH 4 - -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) -#define HASH_VALUE(p) HASH_FUNCTION(A32(p)) -#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base) -#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] -#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) - - -/************************************** - Private functions -**************************************/ -#if LZ4_ARCH64 - -FORCE_INLINE int LZ4_NbCommonBytes (register U64 val) -{ -#if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); -# else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif -#else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; -# endif -#endif -} - -#else - -FORCE_INLINE int LZ4_NbCommonBytes (register U32 val) -{ -#if defined(LZ4_BIG_ENDIAN) -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanReverse( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); -# else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif -#else -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, val ); - return (int)(r>>3); -# elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif -#endif -} - -#endif - - -int LZ4_sizeofStreamStateHC() -{ - return sizeof(LZ4HC_Data_Structure); -} - -FORCE_INLINE void LZ4_initHC (LZ4HC_Data_Structure* hc4, const BYTE* base) -{ - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); - MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = base + 1; - hc4->base = base; - hc4->inputBuffer = base; - hc4->end = base; -} - -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer) -{ - if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ - LZ4_initHC((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); - return 0; -} - - -void* LZ4_createHC (const char* inputBuffer) -{ - void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); - LZ4_initHC ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); - return hc4; -} - - -int LZ4_freeHC (void* LZ4HC_Data) -{ - FREEMEM(LZ4HC_Data); - return (0); -} - - -/* Update chains up to ip (excluded) */ -FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) -{ - U16* chainTable = hc4->chainTable; - HTYPE* HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - - while(hc4->nextToUpdate < ip) - { - const BYTE* const p = hc4->nextToUpdate; - size_t delta = (p) - HASH_POINTER(p); - if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; - DELTANEXT(p) = (U16)delta; - HashTable[HASH_VALUE(p)] = (HTYPE)((p) - base); - hc4->nextToUpdate++; - } -} - - -char* LZ4_slideInputBufferHC(void* LZ4HC_Data) -{ - LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; - U32 distance = (U32)(hc4->end - hc4->inputBuffer) - 64 KB; - distance = (distance >> 16) << 16; /* Must be a multiple of 64 KB */ - LZ4HC_Insert(hc4, hc4->end - MINMATCH); - memcpy((void*)(hc4->end - 64 KB - distance), (const void*)(hc4->end - 64 KB), 64 KB); - hc4->nextToUpdate -= distance; - hc4->base -= distance; - if ((U32)(hc4->inputBuffer - hc4->base) > 1 GB + 64 KB) /* Avoid overflow */ - { - int i; - hc4->base += 1 GB; - for (i=0; ihashTable[i] -= 1 GB; - } - hc4->end -= distance; - return (char*)(hc4->end); -} - - -FORCE_INLINE size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) -{ - const BYTE* p1t = p1; - - while (p1tchainTable; - HTYPE* const HashTable = hc4->hashTable; - const BYTE* ref; - INITBASE(base,hc4->base); - int nbAttempts=maxNbAttempts; - size_t repl=0, ml=0; - U16 delta=0; /* useless assignment, to remove an uninitialization warning */ - - /* HC4 match finder */ - LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); - -#define REPEAT_OPTIMIZATION -#ifdef REPEAT_OPTIMIZATION - /* Detect repetitive sequences of length <= 4 */ - if ((U32)(ip-ref) <= 4) /* potential repetition */ - { - if (A32(ref) == A32(ip)) /* confirmed */ - { - delta = (U16)(ip-ref); - repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; - *matchpos = ref; - } - ref = GETNEXT(ref); - } -#endif - - while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) - { - nbAttempts--; - if (*(ref+ml) == *(ip+ml)) - if (A32(ref) == A32(ip)) - { - size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; - if (mlt > ml) { ml = mlt; *matchpos = ref; } - } - ref = GETNEXT(ref); - } - -#ifdef REPEAT_OPTIMIZATION - /* Complete table */ - if (repl) - { - const BYTE* ptr = ip; - const BYTE* end; - - end = ip + repl - (MINMATCH-1); - while(ptr < end-delta) - { - DELTANEXT(ptr) = delta; /* Pre-Load */ - ptr++; - } - do - { - DELTANEXT(ptr) = delta; - HashTable[HASH_VALUE(ptr)] = (HTYPE)((ptr) - base); /* Head of chain */ - ptr++; - } while(ptr < end); - hc4->nextToUpdate = end; - } -#endif - - return (int)ml; -} - - -FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts) -{ - U16* const chainTable = hc4->chainTable; - HTYPE* const HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - const BYTE* ref; - int nbAttempts = maxNbAttempts; - int delta = (int)(ip-startLimit); - - /* First Match */ - LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); - - while (((U32)(ip-ref) <= MAX_DISTANCE) && (nbAttempts)) - { - nbAttempts--; - if (*(startLimit + longest) == *(ref - delta + longest)) - if (A32(ref) == A32(ip)) - { -#if 1 - const BYTE* reft = ref+MINMATCH; - const BYTE* ipt = ip+MINMATCH; - const BYTE* startt = ip; - - while (iptstartLimit) && (reft > hc4->inputBuffer) && (startt[-1] == reft[-1])) {startt--; reft--;} - - if ((ipt-startt) > longest) - { - longest = (int)(ipt-startt); - *matchpos = reft; - *startpos = startt; - } - } - ref = GETNEXT(ref); - } - - return longest; -} - - -typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; - -FORCE_INLINE int LZ4HC_encodeSequence ( - const BYTE** ip, - BYTE** op, - const BYTE** anchor, - int matchLength, - const BYTE* ref, - limitedOutput_directive limitedOutputBuffer, - BYTE* oend) -{ - int length; - BYTE* token; - - /* Encode Literal length */ - length = (int)(*ip - *anchor); - token = (*op)++; - if ((limitedOutputBuffer) && ((*op + length + (2 + 1 + LASTLITERALS) + (length>>8)) > oend)) return 1; /* Check output limit */ - if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } - else *token = (BYTE)(length<>8) > oend)) return 1; /* Check output limit */ - if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } - else *token += (BYTE)(length); - - /* Prepare next loop */ - *ip += matchLength; - *anchor = *ip; - - return 0; -} - - -#define MAX_COMPRESSION_LEVEL 16 -static int LZ4HC_compress_generic ( - void* ctxvoid, - const char* source, - char* dest, - int inputSize, - int maxOutputSize, - int compressionLevel, - limitedOutput_directive limit - ) -{ - LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + inputSize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - BYTE* op = (BYTE*) dest; - BYTE* const oend = op + maxOutputSize; - - const int maxNbAttempts = compressionLevel > MAX_COMPRESSION_LEVEL ? 1 << MAX_COMPRESSION_LEVEL : compressionLevel ? 1<<(compressionLevel-1) : 1<end) return 0; - ctx->end += inputSize; - - ip++; - - /* Main Loop */ - while (ip < mflimit) - { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); - if (!ml) { ip++; continue; } - - /* saved, in case we would skip too much */ - start0 = ip; - ref0 = ref; - ml0 = ml; - -_Search2: - if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); - else ml2 = ml; - - if (ml2 == ml) /* No better match */ - { - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - continue; - } - - if (start0 < ip) - { - if (start2 < ip + ml0) /* empirical */ - { - ip = start0; - ref = ref0; - ml = ml0; - } - } - - /* Here, start0==ip */ - if ((start2 - ip) < 3) /* First Match too small : removed */ - { - ml = ml2; - ip = start2; - ref =ref2; - goto _Search2; - } - -_Search3: - /* - * Currently we have : - * ml2 > ml1, and - * ip1+3 <= ip2 (usually < ip1+ml1) - */ - if ((start2 - ip) < OPTIMAL_ML) - { - int correction; - int new_ml = ml; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = new_ml - (int)(start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ - - if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); - else ml3 = ml2; - - if (ml3 == ml2) /* No better match : 2 sequences to encode */ - { - /* ip & ref are known; Now for ml */ - if (start2 < ip+ml) ml = (int)(start2 - ip); - /* Now, encode 2 sequences */ - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - ip = start2; - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; - continue; - } - - if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ - { - if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ - { - if (start2 < ip+ml) - { - int correction = (int)(ip+ml - start2); - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < MINMATCH) - { - start2 = start3; - ref2 = ref3; - ml2 = ml3; - } - } - - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - goto _Search3; - } - - /* - * OK, now we have 3 ascending matches; let's write at least the first one - * ip & ref are known; Now for ml - */ - if (start2 < ip+ml) - { - if ((start2 - ip) < (int)ML_MASK) - { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; - correction = ml - (int)(start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - else - { - ml = (int)(start2 - ip); - } - } - if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; - - ip = start2; - ref = ref2; - ml = ml2; - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - - goto _Search3; - - } - - /* Encode Last Literals */ - { - int lastRun = (int)(iend - anchor); - if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (BYTE)(lastRun< The memory position where the next input data block must start is provided as the result of the function. - -Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual. - -When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure. -*/ - -int LZ4_sizeofStreamStateHC(void); -int LZ4_resetStreamStateHC(void* state, const char* inputBuffer); - -/* -These functions achieve the same result as : -void* LZ4_createHC (const char* inputBuffer); - -They are provided here to allow the user program to allocate memory using its own routines. - -To know how much space must be allocated, use LZ4_sizeofStreamStateHC(); -Note also that space must be aligned for pointers (32 or 64 bits). - -Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer); -void* state is a pointer to the space allocated. -It must be aligned for pointers (32 or 64 bits), and be large enough. -The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer. -The input buffer must be already allocated, and size at least 192KB. -'inputBuffer' will also be the 'const char* source' of the first block. - -The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState(). -return value of LZ4_resetStreamStateHC() must be 0 is OK. -Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)). -*/ - - -#if defined (__cplusplus) -} -#endif diff --git a/make-mac.mk b/make-mac.mk index db7ebfb68..784f4c6e6 100644 --- a/make-mac.mk +++ b/make-mac.mk @@ -42,7 +42,7 @@ CXXFLAGS=$(CFLAGS) -fno-rtti include objects.mk -OBJS+=node/BSDRoutingTable.o +OBJS+=osnet/BSDRoutingTable.o all: one diff --git a/node/Constants.hpp b/node/Constants.hpp index 524a8428d..7650de8be 100644 --- a/node/Constants.hpp +++ b/node/Constants.hpp @@ -155,6 +155,11 @@ */ #define ZT_IF_MTU 2800 +/** + * Default interface metric for ZeroTier taps -- should be higher than physical ports + */ +#define ZT_DEFAULT_IF_METRIC 32768 + /** * Maximum number of packet fragments we'll support * diff --git a/node/Defaults.cpp b/node/Defaults.cpp index 4387c1ae4..9176a414e 100644 --- a/node/Defaults.cpp +++ b/node/Defaults.cpp @@ -52,57 +52,6 @@ static inline std::map< Identity,std::vector< std::pair > > _m // Nothing special about a supernode... except that they are // designated as such and trusted to provide WHOIS lookup. -#ifdef ZT_USE_TESTNET - - // If ZT_USE_TESTNET is defined we talk to test rather than live supernode - // instances. The testnet may not always be running, so this is probably not - // of any interest to users. Testnet servers run on port 7773 (UDP) and - // 773 (TCP). - - // cthulhu.zerotier.com - New York, New York, USA - addrs.clear(); - if (!id.fromString("0bfa76f104:0:aff4d4604f2a2538d414a1d69fc722a28bea049d52192aded117c28b0f6c1052db9d36c488c5fe5e2071f2def8f86b6db64db09e819f90fdaedbfcb9f3bcdef9")) - throw std::runtime_error("invalid identity in Defaults"); - addrs.push_back(std::pair(InetAddress("162.243.77.111",7773),false)); - addrs.push_back(std::pair(InetAddress("162.243.77.111",773),true)); - sn[id] = addrs; - - // nyarlathotep.zerotier.com - San Francisco, California, USA - addrs.clear(); - if (!id.fromString("9f2b042cdb:0:8993f9348bb9642afa9a60995a35ef19817894fd0b6859201c0e56e399288867c8f0d01ae2858f9dc6f95eee6d42e2f6d08c44551404906b25679aa6db1faee7")) - throw std::runtime_error("invalid identity in Defaults"); - addrs.push_back(std::pair(InetAddress("198.199.97.220",7773),false)); - addrs.push_back(std::pair(InetAddress("198.199.97.220",773),true)); - sn[id] = addrs; - - // shub-niggurath.zerotier.com - Amsterdam, Netherlands - addrs.clear(); - if (!id.fromString("916a4ca17d:0:b679a8d6761096ba4958fea0036dc4dbb76cb8cbf1ce9bc352cc594c3c24987bb3b30b5448d1f494f5e90a6cdaac9d28317cb4088780278ef20bc7c366cb214a")) - throw std::runtime_error("invalid identity in Defaults"); - addrs.push_back(std::pair(InetAddress("198.211.127.172",7773),false)); - addrs.push_back(std::pair(InetAddress("198.211.127.172",773),true)); - sn[id] = addrs; - - // yig.zerotier.com - Sydney, Australia - addrs.clear(); - if (!id.fromString("3b62c7a69a:0:d967595a3b96d780151764e6ffb47af2fa8865f8e344fba4a684c10dd2e70014e26312f5b8a1590c13bfeb909a1fd35b96a84a8a43e0704cd8d01d9c2b791359")) - throw std::runtime_error("invalid identity in Defaults"); - addrs.push_back(std::pair(InetAddress("108.61.212.61",7773),false)); - addrs.push_back(std::pair(InetAddress("108.61.212.61",773),true)); - sn[id] = addrs; - - // shoggoth.zerotier.com - Tokyo, Japan - addrs.clear(); - if (!id.fromString("345ad16512:0:9e796aec6e083726f45fbfdc10bcf18c0dc7a7914c9ce29f5eb5abcf41bfcb6b3698b68131d347235ae488804317df9c6102e2753841b973037d1e4685dce9fc")) - throw std::runtime_error("invalid identity in Defaults"); - addrs.push_back(std::pair(InetAddress("108.61.200.101",7773),false)); - addrs.push_back(std::pair(InetAddress("108.61.200.101",773),true)); - sn[id] = addrs; - -#else - - // Normally we use the live supernodes. - // cthulhu.zerotier.com - New York, New York, USA addrs.clear(); if (!id.fromString("8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5")) @@ -143,8 +92,6 @@ static inline std::map< Identity,std::vector< std::pair > > _m addrs.push_back(std::pair(InetAddress("108.61.200.101",443),true)); sn[id] = addrs; -#endif - return sn; } diff --git a/node/EthernetTap.hpp b/node/EthernetTap.hpp index ba534a7e5..87ab607d0 100644 --- a/node/EthernetTap.hpp +++ b/node/EthernetTap.hpp @@ -171,14 +171,16 @@ public: virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) = 0; /** - * @return OS-specific device or connection name + * @return OS-specific device or connection name (e.g. zt0, tap0, etc.) */ virtual std::string deviceName() const = 0; /** - * @return OS-internal persistent device ID or empty string if not applicable to this platform or not persistent + * Change this device's user-visible name (if supported) + * + * @param friendlyName New name */ - virtual std::string persistentId() const = 0; + virtual void setFriendlyName(const char *friendlyName) = 0; /** * Fill or modify a set to contain multicast groups for this device diff --git a/node/EthernetTapFactory.hpp b/node/EthernetTapFactory.hpp index fde458f98..17d697f51 100644 --- a/node/EthernetTapFactory.hpp +++ b/node/EthernetTapFactory.hpp @@ -89,11 +89,12 @@ public: void *arg) = 0; /** - * Close an ethernet tap device + * Close an ethernet tap device and delete/free the tap object * * @param tap Tap instance + * @param destroyPersistentDevices If true, destroy persistent device (on platforms where applicable) */ - virtual void close(EthernetTap *tap) = 0; + virtual void close(EthernetTap *tap,bool destroyPersistentDevices) = 0; /** * @return All currently open tap device names diff --git a/node/Network.cpp b/node/Network.cpp index f45d6c2f4..43565e9f6 100644 --- a/node/Network.cpp +++ b/node/Network.cpp @@ -37,12 +37,8 @@ #include "Switch.hpp" #include "Packet.hpp" #include "Buffer.hpp" - -#ifdef __WINDOWS__ -#include "WindowsEthernetTap.hpp" -#else -#include "UnixEthernetTap.hpp" -#endif +#include "EthernetTap.hpp" +#include "EthernetTapFactory.hpp" #define ZT_NETWORK_CERT_WRITE_BUF_SIZE 131072 @@ -69,26 +65,13 @@ Network::~Network() { Thread::join(_setupThread); -#ifdef __WINDOWS__ - std::string devPersistentId; - if (_tap) { - devPersistentId = _tap->persistentId(); - delete _tap; - } -#else if (_tap) - delete _tap; -#endif + _r->tapFactory->close(_tap,_destroyOnDelete); if (_destroyOnDelete) { Utils::rm(std::string(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".conf")); Utils::rm(std::string(_r->homePath + ZT_PATH_SEPARATOR_S + "networks.d" + ZT_PATH_SEPARATOR_S + idString() + ".mcerts")); -#ifdef __WINDOWS__ - if (devPersistentId.length()) - WindowsEthernetTap::deletePersistentTapDevice(_r,devPersistentId.c_str()); -#endif } else { - // Causes flush of membership certs to disk clean(); _dumpMulticastCerts(); } @@ -113,10 +96,16 @@ SharedPtr Network::newInstance(const RuntimeEnvironment *renv,NodeConfi nw->_destroyOnDelete = false; nw->_netconfFailure = NETCONF_FAILURE_NONE; - if (nw->controller() == renv->identity.address()) // netconf masters can't really join networks + if (nw->controller() == renv->identity.address()) // TODO: fix Switch to allow packets to self throw std::runtime_error("cannot join a network for which I am the netconf master"); - nw->_setupThread = Thread::start(nw.ptr()); + try { + nw->_restoreState(); + nw->requestConfiguration(); + } catch ( ... ) { + TRACE("exception in network setup thread in _restoreState() or requestConfiguration()!"); + nw->_lastConfigUpdate = 0; // call requestConfiguration() again + } return nw; } @@ -127,7 +116,7 @@ bool Network::updateMulticastGroups() EthernetTap *t = _tap; if (t) { // Grab current groups from the local tap - bool updated = _tap->updateMulticastGroups(_multicastGroups); + bool updated = t->updateMulticastGroups(_multicastGroups); // Merge in learned groups from any hosts bridged in behind us for(std::map::const_iterator mg(_bridgedMulticastGroups.begin());mg!=_bridgedMulticastGroups.end();++mg) @@ -154,21 +143,11 @@ bool Network::setConfiguration(const Dictionary &conf,bool saveToDisk) { Mutex::Lock _l(_lock); - EthernetTap *t = _tap; - if (!t) { - TRACE("BUG: setConfiguration() called while tap is null!"); - return false; // can't accept config in initialization state - } - try { - SharedPtr newConfig(new NetworkConfig(conf)); + SharedPtr newConfig(new NetworkConfig(conf)); // throws if invalid if ((newConfig->networkId() == _id)&&(newConfig->issuedTo() == _r->identity.address())) { _config = newConfig; - if (newConfig->staticIps().size()) - t->setIps(newConfig->staticIps()); - t->setDisplayName((std::string("ZeroTier One [") + newConfig->name() + "]").c_str()); - _lastConfigUpdate = Utils::now(); _netconfFailure = NETCONF_FAILURE_NONE; @@ -181,6 +160,17 @@ bool Network::setConfiguration(const Dictionary &conf,bool saveToDisk) } } + EthernetTap *t = _tap; + if (t) { + char fname[1024]; + _mkNetworkFriendlyName(fname,sizeof(fname)); + t->setIps(newConfig->staticIps()); + t->setFriendlyName(fname); + } else { + if (!_setupThread) + _setupThread = Thread::start(this); + } + return true; } else { LOG("ignored invalid configuration for network %.16llx (configuration contains mismatched network ID or issued-to address)",(unsigned long long)_id); @@ -196,9 +186,6 @@ bool Network::setConfiguration(const Dictionary &conf,bool saveToDisk) void Network::requestConfiguration() { - if (!_tap) - return; // don't bother requesting until we are initialized - if (controller() == _r->identity.address()) { // netconf master cannot be a member of its own nets LOG("unable to request network configuration for network %.16llx: I am the network master, cannot query self",(unsigned long long)_id); @@ -346,51 +333,46 @@ void Network::_pushMembershipCertificate(const Address &peer,bool force,uint64_t } } +// Ethernet tap creation thread -- required on some platforms where tap +// creation may be time consuming (e.g. Windows). void Network::threadMain() throw() { - // Setup thread -- this exits when tap is constructed. It's here - // because opening the tap can take some time on some platforms. + char fname[1024],lcentry[128]; + Utils::snprintf(lcentry,sizeof(lcentry),"_dev_for_%.16llx",(unsigned long long)_id); + EthernetTap *t; try { -#ifdef __WINDOWS__ - // Windows tags interfaces by their network IDs, which are shoved into the - // registry to mark persistent instance of the tap device. - char tag[24]; - Utils::snprintf(tag,sizeof(tag),"%.16llx",(unsigned long long)_id); - _tap = new WindowsEthernetTap(_r,tag,_mac,ZT_IF_MTU,&_CBhandleTapData,this); -#else - // Unix tries to get the same device name next time, if possible. - std::string tagstr; - char lcentry[128]; - Utils::snprintf(lcentry,sizeof(lcentry),"_dev_for_%.16llx",(unsigned long long)_id); - tagstr = _nc->getLocalConfig(lcentry); + std::string desiredDevice(_nc->getLocalConfig(lcentry)); + _mkNetworkFriendlyName(fname,sizeof(fname)); - const char *tag = (tagstr.length() > 0) ? tagstr.c_str() : (const char *)0; - _tap = new UnixEthernetTap(_r,tag,_mac,ZT_IF_MTU,&_CBhandleTapData,this); + t = _r->tapFactory->open(_mac,ZT_IF_MTU,ZT_DEFAULT_IF_METRIC,_id,(desiredDevice.length() > 0) ? desiredDevice.c_str() : (const char *)0,fname,_CBhandleTapData,this); - std::string dn(_tap->deviceName()); - if ((!tag)||(dn != tag)) + std::string dn(t->deviceName()); + if ((dn.length())&&(dn != desiredDevice)) _nc->putLocalConfig(lcentry,dn); -#endif } catch (std::exception &exc) { - delete _tap; - _tap = (EthernetTap *)0; + delete t; + t = (EthernetTap *)0; LOG("network %.16llx failed to initialize: %s",_id,exc.what()); _netconfFailure = NETCONF_FAILURE_INIT_FAILED; } catch ( ... ) { - delete _tap; - _tap = (EthernetTap *)0; + delete t; + t = (EthernetTap *)0; LOG("network %.16llx failed to initialize: unknown error",_id); _netconfFailure = NETCONF_FAILURE_INIT_FAILED; } - try { - _restoreState(); - requestConfiguration(); - } catch ( ... ) { - TRACE("BUG: exception in network setup thread in _restoreState() or requestConfiguration()!"); - _lastConfigUpdate = 0; // call requestConfiguration() again + { + Mutex::Lock _l(_lock); + if (_tap) // the tap creation thread can technically be re-launched, though this isn't done right now + _r->tapFactory->close(_tap,_destroyOnDelete); + _tap = t; + if (t) { + if (_config) + t->setIps(_config->staticIps()); + t->setEnabled(_enabled); + } } } @@ -423,14 +405,12 @@ void Network::setEnabled(bool enabled) { Mutex::Lock _l(_lock); _enabled = enabled; - // TODO: bring OS network device to "down" state if enabled == false + if (_tap) + _tap->setEnabled(enabled); } void Network::_restoreState() { - if (!_id) - return; // sanity check - Buffer buf; std::string idstr(idString()); @@ -448,7 +428,7 @@ void Network::_restoreState() } else { // If the conf file isn't present, "touch" it so we'll remember // the existence of this network. - FILE *tmp = fopen(confPath.c_str(),"wb"); + FILE *tmp = fopen(confPath.c_str(),"w"); if (tmp) fclose(tmp); } diff --git a/node/Network.hpp b/node/Network.hpp index 03b011f35..1d62bdf12 100644 --- a/node/Network.hpp +++ b/node/Network.hpp @@ -426,6 +426,14 @@ private: void _restoreState(); void _dumpMulticastCerts(); + inline void _mkNetworkFriendlyName(char *buf,unsigned int len) + { + // assumes _lock is locked + if (_config) + Utils::snprintf(buf,len,"ZeroTier One [%s]",_config->name().c_str()); + else Utils::snprintf(buf,len,"ZeroTier One [%.16llx]",(unsigned long long)_id); + } + uint64_t _id; NodeConfig *_nc; // parent NodeConfig object MAC _mac; // local MAC address @@ -439,8 +447,8 @@ private: std::map _membershipCertificates; std::map _lastPushedMembershipCertificate; - std::map _bridgeRoutes; - std::map _bridgedMulticastGroups; + std::map _bridgeRoutes; // remote addresses where given MACs are reachable + std::map _bridgedMulticastGroups; // multicast groups of interest on our side of the bridge SharedPtr _config; volatile uint64_t _lastConfigUpdate; diff --git a/node/Node.cpp b/node/Node.cpp index 92b49d769..f2e6b3e4a 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -67,7 +67,6 @@ #include "EthernetTap.hpp" #include "CMWC4096.hpp" #include "NodeConfig.hpp" -#include "SysEnv.hpp" #include "Network.hpp" #include "MulticastGroup.hpp" #include "Mutex.hpp" @@ -77,6 +76,7 @@ #include "Buffer.hpp" #include "IpcConnection.hpp" #include "AntiRecursion.hpp" +#include "RoutingTable.hpp" namespace ZeroTier { @@ -218,6 +218,7 @@ const char *Node::NodeControlClient::authTokenDefaultSystemPath() struct _NodeImpl { RuntimeEnvironment renv; + unsigned int udpPort,tcpPort; std::string reasonForTerminationStr; volatile Node::ReasonForTermination reasonForTermination; @@ -225,6 +226,7 @@ struct _NodeImpl volatile bool running; volatile bool resynchronize; + // This function performs final node tear-down inline Node::ReasonForTermination terminate() { RuntimeEnvironment *_r = &renv; @@ -238,16 +240,15 @@ struct _NodeImpl #ifndef __WINDOWS__ delete renv.netconfService; #endif - delete renv.updater; - delete renv.nc; - delete renv.sysEnv; - delete renv.topology; - delete renv.sm; - delete renv.sw; - delete renv.mc; - delete renv.antiRec; - delete renv.prng; - delete renv.log; + delete renv.updater; renv.updater = (SoftwareUpdater *)0; + delete renv.nc; renv.nc = (NodeConfig *)0; // shut down all networks, close taps, etc. + delete renv.topology; renv.topology = (Topology *)0; // now we no longer need routing info + delete renv.sm; renv.sm = (SocketManager *)0; // close all sockets + delete renv.sw; renv.sw = (Switch *)0; // order matters less from here down + delete renv.mc; renv.mc = (Multicaster *)0; + delete renv.antiRec; renv.antiRec = (AntiRecursion *)0; + delete renv.prng; renv.prng = (CMWC4096 *)0; + delete renv.log; renv.log = (Logger *)0; // but stop logging last of all return reasonForTermination; } @@ -260,7 +261,7 @@ struct _NodeImpl } }; -#ifndef __WINDOWS__ +#ifndef __WINDOWS__ // "services" are not supported on Windows static void _netconfServiceMessageHandler(void *renv,Service &svc,const Dictionary &msg) { if (!renv) @@ -347,7 +348,13 @@ static void _netconfServiceMessageHandler(void *renv,Service &svc,const Dictiona } #endif // !__WINDOWS__ -Node::Node(const char *hp,unsigned int udpPort,unsigned int tcpPort,bool resetIdentity) +Node::Node( + const char *hp, + EthernetTapFactory *tf, + RoutingTable *rt, + unsigned int udpPort, + unsigned int tcpPort, + bool resetIdentity) throw() : _impl(new _NodeImpl) { @@ -357,6 +364,9 @@ Node::Node(const char *hp,unsigned int udpPort,unsigned int tcpPort,bool resetId impl->renv.homePath = hp; else impl->renv.homePath = ZT_DEFAULTS.defaultHomePath; + impl->renv.tapFactory = tf; + impl->renv.routingTable = rt; + if (resetIdentity) { // Forget identity and peer database, peer keys, etc. Utils::rm((impl->renv.homePath + ZT_PATH_SEPARATOR_S + "identity.public").c_str()); @@ -364,13 +374,14 @@ Node::Node(const char *hp,unsigned int udpPort,unsigned int tcpPort,bool resetId Utils::rm((impl->renv.homePath + ZT_PATH_SEPARATOR_S + "peers.persist").c_str()); // Truncate network config information in networks.d but leave the files since we - // still want to remember any networks we have joined. This will force re-config. + // still want to remember any networks we have joined. This will force those networks + // to be reconfigured with our newly regenerated identity after startup. std::string networksDotD(impl->renv.homePath + ZT_PATH_SEPARATOR_S + "networks.d"); std::map< std::string,bool > nwfiles(Utils::listDirectory(networksDotD.c_str())); for(std::map::iterator nwf(nwfiles.begin());nwf!=nwfiles.end();++nwf) { - FILE *foo = fopen((networksDotD + ZT_PATH_SEPARATOR_S + nwf->first).c_str(),"w"); - if (foo) - fclose(foo); + FILE *trun = fopen((networksDotD + ZT_PATH_SEPARATOR_S + nwf->first).c_str(),"w"); + if (trun) + fclose(trun); } } @@ -470,13 +481,11 @@ Node::ReasonForTermination Node::run() } Utils::lockDownFile(configAuthTokenPath.c_str(),false); - // Create the objects that make up runtime state. _r->antiRec = new AntiRecursion(); _r->mc = new Multicaster(); _r->sw = new Switch(_r); _r->sm = new SocketManager(impl->udpPort,impl->tcpPort,&_CBztTraffic,_r); _r->topology = new Topology(_r,Utils::fileExists((_r->homePath + ZT_PATH_SEPARATOR_S + "iddb.d").c_str())); - _r->sysEnv = new SysEnv(); try { _r->nc = new NodeConfig(_r,configAuthToken.c_str()); } catch (std::exception &exc) { @@ -568,7 +577,7 @@ Node::ReasonForTermination Node::run() // If our network environment looks like it changed, resynchronize. if ((resynchronize)||((now - lastNetworkFingerprintCheck) >= ZT_NETWORK_FINGERPRINT_CHECK_DELAY)) { lastNetworkFingerprintCheck = now; - uint64_t fp = _r->sysEnv->getNetworkConfigurationFingerprint(_r->nc->networkTapDeviceNames()); + uint64_t fp = _r->routingTable->networkEnvironmentFingerprint(_r->nc->networkTapDeviceNames()); if (fp != networkConfigurationFingerprint) { LOG("netconf fingerprint change: %.16llx != %.16llx, resyncing with network",networkConfigurationFingerprint,fp); networkConfigurationFingerprint = fp; @@ -588,7 +597,7 @@ Node::ReasonForTermination Node::run() } if (resynchronize) { - _r->tcpTunnelingEnabled = false; // turn off TCP tunneling master switch at first + _r->tcpTunnelingEnabled = false; // turn off TCP tunneling master switch at first, will be reenabled on persistent UDP failure _r->timeOfLastResynchronize = now; } @@ -643,17 +652,15 @@ Node::ReasonForTermination Node::run() /* Periodically ping all our non-stale direct peers unless we're a supernode. * Supernodes only ping each other (which is done above). */ - if (!_r->topology->amSupernode()) { - if ((now - lastPingCheck) >= ZT_PING_CHECK_DELAY) { - lastPingCheck = now; - try { - _r->topology->eachPeer(Topology::PingPeersThatNeedPing(_r,now)); - _r->topology->eachPeer(Topology::OpenPeersThatNeedFirewallOpener(_r,now)); - } catch (std::exception &exc) { - LOG("unexpected exception running ping check cycle: %s",exc.what()); - } catch ( ... ) { - LOG("unexpected exception running ping check cycle: (unkonwn)"); - } + if ((!_r->topology->amSupernode())&&((now - lastPingCheck) >= ZT_PING_CHECK_DELAY)) { + lastPingCheck = now; + try { + _r->topology->eachPeer(Topology::PingPeersThatNeedPing(_r,now)); + _r->topology->eachPeer(Topology::OpenPeersThatNeedFirewallOpener(_r,now)); + } catch (std::exception &exc) { + LOG("unexpected exception running ping check cycle: %s",exc.what()); + } catch ( ... ) { + LOG("unexpected exception running ping check cycle: (unkonwn)"); } } } diff --git a/node/Node.hpp b/node/Node.hpp index cc2886e7e..6cc3ef40c 100644 --- a/node/Node.hpp +++ b/node/Node.hpp @@ -33,6 +33,9 @@ namespace ZeroTier { +class EthernetTapFactory; +class RoutingTable; + /** * A ZeroTier One node * @@ -150,14 +153,25 @@ public: /** * Create a new node * - * The node is not executed until run() is called. + * The node is not executed until run() is called. The supplied tap factory + * and routing table must not be freed until the node is no longer + * executing. Node does not delete these objects, so the caller still owns + * them. * - * @param hp Home directory path or NULL for system-wide default for this platform (default: NULL) - * @param udpPort UDP port or 0 to disable (default: 9993) - * @param tcpPort TCP port or 0 to disable (default: 0) - * @param resetIdentity If true, delete identity before starting and regenerate (default: false) + * @param hp Home directory path or NULL for system-wide default for this platform + * @param tf Ethernet tap factory for platform network stack + * @param rt Routing table interface for platform network stack + * @param udpPort UDP port or 0 to disable + * @param tcpPort TCP port or 0 to disable + * @param resetIdentity If true, delete identity before starting and regenerate */ - Node(const char *hp = (const char *)0,unsigned int udpPort = 9993,unsigned int tcpPort = 0,bool resetIdentity = false) + Node( + const char *hp, + EthernetTapFactory *tf, + RoutingTable *rt, + unsigned int udpPort, + unsigned int tcpPort, + bool resetIdentity) throw(); ~Node(); diff --git a/node/NodeConfig.hpp b/node/NodeConfig.hpp index ada135ac5..b9bc8f5f4 100644 --- a/node/NodeConfig.hpp +++ b/node/NodeConfig.hpp @@ -120,16 +120,16 @@ public: } /** - * @return Set of network tap device names from our virtual networks (not other taps on system) + * @return Sorted vector of network tap device names from our virtual networks (not other taps on system) */ - inline std::set networkTapDeviceNames() const + inline std::vector networkTapDeviceNames() const { - std::set tapDevs; + std::vector tapDevs; Mutex::Lock _l(_networks_m); for(std::map< uint64_t,SharedPtr >::const_iterator n(_networks.begin());n!=_networks.end();++n) { std::string dn(n->second->tapDeviceName()); if (dn.length()) - tapDevs.insert(dn); + tapDevs.push_back(dn); } return tapDevs; } diff --git a/node/RuntimeEnvironment.hpp b/node/RuntimeEnvironment.hpp index 8887b0817..78d950ce0 100644 --- a/node/RuntimeEnvironment.hpp +++ b/node/RuntimeEnvironment.hpp @@ -39,7 +39,6 @@ class NodeConfig; class Logger; class Switch; class Topology; -class SysEnv; class CMWC4096; class Service; class Node; @@ -47,6 +46,8 @@ class Multicaster; class SoftwareUpdater; class SocketManager; class AntiRecursion; +class EthernetTapFactory; +class RoutingTable; /** * Holds global state for an instance of ZeroTier::Node @@ -68,6 +69,8 @@ public: tcpTunnelingEnabled(false), timeOfLastResynchronize(0), timeOfLastPacketReceived(0), + tapFactory((EthernetTapFactory *)0), + routingTable((RoutingTable *)0), log((Logger *)0), prng((CMWC4096 *)0), antiRec((AntiRecursion *)0), @@ -75,7 +78,6 @@ public: sw((Switch *)0), sm((SocketManager *)0), topology((Topology *)0), - sysEnv((SysEnv *)0), nc((NodeConfig *)0), updater((SoftwareUpdater *)0) #ifndef __WINDOWS__ @@ -103,6 +105,10 @@ public: // via an ugly const_cast<>. volatile uint64_t timeOfLastPacketReceived; + // These are passed in from outside and are not created or deleted by the ZeroTier node core + EthernetTapFactory *tapFactory; + RoutingTable *routingTable; + /* * Order matters a bit here. These are constructed in this order * and then deleted in the opposite order on Node exit. The order ensures @@ -118,7 +124,6 @@ public: Switch *sw; SocketManager *sm; Topology *topology; - SysEnv *sysEnv; NodeConfig *nc; Node *node; SoftwareUpdater *updater; // null if software updates are not enabled diff --git a/node/SysEnv.cpp b/node/SysEnv.cpp deleted file mode 100644 index fdf52f113..000000000 --- a/node/SysEnv.cpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * ZeroTier One - Global Peer to Peer Ethernet - * Copyright (C) 2011-2014 ZeroTier Networks LLC - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "Constants.hpp" -#include "SysEnv.hpp" -#include "Utils.hpp" -#include "RuntimeEnvironment.hpp" -#include "NodeConfig.hpp" - -#ifdef __UNIX_LIKE__ -#include -#include -#include -#include -#endif - -#ifdef __APPLE__ -#include -#include -#include -#include -#endif - -#ifdef __WINDOWS__ -#include -#include -#endif - -namespace ZeroTier { - -SysEnv::SysEnv() -{ -} - -SysEnv::~SysEnv() -{ -} - -#ifdef __APPLE__ - -uint64_t SysEnv::getNetworkConfigurationFingerprint(const std::set &ignoreDevices) -{ - int mib[6]; - size_t needed; - uint64_t fingerprint = 5381; // djb2 hash algorithm is used below - - // Right now this just scans for changes in default routes. This is not - // totally robust -- it will miss cases where we switch from one 10.0.0.0/24 - // network with gateway .1 to another -- but most of the time it'll pick - // up shifts in connectivity. Combined with sleep/wake detection this seems - // pretty solid so far on Mac for detecting when you change locations. - - mib[0] = CTL_NET; - mib[1] = PF_ROUTE; - mib[2] = 0; - mib[3] = AF_UNSPEC; - mib[4] = NET_RT_DUMP; - mib[5] = 0; - if (!sysctl(mib,6,NULL,&needed,NULL,0)) { - char *buf = (char *)malloc(needed); - if (buf) { - if (!sysctl(mib,6,buf,&needed,NULL,0)) { - struct rt_msghdr *rtm; - for(char *next=buf,*end=buf+needed;nextrtm_msglen; - if (((rtm->rtm_addrs & RTA_DST))&&((rtm->rtm_addrs & RTA_GATEWAY))) { - int sano = 0; - struct sockaddr *dst = (struct sockaddr *)0; - struct sockaddr *gateway = (struct sockaddr *)0; - while (saptr < saend) { - struct sockaddr *sa = (struct sockaddr *)saptr; - if (!sa->sa_len) - break; - if (sano == 0) - dst = sa; - else if (sano == 1) - gateway = sa; - else if (sano > 1) - break; - ++sano; - saptr += sa->sa_len; - } - if ((dst)&&(gateway)) { - if ((dst->sa_family == AF_INET)&&(gateway->sa_family == AF_INET)&&(!((struct sockaddr_in *)dst)->sin_addr.s_addr)) { - fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)((struct sockaddr_in *)gateway)->sin_addr.s_addr; - } else if ((dst->sa_family == AF_INET6)&&(gateway->sa_family == AF_INET6)&&(Utils::isZero(((struct sockaddr_in6 *)dst)->sin6_addr.s6_addr,16))) { - for(unsigned int i=0;i<16;++i) - fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)((struct sockaddr_in6 *)gateway)->sin6_addr.s6_addr[i]; - } - } - } - next = saend; - } - } - free(buf); - } - } - - return fingerprint; -} - -#endif // __APPLE__ - -#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux) - -uint64_t SysEnv::getNetworkConfigurationFingerprint(const std::set &ignoreDevices) -{ - char buf[16384]; - uint64_t fingerprint = 5381; // djb2 hash algorithm is used below - char *t1,*t2; - - try { - // Include default IPv4 route if available - int fd = open("/proc/net/route",O_RDONLY); - if (fd > 0) { - long n = read(fd,buf,sizeof(buf) - 1); - ::close(fd); - if (n > 0) { - buf[n] = 0; - for(char *line=strtok_r(buf,"\r\n",&t1);(line);line=strtok_r((char *)0,"\r\n",&t1)) { - int fno = 0; - for(char *field=strtok_r(line," \t",&t2);(field);field=strtok_r((char *)0," \t",&t2)) { - if (fno == 0) { // device name - if ((ignoreDevices.count(std::string(field)))||(!strcmp(field,"lo"))) - break; - } else if ((fno == 1)||(fno == 2)) { // destination, gateway - if (strlen(field) == 8) { // ignore header junk, use only hex route info - while (*field) - fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)*(field++); - } - } else if (fno > 2) - break; - ++fno; - } - } - } - } - - // Include IPs of IPv6 enabled interfaces if available - fd = open("/proc/net/if_inet6",O_RDONLY); - if (fd > 0) { - long n = read(fd,buf,sizeof(buf) - 1); - ::close(fd); - if (n > 0) { - buf[n] = 0; - for(char *line=strtok_r(buf,"\r\n",&t1);(line);line=strtok_r((char *)0,"\r\n",&t1)) { - int fno = 0; - const char *v6ip = (const char *)0; - const char *devname = (const char *)0; - for(char *field=strtok_r(line," \t",&t2);(field);field=strtok_r((char *)0," \t",&t2)) { - switch(fno) { - case 0: - v6ip = field; - break; - case 5: - devname = field; - break; - } - ++fno; - } - - if ((v6ip)&&(devname)) { - if ((!(ignoreDevices.count(std::string(devname))))&&(strcmp(devname,"lo"))) { - while (*v6ip) - fingerprint = ((fingerprint << 5) + fingerprint) + (uint64_t)*(v6ip++); - } - } - } - } - } - } catch ( ... ) {} - - return fingerprint; -} - -#endif // __linux__ - -#ifdef __WINDOWS__ - -uint64_t SysEnv::getNetworkConfigurationFingerprint(const std::set &ignoreDevices) -{ - // TODO: windows version - return 1; -} - -#endif // __WINDOWS__ - -} // namespace ZeroTier diff --git a/node/SysEnv.hpp b/node/SysEnv.hpp deleted file mode 100644 index dc73101b7..000000000 --- a/node/SysEnv.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * ZeroTier One - Global Peer to Peer Ethernet - * Copyright (C) 2011-2014 ZeroTier Networks LLC - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * -- - * - * ZeroTier may be used and distributed under the terms of the GPLv3, which - * are available at: http://www.gnu.org/licenses/gpl-3.0.html - * - * If you would like to embed ZeroTier into a commercial application or - * redistribute it in a modified binary form, please contact ZeroTier Networks - * LLC. Start here: http://www.zerotier.com/ - */ - -#ifndef ZT_SYSENV_HPP -#define ZT_SYSENV_HPP - -#include - -#include - -#include "NonCopyable.hpp" - -namespace ZeroTier { - -class RuntimeEnvironment; - -/** - * Local system environment monitoring utilities - */ -class SysEnv : NonCopyable -{ -public: - SysEnv(); - ~SysEnv(); - - /** - * This computes a CRC-type code from gathered information about your network settings - * - * @param ignoreDevices Ignore these local network devices by OS-specific name (e.g. our taps) - * @return Fingerprint of currently running network environment - */ - uint64_t getNetworkConfigurationFingerprint(const std::set &ignoreDevices); -}; - -} // namespace ZeroTier - -#endif diff --git a/node/Thread.hpp b/node/Thread.hpp index 6426d5bb1..a796af33f 100644 --- a/node/Thread.hpp +++ b/node/Thread.hpp @@ -87,6 +87,8 @@ public: CancelSynchronousIo(t._th); } + inline operator bool() const throw() { return (_th != NULL); } + private: HANDLE _th; DWORD _tid; @@ -123,18 +125,21 @@ public: throw() { memset(&_tid,0,sizeof(_tid)); + _started = false; } Thread(const Thread &t) throw() { memcpy(&_tid,&(t._tid),sizeof(_tid)); + _started = t._started; } inline Thread &operator=(const Thread &t) throw() { memcpy(&_tid,&(t._tid),sizeof(_tid)); + _started = t._started; return *this; } @@ -151,19 +156,21 @@ public: throw(std::runtime_error) { Thread t; + t._started = true; if (pthread_create(&t._tid,(const pthread_attr_t *)0,&___zt_threadMain,instance)) throw std::runtime_error("pthread_create() failed, unable to create thread"); return t; } /** - * Join to a thread, waiting for it to terminate + * Join to a thread, waiting for it to terminate (does nothing on null Thread values) * * @param t Thread to join */ static inline void join(const Thread &t) { - pthread_join(t._tid,(void **)0); + if (t._started) + pthread_join(t._tid,(void **)0); } /** @@ -171,13 +178,13 @@ public: * * @param ms Number of milliseconds to sleep */ - static inline void sleep(unsigned long ms) - { - usleep(ms * 1000); - } + static inline void sleep(unsigned long ms) { usleep(ms * 1000); } + + inline operator bool() const throw() { return (_started); } private: pthread_t _tid; + volatile bool _started; }; } // namespace ZeroTier diff --git a/objects.mk b/objects.mk index 4a6e7b83f..24a54683a 100644 --- a/objects.mk +++ b/objects.mk @@ -1,5 +1,4 @@ OBJS=\ - ext/lz4/lz4hc.o \ ext/lz4/lz4.o \ node/C25519.o \ node/CertificateOfMembership.o \ @@ -26,7 +25,6 @@ OBJS=\ node/SoftwareUpdater.o \ node/SHA512.o \ node/Switch.o \ - node/SysEnv.o \ node/TcpSocket.o \ node/Topology.o \ node/UdpSocket.o \ diff --git a/osnet/BSDRoutingTable.cpp b/osnet/BSDRoutingTable.cpp index fede9cce5..63dd5cef5 100644 --- a/osnet/BSDRoutingTable.cpp +++ b/osnet/BSDRoutingTable.cpp @@ -43,7 +43,7 @@ #include #include -#include "../Constants.hpp" +#include "../node/Constants.hpp" #include "BSDRoutingTable.hpp" // All I wanted was the bloody rounting table. I didn't expect the Spanish inquisition. diff --git a/osnet/BSDRoutingTable.hpp b/osnet/BSDRoutingTable.hpp index c0e989438..b498d5ce2 100644 --- a/osnet/BSDRoutingTable.hpp +++ b/osnet/BSDRoutingTable.hpp @@ -28,7 +28,7 @@ #ifndef ZT_BSDROUTINGTABLE_HPP #define ZT_BSDROUTINGTABLE_HPP -#include "../RoutingTable.hpp" +#include "../node/RoutingTable.hpp" namespace ZeroTier { diff --git a/osnet/LinuxEthernetTap.cpp b/osnet/LinuxEthernetTap.cpp index 0b0c1c4fa..d77d56c0d 100644 --- a/osnet/LinuxEthernetTap.cpp +++ b/osnet/LinuxEthernetTap.cpp @@ -355,9 +355,8 @@ std::string LinuxEthernetTap::deviceName() const return _dev; } -std::string LinuxEthernetTap::persistentId() const +void LinuxEthernetTap::setFriendlyName(const char *friendlyName) { - return std::string(); } bool LinuxEthernetTap::updateMulticastGroups(std::set &groups) diff --git a/osnet/LinuxEthernetTap.hpp b/osnet/LinuxEthernetTap.hpp index e6aea2ec9..f74fdf867 100644 --- a/osnet/LinuxEthernetTap.hpp +++ b/osnet/LinuxEthernetTap.hpp @@ -64,7 +64,7 @@ public: virtual std::set ips() const; virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len); virtual std::string deviceName() const; - virtual std::string persistentId() const; + virtual void setFriendlyName(const char *friendlyName); virtual bool updateMulticastGroups(std::set &groups); void threadMain() diff --git a/osnet/LinuxEthernetTapFactory.cpp b/osnet/LinuxEthernetTapFactory.cpp index fccb6bd54..01633abbc 100644 --- a/osnet/LinuxEthernetTapFactory.cpp +++ b/osnet/LinuxEthernetTapFactory.cpp @@ -57,7 +57,7 @@ EthernetTap *LinuxEthernetTapFactory::open( return t; } -void LinuxEthernetTapFactory::close(EthernetTap *tap) +void LinuxEthernetTapFactory::close(EthernetTap *tap,bool destroyPersistentDevices) { { Mutex::Lock _l(_devices_m); diff --git a/osnet/LinuxEthernetTapFactory.hpp b/osnet/LinuxEthernetTapFactory.hpp index a5d13216b..695fe52ec 100644 --- a/osnet/LinuxEthernetTapFactory.hpp +++ b/osnet/LinuxEthernetTapFactory.hpp @@ -51,7 +51,7 @@ public: const char *friendlyName, void (*handler)(void *,const MAC &,const MAC &,unsigned int,const Buffer<4096> &), void *arg); - virtual void close(EthernetTap *tap); + virtual void close(EthernetTap *tap,bool destroyPersistentDevices); virtual std::vector allTapDeviceNames() const; private: diff --git a/osnet/OSXEthernetTap.cpp b/osnet/OSXEthernetTap.cpp index 1e2ddbed4..164b24a23 100644 --- a/osnet/OSXEthernetTap.cpp +++ b/osnet/OSXEthernetTap.cpp @@ -529,9 +529,8 @@ std::string OSXEthernetTap::deviceName() const return _dev; } -std::string OSXEthernetTap::persistentId() const +void OSXEthernetTap::setFriendlyName(const char *friendlyName) { - return std::string(); } bool OSXEthernetTap::updateMulticastGroups(std::set &groups) diff --git a/osnet/OSXEthernetTap.hpp b/osnet/OSXEthernetTap.hpp index 7ca637301..680fef150 100644 --- a/osnet/OSXEthernetTap.hpp +++ b/osnet/OSXEthernetTap.hpp @@ -68,7 +68,7 @@ public: virtual std::set ips() const; virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len); virtual std::string deviceName() const; - virtual std::string persistentId() const; + virtual void setFriendlyName(const char *friendlyName); virtual bool updateMulticastGroups(std::set &groups); void threadMain() diff --git a/osnet/OSXEthernetTapFactory.cpp b/osnet/OSXEthernetTapFactory.cpp index 4d0f20691..223cec7f2 100644 --- a/osnet/OSXEthernetTapFactory.cpp +++ b/osnet/OSXEthernetTapFactory.cpp @@ -100,7 +100,7 @@ EthernetTap *OSXEthernetTapFactory::open( return t; } -void OSXEthernetTapFactory::close(EthernetTap *tap) +void OSXEthernetTapFactory::close(EthernetTap *tap,bool destroyPersistentDevices) { { Mutex::Lock _l(_devices_m); diff --git a/osnet/OSXEthernetTapFactory.hpp b/osnet/OSXEthernetTapFactory.hpp index 720e038af..2368d982a 100644 --- a/osnet/OSXEthernetTapFactory.hpp +++ b/osnet/OSXEthernetTapFactory.hpp @@ -62,7 +62,7 @@ public: const char *friendlyName, void (*handler)(void *,const MAC &,const MAC &,unsigned int,const Buffer<4096> &), void *arg); - virtual void close(EthernetTap *tap); + virtual void close(EthernetTap *tap,bool destroyPersistentDevices); virtual std::vector allTapDeviceNames() const; private: