From 39e1021f629188adec64bfe47170f4a874b490a2 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 21 Mar 2019 16:18:49 -0700 Subject: [PATCH] Replace certificate based gating of multicast like/gather with a simpler more efficient method, fix some minor issues with request based com/cert push, and clean up some other random stuff. --- node/Address.hpp | 51 +-- node/C25519.cpp | 70 ----- node/IncomingPacket.cpp | 19 +- node/Membership.hpp | 2 +- node/Node.cpp | 2 + node/Packet.cpp | 672 ++++++++++++++++++++-------------------- 6 files changed, 368 insertions(+), 448 deletions(-) diff --git a/node/Address.hpp b/node/Address.hpp index a2214c9e0..ae0566fe4 100644 --- a/node/Address.hpp +++ b/node/Address.hpp @@ -54,28 +54,16 @@ public: * @param bits Raw address -- 5 bytes, big-endian byte order * @param len Length of array */ - Address(const void *bits,unsigned int len) - { - setTo(bits,len); - } + Address(const void *bits,unsigned int len) { setTo(bits,len); } - inline Address &operator=(const Address &a) - { - _a = a._a; - return *this; - } - - inline Address &operator=(const uint64_t a) - { - _a = (a & 0xffffffffffULL); - return *this; - } + inline Address &operator=(const Address &a) { _a = a._a; return *this; } + inline Address &operator=(const uint64_t a) { _a = (a & 0xffffffffffULL); return *this; } /** * @param bits Raw address -- 5 bytes, big-endian byte order * @param len Length of array */ - inline void setTo(const void *bits,unsigned int len) + inline void setTo(const void *bits,const unsigned int len) { if (len < ZT_ADDRESS_LENGTH) { _a = 0; @@ -94,7 +82,7 @@ public: * @param bits Buffer to hold 5-byte address in big-endian byte order * @param len Length of array */ - inline void copyTo(void *bits,unsigned int len) const + inline void copyTo(void *const bits,const unsigned int len) const { if (len < ZT_ADDRESS_LENGTH) return; @@ -125,37 +113,23 @@ public: /** * @return Integer containing address (0 to 2^40) */ - inline uint64_t toInt() const - { - return _a; - } + inline uint64_t toInt() const { return _a; } /** * @return Hash code for use with Hashtable */ - inline unsigned long hashCode() const - { - return (unsigned long)_a; - } + inline unsigned long hashCode() const { return (unsigned long)_a; } /** * @return Hexadecimal string */ - inline char *toString(char buf[11]) const - { - return Utils::hex10(_a,buf); - } + inline char *toString(char buf[11]) const { return Utils::hex10(_a,buf); } /** * @return True if this address is not zero */ inline operator bool() const { return (_a != 0); } - /** - * Set to null/zero - */ - inline void zero() { _a = 0; } - /** * Check if this address is reserved * @@ -165,16 +139,15 @@ public: * * @return True if address is reserved and may not be used */ - inline bool isReserved() const - { - return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); - } + inline bool isReserved() const { return ((!_a)||((_a >> 32) == ZT_ADDRESS_RESERVED_PREFIX)); } /** * @param i Value from 0 to 4 (inclusive) * @return Byte at said position (address interpreted in big-endian order) */ - inline unsigned char operator[](unsigned int i) const { return (unsigned char)((_a >> (32 - (i * 8))) & 0xff); } + inline uint8_t operator[](unsigned int i) const { return (uint8_t)(_a >> (32 - (i * 8))); } + + inline void zero() { _a = 0; } inline bool operator==(const uint64_t &a) const { return (_a == (a & 0xffffffffffULL)); } inline bool operator!=(const uint64_t &a) const { return (_a != (a & 0xffffffffffULL)); } diff --git a/node/C25519.cpp b/node/C25519.cpp index 209ac445b..cab7b9d53 100644 --- a/node/C25519.cpp +++ b/node/C25519.cpp @@ -2003,25 +2003,6 @@ extern "C" void ed25519_amd64_asm_sign(const unsigned char *sk,const unsigned ch namespace ZeroTier { -#ifdef ZT_CONTROLLER -struct C25519CacheKey -{ - uint64_t messageDigest[4]; - uint64_t publicKey[4]; - inline unsigned long hashCode() const { return (unsigned long)(messageDigest[0] ^ publicKey[0]); } - inline bool operator==(const C25519CacheKey &k) const { return (memcmp(this,&k,sizeof(C25519CacheKey)) == 0); } - inline bool operator!=(const C25519CacheKey &k) const { return (memcmp(this,&k,sizeof(C25519CacheKey)) != 0); } -}; -struct C25519CacheValue -{ - uint64_t signature[12]; - uint64_t timestamp; -}; -static uint64_t _ed25519TimestampCounter = 0; -static Hashtable _ed25519Cache; -static Mutex _ed25519CacheLock; -#endif - void C25519::agree(const C25519::Private &mine,const C25519::Public &their,void *keybuf,unsigned int keylen) { unsigned char rawkey[32]; @@ -2043,21 +2024,6 @@ void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPubli unsigned char digest[64]; // we sign the first 32 bytes of SHA-512(msg) SHA512::hash(digest,msg,len); -#ifdef ZT_CONTROLLER - C25519CacheKey ck; - ZT_FAST_MEMCPY(ck.messageDigest,digest,32); - ZT_FAST_MEMCPY(ck.publicKey,myPublic.data + 32,32); - C25519CacheValue *cv = (C25519CacheValue *)0; - { - Mutex::Lock l(_ed25519CacheLock); - cv = _ed25519Cache.get(ck); - } - if (cv) { - ZT_FAST_MEMCPY(signature,cv->signature,ZT_C25519_SIGNATURE_LEN); - return; - } -#endif - #ifdef ZT_USE_FAST_X64_ED25519 ed25519_amd64_asm_sign(myPrivate.data + 32,myPublic.data + 32,digest,(unsigned char *)signature); #else @@ -2103,28 +2069,6 @@ void C25519::sign(const C25519::Private &myPrivate,const C25519::Public &myPubli for(unsigned int i=0;i<32;i++) sig[32 + i] = s[i]; #endif - -#ifdef ZT_CONTROLLER - C25519CacheValue cvn; - memcpy(cvn.signature,signature,ZT_C25519_SIGNATURE_LEN); - { - Mutex::Lock l(_ed25519CacheLock); - - if (_ed25519Cache.size() > 1048576) { - const uint64_t before = _ed25519TimestampCounter - ((1048576 / 3) * 2); - Hashtable< C25519CacheKey,C25519CacheValue >::Iterator i(_ed25519Cache); - C25519CacheKey *ik = (C25519CacheKey *)0; - C25519CacheValue *iv = (C25519CacheValue *)0; - while (i.next(ik,iv)) { - if (iv->timestamp < before) - _ed25519Cache.erase(*ik); - } - } - - cvn.timestamp = ++_ed25519TimestampCounter; - _ed25519Cache.set(ck,cvn); - } -#endif } bool C25519::verify(const C25519::Public &their,const void *msg,unsigned int len,const void *signature) @@ -2135,20 +2079,6 @@ bool C25519::verify(const C25519::Public &their,const void *msg,unsigned int len if (!Utils::secureEq(sig + 64,digest,32)) return false; -#ifdef ZT_CONTROLLER - C25519CacheKey ck; - ZT_FAST_MEMCPY(ck.messageDigest,digest,32); - ZT_FAST_MEMCPY(ck.publicKey,their.data + 32,32); - C25519CacheValue *cv = (C25519CacheValue *)0; - { - Mutex::Lock l(_ed25519CacheLock); - cv = _ed25519Cache.get(ck); - } - if (cv) { - return Utils::secureEq(cv->signature,signature,ZT_C25519_SIGNATURE_LEN); - } -#endif - unsigned char t2[32]; ge25519 get1, get2; sc25519 schram, scs; diff --git a/node/IncomingPacket.cpp b/node/IncomingPacket.cpp index aec37a6bc..9caa30930 100644 --- a/node/IncomingPacket.cpp +++ b/node/IncomingPacket.cpp @@ -761,9 +761,24 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr &peer) { const int64_t now = RR->node->now(); + bool authorized = false; + uint64_t lastNwid = 0; + // Packet contains a series of 18-byte network,MAC,ADI tuples - for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptrmc->add(tPtr,now,at(ptr),MulticastGroup(MAC(field(ptr + 8,6),6),at(ptr + 14)),peer->address()); + for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr(ptr); + if (nwid != lastNwid) { + lastNwid = nwid; + SharedPtr network(RR->node->network(nwid)); + if (network) + authorized = network->gate(tPtr,peer); + if (!authorized) + authorized = ((RR->topology->amUpstream())||(RR->node->localControllerHasAuthorized(now,nwid,peer->address()))); + } + if (authorized) + RR->mc->add(tPtr,now,nwid,MulticastGroup(MAC(field(ptr + 8,6),6),at(ptr + 14)),peer->address()); + } + peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false,0); return true; } diff --git a/node/Membership.hpp b/node/Membership.hpp index 55ba53734..90b6be6a7 100644 --- a/node/Membership.hpp +++ b/node/Membership.hpp @@ -196,7 +196,7 @@ private: } template - void _cleanCredImpl(const NetworkConfig &nconf,Hashtable &remoteCreds) + inline void _cleanCredImpl(const NetworkConfig &nconf,Hashtable &remoteCreds) { uint32_t *k = (uint32_t *)0; C *v = (C *)0; diff --git a/node/Node.cpp b/node/Node.cpp index c96ae8fd9..3342e6cae 100644 --- a/node/Node.cpp +++ b/node/Node.cpp @@ -270,11 +270,13 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64 RR->topology->getUpstreamsToContact(alwaysContact); // Uncomment to dump stats + /* for(unsigned int i=0;i<32;i++) { if (_stats.inVerbCounts[i] > 0) printf("%.2x\t%12lld %lld\n",i,(unsigned long long)_stats.inVerbCounts[i],(unsigned long long)_stats.inVerbBytes[i]); } printf("\n"); + */ // Check last receive time on designated upstreams to see if we seem to be online int64_t lastReceivedFromUpstream = 0; diff --git a/node/Packet.cpp b/node/Packet.cpp index 3978cc485..448b421c8 100644 --- a/node/Packet.cpp +++ b/node/Packet.cpp @@ -42,8 +42,8 @@ #ifdef _MSC_VER #define FORCE_INLINE static __forceinline #include -#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #else #define FORCE_INLINE static inline #endif @@ -155,16 +155,16 @@ namespace { A library is provided to take care of it, see lz4frame.h. */ -#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ -#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ -#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE #define LZ4_QUOTE(str) #str #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) #define LZ4_MEMORY_USAGE 14 -#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ @@ -173,7 +173,7 @@ static inline void LZ4_resetStream (LZ4_stream_t* streamPtr); #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) -#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ +#define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ typedef struct { uint32_t hashTable[LZ4_HASH_SIZE_U32]; @@ -192,14 +192,14 @@ typedef struct { } LZ4_streamDecode_t_internal; #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) -#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) union LZ4_stream_u { unsigned long long table[LZ4_STREAMSIZE_U64]; LZ4_stream_t_internal internal_donotuse; } ; /* previously typedef'd to LZ4_stream_t */ #define LZ4_STREAMDECODESIZE_U64 4 -#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) union LZ4_streamDecode_u { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; LZ4_streamDecode_t_internal internal_donotuse; @@ -224,8 +224,8 @@ union LZ4_streamDecode_u { #endif #define ALLOCATOR(n,s) calloc(n,s) -#define FREEMEM free -#define MEM_INIT memset +#define FREEMEM free +#define MEM_INIT memset typedef uint8_t BYTE; typedef uint16_t U16; @@ -280,21 +280,21 @@ static inline void LZ4_write32(void* memPtr, U32 value) static inline U16 LZ4_readLE16(const void* memPtr) { if (LZ4_isLittleEndian()) { - return LZ4_read16(memPtr); + return LZ4_read16(memPtr); } else { - const BYTE* p = (const BYTE*)memPtr; - return (U16)((U16)p[0] + (p[1]<<8)); + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); } } static inline void LZ4_writeLE16(void* memPtr, U16 value) { if (LZ4_isLittleEndian()) { - LZ4_write16(memPtr, value); + LZ4_write16(memPtr, value); } else { - BYTE* p = (BYTE*)memPtr; - p[0] = (BYTE) value; - p[1] = (BYTE)(value>>8); + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); } } @@ -330,63 +330,63 @@ static const int LZ4_minLength = (MFLIMIT+1); #define RUN_BITS (8-ML_BITS) #define RUN_MASK ((1U<>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r; - _BitScanForward( &r, (U32)val ); - return (int)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } } else /* Big Endian CPU */ { - if (sizeof(val)==8) { -# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll((U64)val) >> 3); -# else - unsigned r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else /* 32 bits */ { -# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, (unsigned long)val ); - return (unsigned)(r>>3); -# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } + if (sizeof(val)==8) { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else /* 32 bits */ { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } } @@ -396,10 +396,10 @@ static inline unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE const BYTE* const pStart = pIn; while (likely(pIn> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); else - return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); + return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } static inline U32 LZ4_hash5(U64 sequence, tableType_t const tableType) @@ -436,9 +436,9 @@ static inline U32 LZ4_hash5(U64 sequence, tableType_t const tableType) static const U64 prime8bytes = 11400714785074694791ULL; const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; if (LZ4_isLittleEndian()) - return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); + return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); else - return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); + return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); } FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) @@ -477,16 +477,16 @@ FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableTy } FORCE_INLINE int LZ4_compress_generic( - LZ4_stream_t_internal* const cctx, - const char* const source, - char* const dest, - const int inputSize, - const int maxOutputSize, - const limitedOutput_directive outputLimited, - const tableType_t tableType, - const dict_directive dict, - const dictIssue_directive dictIssue, - const U32 acceleration) + LZ4_stream_t_internal* const cctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) { const BYTE* ip = (const BYTE*) source; const BYTE* base; @@ -511,20 +511,20 @@ FORCE_INLINE int LZ4_compress_generic( { case noDict: default: - base = (const BYTE*)source; - lowLimit = (const BYTE*)source; - break; + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; case withPrefix64k: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source - cctx->dictSize; - break; + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source - cctx->dictSize; + break; case usingExtDict: - base = (const BYTE*)source - cctx->currentOffset; - lowLimit = (const BYTE*)source; - break; + base = (const BYTE*)source - cctx->currentOffset; + lowLimit = (const BYTE*)source; + break; } if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ - if (inputSizehashTable, tableType, base); @@ -532,143 +532,143 @@ FORCE_INLINE int LZ4_compress_generic( /* Main Loop */ for ( ; ; ) { - ptrdiff_t refDelta = 0; - const BYTE* match; - BYTE* token; + ptrdiff_t refDelta = 0; + const BYTE* match; + BYTE* token; - /* Find a match */ - { const BYTE* forwardIp = ip; - unsigned step = 1; - unsigned searchMatchNb = acceleration << LZ4_skipTrigger; - do { - U32 const h = forwardH; - ip = forwardIp; - forwardIp += step; - step = (searchMatchNb++ >> LZ4_skipTrigger); + /* Find a match */ + { const BYTE* forwardIp = ip; + unsigned step = 1; + unsigned searchMatchNb = acceleration << LZ4_skipTrigger; + do { + U32 const h = forwardH; + ip = forwardIp; + forwardIp += step; + step = (searchMatchNb++ >> LZ4_skipTrigger); - if (unlikely(forwardIp > mflimit)) goto _last_literals; + if (unlikely(forwardIp > mflimit)) goto _last_literals; - match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - forwardH = LZ4_hashPosition(forwardIp, tableType); - LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); + match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); - } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) - || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) - || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); - } + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } - /* Catch up */ - while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + /* Catch up */ + while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } - /* Encode Literals */ - { unsigned const litLength = (unsigned)(ip - anchor); - token = op++; - if ((outputLimited) && /* Check output buffer overflow */ - (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) - return 0; - if (litLength >= RUN_MASK) { - int len = (int)litLength-RUN_MASK; - *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; - *op++ = (BYTE)len; - } - else *token = (BYTE)(litLength< olimit))) + return 0; + if (litLength >= RUN_MASK) { + int len = (int)litLength-RUN_MASK; + *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); - ip += MINMATCH + matchCode; - if (ip==limit) { - unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); - matchCode += more; - ip += more; - } - } else { - matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); - ip += MINMATCH + matchCode; - } + if ((dict==usingExtDict) && (lowLimit==dictionary)) { + const BYTE* limit; + match += refDelta; + limit = ip + (dictEnd-match); + if (limit > matchlimit) limit = matchlimit; + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchCode; + if (ip==limit) { + unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchCode += more; + ip += more; + } + } else { + matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchCode; + } - if ( outputLimited && /* Check output buffer overflow */ - (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) - return 0; - if (matchCode >= ML_MASK) { - *token += ML_MASK; - matchCode -= ML_MASK; - LZ4_write32(op, 0xFFFFFFFF); - while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; - op += matchCode / 255; - *op++ = (BYTE)(matchCode % 255); - } else - *token += (BYTE)(matchCode); - } + if ( outputLimited && /* Check output buffer overflow */ + (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) + return 0; + if (matchCode >= ML_MASK) { + *token += ML_MASK; + matchCode -= ML_MASK; + LZ4_write32(op, 0xFFFFFFFF); + while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; + op += matchCode / 255; + *op++ = (BYTE)(matchCode % 255); + } else + *token += (BYTE)(matchCode); + } - anchor = ip; + anchor = ip; - /* Test end of chunk */ - if (ip > mflimit) break; + /* Test end of chunk */ + if (ip > mflimit) break; - /* Fill table */ - LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); + /* Fill table */ + LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); - /* Test next position */ - match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); - if (dict==usingExtDict) { - if (match < (const BYTE*)source) { - refDelta = dictDelta; - lowLimit = dictionary; - } else { - refDelta = 0; - lowLimit = (const BYTE*)source; - } } - LZ4_putPosition(ip, cctx->hashTable, tableType, base); - if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) - && (match+MAX_DISTANCE>=ip) - && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) - { token=op++; *token=0; goto _next_match; } + /* Test next position */ + match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); + if (dict==usingExtDict) { + if (match < (const BYTE*)source) { + refDelta = dictDelta; + lowLimit = dictionary; + } else { + refDelta = 0; + lowLimit = (const BYTE*)source; + } } + LZ4_putPosition(ip, cctx->hashTable, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } - /* Prepare next loop */ - forwardH = LZ4_hashPosition(++ip, tableType); + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); } _last_literals: /* Encode Last Literals */ { size_t const lastRun = (size_t)(iend - anchor); - if ( (outputLimited) && /* Check output buffer overflow */ - ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) - return 0; - if (lastRun >= RUN_MASK) { - size_t accumulator = lastRun - RUN_MASK; - *op++ = RUN_MASK << ML_BITS; - for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; - *op++ = (BYTE) accumulator; - } else { - *op++ = (BYTE)(lastRun< (U32)maxOutputSize) ) + return 0; + if (lastRun >= RUN_MASK) { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } else { + *op++ = (BYTE)(lastRun<= LZ4_compressBound(inputSize)) { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { - if (inputSize < LZ4_64Klimit) - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); - else - return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); } } @@ -717,19 +717,19 @@ static inline void LZ4_resetStream (LZ4_stream_t* LZ4_stream) } FORCE_INLINE int LZ4_decompress_generic( - const char* const source, - char* const dest, - int inputSize, - int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ - int endOnInput, /* endOnOutputSize, endOnInputSize */ - int partialDecoding, /* full, partial */ - int targetOutputSize, /* only used if partialDecoding==partial */ - int dict, /* noDict, withPrefix64k, usingExtDict */ - const BYTE* const lowPrefix, /* == dest when no prefix */ - const BYTE* const dictStart, /* only if dict==usingExtDict */ - const size_t dictSize /* note : = 0 if noDict */ - ) + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest when no prefix */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) { /* Local Variables */ const BYTE* ip = (const BYTE*) source; @@ -750,125 +750,125 @@ FORCE_INLINE int LZ4_decompress_generic( /* Special cases */ - if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); /* Main Loop : decode sequences */ while (1) { - size_t length; - const BYTE* match; - size_t offset; + size_t length; + const BYTE* match; + size_t offset; - /* get literal length */ - unsigned const token = *ip++; - if ((length=(token>>ML_BITS)) == RUN_MASK) { - unsigned s; - do { - s = *ip++; - length += s; - } while ( likely(endOnInput ? ip>ML_BITS)) == RUN_MASK) { + unsigned s; + do { + s = *ip++; + length += s; + } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) - || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) - { - if (partialDecoding) { - if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ - if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ - } else { - if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ - if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ - } - ZT_FAST_MEMCPY(op, ip, length); - ip += length; - op += length; - break; /* Necessarily EOF, due to parsing restrictions */ - } - LZ4_wildCopy(op, ip, cpy); - ip += length; op = cpy; + /* copy literals */ + cpy = op+length; + if ( ((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) + { + if (partialDecoding) { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } else { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + ZT_FAST_MEMCPY(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; - /* get offset */ - offset = LZ4_readLE16(ip); ip+=2; - match = op - offset; - if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ - LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ + /* get offset */ + offset = LZ4_readLE16(ip); ip+=2; + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ - /* get matchlength */ - length = token & ML_MASK; - if (length == ML_MASK) { - unsigned s; - do { - s = *ip++; - if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; - length += s; - } while (s==255); - if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ - } - length += MINMATCH; + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) { + unsigned s; + do { + s = *ip++; + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; - /* check external dictionary */ - if ((dict==usingExtDict) && (match < lowPrefix)) { - if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ - if (length <= (size_t)(lowPrefix-match)) { - /* match can be copied as a single segment from external dictionary */ - memmove(op, dictEnd - (lowPrefix-match), length); - op += length; - } else { - /* match encompass external dictionary and current block */ - size_t const copySize = (size_t)(lowPrefix-match); - size_t const restSize = length - copySize; - ZT_FAST_MEMCPY(op, dictEnd - copySize, copySize); - op += copySize; - if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ - BYTE* const endOfMatch = op + restSize; - const BYTE* copyFrom = lowPrefix; - while (op < endOfMatch) *op++ = *copyFrom++; - } else { - ZT_FAST_MEMCPY(op, lowPrefix, restSize); - op += restSize; - } } - continue; - } + if (length <= (size_t)(lowPrefix-match)) { + /* match can be copied as a single segment from external dictionary */ + memmove(op, dictEnd - (lowPrefix-match), length); + op += length; + } else { + /* match encompass external dictionary and current block */ + size_t const copySize = (size_t)(lowPrefix-match); + size_t const restSize = length - copySize; + ZT_FAST_MEMCPY(op, dictEnd - copySize, copySize); + op += copySize; + if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ + BYTE* const endOfMatch = op + restSize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } else { + ZT_FAST_MEMCPY(op, lowPrefix, restSize); + op += restSize; + } } + continue; + } - /* copy match within block */ - cpy = op + length; - if (unlikely(offset<8)) { - const int dec64 = dec64table[offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[offset]; - ZT_FAST_MEMCPY(op+4, match, 4); - match -= dec64; - } else { LZ4_copy8(op, match); match+=8; } - op += 8; + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + ZT_FAST_MEMCPY(op+4, match, 4); + match -= dec64; + } else { LZ4_copy8(op, match); match+=8; } + op += 8; - if (unlikely(cpy>oend-12)) { - BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); - if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ - if (op < oCopyLimit) { - LZ4_wildCopy(op, match, oCopyLimit); - match += oCopyLimit - op; - op = oCopyLimit; - } - while (op16) LZ4_wildCopy(op+8, match+8, cpy); - } - op=cpy; /* correction */ + if (unlikely(cpy>oend-12)) { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) { + LZ4_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (op16) LZ4_wildCopy(op+8, match+8, cpy); + } + op=cpy; /* correction */ } /* end of decoding */ if (endOnInput) - return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ + return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ else return (int) (((const char*)ip)-source); /* Nb of input bytes read */ @@ -989,7 +989,7 @@ bool Packet::compress() if ((!compressed())&&(size() > (ZT_PACKET_IDX_PAYLOAD + 64))) { // don't bother compressing tiny packets int pl = (int)(size() - ZT_PACKET_IDX_PAYLOAD); - int cl = LZ4_compress_fast(data + ZT_PACKET_IDX_PAYLOAD,buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,2); + int cl = LZ4_compress_fast(data + ZT_PACKET_IDX_PAYLOAD,buf,pl,ZT_PROTO_MAX_PACKET_LENGTH * 2,1); if ((cl > 0)&&(cl < pl)) { data[ZT_PACKET_IDX_VERB] |= (char)ZT_PROTO_VERB_FLAG_COMPRESSED; setSize((unsigned int)cl + ZT_PACKET_IDX_PAYLOAD);