From 4bd58d47cf5d38c5fc38aee9a62382a1ecc94017 Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@zerotier.com>
Date: Tue, 17 Nov 2020 12:52:48 -0800
Subject: [PATCH] Revert "Try factoring out a mutex to see if its faster."

This reverts commit 982c32322bd78bb8485f8a8819b65922d259bb5c.
---
 osdep/LinuxEthernetTap.cpp | 41 +++++++++++++-------------------------
 osdep/LinuxEthernetTap.hpp |  6 +-----
 2 files changed, 15 insertions(+), 32 deletions(-)

diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp
index b7ecdfe10..9306dc34c 100644
--- a/osdep/LinuxEthernetTap.cpp
+++ b/osdep/LinuxEthernetTap.cpp
@@ -92,17 +92,12 @@ LinuxEthernetTap::LinuxEthernetTap(
 	_homePath(homePath),
 	_mtu(mtu),
 	_fd(0),
-	_enabled(true),
-	_bufferReadPtr(0),
-	_bufferWritePtr(0)
+	_enabled(true)
 {
 	static std::mutex s_tapCreateLock;
 	char procpath[128],nwids[32];
 	struct stat sbuf;
 
-	for(unsigned long i=0;i<ZT_BUFFER_POOL_SIZE;++i)
-		_buffers[i] = 0;
-
 	// Create only one tap at a time globally.
 	std::lock_guard<std::mutex> tapCreateLock(s_tapCreateLock);
 
@@ -257,14 +252,15 @@ LinuxEthernetTap::LinuxEthernetTap(
 			if (FD_ISSET(_fd,&readfds)) {
 				for(;;) { // read until there are no more packets, then return to outer select() loop
 					if (!buf) {
-						for(unsigned int k=0;k<ZT_BUFFER_POOL_SIZE;++k) {
-							buf = reinterpret_cast<void *>(_buffers[_bufferReadPtr++ & ZT_BUFFER_POOL_MASK].exchange(0));
-							if (buf)
+						std::lock_guard<std::mutex> l(_buffers_l);
+						if (_buffers.empty()) {
+							buf = malloc(ZT_TAP_BUF_SIZE);
+							if (!buf)
 								break;
+						} else {
+							buf = _buffers.back();
+							_buffers.pop_back();
 						}
-						buf = malloc(ZT_TAP_BUF_SIZE);
-						if (!buf)
-							continue;
 					}
 
 					n = (int)::read(_fd,reinterpret_cast<uint8_t *>(buf) + r,ZT_TAP_BUF_SIZE - r);
@@ -298,22 +294,16 @@ LinuxEthernetTap::LinuxEthernetTap(
 		MAC to,from;
 		std::pair<void *,int> qi;
 		while (_tapq.get(qi)) {
-			uint8_t *b = reinterpret_cast<uint8_t *>(qi.first);
+			uint8_t *const b = reinterpret_cast<uint8_t *>(qi.first);
 			if (b) {
 				to.setTo(b, 6);
 				from.setTo(b + 6, 6);
 				unsigned int etherType = Utils::ntoh(((const uint16_t *)b)[6]);
 				_handler(_arg, nullptr, _nwid, from, to, etherType, 0, (const void *)(b + 14),(unsigned int)(qi.second - 14));
-
-				for(unsigned int k=0;k<ZT_BUFFER_POOL_SIZE;++k) {
-					uintptr_t zero = 0;
-					if (_buffers[_bufferWritePtr++ & ZT_BUFFER_POOL_MASK].compare_exchange_strong(zero,reinterpret_cast<uintptr_t>(b))) {
-						b = nullptr;
-						break;
-					}
+				{
+					std::lock_guard<std::mutex> l(_buffers_l);
+					_buffers.push_back(qi.first);
 				}
-				if (b)
-					free(b);
 			} else break;
 		}
 	});
@@ -331,11 +321,8 @@ LinuxEthernetTap::~LinuxEthernetTap()
 	_tapReaderThread.join();
 	_tapProcessorThread.join();
 
-	for(unsigned int k=0;k<ZT_BUFFER_POOL_SIZE;++k) {
-		void *p = reinterpret_cast<void *>(_buffers[k].load());
-		if (p)
-			free(p);
-	}
+	for(std::vector<void *>::iterator i(_buffers.begin());i!=_buffers.end();++i)
+		free(*i);
 	std::vector< std::pair<void *,int> > dv(_tapq.drain());
 	for(std::vector< std::pair<void *,int> >::iterator i(dv.begin());i!=dv.end();++i) {
 		if (i->first)
diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp
index f56477af6..9e9206ead 100644
--- a/osdep/LinuxEthernetTap.hpp
+++ b/osdep/LinuxEthernetTap.hpp
@@ -28,9 +28,6 @@
 #include "EthernetTap.hpp"
 #include "BlockingQueue.hpp"
 
-#define ZT_BUFFER_POOL_SIZE 64
-#define ZT_BUFFER_POOL_MASK 63U
-
 namespace ZeroTier {
 
 class LinuxEthernetTap : public EthernetTap
@@ -76,8 +73,7 @@ private:
 	std::thread _tapReaderThread;
 	std::thread _tapProcessorThread;
 	std::mutex _buffers_l;
-	std::atomic<uintptr_t> _buffers[ZT_BUFFER_POOL_SIZE];
-	std::atomic<uintptr_t> _bufferReadPtr,_bufferWritePtr;
+	std::vector<void *> _buffers;
 	BlockingQueue< std::pair<void *,int> > _tapq;
 };