From a390629371970be9dffa9db7cd8af2a7004ee50c Mon Sep 17 00:00:00 2001
From: Adam Ierymenko <adam.ierymenko@zerotier.com>
Date: Wed, 18 Nov 2020 22:06:17 -0500
Subject: [PATCH] Try another optimization in LinuxEthernetTap.

---
 osdep/LinuxEthernetTap.cpp | 112 ++++++++++++++-----------------------
 osdep/LinuxEthernetTap.hpp |   1 -
 2 files changed, 43 insertions(+), 70 deletions(-)

diff --git a/osdep/LinuxEthernetTap.cpp b/osdep/LinuxEthernetTap.cpp
index 25ed52edf..4659df556 100644
--- a/osdep/LinuxEthernetTap.cpp
+++ b/osdep/LinuxEthernetTap.cpp
@@ -182,14 +182,7 @@ LinuxEthernetTap::LinuxEthernetTap(
 	_dev = ifr.ifr_name;
 	::fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC);
 
-	(void)::pipe(_shutdownSignalPipe);
-
 	_tapReaderThread = std::thread([this]{
-		fd_set readfds,nullfds;
-		int n,nfds,r;
-		void *buf = nullptr;
-		std::vector<void *> buffers;
-
 		{
 			struct ifreq ifr;
 			memset(&ifr,0,sizeof(ifr));
@@ -232,67 +225,51 @@ LinuxEthernetTap::LinuxEthernetTap(
 				return;
 			}
 
-			fcntl(_fd,F_SETFL,O_NONBLOCK);
-
 			::close(sock);
 		}
 
-		FD_ZERO(&readfds);
-		FD_ZERO(&nullfds);
-		nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
-
-		r = 0;
-		for(;;) {
-			FD_SET(_shutdownSignalPipe[0],&readfds);
-			FD_SET(_fd,&readfds);
-			select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
-
-			if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
-				break;
-
-			if (FD_ISSET(_fd,&readfds)) {
-				for(;;) { // read until there are no more packets, then return to outer select() loop
-					if (!buf) {
-						// To reduce use of the mutex, we keep a local buffer vector and
-						// swap (which is a pointer swap) with the global one when it's
-						// empty. This retrieves a batch of buffers to use.
-						if (buffers.empty()) {
-							std::lock_guard<std::mutex> l(_buffers_l);
-							buffers.swap(_buffers);
-						}
-						if (buffers.empty()) {
-							buf = malloc(ZT_TAP_BUF_SIZE);
-							if (!buf)
-								break;
-						} else {
-							buf = buffers.back();
-							buffers.pop_back();
-						}
-					}
-
-					n = (int)::read(_fd,reinterpret_cast<uint8_t *>(buf) + r,ZT_TAP_BUF_SIZE - r);
-
-					if (n > 0) {
-						// Some tap drivers like to send the ethernet frame and the
-						// payload in two chunks, so handle that by accumulating
-						// data until we have at least a frame.
-						r += n;
-						if (r > 14) {
-							if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
-								r = _mtu + 14;
-
-							if (_enabled) {
-								_tapq.post(std::pair<void *,int>(buf,r));
-								buf = nullptr;
-							}
-
-							r = 0;
-						}
-					} else {
-						r = 0;
-						break;
-					}
+		std::vector<void *> buffers;
+		void *buf = nullptr;
+		for(int r=0;;) {
+			if (!buf) {
+				// To reduce use of the mutex, we keep a local buffer vector and
+				// swap (which is a pointer swap) with the global one when it's
+				// empty. This retrieves a batch of buffers to use.
+				if (buffers.empty()) {
+					std::lock_guard<std::mutex> l(_buffers_l);
+					buffers.swap(_buffers);
 				}
+				if (buffers.empty()) {
+					buf = malloc(ZT_TAP_BUF_SIZE);
+					if (!buf)
+						break;
+				} else {
+					buf = buffers.back();
+					buffers.pop_back();
+				}
+			}
+
+			const int n = (int)::read(_fd,reinterpret_cast<uint8_t *>(buf) + r,ZT_TAP_BUF_SIZE - r);
+
+			if (n > 0) {
+				// Some tap drivers like to send the ethernet frame and the
+				// payload in two chunks, so handle that by accumulating
+				// data until we have at least a frame.
+				r += n;
+				if (r > 14) {
+					if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
+						r = _mtu + 14;
+
+					if (_enabled) {
+						_tapq.post(std::pair<void *,int>(buf,r));
+						buf = nullptr;
+					}
+
+					r = 0;
+				}
+			} else {
+				r = 0;
+				break;
 			}
 		}
 	});
@@ -320,12 +297,9 @@ LinuxEthernetTap::LinuxEthernetTap(
 
 LinuxEthernetTap::~LinuxEthernetTap()
 {
-	(void)::write(_shutdownSignalPipe[1],"\0",1); // causes reader thread to exit
-	_tapq.post(std::pair<void *,int>(nullptr,0)); // causes processor thread to exit
-
+	_tapq.post(std::pair<void *,int>(nullptr,0));
+	::shutdown(_fd, SHUT_RDWR);
 	::close(_fd);
-	::close(_shutdownSignalPipe[0]);
-	::close(_shutdownSignalPipe[1]);
 
 	_tapReaderThread.join();
 	_tapProcessorThread.join();
diff --git a/osdep/LinuxEthernetTap.hpp b/osdep/LinuxEthernetTap.hpp
index 9e9206ead..f296ec693 100644
--- a/osdep/LinuxEthernetTap.hpp
+++ b/osdep/LinuxEthernetTap.hpp
@@ -68,7 +68,6 @@ private:
 	std::vector<MulticastGroup> _multicastGroups;
 	unsigned int _mtu;
 	int _fd;
-	int _shutdownSignalPipe[2];
 	std::atomic_bool _enabled;
 	std::thread _tapReaderThread;
 	std::thread _tapProcessorThread;