Patch tap-mac to latest tuntaposx upstream changes, and add updated build for 10.8 or later (keep old one for 10.7).

ext/bin/tap-mac/com.zerotier.tap.kext/Contents/Info.plist

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>English</string>
+	<key>CFBundleExecutable</key>
+	<string>tap</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.zerotier.tap</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>tap</string>
+	<key>CFBundlePackageType</key>
+	<string>KEXT</string>
+	<key>CFBundleShortVersionString</key>
+	<string>20150118</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>1.0</string>
+	<key>OSBundleLibraries</key>
+	<dict>
+		<key>com.apple.kpi.mach</key>
+		<string>8.0</string>
+		<key>com.apple.kpi.bsd</key>
+		<string>8.0</string>
+		<key>com.apple.kpi.libkern</key>
+		<string>8.0</string>
+		<key>com.apple.kpi.unsupported</key>
+		<string>8.0</string>
+	</dict>
+</dict>
+</plist>
+

ext/bin/tap-mac/com.zerotier.tap.kext/Contents/_CodeSignature/CodeResources

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>files</key>
+	<dict/>
+	<key>files2</key>
+	<dict/>
+	<key>rules</key>
+	<dict>
+		<key>^Resources/</key>
+		<true/>
+		<key>^Resources/.*\.lproj/</key>
+		<dict>
+			<key>optional</key>
+			<true/>
+			<key>weight</key>
+			<real>1000</real>
+		</dict>
+		<key>^Resources/.*\.lproj/locversion.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>1100</real>
+		</dict>
+		<key>^version.plist$</key>
+		<true/>
+	</dict>
+	<key>rules2</key>
+	<dict>
+		<key>.*\.dSYM($|/)</key>
+		<dict>
+			<key>weight</key>
+			<real>11</real>
+		</dict>
+		<key>^(.*/)?\.DS_Store$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>2000</real>
+		</dict>
+		<key>^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/</key>
+		<dict>
+			<key>nested</key>
+			<true/>
+			<key>weight</key>
+			<real>10</real>
+		</dict>
+		<key>^.*</key>
+		<true/>
+		<key>^Info\.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^PkgInfo$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^Resources/</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^Resources/.*\.lproj/</key>
+		<dict>
+			<key>optional</key>
+			<true/>
+			<key>weight</key>
+			<real>1000</real>
+		</dict>
+		<key>^Resources/.*\.lproj/locversion.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>1100</real>
+		</dict>
+		<key>^[^/]+$</key>
+		<dict>
+			<key>nested</key>
+			<true/>
+			<key>weight</key>
+			<real>10</real>
+		</dict>
+		<key>^embedded\.provisionprofile$</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^version\.plist$</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+	</dict>
+</dict>
+</plist>

ext/tap-mac/tuntap/Makefile

@@ -1,12 +1,95 @@
-TUNTAP_VERSION = 20131028
+# Lets have a version, at last!
+TUNTAP_VERSION = 20150118
+
+# BASE install directory
 BASE=
 
 all: tap.kext
 
+keysetup:
+	-security delete-keychain net.sf.tuntaposx.tmp
+	security create-keychain -p $$(head -c 32 /dev/urandom | hexdump -e '"%02x"') \
+		net.sf.tuntaposx.tmp
+	security set-keychain-settings -lut 60 net.sf.tuntaposx.tmp
+	security import identity.p12 -k net.sf.tuntaposx.tmp -f pkcs12 \
+		-P $$(read -sp 'identity passphrase: ' pw && echo "$$pw") -A
+	security find-identity -v net.sf.tuntaposx.tmp | \
+		awk -F \" '$$2 ~ /^Developer ID Application:/ { print $$2 }' > .signing_identity
+	security find-identity -v net.sf.tuntaposx.tmp | \
+		awk -F \" '$$2 ~ /^Developer ID Installer:/ { print $$2 }' > .installer_identity
+
+pkgbuild/%.pkg: %.kext
+	mkdir -p pkgbuild/$*_root/Library/Extensions
+	cp -pR $*.kext pkgbuild/$*_root/Library/Extensions
+	mkdir -p pkgbuild/$*_root/Library/LaunchDaemons
+	cp pkg/launchd/net.sf.tuntaposx.$*.plist pkgbuild/$*_root/Library/LaunchDaemons
+	pkgbuild --root pkgbuild/$*_root \
+		--component-plist pkg/components/$*.plist \
+		--scripts pkg/scripts/$* pkgbuild/$*.pkg
+
+tuntap_$(TUNTAP_VERSION).pkg: pkgbuild/tap.pkg pkgbuild/tun.pkg
+	productbuild --distribution pkg/distribution.xml --package-path pkgbuild \
+		--resources pkg/res.dummy \
+		tuntap_$(TUNTAP_VERSION).pkg ; \
+	pkgutil --expand tuntap_$(TUNTAP_VERSION).pkg pkgbuild/tuntap_pkg.d
+	cp -pR pkg/res/ pkgbuild/tuntap_pkg.d/Resources
+	pkgutil --flatten pkgbuild/tuntap_pkg.d tuntap_$(TUNTAP_VERSION).pkg
+	if test -s ".installer_identity"; then \
+		productsign --sign "$$(cat .installer_identity)" --keychain net.sf.tuntaposx.tmp \
+			tuntap_$(TUNTAP_VERSION).pkg tuntap_$(TUNTAP_VERSION).pkg.signed ; \
+		mv tuntap_$(TUNTAP_VERSION).pkg.signed tuntap_$(TUNTAP_VERSION).pkg ; \
+	fi
+
+pkg: tuntap_$(TUNTAP_VERSION).pkg
+	tar czf tuntap_$(TUNTAP_VERSION).tar.gz \
+		README.installer README tuntap_$(TUNTAP_VERSION).pkg 
+
+# Install targets
+# They are provided for the gentoo ebuild, but should work just fine for other people as well.
+install_%_kext: %.kext
+	mkdir -p $(BASE)/Library/Extensions
+	cp -pR $*.kext $(BASE)/Library/Extensions/
+	chown -R root:wheel $(BASE)/Library/Extensions/$*.kext
+	mkdir -p $(BASE)/Library/LaunchDaemons
+	cp pkg/launchd/net.sf.tuntaposx.$*.plist $(BASE)/Library/LaunchDaemons
+	chown -R root:wheel $(BASE)/Library/LaunchDaemons/net.sf.tuntaposx.$*.plist
+
+install: install_tap_kext install_tun_kext
+
+tarball: clean
+	touch tuntap_$(TUNTAP_VERSION)_src.tar.gz
+	tar czf tuntap_$(TUNTAP_VERSION)_src.tar.gz \
+		-C .. \
+		--exclude "tuntap/identity.p12" \
+		--exclude "tuntap/tuntap_$(TUNTAP_VERSION)_src.tar.gz" \
+		--exclude "tuntap/tuntap_$(TUNTAP_VERSION).tar.gz" \
+		--exclude "tuntap/tuntap_$(TUNTAP_VERSION).pkg" \
+		--exclude "*/.*" \
+		tuntap
+
 clean:
 	cd src/tap && make -f Makefile clean
+	cd src/tun && make -f Makefile clean
+	-rm -rf pkgbuild
+	-rm -rf tuntap_$(TUNTAP_VERSION).pkg
+	-rm -f tuntap_$(TUNTAP_VERSION).tar.gz
+	-rm -f tuntap_$(TUNTAP_VERSION)_src.tar.gz
 
-tap.kext:
+%.kext:
-	cd src/tap && make TUNTAP_VERSION=$(TUNTAP_VERSION) -f Makefile all
+	cd src/$* && make TUNTAP_VERSION=$(TUNTAP_VERSION) -f Makefile all
+	if test -s ".signing_identity"; then \
+		codesign -fv --keychain net.sf.tuntaposx.tmp -s "$$(cat .signing_identity)" \
+			$*.kext ; \
+	fi
+
+test:
+	# configd messes with interface flags, issuing SIOCSIFFLAGS ioctls upon receiving kernel
+	# events indicating protocols have been attached and detached. Unfortunately, configd does
+	# this asynchronously, making the SIOCSIFFLAGS changes totally unpredictable when we bring
+	# our interfaces up and down in rapid succession during our tests. I haven't found a good
+	# way to suppress or handle this mess other than disabling configd temporarily.
+	killall -STOP configd
+	-PYTHONPATH=test python test/tuntap/tuntap_tests.py --tests='$(TESTS)'
+	killall -CONT configd
 
 .PHONY: test

ext/tap-mac/tuntap/src/lock.cc

@@ -31,6 +31,8 @@
 
 extern "C" {
 
+#include <kern/clock.h>
+
 #include <sys/syslog.h>
 #include <sys/proc.h>
 
@@ -120,10 +122,13 @@ tt_mutex::sleep(void *cond)
 }
 
 void
-tt_mutex::sleep(void *cond, uint64_t timeout)
+tt_mutex::sleep(void *cond, uint64_t nanoseconds)
 {
-	if (lck != NULL)
+	if (lck != NULL) {
-		lck_rw_sleep_deadline(lck, LCK_SLEEP_DEFAULT, cond, THREAD_INTERRUPTIBLE, timeout);
+		uint64_t abstime;
+		nanoseconds_to_absolutetime(nanoseconds, &abstime);
+		lck_rw_sleep_deadline(lck, LCK_SLEEP_DEFAULT, cond, THREAD_INTERRUPTIBLE, abstime);
+	}
 }
 
 void
@@ -188,9 +193,9 @@ tt_gate::sleep(void* cond)
 }
 
 void
-tt_gate::sleep(void* cond, uint64_t timeout)
+tt_gate::sleep(void* cond, uint64_t nanoseconds)
 {
-	slock.sleep(cond, timeout);
+	slock.sleep(cond, nanoseconds);
 }
 
 void

ext/tap-mac/tuntap/src/tap/Makefile

@@ -19,18 +19,18 @@ BUNDLE_SIGNATURE = ????
 BUNDLE_PACKAGETYPE = KEXT
 BUNDLE_VERSION = $(TAP_KEXT_VERSION)
 
-INCLUDE = -I.. -I/System/Library/Frameworks/Kernel.framework/Headers -I/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.8.sdk/System/Library/Frameworks/Kernel.framework/Headers
+INCLUDE = -I.. -I/System/Library/Frameworks/Kernel.framework/Headers
-CFLAGS = -Wall -mkernel -force_cpusubtype_ALL \
+CFLAGS = -Wall -Werror -mkernel -force_cpusubtype_ALL \
-	-fno-builtin -fno-stack-protector -arch i386 -arch x86_64 \
+	-nostdinc -fno-builtin -fno-stack-protector -msoft-float -fno-common \
-	-DKERNEL -D__APPLE__ -DKERNEL_PRIVATE -DTUNTAP_VERSION=\"$(TUNTAP_VERSION)\" \
+	-arch x86_64 \
+	-DKERNEL -DAPPLE -DKERNEL_PRIVATE -DTUNTAP_VERSION=\"$(TUNTAP_VERSION)\" \
 	-DTAP_KEXT_VERSION=\"$(TAP_KEXT_VERSION)\"
 CCFLAGS = $(CFLAGS)
-LDFLAGS = -Wall -mkernel -nostdlib -r -lcc_kext -arch i386 -arch x86_64 -Xlinker -kext
+LDFLAGS = -Wall -Werror -arch x86_64 -Xlinker -kext -nostdlib -lkmodc++ -lkmod -lcc_kext
 
-#CCP = g++
+CCP = clang -x c++
-#CC = gcc
+CC = clang -x c
-CCP = ../../../../llvm-g++-Xcode4.6.2/bin/llvm-g++
+LD = clang
-CC = ../../../../llvm-g++-Xcode4.6.2/bin/llvm-gcc
 
 all: $(KMOD_BIN) bundle
 
@@ -40,7 +40,7 @@ all: $(KMOD_BIN) bundle
 	$(CCP) $(CCFLAGS) $(INCLUDE) -c $< -o $@
 
 $(KMOD_BIN):	$(OBJS)
-	$(CCP) $(LDFLAGS) -o $(KMOD_BIN) $(OBJS)
+	$(LD) $(LDFLAGS) -o $(KMOD_BIN) $(OBJS)
 
 bundle:	$(KMOD_BIN)
 	rm -rf $(BUNDLE_DIR)/$(BUNDLE_NAME)

ext/tap-mac/tuntap/src/tap/tap.cc

@@ -38,6 +38,8 @@ extern "C" {
 #include <sys/random.h>
 #include <sys/kern_event.h>
 
+#include <mach/thread_policy.h>
+
 #include <net/if_types.h>
 #include <net/if_arp.h>
 #include <net/if_dl.h>
@@ -89,10 +91,25 @@ struct ifmediareq32 {
 #define	SIOCGIFMEDIA32	_IOWR('i', 56, struct ifmediareq32) /* get net media */
 #define	SIOCGIFMEDIA64	_IOWR('i', 56, struct ifmediareq64) /* get net media (64-bit) */
 
+/* thread_policy_set is exported in Mach.kext, but commented in mach/thread_policy.h in the
+ * Kernel.Framework headers (why?). Add a local declaration to work around that.
+ */
+extern "C" {
+kern_return_t thread_policy_set(
+	thread_t thread,
+	thread_policy_flavor_t flavor,
+	thread_policy_t policy_info,
+	mach_msg_type_number_t count);
+}
 
 static unsigned char ETHER_BROADCAST_ADDR[] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 /* members */
+tap_interface::tap_interface() {
+	bzero(attached_protos, sizeof(attached_protos));
+	input_thread = THREAD_NULL;
+}
+
 bool
 tap_interface::initialize(unsigned short major, unsigned short unit)
 {
@@ -166,6 +183,30 @@ tap_interface::initialize_interface()
 	 */
 	bpfattach(ifp, DLT_EN10MB, ifnet_hdrlen(ifp));
 
+	/* Inject an empty packet to trigger the input thread calling demux(), which will unblock
+	 * thread_sync_lock. This is part of a hack to avoid a kernel crash on re-attaching
+	 * interfaces, see comment in shutdown_interface for more information.
+	 */
+	mbuf_t empty_mbuf;
+	mbuf_gethdr(MBUF_WAITOK, MBUF_TYPE_DATA, &empty_mbuf);
+	if (empty_mbuf != NULL) {
+		mbuf_pkthdr_setrcvif(empty_mbuf, ifp);
+		mbuf_pkthdr_setlen(empty_mbuf, 0);
+		mbuf_pkthdr_setheader(empty_mbuf, mbuf_data(empty_mbuf));
+		mbuf_set_csum_performed(empty_mbuf, 0, 0);
+		if (ifnet_input(ifp, empty_mbuf, NULL) == 0) {
+			auto_lock l(&thread_sync_lock);
+			for (int i = 0; i < 100 && input_thread == THREAD_NULL; ++i) {
+				dprintf("input thread not found, waiting...\n");
+				thread_sync_lock.sleep(&input_thread, 10000000);
+			}
+		} else {
+			mbuf_freem(empty_mbuf);
+		}
+	}
+	if (input_thread == THREAD_NULL)
+		dprintf("Failed to determine input thread!\n");
+
 	return 0;
 }
 
@@ -186,6 +227,36 @@ tap_interface::shutdown_interface()
 
 	cleanup_interface();
 	unregister_interface();
+
+	/* There's a race condition in the kernel that may cause crashes when quickly re-attaching
+	 * interfaces. The crash happens when the interface gets re-attached before the input thread
+	 * for the interface managed to terminate, in which case an assert on the input_waiting flag
+	 * to be clear triggers in ifnet_attach. The bug is really that there's no synchronization
+	 * for terminating the input thread. To work around this, the following code does add the
+	 * missing synchronization to wait for the input thread to terminate. Of course, threading
+	 * primitives available to kexts are few, and I'm not aware of a way to wait for a thread to
+	 * terminate. Hence, the code calls thread_policy_set (passing bogus parameters) in a loop,
+	 * until it returns KERN_TERMINATED. Since this is all rather fragile, there's an upper
+	 * limit on the loop iteratations we're willing to make, so this terminates eventually even
+	 * if things change on the kernel side eventually.
+	 */
+	if (input_thread != THREAD_NULL) {
+		dprintf("Waiting for input thread...\n");
+		kern_return_t result = 0;
+		for (int i = 0; i < 100; ++i) {
+			result = thread_policy_set(input_thread, -1, NULL, 0);
+			dprintf("thread_policy_set result: %d\n", result);
+			if (result == KERN_TERMINATED) {
+				dprintf("Input thread terminated.\n");
+				thread_deallocate(input_thread);
+				input_thread = THREAD_NULL;
+				break;
+			}
+
+			auto_lock l(&thread_sync_lock);
+			thread_sync_lock.sleep(&input_thread, 10000000);
+		}
+	}
 }
 
 errno_t
@@ -263,6 +334,16 @@ tap_interface::if_demux(mbuf_t m, char *header, protocol_family_t *proto)
 
 	dprintf("tap: if_demux\n");
 
+	/* Make note of what input thread this interface is running on. This is part of a hack to
+	 * avoid a crash on re-attaching interfaces, see comment in shutdown_interface for details.
+	 */
+	if (input_thread == THREAD_NULL) {
+		auto_lock l(&thread_sync_lock);
+		input_thread = current_thread();
+		thread_reference(input_thread);
+		thread_sync_lock.wakeup(&input_thread);
+	}
+
 	/* size check */
 	if (mbuf_len(m) < sizeof(struct ether_header))
 		return ENOENT;

ext/tap-mac/tuntap/src/tap/tap.h

@@ -30,12 +30,15 @@
 
 #include "tuntap.h"
 
+extern "C" {
+
+#include <kern/thread.h>
+
+}
+
 #define TAP_FAMILY_NAME			((char *) "zt")
-
 #define TAP_IF_COUNT			32	/* max number of tap interfaces */
-
 #define TAP_MTU				2800
-
 #define TAP_LLADDR			tap_lladdr
 
 /* the mac address of our interfaces. note that the last byte will be replaced by the unit number */
@@ -52,6 +55,8 @@ class tap_manager : public tuntap_manager {
 
 /* the tap network interface */
 class tap_interface : public tuntap_interface {
+        public:
+	   	tap_interface();
 
 	protected:
 		/* maximum number of protocols that can be attached */
@@ -67,6 +72,9 @@ class tap_interface : public tuntap_interface {
 			protocol_family_t proto;
 		} attached_protos[MAX_ATTACHED_PROTOS];
 
+		/* The input thread for the network interface. */
+		thread_t input_thread;
+
 		/* initializes the interface */
 		virtual bool initialize(unsigned short major, unsigned short unit);
 

ext/tap-mac/tuntap/src/tuntap.cc

@@ -384,10 +384,10 @@ tuntap_interface::unregister_interface()
 		dprintf("interface detaching\n");
 
 		/* Wait until the interface has completely been detached. */
-		detach_lock.lock();
+		thread_sync_lock.lock();
 		while (!interface_detached)
-			detach_lock.sleep(&interface_detached);
+			thread_sync_lock.sleep(&interface_detached);
-		detach_lock.unlock();
+		thread_sync_lock.unlock();
 
 		dprintf("interface detached\n");
 
@@ -642,15 +642,14 @@ tuntap_interface::cdev_write(uio_t uio, int ioflag)
 	unsigned int mlen = mbuf_maxlen(first);
 	unsigned int chunk_len;
 	unsigned int copied = 0;
+	unsigned int max_data_len = ifnet_mtu(ifp) + ifnet_hdrlen(ifp);
 	int error;
 
 	/* stuff the data into the mbuf(s) */
 	mb = first;
 	while (uio_resid(uio) > 0) {
 		/* copy a chunk. enforce mtu (don't know if this is correct behaviour) */
-		// ... evidently not :) -- Adam Ierymenko <adam.ierymenko@zerotier.com>
+		chunk_len = min(max_data_len - copied, min(uio_resid(uio), mlen));
-		//chunk_len = min(ifnet_mtu(ifp), min(uio_resid(uio), mlen));
-		chunk_len = min(uio_resid(uio),mlen);
 		error = uiomove((caddr_t) mbuf_data(mb), chunk_len, uio);
 		if (error) {
 			log(LOG_ERR, "tuntap: could not copy data from userspace: %d\n", error);
@@ -666,9 +665,7 @@ tuntap_interface::cdev_write(uio_t uio, int ioflag)
 		copied += chunk_len;
 
 		/* if done, break the loop */
-		//if (uio_resid(uio) <= 0 || copied >= ifnet_mtu(ifp))
+		if (uio_resid(uio) <= 0 || copied >= max_data_len)
-		//	break;
-		if (uio_resid(uio) <= 0)
 			break;
 
 		/* allocate a new mbuf if the current is filled */
@@ -956,10 +953,10 @@ tuntap_interface::if_detached()
 	dprintf("tuntap: if_detached\n");
 
 	/* wake unregister_interface() */
-	detach_lock.lock();
+	thread_sync_lock.lock();
 	interface_detached = true;
-	detach_lock.wakeup(&interface_detached);
+	thread_sync_lock.wakeup(&interface_detached);
-	detach_lock.unlock();
+	thread_sync_lock.unlock();
 
 	dprintf("if_detached done\n");
 }

ext/tap-mac/tuntap/src/tuntap.h

@@ -197,7 +197,7 @@ class tuntap_interface {
 		/* synchronization */
 		tt_mutex lock;
 		tt_mutex bpf_lock;
-		tt_mutex detach_lock;
+		tt_mutex thread_sync_lock;
 
 		/* the interface structure registered */
 		ifnet_t ifp;