Merge pull request #2451 from zerotier/dependabot/cargo/rustybits/crossbeam-channel-0.5.15

Bump crossbeam-channel from 0.5.13 to 0.5.15 in /rustybits

.clangd

@@ -0,0 +1,6 @@
+CompileFlags:
+  Add:
+    - "-std=c++17"
+    - "-I../ext"
+    - "-I../ext/prometheus-cpp-lite-1.0/core/include"
+    - "-I../ext/prometheus-cpp-lite-1.0/simpleapi/include"

.github/workflows/build.yml

@@ -1,4 +1,7 @@
-on: [ push ]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 
 jobs:
   build_ubuntu:
@@ -9,23 +12,22 @@ jobs:
         git config --global core.autocrlf input
       #        git config --global core.eol lf
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
-        target: x86_64-unknown-linux-gnu
-        override: true
+        targets: x86_64-unknown-linux-gnu
         components: rustfmt, clippy
 
     - name: Set up cargo cache
       uses: Swatinem/rust-cache@v2
       continue-on-error: false
       with:
-        key: ${{ runner.os }}-cargo-${{ hashFiles('zeroidc//Cargo.lock') }}
+        key: ${{ runner.os }}-cargo-${{ hashFiles('rustybits//Cargo.lock') }}
         shared-key: ${{ runner.os }}-cargo-
         workspaces: |
-          zeroidc/
+          rustybits/
 
     - name: make
       run: make
@@ -33,6 +35,14 @@ jobs:
       run: |
         make selftest
         ./zerotier-selftest
+    - name: 'Tar files' # keeps permissions (execute)
+      run: tar -cvf zerotier-one.tar zerotier-one
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: zerotier-one-ubuntu-x64
+        path: zerotier-one.tar
+        retention-days: 7
 
   build_macos:
     runs-on: macos-latest
@@ -42,29 +52,42 @@ jobs:
         git config --global core.autocrlf input
       #        git config --global core.eol lf
     - name: checkout
-      uses: actions/checkout@v3
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: actions/checkout@v4
+    - name: Install Rust aarch64
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
         target: aarch64-apple-darwin
-        override: true
+        components: rustfmt, clippy
+    - name: Install Rust x86_64
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        toolchain: stable
+        target: x86_64-apple-darwin
         components: rustfmt, clippy
     - name: Set up cargo cache
       uses: Swatinem/rust-cache@v2
       continue-on-error: false
       with:
-        key: ${{ runner.os }}-cargo-${{ hashFiles('zeroidc//Cargo.lock') }}
+        key: ${{ runner.os }}-cargo-${{ hashFiles('rustybits//Cargo.lock') }}
         shared-key: ${{ runner.os }}-cargo-
         workspaces: |
-          zeroidc/
-
+          rustybits/
     - name: make
       run: make
     - name: selftest
       run: |
         make selftest
         ./zerotier-selftest
+    - name: 'Tar files' # keeps permissions (execute)
+      run: tar -cvf zerotier-one.tar zerotier-one
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: zerotier-one-mac
+        path: zerotier-one.tar
+        retention-days: 7
+
 
   build_windows:
     runs-on: windows-latest
@@ -74,25 +97,30 @@ jobs:
         git config --global core.autocrlf true
       #        git config --global core.eol lf
     - name: checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Install Rust
-      uses: actions-rs/toolchain@v1
+      uses: dtolnay/rust-toolchain@stable
       with:
         toolchain: stable
         target: aarch64-apple-darwin
-        override: true
         components: rustfmt, clippy
     - name: Set up cargo cache
       uses: Swatinem/rust-cache@v2
       continue-on-error: false
       with:
-        key: ${{ runner.os }}-cargo-${{ hashFiles('zeroidc//Cargo.lock') }}
+        key: ${{ runner.os }}-cargo-${{ hashFiles('rustybits//Cargo.lock') }}
         shared-key: ${{ runner.os }}-cargo-
         workspaces: |
-          zeroidc/
+          rustybits/
 
     - name: setup msbuild
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@v2
     - name: msbuild
       run: |
         msbuild windows\ZeroTierOne.sln /m /p:Configuration=Release  /property:Platform=x64 /t:ZeroTierOne
+    - name: Archive production artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: zerotier-one-windows
+        path: windows/Build
+        retention-days: 7

.github/workflows/validate.yml

@@ -1,4 +1,5 @@
 on:
+  pull_request:
   push:
   workflow_dispatch:
 
@@ -44,7 +45,7 @@ jobs:
         sudo ./.github/workflows/validate-linux.sh
 
     - name: Archive test results
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: ${{github.sha}}-test-results
         path: "*test-results*"

.gitignore

@@ -124,6 +124,7 @@ attic/world/mkworld
 workspace/
 workspace2/
 zeroidc/target/
+tcp-proxy/target
 
 #snapcraft specifics
 /parts/
@@ -138,3 +139,4 @@ __pycache__
 *_source.tar.bz2
 snap/.snapcraft
 tcp-proxy/tcp-proxy
+rustybits/target

Dockerfile.release

@@ -1,6 +1,6 @@
 # vim: ft=dockerfile
 
-FROM debian:bullseye
+FROM debian:bookworm
 
 ARG VERSION
 
@@ -9,9 +9,9 @@ RUN mkdir -p /usr/share/zerotier && \
     curl -o /usr/share/zerotier/tmp.asc "https://download.zerotier.com/contact%40zerotier.com.gpg" && \
     gpg --no-default-keyring --keyring /usr/share/zerotier/zerotier.gpg --import /usr/share/zerotier/tmp.asc && \
     rm -f /usr/share/zerotier/tmp.asc && \
-    echo "deb [signed-by=/usr/share/zerotier/zerotier.gpg] http://download.zerotier.com/debian/bullseye bullseye main" > /etc/apt/sources.list.d/zerotier.list
+    echo "deb [signed-by=/usr/share/zerotier/zerotier.gpg] http://download.zerotier.com/debian/bookworm bookworm main" > /etc/apt/sources.list.d/zerotier.list
 
-RUN apt-get update -qq && apt-get install zerotier-one=${VERSION} curl iproute2 net-tools iputils-ping openssl libssl1.1 -y
+RUN apt-get update -qq && apt-get install zerotier-one=${VERSION} curl iproute2 net-tools iputils-ping openssl libssl3 -y
 RUN rm -rf /var/lib/zerotier-one
 
 COPY entrypoint.sh.release /entrypoint.sh

LICENSE.txt

@@ -26,7 +26,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided you
                       ZeroTier behind the scenes to operate a service not
                       related to ZeroTier network administration.
 
-                      * Create Non-Open-Source Commercial Derviative Works
+                      * Create Non-Open-Source Commercial Derivative Works
 
                       (2) Link or directly include the Licensed Work in a
                       commercial or for-profit application or other product
@@ -47,7 +47,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided you
                       services, social welfare, senior care, child care, and
                       the care of persons with disabilities.
 
-Change Date:          2025-01-01
+Change Date:          2026-01-01
 
 Change License:       Apache License version 2.0 as published by the Apache
                       Software Foundation

OFFICIAL-RELEASE-STEPS.md

@@ -14,6 +14,7 @@ The version must be incremented in all of the following files:
     /debian/changelog
     /ext/installfiles/mac/ZeroTier One.pkgproj
     /ext/installfiles/windows/ZeroTier One.aip
+  ../DesktopUI/mac-app-template/ZeroTier.app/Contents/Info.plist
 
 The final .AIP file can only be edited on Windows with [Advanced Installer Enterprise](http://www.advancedinstaller.com/). In addition to incrementing the version be sure that a new product code is generated. (The "upgrade code" GUID on the other hand must never change.)
 

README.docker.md

@@ -64,6 +64,7 @@ You can control a few settings including the identity used and the authtoken use
 - `ZEROTIER_API_SECRET`: replaces the `authtoken.secret` before booting and allows you to manage the control socket's authentication key.
 - `ZEROTIER_IDENTITY_PUBLIC`: the `identity.public` file for zerotier-one. Use `zerotier-idtool` to generate one of these for you.
 - `ZEROTIER_IDENTITY_SECRET`: the `identity.secret` file for zerotier-one. Use `zerotier-idtool` to generate one of these for you.
+- `ZEROTIER_LOCAL_CONF`: Sets the the `local.conf` file content for zerotier-one
 
 ### Tips
 

README.md

@@ -44,6 +44,12 @@ The base path contains the ZeroTier One service main entry point (`one.cpp`), se
  - `windows/`: Visual Studio solution files, Windows service code, and the Windows task bar app UI.
  - `zeroidc/`: OIDC implementation used by ZeroTier service to log into SSO-enabled networks. (This part is written in Rust, and more Rust will be appearing in this repository in the future.)
 
+### Contributing
+
+Please do pull requests off of the `dev` branch.
+
+Releases are done by merging `dev` into `main` and then tagging and doing builds. 
+
 ### Build and Platform Notes
 
 To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU make) is required and can be installed from packages or ports. For Windows there is a Visual Studio solution in `windows/`.
@@ -52,7 +58,7 @@ To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU
    - Xcode command line tools for macOS 10.13 or newer are required.
    - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **Linux**
-   - The minimum compiler versions required are GCC/G++ 4.9.3 or CLANG/CLANG++ 3.4.2. (Install `clang` on CentOS 7 as G++ is too old.)
+   - The minimum compiler versions required are GCC/G++ 8.x or CLANG/CLANG++ 5.x.
    - Linux makefiles automatically detect and prefer clang/clang++ if present as it produces smaller and slightly faster binaries in most cases. You can override by supplying CC and CXX variables on the make command line.
    - Rust for x86_64 and ARM64 targets *if SSO is enabled in the build*.
  - **Windows**
@@ -81,7 +87,7 @@ On most distributions, macOS, and Windows, the installer will start the service
 
 A home folder for your system will automatically be created.
 
-The service is controlled via the JSON API, which by default is available at 127.0.0.1 port 9993. We include a *zerotier-cli* command line utility to make API calls for standard things like joining and leaving networks. The *authtoken.secret* file in the home folder contains the secret token for accessing this API. See [service/README.md](service/README.md) for API documentation.
+The service is controlled via the JSON API, which by default is available at `127.0.0.1:9993`. It also listens on `0.0.0.0:9993` which is only usable if `allowManagementFrom` is properly configured in `local.conf`. We include a *zerotier-cli* command line utility to make API calls for standard things like joining and leaving networks. The *authtoken.secret* file in the home folder contains the secret token for accessing this API. See [service/README.md](service/README.md) for API documentation.
 
 Here's where home folders live (by default) on each OS:
 
@@ -169,3 +175,23 @@ Metrics are also available on disk in ZeroTier's working directory:
 | zt_peer_packet_errors | node_id | Counter | number of incoming packet errors from a peer |
 
 If there are other metrics you'd like to see tracked, ask us in an Issue or send us a Pull Request!
+
+### HTTP / App server
+
+There is a static http file server suitable for hosting Single Page Apps at http://localhost:9993/app/<app-path>
+
+Use `zerotier-cli info -j` to find your zerotier-one service's homeDir
+
+``` sh
+cd $ZT_HOME
+sudo mkdir -p app/app1
+sudo mkdir -p app/appB
+echo '<html><meta charset=utf-8><title>appA</title><body><h1>hello world A' | sudo tee app/appA/index.html 
+echo '<html><meta charset=utf-8><title>app2</title><body><h1>hello world 2' | sudo tee app/app2/index.html 
+curl -sL http://localhost:9993/app/appA http://localhost:9993/app/app2 
+```
+
+Then visit [http://localhost:9993/app/app1/](http://localhost:9993/app/app1/) and [http://localhost:9993/app/appB/](http://localhost:9993/app/appB/)
+
+Requests to paths don't exist return the app root index.html, as is customary for SPAs. 
+If you want, you can write some javascript that talks to the service or controller [api](https://docs.zerotier.com/service/v1).

RELEASE-NOTES.md

@@ -1,6 +1,38 @@
 ZeroTier Release Notes
 ======
 
+# 2024-10-23 -- Version 1.14.2
+
+  * Fix for missing entitlement on macOS Sequoia.
+  * Fix for a problem correctly parsing local.conf to enable low bandwidth mode.
+  * Increment versions of some dependent libraries.
+  * Other fixes.
+
+# 2024-09-12 -- Version 1.14.1
+
+  * Multithreaded packet I/O support! Currently this is just for Linux and must
+    be enabled in local.conf. It will likely make the largest difference on small
+    multi-core devices where CPU is a bottleneck and high throughput is desired.
+    It may be enabled by default in the future but we want it to be thoroughly
+    tested. It's a little harder than it seems at first glance due to the need
+    to keep packets in sequence and balance load.
+  * Several multipath bug fixes.
+  * Updated the versions on a number of libraries related to OIDC support and HTTP.
+  * MacOS .app now shows the correct version in its Info.plist manifest.
+  * Sanitize MAC addresses in JSON format rules parser.
+  * Some basic information about the platform (OS, CPU architecture) is now reported
+    to network controllers when networks are joined so it can be displayed to
+    network admins and in the future used in policy checking and inventory operations.
+
+# 2024-05-02 -- Version 1.14.0
+
+  * Linux I/O performance improvements under heavy load
+  * Improvements to multipath
+  * Fix for port rebinding "coma" bug after periods offline (some laptop users)
+  * Fixed a rules engine quirk/ambiguity (GitHub Issue #2200)
+  * Controller API enhancements: node names and other node meta-data
+  * Other bug fixes
+
 # 2023-09-12 -- Version 1.12.2
 
   * More improvements to macOS full tunnel mode.
@@ -89,7 +121,7 @@ Note that releases are coming few and far between because most of our dev effort
 # 2022-04-25 -- Version 1.8.9
 
  * Fixed a long-standing and strange bug that was causing sporadic "phantom" packet authentication failures. Not a security problem but could be behind sporadic reports of link failures under some conditions.
- * Fized a memory leak in SSO/OIDC support.
+ * Fixed a memory leak in SSO/OIDC support.
  * Fixed SSO/OIDC display error on CLI.
  * Fixed a bug causing nodes to sometimes fail to push certs to each other (primarily affects SSO/OIDC use cases).
  * Fixed a deadlock bug on leaving SSO/OIDC managed networks.
@@ -340,7 +372,7 @@ We're trying to fix all these issues before the 1.6.0 release. Stay tuned.
 # 2017-04-20 -- Version 1.2.4
 
  * Managed routes are now only bifurcated for the default route. This is a change in behavior, though few people will probably notice. Bifurcating all managed routes was causing more trouble than it was worth for most users.
- * Up to 2X crypto speedup on x86-64 (except Windows, which will take some porting) and 32-bit ARM platforms due to integration of fast assembly language implementations of Salsa20/12 from the [supercop](http://bench.cr.yp.to/supercop.html) code base. These were written by Daniel J. Bernstein and are in the public domain. My Macbook Pro (Core i5 2.8ghz) now does almost 1.5GiB/sec Salsa20/12 per core and a Raspberry Pi got a 2X boost. 64-bit ARM support and Windows support will take some work but should not be too hard.
+ * Up to 2X crypto speedup on x86-64 (except Windows, which will take some porting) and 32-bit ARM platforms due to integration of fast assembly language implementations of Salsa20/12 from the [supercop](http://bench.cr.yp.to/supercop.html) code base. These were written by Daniel J. Bernstein and are in the public domain. My MacBook Pro (Core i5 2.8ghz) now does almost 1.5GiB/sec Salsa20/12 per core and a Raspberry Pi got a 2X boost. 64-bit ARM support and Windows support will take some work but should not be too hard.
  * Refactored code that manages credentials to greatly reduce memory use in most cases. This may also result in a small performance improvement.
  * Reworked and simplified path selection and priority logic to fix path instability and dead path persistence edge cases. There have been some sporadic reports of persistent path instabilities and dead paths hanging around that take minutes to resolve. These have proven difficult to reproduce in house, but hopefully this will fix them. In any case it seems to speed up path establishment in our tests and it makes the code simpler and more readable.
  * Eliminated some unused cruft from the code around path management and in the peer class.

SECURITY.md

@@ -7,11 +7,11 @@ includes all source code repositories managed through our GitHub organization.
 
 The following versions of ZeroTier One receive security updates
 
-| Version | Supported          |
-| ------- | ------------------ |
-| 1.10.x   | :white_check_mark: |
-| 1.8.x   | :white_check_mark:  |
-| < 1.8.0 | :x: |
+| Version  | Supported          |
+| -------- | ------------------ |
+| 1.14.x   | :white_check_mark: |
+| 1.12.x   | :white_check_mark: |
+| < 1.12.0 | :x:                |
 
 ## Reporting a Vulnerability
 
@@ -23,7 +23,6 @@ please encrypt with our PGP key (see below).
 Please include the following information, or as much as you can provide to help us 
 understand the nature and scope of the issue:
 
-
   * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
   * Full paths of source file(s) related to the manifestation of the issue
   * The location of the affected source code (tag/branch/commit or direct URL)
@@ -32,7 +31,6 @@ understand the nature and scope of the issue:
   * Proof-of-concept or exploit code (if possible)
   * Impact of the issue, including how an attacker might exploit the issue
 
-
 ## Preferred Languages
 
 We prefer all communications to be in English.

controller/ConnectionPool.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/DB.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -196,7 +196,7 @@ bool DB::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohma
 void DB::networks(std::set<uint64_t> &networks)
 {
 	waitForReady();
-	Metrics::db_get_member_list++;
+	Metrics::db_get_network_list++;
 	std::shared_lock<std::shared_mutex> l(_networks_l);
 	for(auto n=_networks.begin();n!=_networks.end();++n)
 		networks.insert(n->first);
@@ -382,6 +382,24 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool
 		const std::string ids = old["id"];
 		const uint64_t networkId = Utils::hexStrToU64(ids.c_str());
 		if (networkId) {
+			try {
+				// deauth all members on the network
+				nlohmann::json network;
+				std::vector<nlohmann::json> members;
+				this->get(networkId, network, members);
+				for(auto i=members.begin();i!=members.end();++i) {
+					const std::string nodeID = (*i)["id"];
+					const uint64_t memberId = Utils::hexStrToU64(nodeID.c_str());
+					std::unique_lock<std::shared_mutex> ll(_changeListeners_l);
+					for(auto j=_changeListeners.begin();j!=_changeListeners.end();++j) {
+						(*j)->onNetworkMemberDeauthorize(this,networkId,memberId);
+					}
+				}
+			} catch (std::exception &e) {
+				std::cerr << "Error deauthorizing members on network delete: " << e.what() << std::endl;
+			}
+
+			// delete the network
 			std::unique_lock<std::shared_mutex> l(_networks_l);
 			_networks.erase(networkId);
 		}

controller/DB.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/DBMirrorSet.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/DBMirrorSet.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/EmbeddedNetworkController.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -315,12 +315,14 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
 		return true;
 	} else if (t == "MATCH_MAC_SOURCE") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_MAC_SOURCE;
-		const std::string mac(OSUtils::jsonString(r["mac"],"0"));
+		std::string mac(OSUtils::jsonString(r["mac"],"0"));
+		Utils::cleanMac(mac);
 		Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6);
 		return true;
 	} else if (t == "MATCH_MAC_DEST") {
 		rule.t |= ZT_NETWORK_RULE_MATCH_MAC_DEST;
-		const std::string mac(OSUtils::jsonString(r["mac"],"0"));
+		std::string mac(OSUtils::jsonString(r["mac"],"0"));
+		Utils::cleanMac(mac);
 		Utils::unhex(mac.c_str(),(unsigned int)mac.length(),rule.v.mac,6);
 		return true;
 	} else if (t == "MATCH_IPV4_SOURCE") {
@@ -869,11 +871,14 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	// Control plane Endpoints
 	std::string controllerPath = "/controller";
 	std::string networkListPath = "/controller/network";
+	std::string networkListPath2 = "/unstable/controller/network";
 	std::string networkPath = "/controller/network/([0-9a-fA-F]{16})";
 	std::string oldAndBustedNetworkCreatePath = "/controller/network/([0-9a-fA-F]{10})______";
 	std::string memberListPath = "/controller/network/([0-9a-fA-F]{16})/member";
+	std::string memberListPath2 = "/unstable/controller/network/([0-9a-fA-F]{16})/member";
 	std::string memberPath = "/controller/network/([0-9a-fA-F]{16})/member/([0-9a-fA-F]{10})";
 
+
 	auto controllerGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
 		char tmp[4096];
 		const bool dbOk = _db.isReady();
@@ -885,11 +890,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 			(unsigned long long)OSUtils::now(),
 			dbOk ? "true" : "false");
 
-			if (!dbOk) {
-				res.status = 503;
-			}
+		if (!dbOk) {
+			res.status = 503;
+		}
 
-			setContent(req, res, tmp);
+		setContent(req, res, tmp);
 	};
 	s.Get(controllerPath, controllerGet);
 	sv6.Get(controllerPath, controllerGet);
@@ -910,6 +915,52 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	s.Get(networkListPath, networkListGet);
 	sv6.Get(networkListPath, networkListGet);
 
+	auto networkListGet2 = [&, setContent](const httplib::Request &req, httplib::Response &res) {
+		std::set<uint64_t> networkIds;
+		_db.networks(networkIds);
+
+		auto meta = json::object();
+		auto data = json::array();
+		uint64_t networkCount = 0;
+
+		for(std::set<uint64_t>::const_iterator nwid(networkIds.begin()); nwid != networkIds.end(); ++nwid) {
+			json network;
+			if (!_db.get(*nwid, network)) {
+				continue;
+			}
+
+			std::vector<json> memTmp;
+			if (_db.get(*nwid, network, memTmp)) {
+				if (!network.is_null()) {
+					uint64_t authorizedCount = 0;
+					uint64_t totalCount = memTmp.size();
+					networkCount++;
+
+					for (auto m = memTmp.begin(); m != memTmp.end(); ++m) {
+						bool a = OSUtils::jsonBool((*m)["authorized"], 0);
+						if (a) { authorizedCount++; }
+					}
+
+					auto nwMeta = json::object();
+					nwMeta["totalMemberCount"] = totalCount;
+					nwMeta["authorizedMemberCount"] = authorizedCount;
+					network["meta"] = nwMeta;
+
+					data.push_back(network);
+				}
+			}
+		}
+		meta["networkCount"] = networkCount;
+
+		auto out = json::object();
+		out["data"] = data;
+		out["meta"] = meta;
+
+		setContent(req, res, out.dump());
+	};
+	s.Get(networkListPath2, networkListGet2);
+	sv6.Get(networkListPath2, networkListGet2);
+
 	auto networkGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
 		auto networkID = req.matches[1];
 		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
@@ -925,7 +976,7 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	sv6.Get(networkPath, networkGet);
 
 	auto createNewNetwork = [&, setContent](const httplib::Request &req, httplib::Response &res) {
-		fprintf(stderr, "creating new network (new style)\n");
+		// fprintf(stderr, "creating new network (new style)\n");
 		uint64_t nwid = 0;
 		uint64_t nwidPrefix = (Utils::hexStrToU64(_signingIdAddressString.c_str()) << 24) & 0xffffffffff000000ULL;
 		uint64_t nwidPostfix = 0;
@@ -1035,6 +1086,41 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	s.Get(memberListPath, memberListGet);
 	sv6.Get(memberListPath, memberListGet);
 
+	auto memberListGet2 = [&, setContent](const httplib::Request &req, httplib::Response &res) {
+		auto networkID = req.matches[1];
+		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
+		json network;
+		if (!_db.get(nwid, network)) {
+			res.status = 404;
+			return;
+		}
+
+		auto out = nlohmann::json::object();
+		auto meta = nlohmann::json::object();
+		std::vector<json> memTmp;
+		if (_db.get(nwid, network, memTmp)) {
+			uint64_t authorizedCount = 0;
+			uint64_t totalCount = memTmp.size();
+			for (auto m = memTmp.begin(); m != memTmp.end(); ++m) {
+				bool a = OSUtils::jsonBool((*m)["authorized"], 0);
+				if (a) { authorizedCount++; }
+			}
+
+			meta["totalCount"] = totalCount;
+			meta["authorizedCount"] = authorizedCount;
+
+			out["data"] = memTmp;
+			out["meta"] = meta;
+
+			setContent(req, res, out.dump());
+		} else {
+			res.status = 404;
+			return;
+		}
+	};
+	s.Get(memberListPath2, memberListGet2);
+	sv6.Get(memberListPath2, memberListGet2);
+
 	auto memberGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
 		auto networkID = req.matches[1];
 		auto memberID = req.matches[2];
@@ -1057,6 +1143,12 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 		auto memberID = req.matches[2].str();
 		uint64_t nwid = Utils::hexStrToU64(networkID.c_str());
 		uint64_t memid = Utils::hexStrToU64(memberID.c_str());
+
+		if (!_db.hasNetwork(nwid)) {
+			res.status = 404;
+			return;
+		}
+
 		json network;
 		json member;
 		_db.get(nwid, network, memid, member);
@@ -1068,6 +1160,7 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 		if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"], false);
 		if (b.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = (uint64_t)OSUtils::jsonInt(b["authenticationExpiryTime"], 0ULL);
 		if (b.count("authenticationURL")) member["authenticationURL"] = OSUtils::jsonString(b["authenticationURL"], "");
+		if (b.count("name")) member["name"] = OSUtils::jsonString(b["name"], "");
 
 		if (b.count("remoteTraceTarget")) {
 			const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],""));
@@ -1456,6 +1549,7 @@ void EmbeddedNetworkController::_request(
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_NONCE, info.ssoNonce.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_STATE, info.ssoState.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_CLIENT_ID, info.ssoClientID.c_str()); 
+				authInfo.add(ZT_AUTHINFO_DICT_KEY_SSO_PROVIDER, info.ssoProvider.c_str());
 				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
 			}
 			DB::cleanMember(member);

controller/EmbeddedNetworkController.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/FileDB.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -136,7 +136,7 @@ void FileDB::eraseNetwork(const uint64_t networkId)
 	char p[16384];
 	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),networkId);
 	OSUtils::rm(p);
-	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)networkId);
+	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx",_networksPath.c_str(),(unsigned long long)networkId);
 	OSUtils::rmDashRf(p);
 	_networkChanged(network,nullJson,true);
 	std::lock_guard<std::mutex> l(this->_online_l);

controller/FileDB.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/LFDB.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/LFDB.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

controller/PostgreSQL.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -21,6 +21,8 @@
 #include "../version.h"
 #include "Redis.hpp"
 
+#include <smeeclient.h>
+
 #include <libpq-fe.h>
 #include <sstream>
 #include <iomanip>
@@ -159,6 +161,8 @@ using Attrs = std::vector<std::pair<std::string, std::string>>;
 using Item = std::pair<std::string, Attrs>;
 using ItemStream = std::vector<Item>;
 
+
+
 PostgreSQL::PostgreSQL(const Identity &myId, const char *path, int listenPort, RedisConfig *rc)
 	: DB()
 	, _pool()
@@ -173,6 +177,7 @@ PostgreSQL::PostgreSQL(const Identity &myId, const char *path, int listenPort, R
 	, _redis(NULL)
 	, _cluster(NULL)
 	, _redisMemberStatus(false)
+	, _smee(NULL)
 {
 	char myAddress[64];
 	_myAddressStr = myId.address().toString(myAddress);
@@ -248,10 +253,17 @@ PostgreSQL::PostgreSQL(const Identity &myId, const char *path, int listenPort, R
 		_commitThread[i] = std::thread(&PostgreSQL::commitThread, this);
 	}
 	_onlineNotificationThread = std::thread(&PostgreSQL::onlineNotificationThread, this);
+
+	configureSmee();
 }
 
 PostgreSQL::~PostgreSQL()
 {
+	if (_smee != NULL) {
+		smeeclient::smee_client_delete(_smee);
+		_smee = NULL;
+	}
+
 	_run = 0;
 	std::this_thread::sleep_for(std::chrono::milliseconds(100));
 
@@ -265,6 +277,31 @@ PostgreSQL::~PostgreSQL()
 	_onlineNotificationThread.join();
 }
 
+void PostgreSQL::configureSmee() 
+{
+	const char *TEMPORAL_SCHEME = "ZT_TEMPORAL_SCHEME";
+	const char *TEMPORAL_HOST = "ZT_TEMPORAL_HOST";
+	const char *TEMPORAL_PORT = "ZT_TEMPORAL_PORT";
+	const char *TEMPORAL_NAMESPACE = "ZT_TEMPORAL_NAMESPACE";
+	const char *SMEE_TASK_QUEUE = "ZT_SMEE_TASK_QUEUE";
+
+	const char *scheme = getenv(TEMPORAL_SCHEME);
+	if (scheme == NULL) {
+		scheme = "http";
+	}
+	const char *host = getenv(TEMPORAL_HOST);
+	const char *port = getenv(TEMPORAL_PORT);
+	const char *ns = getenv(TEMPORAL_NAMESPACE);
+	const char *task_queue = getenv(SMEE_TASK_QUEUE);
+
+	if (scheme != NULL && host != NULL && port != NULL && ns != NULL && task_queue != NULL) {
+		fprintf(stderr, "creating smee client\n");
+		std::string hostPort = std::string(scheme) + std::string("://") + std::string(host) + std::string(":") + std::string(port);
+		this->_smee = smeeclient::smee_client_new(hostPort.c_str(), ns, task_queue);
+	} else {
+		fprintf(stderr, "Smee client not configured\n");
+	}
+}
 
 bool PostgreSQL::waitForReady()
 {
@@ -743,11 +780,25 @@ void PostgreSQL::initializeNetworks()
 				fprintf(stderr, "adding networks to redis...\n");
 				if (_rc->clusterMode) {
 					auto tx = _cluster->transaction(_myAddressStr, true, false);
-					tx.sadd(setKey, networkSet.begin(), networkSet.end());
+					uint64_t count = 0;
+					for (std::string nwid : networkSet) {
+						tx.sadd(setKey, nwid);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _cluster->transaction(_myAddressStr, true, false);
+						}
+					}
 					tx.exec();
 				} else {
 					auto tx = _redis->transaction(true, false);
-					tx.sadd(setKey, networkSet.begin(), networkSet.end());
+					uint64_t count = 0;
+					for (std::string nwid : networkSet) {
+						tx.sadd(setKey, nwid);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _redis->transaction(true, false);
+						}
+					}
 					tx.exec();
 				}
 				fprintf(stderr, "done.\n");
@@ -968,14 +1019,24 @@ void PostgreSQL::initializeMembers()
 				fprintf(stderr, "Load member data into redis...\n");
 				if (_rc->clusterMode) {
 					auto tx = _cluster->transaction(_myAddressStr, true, false);
+					uint64_t count = 0;
 					for (auto it : networkMembers) {
 						tx.sadd(it.first, it.second);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _cluster->transaction(_myAddressStr, true, false);
+						}
 					}
 					tx.exec();
 				} else {
 					auto tx = _redis->transaction(true, false);
+					uint64_t count = 0;
 					for (auto it : networkMembers) {
 						tx.sadd(it.first, it.second);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _redis->transaction(true, false);
+						}
 					}
 					tx.exec();
 				}
@@ -1143,7 +1204,7 @@ void PostgreSQL::_membersWatcher_Redis() {
 									_memberChanged(oldConfig,newConfig,(this->_ready >= 2));
 								}
 							} catch (...) {
-								fprintf(stderr, "json parse error in networkWatcher_Redis\n");
+								fprintf(stderr, "json parse error in _membersWatcher_Redis: %s\n", a.second.c_str());
 							}
 						}
 						if (_rc->clusterMode) {
@@ -1232,8 +1293,8 @@ void PostgreSQL::_networksWatcher_Redis() {
 								if (oldConfig.is_object()||newConfig.is_object()) {
 									_networkChanged(oldConfig,newConfig,(this->_ready >= 2));
 								}
-							} catch (...) {
-								fprintf(stderr, "json parse error in networkWatcher_Redis\n");
+							} catch (std::exception &e) {
+								fprintf(stderr, "json parse error in networkWatcher_Redis: what: %s json: %s\n", e.what(), a.second.c_str());
 							}
 						}
 						if (_rc->clusterMode) {
@@ -1306,40 +1367,72 @@ void PostgreSQL::commitThread()
 						continue;
 					}
 
-					pqxx::result res = w.exec_params0(
-						"INSERT INTO ztc_member (id, network_id, active_bridge, authorized, capabilities, "
-						"identity, last_authorized_time, last_deauthorized_time, no_auto_assign_ips, "
-						"remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, v_proto) "
-						"VALUES ($1, $2, $3, $4, $5, $6, "
-						"TO_TIMESTAMP($7::double precision/1000), TO_TIMESTAMP($8::double precision/1000), "
-						"$9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (network_id, id) DO UPDATE SET "
-						"active_bridge = EXCLUDED.active_bridge, authorized = EXCLUDED.authorized, capabilities = EXCLUDED.capabilities, "
-						"identity = EXCLUDED.identity, last_authorized_time = EXCLUDED.last_authorized_time, "
-						"last_deauthorized_time = EXCLUDED.last_deauthorized_time, no_auto_assign_ips = EXCLUDED.no_auto_assign_ips, "
-						"remote_trace_level = EXCLUDED.remote_trace_level, remote_trace_target = EXCLUDED.remote_trace_target, "
-						"revision = EXCLUDED.revision+1, tags = EXCLUDED.tags, v_major = EXCLUDED.v_major, "
-						"v_minor = EXCLUDED.v_minor, v_rev = EXCLUDED.v_rev, v_proto = EXCLUDED.v_proto",
-						memberId,
-						networkId,
-						(bool)config["activeBridge"],
-						(bool)config["authorized"],
-						OSUtils::jsonDump(config["capabilities"], -1),
-						OSUtils::jsonString(config["identity"], ""),
-						(uint64_t)config["lastAuthorizedTime"],
-						(uint64_t)config["lastDeauthorizedTime"],
-						(bool)config["noAutoAssignIps"],
-						(int)config["remoteTraceLevel"],
-						target,
-						(uint64_t)config["revision"],
-						OSUtils::jsonDump(config["tags"], -1),
-						(int)config["vMajor"],
-						(int)config["vMinor"],
-						(int)config["vRev"],
-						(int)config["vProto"]);
 
+					pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM ztc_member WHERE id = $1 AND network_id = $2", memberId, networkId);
+					int membercount = mrow[0].as<int>();
 
-					res = w.exec_params0("DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2",
-						memberId, networkId);
+					bool isNewMember = false;
+					if (membercount == 0) {
+						// new member
+						isNewMember = true;
+						pqxx::result res = w.exec_params0(
+							"INSERT INTO ztc_member (id, network_id, active_bridge, authorized, capabilities, "
+							"identity, last_authorized_time, last_deauthorized_time, no_auto_assign_ips, "
+							"remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, v_proto) "
+							"VALUES ($1, $2, $3, $4, $5, $6, "
+							"TO_TIMESTAMP($7::double precision/1000), TO_TIMESTAMP($8::double precision/1000), "
+							"$9, $10, $11, $12, $13, $14, $15, $16, $17)",
+							memberId,
+							networkId,
+							(bool)config["activeBridge"],
+							(bool)config["authorized"],
+							OSUtils::jsonDump(config["capabilities"], -1),
+							OSUtils::jsonString(config["identity"], ""),
+							(uint64_t)config["lastAuthorizedTime"],
+							(uint64_t)config["lastDeauthorizedTime"],
+							(bool)config["noAutoAssignIps"],
+							(int)config["remoteTraceLevel"],
+							target,
+							(uint64_t)config["revision"],
+							OSUtils::jsonDump(config["tags"], -1),
+							(int)config["vMajor"],
+							(int)config["vMinor"],
+							(int)config["vRev"],
+							(int)config["vProto"]);
+					} else {
+						// existing member
+						pqxx::result res = w.exec_params0(
+							"UPDATE ztc_member "
+							"SET active_bridge = $3, authorized = $4, capabilities = $5, identity = $6, "
+							"last_authorized_time = TO_TIMESTAMP($7::double precision/1000), "
+							"last_deauthorized_time = TO_TIMESTAMP($8::double precision/1000), "
+							"no_auto_assign_ips = $9, remote_trace_level = $10, remote_trace_target= $11, "
+							"revision = $12, tags = $13, v_major = $14, v_minor = $15, v_rev = $16, v_proto = $17 "
+							"WHERE id = $1 AND network_id = $2",
+							memberId,
+							networkId,
+							(bool)config["activeBridge"],
+							(bool)config["authorized"],
+							OSUtils::jsonDump(config["capabilities"], -1),
+							OSUtils::jsonString(config["identity"], ""),
+							(uint64_t)config["lastAuthorizedTime"],
+							(uint64_t)config["lastDeauthorizedTime"],
+							(bool)config["noAutoAssignIps"],
+							(int)config["remoteTraceLevel"],
+							target,
+							(uint64_t)config["revision"],
+							OSUtils::jsonDump(config["tags"], -1),
+							(int)config["vMajor"],
+							(int)config["vMinor"],
+							(int)config["vRev"],
+							(int)config["vProto"]
+						);
+					}
+
+					if (!isNewMember) {
+						pqxx::result res = w.exec_params0("DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2",
+							memberId, networkId);
+					}
 
 					std::vector<std::string> assignments;
 					bool ipAssignError = false;
@@ -1350,7 +1443,7 @@ void PostgreSQL::commitThread()
 							continue;
 						}
 
-						res = w.exec_params0(
+						pqxx::result res = w.exec_params0(
 							"INSERT INTO ztc_member_ip_assignment (member_id, network_id, address) VALUES ($1, $2, $3) ON CONFLICT (network_id, member_id, address) DO NOTHING",
 							memberId, networkId, addr);
 
@@ -1366,6 +1459,24 @@ void PostgreSQL::commitThread()
 
 					w.commit();
 
+					if (_smee != NULL && isNewMember) {
+						pqxx::row row = w.exec_params1(
+							"SELECT "
+							"	count(h.hook_id) "
+							"FROM "
+							"	ztc_hook h "
+							"	INNER JOIN ztc_org o ON o.org_id = h.org_id "
+							"   INNER JOIN ztc_network n ON n.owner_id = o.owner_id "
+							" WHERE "
+							"n.id = $1 ",
+							networkId
+						);
+						int64_t hookCount = row[0].as<int64_t>();
+						if (hookCount > 0) {
+							notifyNewMember(networkId, memberId);
+						}
+					}
+
 					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
 					const uint64_t memberidInt = OSUtils::jsonIntHex(config["id"], 0ULL);
 					if (nwidInt && memberidInt) {
@@ -1609,6 +1720,13 @@ void PostgreSQL::commitThread()
 	fprintf(stderr, "%s commitThread finished\n", _myAddressStr.c_str());
 }
 
+void PostgreSQL::notifyNewMember(const std::string &networkID, const std::string &memberID) {
+	smeeclient::smee_client_notify_network_joined(
+		_smee,
+		networkID.c_str(),
+		memberID.c_str());
+}
+
 void PostgreSQL::onlineNotificationThread()
 {
  	waitForReady();

controller/PostgreSQL.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -32,6 +32,10 @@ extern "C" {
 typedef struct pg_conn PGconn;
 }
 
+namespace smeeclient {
+	struct SmeeClient;
+}
+
 namespace ZeroTier {
 
 struct RedisConfig;
@@ -144,6 +148,9 @@ private:
 	uint64_t _doRedisUpdate(sw::redis::Transaction &tx, std::string &controllerId,
 		std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > &lastOnline);
 
+	void configureSmee();
+	void notifyNewMember(const std::string &networkID, const std::string &memberID);
+
 	enum OverrideMode {
 		ALLOW_PGBOUNCER_OVERRIDE = 0,
 		NO_OVERRIDE = 1
@@ -178,6 +185,8 @@ private:
 	std::shared_ptr<sw::redis::Redis> _redis;
 	std::shared_ptr<sw::redis::RedisCluster> _cluster;
     bool _redisMemberStatus;
+
+	smeeclient::SmeeClient *_smee;
 };
 
 } // namespace ZeroTier

debian/changelog

@@ -1,3 +1,21 @@
+zerotier-one (1.14.2) unstable; urgency=medium
+
+  * See RELEASE-NOTES.md for release notes.
+
+ -- Adam Ierymenko <adam.ierymenko@zerotier.com>  Wed, 23 Oct 2024 01:00:00 -0700
+
+zerotier-one (1.14.1) unstable; urgency=medium
+
+  * See RELEASE-NOTES.md for release notes.
+
+ -- Adam Ierymenko <adam.ierymenko@zerotier.com>  Wed, 11 Sep 2024 01:00:00 -0700
+
+zerotier-one (1.14.0) unstable; urgency=medium
+
+  * See RELEASE-NOTES.md for release notes.
+
+ -- Adam Ierymenko <adam.ierymenko@zerotier.com>  Tue, 19 Mar 2024 01:00:00 -0700
+
 zerotier-one (1.12.2) unstable; urgency=medium
 
   * See RELEASE-NOTES.md for release notes.

debian/copyright

@@ -12,7 +12,7 @@ License: ZeroTier BSL 1.1
  Use of this software is governed by the Business Source License included
  in the LICENSE.TXT file in the project's root directory.
 
- Change Date: 2025-01-01
+ Change Date: 2026-01-01
 
  On the date above, in accordance with the Business Source License, use
  of this software will be governed by version 2.0 of the Apache License.

doc/README.md

@@ -3,4 +3,4 @@ Manual Pages and Other Documentation
 
 Use "./build.sh" to build the manual pages.
 
-You'll need either NodeJS/npm installed (script will then automatically install the npm *marked-man* package) or */usr/bin/ronn*. The latter is a Ruby program packaged on some distributions as *rubygem-ronn* or *ruby-ronn* or installable as *gem install ronn*. The Node *marked-man* package and *ronn* from rubygems are two roughly equivalent alternatives for compiling MarkDown into roff/man format.
+You'll need either Node.js/npm installed (script will then automatically install the npm *marked-man* package) or */usr/bin/ronn*. The latter is a Ruby program packaged on some distributions as *rubygem-ronn* or *ruby-ronn* or installable as *gem install ronn*. The Node *marked-man* package and *ronn* from RubyGems are two roughly equivalent alternatives for compiling Markdown into roff/man format.

entrypoint.sh.release

@@ -9,15 +9,16 @@ mkztfile() {
   file=$1
   mode=$2
   content=$3
-
+  echo "creating $file"
   mkdir -p /var/lib/zerotier-one
-  echo "$content" > "/var/lib/zerotier-one/$file"
+  echo -n "$content" > "/var/lib/zerotier-one/$file"
   chmod "$mode" "/var/lib/zerotier-one/$file"
 }
 
 if [ "x$ZEROTIER_API_SECRET" != "x" ]
 then
   mkztfile authtoken.secret 0600 "$ZEROTIER_API_SECRET"
+  mkztfile metricstoken.secret 0600 "$ZEROTIER_API_SECRET"
 fi
 
 if [ "x$ZEROTIER_IDENTITY_PUBLIC" != "x" ]
@@ -30,6 +31,11 @@ then
   mkztfile identity.secret 0600 "$ZEROTIER_IDENTITY_SECRET"
 fi
 
+if [ "x$ZEROTIER_LOCAL_CONF" != "x" ]
+then
+  mkztfile local.conf 0644 "$ZEROTIER_LOCAL_CONF"
+fi
+
 mkztfile zerotier-one.port 0600 "9993"
 
 killzerotier() {

ext/central-controller-docker/Dockerfile.builder

@@ -20,6 +20,7 @@ RUN apt -y install \
     postgresql-client-common \
     curl \
     google-perftools \
-    libgoogle-perftools-dev
+    libgoogle-perftools-dev \
+    protobuf-compiler 
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y

ext/central-controller-docker/Dockerfile.run_base

@@ -3,6 +3,7 @@ FROM ubuntu:jammy
 RUN apt update && apt upgrade -y
 
 RUN apt -y install \
+    netcat \
     postgresql-client \
     postgresql-client-common \
     libjemalloc2 \

ext/central-controller-docker/main.sh

@@ -100,6 +100,15 @@ else
     done
 fi
 
+if [ -n "$ZT_TEMPORAL_HOST" ] && [ -n "$ZT_TEMPORAL_PORT" ]; then
+    echo "waiting for temporal..."
+    while ! nc -z ${ZT_TEMPORAL_HOST} ${ZT_TEMPORAL_PORT}; do
+        echo "waiting...";
+        sleep 1;
+    done
+    echo "Temporal is up"
+fi
+
 export GLIBCXX_FORCE_NEW=1
 export GLIBCPP_FORCE_NEW=1
 export LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libjemalloc.so.2"

ext/installfiles/mac/ZeroTier One.pkgproj

@@ -701,7 +701,7 @@
 				<key>USE_HFS+_COMPRESSION</key>
 				<false/>
 				<key>VERSION</key>
-				<string>1.12.2</string>
+				<string>1.14.2</string>
 			</dict>
 			<key>TYPE</key>
 			<integer>0</integer>

ext/installfiles/windows/ZeroTier One.aip

@@ -24,10 +24,10 @@
     <ROW Property="AiFeatIcoZeroTierOne" Value="ZeroTierIcon.exe" Type="8"/>
     <ROW Property="MSIFASTINSTALL" MultiBuildValue="DefaultBuild:2"/>
     <ROW Property="Manufacturer" Value="ZeroTier, Inc."/>
-    <ROW Property="ProductCode" Value="1033:{56528063-D8C2-43F4-97DB-C787E6A2D9DB} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{0143A36C-46C6-458D-AB9B-C8843E089323} " Type="16"/>
     <ROW Property="ProductLanguage" Value="1033"/>
     <ROW Property="ProductName" Value="ZeroTier One"/>
-    <ROW Property="ProductVersion" Value="1.12.2" Options="32"/>
+    <ROW Property="ProductVersion" Value="1.14.2" Options="32"/>
     <ROW Property="REBOOT" MultiBuildValue="DefaultBuild:ReallySuppress"/>
     <ROW Property="SecureCustomProperties" Value="OLDPRODUCTS;AI_NEWERPRODUCTFOUND;AI_SETUPEXEPATH;SETUPEXEDIR"/>
     <ROW Property="UpgradeCode" Value="{B0E2A5F3-88B6-4E77-B922-CB4739B4C4C8}"/>
@@ -62,7 +62,7 @@
     <ROW Directory="regid.201001.com.zerotier_Dir" Directory_Parent="CommonAppDataFolder" DefaultDir="REGID2~1.ZER|regid.2010-01.com.zerotier" DirectoryOptions="12"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiCompsComponent">
-    <ROW Component="AI_CustomARPName" ComponentId="{A0629900-689C-4BD7-9315-85F05804DF03}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{DFE7A60C-C2B9-41F6-9171-8955BA30E556}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_DisableModify" ComponentId="{46FFA8C5-A0CB-4E05-9AD3-911D543DE8CA}" Directory_="APPDIR" Attributes="4" KeyPath="NoModify" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{8E02B36C-7A19-429B-A93E-77A9261AC918}" Directory_="APPDIR" Attributes="4" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{4DD7907D-D7FE-4CD6-B1A0-B5C1625F5133}" Directory_="APPDIR" Attributes="0"/>
@@ -124,7 +124,7 @@
     <ROW Path="&lt;AI_DICTS&gt;ui_en.ail"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.DigCertStoreComponent">
-    <ROW TimeStampUrl="http://timestamp.digicert.com" SignerDescription="ZeroTier One" DescriptionUrl="https://www.zerotier.com/" SignOptions="7" SignTool="5" UseSha256="1" KVTenantId="5300bf3b-0eff-4a5f-a63f-821e22ed1730" KVAppId="5f94d77e-b795-41fd-afe7-ec913b03c1d3" KVName="ZeroTier-CS" KVCertName="ZT-EV-CS" KVCertVersion="442c2d6f77874ff99eed4b36f5cb401c"/>
+    <ROW TimeStampUrl="http://timestamp.digicert.com" SignerDescription="ZeroTier One" DescriptionUrl="https://www.zerotier.com/" SignOptions="7" SignTool="5" UseSha256="1" KVTenantId="5300bf3b-0eff-4a5f-a63f-821e22ed1730" KVAppId="5f94d77e-b795-41fd-afe7-ec913b03c1d3" KVName="ZeroTier-CS" KVCertName="ZT-EV-CS-2024" KVCertVersion="64807be24d57468e895e2e577f430de2"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.FirewallExceptionComponent">
     <ROW FirewallException="ZeroTierOneUDP9993" Direction="1" Action="1" DisplayName="ZeroTier UDP/9993 In" GroupName="ZeroTierOne" Enabled="1" Scope="*" Condition="1" Profiles="7" Port="9993" Protocol="UDP"/>
@@ -501,7 +501,7 @@
     <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="2" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidentitlement_required_indicator" ParentElement="swidsoftware_identification_tag" Name="swid:entitlement_required_indicator" Condition="1" Order="0" Flags="14" Text="false" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidmajor" ParentElement="swidnumeric" Name="swid:major" Condition="1" Order="0" Flags="14" Text="1" UpdateIndexInParent="0"/>
-    <ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="12" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="14" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidname" ParentElement="swidproduct_version" Name="swid:name" Condition="1" Order="0" Flags="14" Text="[ProductVersion]" UpdateIndexInParent="0"/>
     <ROW XmlElement="swidname_1" ParentElement="swidsoftware_creator" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc." UpdateIndexInParent="0"/>
     <ROW XmlElement="swidname_2" ParentElement="swidsoftware_licensor" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc." UpdateIndexInParent="0"/>

ext/prometheus-cpp-lite-1.0/core/include/prometheus/client_metric.h

@@ -8,6 +8,7 @@
 namespace prometheus {
 
   // <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>, <20> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  
   struct ClientMetric {
 
     // Label

include/ZeroTierDebug.h

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

include/ZeroTierOne.h

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -84,7 +84,7 @@ extern "C" {
 /**
  * Minimum UDP payload size allowed
  */
-#define ZT_MIN_PHYSMTU 1400
+#define ZT_MIN_PHYSMTU 510
 
 /**
  * Maximum physical interface name length. This number is gigantic because of Windows.
@@ -1347,8 +1347,16 @@ typedef struct
 	 */
 	char ifname[ZT_MAX_PHYSIFNAME];
 
+	/**
+	 * Pointer to PhySocket object for this path
+	 */
 	uint64_t localSocket;
 
+	/**
+	 * Local port corresponding to this path's localSocket
+	 */
+	uint16_t localPort;
+
 	/**
 	 * Is path expired?
 	 */

java/jni/com_zerotierone_sdk_Node.cpp

@@ -111,6 +111,44 @@ namespace {
         bool finishInitializing();
     };
 
+    //
+    // RAII construct for calling AttachCurrentThread and DetachCurrent automatically
+    //
+    struct ScopedJNIThreadAttacher {
+
+        JavaVM *jvm;
+        JNIEnv **env_p;
+        jint getEnvRet;
+
+        ScopedJNIThreadAttacher(JavaVM *jvmIn, JNIEnv **env_pIn, jint getEnvRetIn) :
+        jvm(jvmIn),
+        env_p(env_pIn),
+        getEnvRet(getEnvRetIn) {
+
+            if (getEnvRet != JNI_EDETACHED) {
+                return;
+            }
+
+            jint attachCurrentThreadRet;
+            if ((attachCurrentThreadRet = jvm->AttachCurrentThread(env_p, NULL)) != JNI_OK) {
+                LOGE("Error calling AttachCurrentThread: %d", attachCurrentThreadRet);
+                assert(false && "Error calling AttachCurrentThread");
+            }
+        }
+
+        ~ScopedJNIThreadAttacher() {
+
+            if (getEnvRet != JNI_EDETACHED) {
+                return;
+            }
+
+            jint detachCurrentThreadRet;
+            if ((detachCurrentThreadRet = jvm->DetachCurrentThread()) != JNI_OK) {
+                LOGE("Error calling DetachCurrentThread: %d", detachCurrentThreadRet);
+                assert(false && "Error calling DetachCurrentThread");
+            }
+        }
+    };
 
     /*
     * This must return 0 on success. It can return any OS-dependent error code
@@ -194,7 +232,25 @@ namespace {
         assert(ref);
         assert(ref->node == node);
         JNIEnv *env;
-        GETENV(env, ref->jvm);
+        
+        jint getEnvRet;
+        assert(ref->jvm);
+        getEnvRet = ref->jvm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6);
+
+        if (!(getEnvRet == JNI_OK || getEnvRet == JNI_EDETACHED)) {
+            LOGE("Error calling GetEnv: %d", getEnvRet);
+            assert(false && "Error calling GetEnv");
+        }
+
+        //
+        // Thread might actually be detached.
+        //
+        // e.g:
+        // https://github.com/zerotier/ZeroTierOne/blob/91e7ce87f09ac1cfdeaf6ff22c3cedcd93574c86/node/Switch.cpp#L519
+        //
+        // Make sure to attach if needed
+        //
+        ScopedJNIThreadAttacher attacher{ref->jvm, &env, getEnvRet};
 
         if (env->ExceptionCheck()) {
             LOGE("Unhandled pending exception");

make-linux.mk

@@ -9,10 +9,11 @@ ifeq ($(origin CXX),default)
 	CXX:=$(shell if [ -e /opt/rh/devtoolset-8/root/usr/bin/g++ ]; then echo /opt/rh/devtoolset-8/root/usr/bin/g++; else echo $(CXX); fi)
 endif
 
-INCLUDES?=-Izeroidc/target -isystem ext -Iext/prometheus-cpp-lite-1.0/core/include -Iext-prometheus-cpp-lite-1.0/3rdparty/http-client-lite/include -Iext/prometheus-cpp-lite-1.0/simpleapi/include
+INCLUDES?=-Irustybits/target -isystem ext -Iext/prometheus-cpp-lite-1.0/core/include -Iext-prometheus-cpp-lite-1.0/3rdparty/http-client-lite/include -Iext/prometheus-cpp-lite-1.0/simpleapi/include
 DEFS?=
 LDLIBS?=
 DESTDIR?=
+EXTRA_DEPS?=
 
 include objects.mk
 ONE_OBJS+=osdep/LinuxEthernetTap.o
@@ -61,7 +62,7 @@ ifeq ($(ZT_DEBUG),1)
 	override CFLAGS+=-Wall -Wno-deprecated -g -O -pthread $(INCLUDES) $(DEFS)
 	override CXXFLAGS+=-Wall -Wno-deprecated -g -O -std=c++17 -pthread $(INCLUDES) $(DEFS)
 	ZT_TRACE=1
-	RUSTFLAGS=
+	ZT_CARGO_FLAGS=
 	# The following line enables optimization for the crypto code, since
 	# C25519 in particular is almost UNUSABLE in -O0 even on a 3ghz box!
 node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CXXFLAGS=-Wall -O2 -g -pthread $(INCLUDES) $(DEFS)
@@ -70,8 +71,8 @@ else
 	override CFLAGS+=-Wall -Wno-deprecated -pthread $(INCLUDES) -DNDEBUG $(DEFS)
 	CXXFLAGS?=-O3 -fstack-protector
 	override CXXFLAGS+=-Wall -Wno-deprecated -std=c++17 -pthread $(INCLUDES) -DNDEBUG $(DEFS)
-	LDFLAGS=-pie -Wl,-z,relro,-z,now
-	RUSTFLAGS=--release
+	LDFLAGS?=-pie -Wl,-z,relro,-z,now
+	ZT_CARGO_FLAGS=--release
 endif
 
 ifeq ($(ZT_QNAP), 1)
@@ -295,9 +296,9 @@ ifeq ($(ZT_SSO_SUPPORTED), 1)
 	ifeq ($(ZT_EMBEDDED),)
 		override DEFS+=-DZT_SSO_SUPPORTED=1
 		ifeq ($(ZT_DEBUG),1)
-			LDLIBS+=zeroidc/target/debug/libzeroidc.a -ldl -lssl -lcrypto
+			LDLIBS+=rustybits/target/debug/libzeroidc.a -ldl -lssl -lcrypto
 		else
-			LDLIBS+=zeroidc/target/release/libzeroidc.a -ldl -lssl -lcrypto
+			LDLIBS+=rustybits/target/release/libzeroidc.a -ldl -lssl -lcrypto
 		endif
 	endif
 endif
@@ -324,6 +325,11 @@ ifeq ($(ZT_CONTROLLER),1)
 	override LDLIBS+=-Lext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/lib -lpqxx -lpq ext/hiredis-1.0.2/lib/ubuntu22.04/$(EXT_ARCH)/libhiredis.a ext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/lib/libredis++.a -lssl -lcrypto
 	override DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_NO_PEER_METRICS
 	override INCLUDES+=-I/usr/include/postgresql -Iext/libpqxx-7.7.3/install/ubuntu22.04/$(EXT_ARCH)/include -Iext/hiredis-1.0.2/include/ -Iext/redis-plus-plus-1.3.3/install/ubuntu22.04/$(EXT_ARCH)/include/sw/
+	ifeq ($(ZT_DEBUG),1)
+		override LDLIBS+=rustybits/target/debug/libsmeeclient.a
+	else
+		override LDLIBS+=rustybits/target/release/libsmeeclient.a
+	endif
 endif
 
 # ARM32 hell -- use conservative CFLAGS
@@ -358,7 +364,7 @@ override CFLAGS+=-fPIC -fPIE
 override CXXFLAGS+=-fPIC -fPIE
 
 # Non-executable stack
-override ASFLAGS+=--noexecstack
+override LDFLAGS+=-Wl,-z,noexecstack
 
 .PHONY: all
 all:	one
@@ -370,7 +376,7 @@ from_builder:	FORCE
 	ln -sf zerotier-one zerotier-idtool
 	ln -sf zerotier-one zerotier-cli
 
-zerotier-one:	$(CORE_OBJS) $(ONE_OBJS) one.o
+zerotier-one: $(CORE_OBJS) $(ONE_OBJS) one.o
 	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LDLIBS)
 
 zerotier-idtool: zerotier-one
@@ -379,7 +385,7 @@ zerotier-idtool: zerotier-one
 zerotier-cli: zerotier-one
 	ln -sf zerotier-one zerotier-cli
 
-$(ONE_OBJS): zeroidc
+$(ONE_OBJS): zeroidc smeeclient
 
 libzerotiercore.a:	FORCE
 	make CFLAGS="-O3 -fstack-protector -fPIC" CXXFLAGS="-O3 -std=c++17 -fstack-protector -fPIC" $(CORE_OBJS)
@@ -399,7 +405,7 @@ manpages:	FORCE
 doc:	manpages
 
 clean: FORCE
-	rm -rf *.a *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules ext/misc/*.o debian/.debhelper debian/debhelper-build-stamp docker/zerotier-one zeroidc/target
+	rm -rf *.a *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/miniupnpc/*.o ext/libnatpmp/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest build-* ZeroTierOneInstaller-* *.deb *.rpm .depend debian/files debian/zerotier-one*.debhelper debian/zerotier-one.substvars debian/*.log debian/zerotier-one doc/node_modules ext/misc/*.o debian/.debhelper debian/debhelper-build-stamp docker/zerotier-one rustybits/target
 
 distclean:	clean
 
@@ -432,13 +438,19 @@ debug:	FORCE
 ifeq ($(ZT_SSO_SUPPORTED), 1)
 ifeq ($(ZT_EMBEDDED),)
 zeroidc:	FORCE
-#	export PATH=/root/.cargo/bin:$$PATH; cd zeroidc && cargo build -j1 $(RUSTFLAGS)
-	export PATH=/${HOME}/.cargo/bin:$$PATH; cd zeroidc && cargo build $(RUSTFLAGS)
+	export PATH=/${HOME}/.cargo/bin:$$PATH; cd rustybits && cargo build $(ZT_CARGO_FLAGS) -p zeroidc
 endif
 else
 zeroidc:
 endif
 
+ifeq ($(ZT_CONTROLLER), 1)
+smeeclient:	FORCE
+	export PATH=/${HOME}/.cargo/bin:$$PATH; cd rustybits && cargo build $(ZT_CARGO_FLAGS) -p smeeclient
+else
+smeeclient:
+endif
+
 # Note: keep the symlinks in /var/lib/zerotier-one to the binaries since these
 # provide backward compatibility with old releases where the binaries actually
 # lived here. Folks got scripts.

make-mac.mk

@@ -1,8 +1,8 @@
 CC=clang
 CXX=clang++
-TOPDIR=$(shell PWD)
+TOPDIR=$(shell pwd)
 
-INCLUDES=-I$(shell PWD)/zeroidc/target -isystem $(TOPDIR)/ext  -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/core/include -I$(TOPDIR)/ext-prometheus-cpp-lite-1.0/3rdparty/http-client-lite/include -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/simpleapi/include
+INCLUDES=-I$(shell pwd)/rustybits/target -isystem $(TOPDIR)/ext  -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/core/include -I$(TOPDIR)/ext-prometheus-cpp-lite-1.0/3rdparty/http-client-lite/include -I$(TOPDIR)/ext/prometheus-cpp-lite-1.0/simpleapi/include
 DEFS=
 LIBS=
 ARCH_FLAGS=-arch x86_64 -arch arm64 
@@ -12,7 +12,8 @@ PRODUCTSIGN=echo
 CODESIGN_APP_CERT=
 CODESIGN_INSTALLER_CERT=
 NOTARIZE=echo
-NOTARIZE_USER_ID=null
+NOTARIZE_APPLE_ID=null
+NOTARIZE_TEAM_ID=null
 
 ZT_BUILD_PLATFORM=3
 ZT_BUILD_ARCHITECTURE=2
@@ -38,8 +39,9 @@ ifeq ($(ZT_OFFICIAL_RELEASE),1)
 	PRODUCTSIGN=productsign
 	CODESIGN_APP_CERT="Developer ID Application: ZeroTier, Inc (8ZD9JUCZ4V)"
 	CODESIGN_INSTALLER_CERT="Developer ID Installer: ZeroTier, Inc (8ZD9JUCZ4V)"
-	NOTARIZE=xcrun altool
-	NOTARIZE_USER_ID="adam.ierymenko@gmail.com"
+	NOTARIZE=xcrun notarytool
+	NOTARIZE_APPLE_ID="adam.ierymenko@gmail.com"
+	NOTARIZE_TEAM_ID="8ZD9JUCZ4V"
 else
 	DEFS+=-DZT_SOFTWARE_UPDATE_DEFAULT="\"download\""
 endif
@@ -114,7 +116,7 @@ osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm
 	$(CXX) $(CXXFLAGS) -c osdep/MacDNSHelper.mm -o osdep/MacDNSHelper.o 
 
 one:	zeroidc $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
-	$(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) zeroidc/target/libzeroidc.a
+	$(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) rustybits/target/libzeroidc.a
 	# $(STRIP) zerotier-one
 	ln -sf zerotier-one zerotier-idtool
 	ln -sf zerotier-one zerotier-cli
@@ -122,12 +124,12 @@ one:	zeroidc $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent
 
 zerotier-one: one
 
-zeroidc: zeroidc/target/libzeroidc.a
+zeroidc: rustybits/target/libzeroidc.a
 
-zeroidc/target/libzeroidc.a:	FORCE
-	cd zeroidc && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
-	cd zeroidc && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
-	cd zeroidc && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libzeroidc.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libzeroidc.a -output target/libzeroidc.a
+rustybits/target/libzeroidc.a:	FORCE
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libzeroidc.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libzeroidc.a -output target/libzeroidc.a
 
 central-controller:
 	make ARCH_FLAGS="-arch x86_64" ZT_CONTROLLER=1 one
@@ -149,7 +151,7 @@ core: libzerotiercore.a
 #	$(STRIP) zerotier
 
 selftest: $(CORE_OBJS) $(ONE_OBJS) selftest.o
-	$(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS) $(LIBS) zeroidc/target/libzeroidc.a
+	$(CXX) $(CXXFLAGS) -o zerotier-selftest selftest.o $(CORE_OBJS) $(ONE_OBJS) $(LIBS) rustybits/target/libzeroidc.a
 	$(STRIP) zerotier-selftest
 
 zerotier-selftest: selftest
@@ -166,7 +168,7 @@ mac-dist-pkg: FORCE
 	if [ -f "ZeroTier One Signed.pkg" ]; then mv -f "ZeroTier One Signed.pkg" "ZeroTier One.pkg"; fi
 	rm -f zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_*
 	cat ext/installfiles/mac-update/updater.tmpl.sh "ZeroTier One.pkg" >zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_$(ZT_VERSION_MAJOR).$(ZT_VERSION_MINOR).$(ZT_VERSION_REV)_$(ZT_VERSION_BUILD).exe
-	$(NOTARIZE) -t osx -f "ZeroTier One.pkg" --primary-bundle-id com.zerotier.pkg.ZeroTierOne --output-format xml --notarize-app -u $(NOTARIZE_USER_ID)
+	$(NOTARIZE) submit --apple-id "adam.ierymenko@gmail.com" --team-id "8ZD9JUCZ4V" --wait "ZeroTier One.pkg"
 	echo '*** When Apple notifies that the app is notarized, run: xcrun stapler staple "ZeroTier One.pkg"'
 
 # For ZeroTier, Inc. to build official signed packages
@@ -183,12 +185,21 @@ _buildx:
 	@echo docker buildx create --name multiarch --driver docker-container --use
 	@echo docker buildx inspect --bootstrap
 
+controller-builder: _buildx FORCE
+	docker buildx build --platform linux/arm64,linux/amd64 --no-cache -t registry.zerotier.com/zerotier/ctlbuild:latest -f ext/central-controller-docker/Dockerfile.builder . --push
+
+controller-run: _buildx FORCE
+	docker buildx build --platform linux/arm64,linux/amd64 --no-cache -t registry.zerotier.com/zerotier-central/ctlrun:latest -f ext/central-controller-docker/Dockerfile.run_base . --push
+
 central-controller-docker: _buildx FORCE
 	docker buildx build --platform linux/arm64,linux/amd64 --no-cache -t registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=$(shell git name-rev --name-only HEAD) . --push
 	@echo Image: registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP}
 
+docker-release:	_buildx
+	docker buildx build --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/s390x -t zerotier/zerotier:${RELEASE_DOCKER_TAG} -t zerotier/zerotier:latest --build-arg VERSION=${RELEASE_VERSION} -f Dockerfile.release . --push
+	
 clean:
-	rm -rf MacEthernetTapAgent *.dSYM build-* *.a *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier doc/node_modules zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* zeroidc/target/
+	rm -rf MacEthernetTapAgent *.dSYM build-* *.a *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o $(CORE_OBJS) $(ONE_OBJS) zerotier-one zerotier-idtool zerotier-selftest zerotier-cli zerotier doc/node_modules zt1_update_$(ZT_BUILD_PLATFORM)_$(ZT_BUILD_ARCHITECTURE)_* rustybits/target/
 
 distclean:	clean
 

node/AES.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/AES.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/AES_aesni.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/AES_armcrypto.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Address.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/AtomicCounter.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Bond.cpp

@@ -15,10 +15,10 @@
 
 #include "Switch.hpp"
 
+#include <cinttypes>   // for PRId64, etc. macros
 #include <cmath>
 #include <cstdio>
 #include <string>
-#include <cinttypes> // for PRId64, etc. macros
 
 // FIXME: remove this suppression and actually fix warnings
 #ifdef __GNUC__
@@ -108,7 +108,7 @@ bool Bond::setAllMtuByTuple(uint16_t mtu, const std::string& ifStr, const std::s
 	std::map<int64_t, SharedPtr<Bond> >::iterator bondItr = _bonds.begin();
 	bool found = false;
 	while (bondItr != _bonds.end()) {
-		if (bondItr->second->setMtuByTuple(mtu,ifStr,ipStr)) {
+		if (bondItr->second->setMtuByTuple(mtu, ifStr, ipStr)) {
 			found = true;
 		}
 		++bondItr;
@@ -154,11 +154,13 @@ SharedPtr<Bond> Bond::createBond(const RuntimeEnvironment* renv, const SharedPtr
 				bond = new Bond(renv, _bondPolicyTemplates[_defaultPolicyStr].ptr(), peer);
 				bond->debug("new default custom bond (based on %s)", bond->getPolicyStrByCode(bond->policy()).c_str());
 			}
-		} else {
+		}
+		else {
 			if (! _bondPolicyTemplates[_policyTemplateAssignments[identity]]) {
 				bond = new Bond(renv, _defaultPolicy, peer);
 				bond->debug("peer-specific bond, was specified as %s but the bond definition was not found, using default %s", _policyTemplateAssignments[identity].c_str(), getPolicyStrByCode(_defaultPolicy).c_str());
-			} else {
+			}
+			else {
 				bond = new Bond(renv, _bondPolicyTemplates[_policyTemplateAssignments[identity]].ptr(), peer);
 				bond->debug("new default bond");
 			}
@@ -227,10 +229,12 @@ SharedPtr<Link> Bond::getLinkBySocket(const std::string& policyAlias, uint64_t l
 			SharedPtr<Link> s = new Link(ifnameStr, 0, 0, 0, true, ZT_BOND_SLAVE_MODE_PRIMARY, "");
 			_interfaceToLinkMap[policyAlias].insert(std::pair<std::string, SharedPtr<Link> >(ifnameStr, s));
 			return s;
-		} else {
+		}
+		else {
 			return SharedPtr<Link>();
 		}
-	} else {
+	}
+	else {
 		return search->second;
 	}
 }
@@ -340,6 +344,7 @@ void Bond::nominatePathToBond(const SharedPtr<Path>& path, int64_t now)
 						_paths[i].ipvPref = sl->ipvPref();
 						_paths[i].mode = sl->mode();
 						_paths[i].enabled = sl->enabled();
+						_paths[i].localPort = _phy->getLocalPort((PhySocket*)((uintptr_t)path->localSocket()));
 						_paths[i].onlyPathOnLink = ! bFoundCommonLink;
 					}
 				}
@@ -368,6 +373,7 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
 	 */
 	if (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
 		if (_abPathIdx != ZT_MAX_PEER_NETWORK_PATHS && _paths[_abPathIdx].p) {
+			//fprintf(stderr, "trying to send via (_abPathIdx=%d) %s\n", _abPathIdx, pathToStr(_paths[_abPathIdx].p).c_str());
 			return _paths[_abPathIdx].p;
 		}
 	}
@@ -397,7 +403,8 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
 		_rrPacketsSentOnCurrLink = 0;
 		if (_numBondedPaths == 1 || _rrIdx >= (ZT_MAX_PEER_NETWORK_PATHS - 1)) {
 			_rrIdx = 0;
-		} else {
+		}
+		else {
 			int _tempIdx = _rrIdx;
 			for (int searchCount = 0; searchCount < (_numBondedPaths - 1); searchCount++) {
 				_tempIdx = (_tempIdx == (_numBondedPaths - 1)) ? 0 : _tempIdx + 1;
@@ -427,7 +434,8 @@ SharedPtr<Path> Bond::getAppropriatePath(int64_t now, int32_t flowId)
 		if (likely(it != _flows.end())) {
 			it->second->lastActivity = now;
 			return _paths[it->second->assignedPath].p;
-		} else {
+		}
+		else {
 			unsigned char entropy;
 			Utils::getSecureRandom(&entropy, 1);
 			SharedPtr<Flow> flow = createFlow(ZT_MAX_PEER_NETWORK_PATHS, flowId, entropy, now);
@@ -505,7 +513,8 @@ void Bond::recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId,
 					_paths[pathIdx].qosStatsIn[packetId] = now;
 					++(_paths[pathIdx].packetsReceivedSinceLastQoS);
 					//_paths[pathIdx].packetValiditySamples.push(true);
-				} else {
+				}
+				else {
 					// debug("QoS buffer full, will not record information");
 				}
 				/*
@@ -532,7 +541,8 @@ void Bond::recordIncomingPacket(const SharedPtr<Path>& path, uint64_t packetId,
 		SharedPtr<Flow> flow;
 		if (! _flows.count(flowId)) {
 			flow = createFlow(pathIdx, flowId, 0, now);
-		} else {
+		}
+		else {
 			flow = _flows[flowId];
 		}
 		if (flow) {
@@ -618,7 +628,8 @@ bool Bond::assignFlowToBondedPath(SharedPtr<Flow>& flow, int64_t now, bool reass
 
 		if (reassign) {
 			log("attempting to re-assign out-flow %04x previously on idx %d (%u / %zu flows)", flow->id, flow->assignedPath, _paths[_realIdxMap[flow->assignedPath]].assignedFlowCount, _flows.size());
-		} else {
+		}
+		else {
 			debug("attempting to assign flow for the first time");
 		}
 
@@ -632,7 +643,8 @@ bool Bond::assignFlowToBondedPath(SharedPtr<Flow>& flow, int64_t now, bool reass
 
 			if (reassign) {
 				bondedIdx = (flow->assignedPath + offset) % (_numBondedPaths);
-			} else {
+			}
+			else {
 				bondedIdx = abs((int)((entropy + offset) % (_numBondedPaths)));
 			}
 			// debug("idx=%d, offset=%d, randomCap=%f, actualCap=%f", bondedIdx, offset, randomLinkCapacity, _paths[_realIdxMap[bondedIdx]].relativeLinkCapacity);
@@ -655,7 +667,8 @@ bool Bond::assignFlowToBondedPath(SharedPtr<Flow>& flow, int64_t now, bool reass
 			flow->assignPath(_realIdxMap[bondedIdx], now);
 			++(_paths[_realIdxMap[bondedIdx]].assignedFlowCount);
 			// debug("       ABLE to find optimal link %f idx %d", _paths[_realIdxMap[bondedIdx]].relativeQuality, bondedIdx);
-		} else {
+		}
+		else {
 			// We were (unable) to find a path that didn't violate at least one quality requirement, will choose next best option
 			flow->assignPath(_realIdxMap[nextBestQualIdx], now);
 			++(_paths[_realIdxMap[nextBestQualIdx]].assignedFlowCount);
@@ -715,11 +728,13 @@ void Bond::forgetFlowsWhenNecessary(uint64_t age, bool oldest, int64_t now)
 				debug("forget flow %04x (age %" PRId64 ") (%u / %zu)", it->first, it->second->age(now), _paths[it->second->assignedPath].assignedFlowCount, (_flows.size() - 1));
 				_paths[it->second->assignedPath].assignedFlowCount--;
 				it = _flows.erase(it);
-			} else {
+			}
+			else {
 				++it;
 			}
 		}
-	} else if (oldest) {	 // Remove single oldest by natural expiration
+	}
+	else if (oldest) {	 // Remove single oldest by natural expiration
 		uint64_t maxAge = 0;
 		while (it != _flows.end()) {
 			if (it->second->age(now) > maxAge) {
@@ -766,7 +781,8 @@ void Bond::processIncomingPathNegotiationRequest(uint64_t now, SharedPtr<Path>&
 			if (_peer->_id.address().toInt() > RR->node->identity().address().toInt()) {
 				debug("agree with peer to use alternate link %s/%s\n", link->ifname().c_str(), pathStr);
 				_negotiatedPathIdx = pathIdx;
-			} else {
+			}
+			else {
 				debug("ignore petition from peer to use alternate link %s/%s\n", link->ifname().c_str(), pathStr);
 			}
 		}
@@ -881,7 +897,8 @@ void Bond::sendQOS_MEASUREMENT(void* tPtr, int pathIdx, int64_t localSocket, con
 		if (atAddress) {
 			outp.armor(_peer->key(), false, _peer->aesKeysIfSupported());
 			RR->node->putPacket(tPtr, localSocket, atAddress, outp.data(), outp.size());
-		} else {
+		}
+		else {
 			RR->sw->send(tPtr, outp, false);
 		}
 		Metrics::pkt_qos_out++;
@@ -1016,6 +1033,13 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 		bool satisfiedUpDelay = (now - _paths[i].lastAliveToggle) >= _upDelay;
 		// How long since the last QoS was received (Must be less than ZT_PEER_PATH_EXPIRATION since the remote peer's _qosSendInterval isn't known)
 		bool acceptableQoSAge = (_paths[i].lastQoSReceived == 0 && inTrial) || ((now - _paths[i].lastQoSReceived) < ZT_PEER_EXPIRED_PATH_TRIAL_PERIOD);
+
+		// Allow active-backup to operate without the receipt of QoS records
+		// This may be expanded to the other modes as an option
+		if (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
+			acceptableQoSAge = true;
+		}
+
 		currEligibility = _paths[i].allowed() && ((acceptableAge && satisfiedUpDelay && acceptableQoSAge) || inTrial);
 
 		if (currEligibility) {
@@ -1027,12 +1051,11 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 		 */
 		if (currEligibility != _paths[i].eligible) {
 			if (currEligibility == 0) {
-				log("link %s is no longer eligible", pathToStr(_paths[i].p).c_str());
+				log("link %s is no longer eligible (reason: allowed=%d, age=%d, ud=%d, qos=%d, trial=%d)", pathToStr(_paths[i].p).c_str(), _paths[i].allowed(), acceptableAge, satisfiedUpDelay, acceptableQoSAge, inTrial);
 			}
 			if (currEligibility == 1) {
 				log("link %s is eligible", pathToStr(_paths[i].p).c_str());
 			}
-			debug("\t[%d] allowed=%d, age=%d, qa=%d, ud=%d, trial=%d", i, _paths[i].allowed(), acceptableAge, acceptableQoSAge, satisfiedUpDelay, inTrial);
 			dumpPathStatus(now, i);
 			if (currEligibility) {
 				rebuildBond = true;
@@ -1078,6 +1101,7 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 	 * Curate the set of paths that are part of the bond proper. Select a set of paths
 	 * per logical link according to eligibility and user-specified constraints.
 	 */
+	int updatedBondedPathCount = 0;
 	if ((_policy == ZT_BOND_POLICY_BALANCE_RR) || (_policy == ZT_BOND_POLICY_BALANCE_XOR) || (_policy == ZT_BOND_POLICY_BALANCE_AWARE)) {
 		if (! _numBondedPaths) {
 			rebuildBond = true;
@@ -1089,7 +1113,6 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 				_paths[i].bonded = false;
 			}
 
-			int updatedBondedPathCount = 0;
 			// Build map associating paths with local physical links. Will be selected from in next step
 			std::map<SharedPtr<Link>, std::vector<int> > linkMap;
 			for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) {
@@ -1191,6 +1214,14 @@ void Bond::curateBond(int64_t now, bool rebuildBond)
 			}
 		}
 	}
+	if (_policy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
+		for (int i = 0; i < ZT_MAX_PEER_NETWORK_PATHS; ++i) {
+			if (_paths[i].p && _paths[i].bonded) {
+				updatedBondedPathCount++;
+			}
+		}
+		_numBondedPaths = updatedBondedPathCount;
+	}
 }
 
 void Bond::estimatePathQuality(int64_t now)
@@ -1222,7 +1253,8 @@ void Bond::estimatePathQuality(int64_t now)
 			if ((now - it->second) >= qosRecordTimeout) {
 				it = _paths[i].qosStatsOut.erase(it);
 				++numDroppedQosOutRecords;
-			} else {
+			}
+			else {
 				++it;
 			}
 		}
@@ -1250,7 +1282,8 @@ void Bond::estimatePathQuality(int64_t now)
 			if ((now - it->second) >= qosRecordTimeout) {
 				it = _paths[i].qosStatsIn.erase(it);
 				++numDroppedQosInRecords;
-			} else {
+			}
+			else {
 				++it;
 			}
 		}
@@ -1327,10 +1360,10 @@ void Bond::estimatePathQuality(int64_t now)
 			continue;
 		}
 		// Compute/Smooth average of real-world observations
-		if (_paths[i].latencySamples.count() == ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE) {
+		if (_paths[i].latencySamples.count() >= ZT_QOS_SHORTTERM_SAMPLE_WIN_MIN_REQ_SIZE) {
 			_paths[i].latency = _paths[i].latencySamples.mean();
 		}
-		if (_paths[i].latencySamples.count() == ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE) {
+		if (_paths[i].latencySamples.count() >= ZT_QOS_SHORTTERM_SAMPLE_WIN_MIN_REQ_SIZE) {
 			_paths[i].latencyVariance = _paths[i].latencySamples.stddev();
 		}
 
@@ -1344,6 +1377,7 @@ void Bond::estimatePathQuality(int64_t now)
 		//_paths[i].packetErrorRatio = 1.0 - (_paths[i].packetValiditySamples.count() ? _paths[i].packetValiditySamples.mean() : 1.0);
 		// _valid is written elsewhere
 		_paths[i].p->_relativeQuality = _paths[i].relativeQuality;
+		_paths[i].p->_localPort = _paths[i].localPort;
 	}
 
 	// Flag links for avoidance
@@ -1370,7 +1404,8 @@ void Bond::estimatePathQuality(int64_t now)
 				shouldAvoid = true;
 			}
 			_paths[i].shouldAvoid = shouldAvoid;
-		} else {
+		}
+		else {
 			if (! shouldAvoid) {
 				log("no longer avoiding link %s", pathToStr(_paths[i].p).c_str());
 				_paths[i].shouldAvoid = false;
@@ -1468,7 +1503,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 {
 	int prevActiveBackupPathIdx = _abPathIdx;
 	int nonPreferredPathIdx = ZT_MAX_PEER_NETWORK_PATHS;
-	bool bFoundPrimaryLink = false;
+	bool foundPathOnPrimaryLink = false;
+	bool foundPreferredPath = false;
 
 	if (_abPathIdx != ZT_MAX_PEER_NETWORK_PATHS && ! _paths[_abPathIdx].p) {
 		_abPathIdx = ZT_MAX_PEER_NETWORK_PATHS;
@@ -1482,7 +1518,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 		_lastBondStatusLog = now;
 		if (_abPathIdx == ZT_MAX_PEER_NETWORK_PATHS) {
 			log("no active link");
-		} else if (_paths[_abPathIdx].p) {
+		}
+		else if (_paths[_abPathIdx].p) {
 			log("active link is %s, failover queue size is %zu", pathToStr(_paths[_abPathIdx].p).c_str(), _abFailoverQueue.size());
 		}
 		if (_abFailoverQueue.empty()) {
@@ -1530,15 +1567,16 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 							if (! _paths[i].preferred()) {
 								// Found path on primary link, take note in case we don't find a preferred path
 								nonPreferredPathIdx = i;
-								bFoundPrimaryLink = true;
+								foundPathOnPrimaryLink = true;
 							}
 							if (_paths[i].preferred()) {
 								_abPathIdx = i;
-								bFoundPrimaryLink = true;
+								foundPathOnPrimaryLink = true;
 								if (_paths[_abPathIdx].p) {
 									SharedPtr<Link> abLink = RR->bc->getLinkBySocket(_policyAlias, _paths[_abPathIdx].p->localSocket());
 									if (abLink) {
-										log("found preferred primary link %s", pathToStr(_paths[_abPathIdx].p).c_str());
+										log("found preferred primary link (_abPathIdx=%d), %s", _abPathIdx, pathToStr(_paths[_abPathIdx].p).c_str());
+										foundPreferredPath = true;
 									}
 									break;	 // Found preferred path on primary link
 								}
@@ -1546,8 +1584,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 						}
 					}
 				}
-				if (bFoundPrimaryLink && (nonPreferredPathIdx != ZT_MAX_PEER_NETWORK_PATHS)) {
-					log("found non-preferred primary link");
+				if (!foundPreferredPath && foundPathOnPrimaryLink && (nonPreferredPathIdx != ZT_MAX_PEER_NETWORK_PATHS)) {
+					log("found non-preferred primary link (_abPathIdx=%d)", _abPathIdx);
 					_abPathIdx = nonPreferredPathIdx;
 				}
 			}
@@ -1585,12 +1623,13 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 		}
 		if (_paths[(*it)].p && ! _paths[(*it)].eligible) {
 			SharedPtr<Link> link = RR->bc->getLinkBySocket(_policyAlias, _paths[(*it)].p->localSocket());
-			it = _abFailoverQueue.erase(it);
 			if (link) {
-				log("link %s is ineligible, removing from failover queue (%zu links remain in queue)", pathToStr(_paths[_abPathIdx].p).c_str(), _abFailoverQueue.size());
+				log("link %s is ineligible, removing from failover queue (%zu links remain in queue)", pathToStr(_paths[(*it)].p).c_str(), _abFailoverQueue.size());
 			}
+			it = _abFailoverQueue.erase(it);
 			continue;
-		} else {
+		}
+		else {
 			++it;
 		}
 	}
@@ -1654,9 +1693,9 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 						}
 					}
 					if (! bFoundPathInQueue) {
-						_abFailoverQueue.push_front(i);
+						_abFailoverQueue.push_back(i);
 						log("add link %s to failover queue (%zu links in queue)", pathToStr(_paths[i].p).c_str(), _abFailoverQueue.size());
-						addPathToBond(0, i);
+						addPathToBond(i, 0);
 					}
 				}
 			}
@@ -1704,13 +1743,14 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 					}
 				}
 				if (! bFoundPathInQueue) {
-					_abFailoverQueue.push_front(i);
+					_abFailoverQueue.push_back(i);
 					log("add link %s to failover queue (%zu links in queue)", pathToStr(_paths[i].p).c_str(), _abFailoverQueue.size());
-					addPathToBond(0, i);
+					addPathToBond(i, 0);
 				}
 			}
 		}
 	}
+	/*
 	// Sort queue based on performance
 	if (! _abFailoverQueue.empty()) {
 		for (int i = 0; i < _abFailoverQueue.size(); i++) {
@@ -1722,7 +1762,7 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 			}
 			_abFailoverQueue[hole_position] = value_to_insert;
 		}
-	}
+	}*/
 
 	/**
 	 * Short-circuit if we have no queued paths
@@ -1739,7 +1779,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 		if (! _abFailoverQueue.empty()) {
 			dequeueNextActiveBackupPath(now);
 			log("active link switched to %s", pathToStr(_paths[_abPathIdx].p).c_str());
-		} else {
+		}
+		else {
 			log("failover queue is empty, no links to choose from");
 		}
 	}
@@ -1785,7 +1826,8 @@ void Bond::processActiveBackupTasks(void* tPtr, int64_t now)
 			dequeueNextActiveBackupPath(now);
 			_lastPathNegotiationCheck = now;
 			log("switch negotiated link %s (select mode: optimize)", pathToStr(_paths[_abPathIdx].p).c_str());
-		} else {
+		}
+		else {
 			// Try to find a better path and automatically switch to it -- not too often, though.
 			if ((now - _lastActiveBackupPathChange) > ZT_BOND_OPTIMIZE_INTERVAL) {
 				if (! _abFailoverQueue.empty()) {
@@ -1870,7 +1912,7 @@ void Bond::setBondParameters(int policy, SharedPtr<Bond> templateBond, bool useT
 	 * Policy defaults
 	 */
 	_abPathIdx = ZT_MAX_PEER_NETWORK_PATHS;
-	_abLinkSelectMethod = ZT_BOND_RESELECTION_POLICY_OPTIMIZE;
+	_abLinkSelectMethod = ZT_BOND_RESELECTION_POLICY_ALWAYS;
 	_rrPacketsSentOnCurrLink = 0;
 	_rrIdx = 0;
 	_packetsPerLink = 64;
@@ -1901,7 +1943,7 @@ void Bond::setBondParameters(int policy, SharedPtr<Bond> templateBond, bool useT
 	}
 
 	if (! _isLeaf) {
-		_policy = ZT_BOND_POLICY_ACTIVE_BACKUP;
+		_policy = ZT_BOND_POLICY_NONE;
 	}
 
 	// Timer geometry
@@ -1989,7 +2031,8 @@ void Bond::dumpInfo(int64_t now, bool force)
 	_lastSummaryDump = now;
 	float overhead = (_overheadBytes / (timeSinceLastDump / 1000.0f) / 1000.0f);
 	_overheadBytes = 0;
-	log("bond: bp=%d, fi=%" PRIu64 ", mi=%d, ud=%d, dd=%d, flows=%zu, leaf=%d, overhead=%f KB/s, links=(%d/%d)",
+	log("bond: ready=%d, bp=%d, fi=%" PRIu64 ", mi=%d, ud=%d, dd=%d, flows=%zu, leaf=%d, overhead=%f KB/s, links=(%d/%d)",
+		isReady(),
 		_policy,
 		_failoverInterval,
 		_monitorInterval,

node/Bond.hpp

@@ -315,7 +315,6 @@ class Peer;
 
 class Bond {
   public:
-
 	/**
 	 * Stop bond's internal functions (can be resumed)
 	 */
@@ -909,7 +908,8 @@ class Bond {
 		_lastAckRateCheck = now;
 		if (_ackCutoffCount > numToDrain) {
 			_ackCutoffCount -= numToDrain;
-		} else {
+		}
+		else {
 			_ackCutoffCount = 0;
 		}
 		return (_ackCutoffCount < ZT_ACK_CUTOFF_LIMIT);
@@ -928,7 +928,8 @@ class Bond {
 		uint64_t diff = now - _lastQoSRateCheck;
 		if ((diff) <= (_qosSendInterval / ZT_MAX_PEER_NETWORK_PATHS)) {
 			++_qosCutoffCount;
-		} else {
+		}
+		else {
 			_qosCutoffCount = 0;
 		}
 		_lastQoSRateCheck = now;
@@ -948,7 +949,8 @@ class Bond {
 		int diff = now - _lastPathNegotiationReceived;
 		if ((diff) <= (ZT_PATH_NEGOTIATION_CUTOFF_TIME / ZT_MAX_PEER_NETWORK_PATHS)) {
 			++_pathNegotiationCutoffCount;
-		} else {
+		}
+		else {
 			_pathNegotiationCutoffCount = 0;
 		}
 		_lastPathNegotiationReceived = now;
@@ -1142,6 +1144,7 @@ class Bond {
 		__attribute__((format(printf, 2, 3)))
 #endif
 	{
+		//if (_peerId != 0x0 && _peerId != 0x0) { return; }
 #ifdef ZT_TRACE
 		time_t rawtime;
 		struct tm* timeinfo;
@@ -1173,6 +1176,7 @@ class Bond {
 		__attribute__((format(printf, 2, 3)))
 #endif
 	{
+		//if (_peerId != 0x0 && _peerId != 0x0) { return; }
 #ifdef ZT_DEBUG
 		time_t rawtime;
 		struct tm* timeinfo;
@@ -1230,6 +1234,7 @@ class Bond {
 			, packetsReceivedSinceLastQoS(0)
 			, packetsIn(0)
 			, packetsOut(0)
+			, localPort(0)
 		{
 		}
 
@@ -1245,17 +1250,20 @@ class Bond {
 				unsigned int suggestedRefractoryPeriod = refractoryPeriod ? punishment + (refractoryPeriod * 2) : punishment;
 				refractoryPeriod = std::min(suggestedRefractoryPeriod, (unsigned int)ZT_BOND_MAX_REFRACTORY_PERIOD);
 				lastRefractoryUpdate = 0;
-			} else {
+			}
+			else {
 				uint32_t drainRefractory = 0;
 				if (lastRefractoryUpdate) {
 					drainRefractory = (now - lastRefractoryUpdate);
-				} else {
+				}
+				else {
 					drainRefractory = (now - lastAliveToggle);
 				}
 				lastRefractoryUpdate = now;
 				if (refractoryPeriod > drainRefractory) {
 					refractoryPeriod -= drainRefractory;
-				} else {
+				}
+				else {
 					refractoryPeriod = 0;
 					lastRefractoryUpdate = 0;
 				}
@@ -1292,7 +1300,6 @@ class Bond {
 		 */
 		inline bool needsToSendQoS(int64_t now, uint64_t qosSendInterval)
 		{
-			// fprintf(stderr, "QOS table (%d / %d)\n", packetsReceivedSinceLastQoS, ZT_QOS_TABLE_SIZE);
 			return ((packetsReceivedSinceLastQoS >= ZT_QOS_TABLE_SIZE) || ((now - lastQoSMeasurement) > qosSendInterval)) && packetsReceivedSinceLastQoS;
 		}
 
@@ -1364,6 +1371,8 @@ class Bond {
 		int packetsIn;
 		int packetsOut;
 
+		uint16_t localPort;
+
 		// AtomicCounter __refCount;
 
 		SharedPtr<Path> p;

node/Buffer.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/C25519.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Capability.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Capability.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/CertificateOfMembership.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/CertificateOfMembership.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/CertificateOfOwnership.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/CertificateOfOwnership.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Constants.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -202,6 +202,72 @@
 #define ZT_PACKED_STRUCT(D) D __attribute__((packed))
 #endif
 
+#if defined(_WIN32)
+#define ZT_PLATFORM_NAME "windows" // Windows
+#elif defined(_WIN64)
+#define ZT_PLATFORM_NAME "windows" // Windows
+#elif defined(__CYGWIN__)
+#define ZT_PLATFORM_NAME "windows" // Windows (Cygwin POSIX under Microsoft Window)
+#elif defined(__ANDROID__)
+#define ZT_PLATFORM_NAME "android" // Android (implies Linux, so it must come first)
+#elif defined(__linux__)
+#define ZT_PLATFORM_NAME "linux" // Debian, Ubuntu, Gentoo, Fedora, openSUSE, RedHat, Centos and other
+#elif defined(__unix__) || !defined(__APPLE__) && defined(__MACH__)
+#include <sys/param.h>
+#if defined(BSD)
+#define ZT_PLATFORM_NAME "bsd" // FreeBSD, NetBSD, OpenBSD, DragonFly BSD
+#endif
+#elif defined(__hpux)
+#define ZT_PLATFORM_NAME "hp-ux" // HP-UX
+#elif defined(_AIX)
+#define ZT_PLATFORM_NAME "aix" // IBM AIX
+#elif defined(__APPLE__) && defined(__MACH__) // Apple OSX and iOS (Darwin)
+#include <TargetConditionals.h>
+#if defined(TARGET_IPHONE_SIMULATOR) && TARGET_IPHONE_SIMULATOR == 1
+#define ZT_PLATFORM_NAME "ios_sim" // Apple iOS
+#elif defined(TARGET_OS_IPAD) && TARGET_OS_IPAD == 1
+#define ZT_PLATFORM_NAME "ios_ipad"
+#elif defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1
+#define ZT_PLATFORM_NAME "ios_iphone" // Apple iOS
+#elif defined(TARGET_OS_MAC) && TARGET_OS_MAC == 1
+#define ZT_PLATFORM_NAME "macos" // Apple OSX
+#endif
+#elif defined(__sun) && defined(__SVR4)
+#define ZT_PLATFORM_NAME "solaris" // Oracle Solaris, Open Indiana
+#else
+#define ZT_PLATFORM_NAME "unknown"
+#endif
+#ifndef ZT_PLATFORM_NAME
+#define ZT_PLATFORM_NAME "unknown"
+#endif
+
+#if defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64) || defined(_M_AMD64)
+#define ZT_ARCH_NAME "x86_64"
+#elif defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_X86_) || defined(_M_IX86) || defined(__X86__) || defined(__I86__) || defined(_M_I86)
+#define ZT_ARCH_NAME "x86"
+#elif defined(__aarch64__) || defined(__AARCH64EL__) || defined(_M_ARM64)
+#define ZT_ARCH_NAME "arm64"
+#elif defined(__arm__) || defined(__TARGET_ARCH_ARM) || defined(_ARM) || defined(_M_ARM) || defined(_M_ARMT) || defined(__arm) || defined(__thumb__)
+#define ZT_ARCH_NAME "arm"
+#elif defined(__loongarch__) || defined(_LOONGARCH_ARCH)
+#define ZT_ARCH_NAME "loongarch"
+#elif defined(__mips__) || defined(__MIPS__)
+#define ZT_ARCH_NAME "mips"
+#elif defined(__riscv) || defined(__riscv_xlen)
+#define ZT_ARCH_NAME "riscv"
+#elif defined(__powerpc__) || defined(__powerpc64__) || defined(__ppc__) || defined(__ppc64__) || defined (_M_PPC)
+#define ZT_ARCH_NAME "powerpc"
+#elif defined(__s390__) || defined(__s390x__) || defined(__zarch__)
+#define ZT_ARCH_NAME "s390"
+#else
+#define ZT_ARCH_NAME "unknown"
+#endif
+#ifndef ZT_ARCH_NAME
+#define ZT_ARCH_NAME "unknown"
+#endif
+
+#define ZT_TARGET_NAME (ZT_PLATFORM_NAME "/" ZT_ARCH_NAME)
+
 /**
  * Length of a ZeroTier address in bytes
  */
@@ -399,6 +465,11 @@
  */
 #define ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE 64
 
+/**
+ * Number of samples required before statistics summaries are computed
+ */
+#define ZT_QOS_SHORTTERM_SAMPLE_WIN_MIN_REQ_SIZE 4
+
 /**
  * Max allowable time spent in any queue (in ms)
  */

node/Credential.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/DNS.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Dictionary.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Hashtable.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Identity.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Identity.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/IncomingPacket.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -38,6 +38,7 @@
 #include "Path.hpp"
 #include "Bond.hpp"
 #include "Metrics.hpp"
+#include "PacketMultiplexer.hpp"
 
 namespace ZeroTier {
 
@@ -334,7 +335,6 @@ bool IncomingPacket::_doACK(const RuntimeEnvironment* RR, void* tPtr, const Shar
 bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment* RR, void* tPtr, const SharedPtr<Peer>& peer)
 {
 	Metrics::pkt_qos_in++;
-	SharedPtr<Bond> bond = peer->bond();
 	if (! peer->rateGateQoS(RR->node->now(), _path)) {
 		return true;
 	}
@@ -793,66 +793,65 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 {
 	Metrics::pkt_frame_in++;
 	int32_t _flowId = ZT_QOS_NO_FLOW;
-	if (peer->flowHashingSupported()) {
-		if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
-			const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
-			const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
-			const uint8_t *const frameData = reinterpret_cast<const uint8_t *>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 
-			if (etherType == ZT_ETHERTYPE_IPV4 && (frameLen >= 20)) {
-				uint16_t srcPort = 0;
-				uint16_t dstPort = 0;
-				uint8_t proto = (reinterpret_cast<const uint8_t *>(frameData)[9]);
-				const unsigned int headerLen = 4 * (reinterpret_cast<const uint8_t *>(frameData)[0] & 0xf);
-				switch(proto) {
-					case 0x01: // ICMP
-						//flowId = 0x01;
-						break;
-					// All these start with 16-bit source and destination port in that order
-					case 0x06: // TCP
-					case 0x11: // UDP
-					case 0x84: // SCTP
-					case 0x88: // UDPLite
-						if (frameLen > (headerLen + 4)) {
-							unsigned int pos = headerLen + 0;
-							srcPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
-							srcPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
-							pos++;
-							dstPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
-							dstPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
-							_flowId = dstPort ^ srcPort ^ proto;
-						}
-						break;
-				}
+	if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
+		const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
+		const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
+		const uint8_t *const frameData = reinterpret_cast<const uint8_t *>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
+
+		if (etherType == ZT_ETHERTYPE_IPV4 && (frameLen >= 20)) {
+			uint16_t srcPort = 0;
+			uint16_t dstPort = 0;
+			uint8_t proto = (reinterpret_cast<const uint8_t *>(frameData)[9]);
+			const unsigned int headerLen = 4 * (reinterpret_cast<const uint8_t *>(frameData)[0] & 0xf);
+			switch(proto) {
+				case 0x01: // ICMP
+					//flowId = 0x01;
+					break;
+				// All these start with 16-bit source and destination port in that order
+				case 0x06: // TCP
+				case 0x11: // UDP
+				case 0x84: // SCTP
+				case 0x88: // UDPLite
+					if (frameLen > (headerLen + 4)) {
+						unsigned int pos = headerLen + 0;
+						srcPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
+						srcPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
+						pos++;
+						dstPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
+						dstPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
+						_flowId = dstPort ^ srcPort ^ proto;
+					}
+					break;
 			}
+		}
 
-			if (etherType == ZT_ETHERTYPE_IPV6 && (frameLen >= 40)) {
-				uint16_t srcPort = 0;
-				uint16_t dstPort = 0;
-				unsigned int pos;
-				unsigned int proto;
-				_ipv6GetPayload((const uint8_t *)frameData, frameLen, pos, proto);
-				switch(proto) {
-					case 0x3A: // ICMPv6
-						//flowId = 0x3A;
-						break;
-					// All these start with 16-bit source and destination port in that order
-					case 0x06: // TCP
-					case 0x11: // UDP
-					case 0x84: // SCTP
-					case 0x88: // UDPLite
-						if (frameLen > (pos + 4)) {
-							srcPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
-							srcPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
-							pos++;
-							dstPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
-							dstPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
-							_flowId = dstPort ^ srcPort ^ proto;
-						}
-						break;
-					default:
-						break;
-				}
+		if (etherType == ZT_ETHERTYPE_IPV6 && (frameLen >= 40)) {
+			uint16_t srcPort = 0;
+			uint16_t dstPort = 0;
+			unsigned int pos;
+			unsigned int proto;
+			_ipv6GetPayload((const uint8_t *)frameData, frameLen, pos, proto);
+			switch(proto) {
+				case 0x3A: // ICMPv6
+					//flowId = 0x3A;
+					break;
+				// All these start with 16-bit source and destination port in that order
+				case 0x06: // TCP
+				case 0x11: // UDP
+				case 0x84: // SCTP
+				case 0x88: // UDPLite
+					if (frameLen > (pos + 4)) {
+						srcPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
+						srcPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
+						pos++;
+						dstPort = (reinterpret_cast<const uint8_t *>(frameData)[pos++]) << 8;
+						dstPort |= (reinterpret_cast<const uint8_t *>(frameData)[pos]);
+						_flowId = dstPort ^ srcPort ^ proto;
+					}
+					break;
+				default:
+					break;
 			}
 		}
 	}
@@ -869,7 +868,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 				const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 				const uint8_t *const frameData = reinterpret_cast<const uint8_t *>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 				if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0) {
-					RR->node->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen);
+					RR->pm->putFrame(tPtr,nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen, _flowId);
 				}
 			}
 		} else {
@@ -942,7 +941,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 					}
 					// fall through -- 2 means accept regardless of bridging checks or other restrictions
 				case 2:
-					RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen);
+					RR->pm->putFrame(tPtr,nwid,network->userPtr(),from,to,etherType,0,(const void *)frameData,frameLen, flowId);
 					break;
 			}
 		}

node/IncomingPacket.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/InetAddress.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -132,7 +132,20 @@ InetAddress::IpScope InetAddress::ipScope() const
 					return IP_SCOPE_PRIVATE;        // fc00::/7
 				}
 			}
+
+			// :::ffff:127.0.0.1
+			// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0x7f, 0, 0, 1
 			unsigned int k = 0;
+			while ((!ip[k])&&(k < 9)) {
+				++k;
+			}
+			if (k == 9) {
+				if (ip[10] == 0xff && ip[11] == 0xff && ip[12] == 0x7f) {
+					return IP_SCOPE_LOOPBACK;
+				}
+			}
+
+			k = 0;
 			while ((!ip[k])&&(k < 15)) {
 				++k;
 			}

node/InetAddress.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/MAC.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Membership.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Membership.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Metrics.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Metrics.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/MulticastGroup.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Multicaster.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Multicaster.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Mutex.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Network.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -107,25 +107,50 @@ static _doZtFilterResult _doZtFilter(
 	// The default match state for each set of entries starts as 'true' since an
 	// ACTION with no MATCH entries preceding it is always taken.
 	uint8_t thisSetMatches = 1;
+	uint8_t skipDrop = 0;
 
 	rrl.clear();
 
+	// uncomment for easier debugging fprintf
+	// if (!ztDest) { return DOZTFILTER_ACCEPT; }
+#ifdef ZT_TRACE
+	//char buf[40], buf2[40];
+	//fprintf(stderr, "\nsrc %s dest %s inbound: %d ethertype %u", ztSource.toString(buf), ztDest.toString(buf2), inbound, etherType);
+#endif
+
 	for(unsigned int rn=0;rn<ruleCount;++rn) {
 		const ZT_VirtualNetworkRuleType rt = (ZT_VirtualNetworkRuleType)(rules[rn].t & 0x3f);
+#ifdef ZT_TRACE
+		//fprintf(stderr, "\n%02u %02d", rn, rt);
+#endif
 
 		// First check if this is an ACTION
 		if ((unsigned int)rt <= (unsigned int)ZT_NETWORK_RULE_ACTION__MAX_ID) {
 			if (thisSetMatches) {
 				switch(rt) {
 					case ZT_NETWORK_RULE_ACTION_PRIORITY:
-						qosBucket = (rules[rn].v.qosBucket >= 0 || rules[rn].v.qosBucket <= 8) ? rules[rn].v.qosBucket : 4; // 4 = default bucket (no priority)
+						qosBucket = (rules[rn].v.qosBucket <= 8) ? rules[rn].v.qosBucket : 4; // 4 = default bucket (no priority)
 						return DOZTFILTER_ACCEPT;
 
-					case ZT_NETWORK_RULE_ACTION_DROP:
+					case ZT_NETWORK_RULE_ACTION_DROP: {
+						if (!!skipDrop) {
+#ifdef ZT_TRACE
+							//fprintf(stderr, "\tskip Drop");
+#endif
+							skipDrop = 0; continue;
+						}
+#ifdef ZT_TRACE
+						//fprintf(stderr, "\tDrop\n");
+#endif
 						return DOZTFILTER_DROP;
+					}
 
-					case ZT_NETWORK_RULE_ACTION_ACCEPT:
+					case ZT_NETWORK_RULE_ACTION_ACCEPT: {
+#ifdef ZT_TRACE
+						//fprintf(stderr, "\tAccept\n");
+#endif
 						return (superAccept ? DOZTFILTER_SUPER_ACCEPT : DOZTFILTER_ACCEPT); // match, accept packet
+					}
 
 					// These are initially handled together since preliminary logic is common
 					case ZT_NETWORK_RULE_ACTION_TEE:
@@ -192,6 +217,9 @@ static _doZtFilterResult _doZtFilter(
 		// If this was not an ACTION evaluate next MATCH and update thisSetMatches with (AND [result])
 		uint8_t thisRuleMatches = 0;
 		uint64_t ownershipVerificationMask = 1; // this magic value means it hasn't been computed yet -- this is done lazily the first time it's needed
+		uint8_t hardYes = (rules[rn].t >> 7) ^ 1; // XOR with the NOT bit of the rule
+		uint8_t hardNo = (rules[rn].t >> 7) ^ 0;
+
 		switch(rt) {
 			case ZT_NETWORK_RULE_MATCH_SOURCE_ZEROTIER_ADDRESS:
 				thisRuleMatches = (uint8_t)(rules[rn].v.zt == ztSource.toInt());
@@ -220,28 +248,28 @@ static _doZtFilterResult _doZtFilter(
 				if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 12),4,0)));
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV4_DEST:
 				if ((etherType == ZT_ETHERTYPE_IPV4)&&(frameLen >= 20)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void *)&(rules[rn].v.ipv4.ip),4,rules[rn].v.ipv4.mask).containsAddress(InetAddress((const void *)(frameData + 16),4,0)));
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV6_SOURCE:
 				if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 8),16,0)));
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IPV6_DEST:
 				if ((etherType == ZT_ETHERTYPE_IPV6)&&(frameLen >= 40)) {
 					thisRuleMatches = (uint8_t)(InetAddress((const void *)rules[rn].v.ipv6.ip,16,rules[rn].v.ipv6.mask).containsAddress(InetAddress((const void *)(frameData + 24),16,0)));
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IP_TOS:
@@ -252,7 +280,7 @@ static _doZtFilterResult _doZtFilter(
 					const uint8_t tosMasked = (((frameData[0] << 4) & 0xf0) | ((frameData[1] >> 4) & 0x0f)) & rules[rn].v.ipTos.mask;
 					thisRuleMatches = (uint8_t)((tosMasked >= rules[rn].v.ipTos.value[0])&&(tosMasked <= rules[rn].v.ipTos.value[1]));
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IP_PROTOCOL:
@@ -263,10 +291,10 @@ static _doZtFilterResult _doZtFilter(
 					if (_ipv6GetPayload(frameData,frameLen,pos,proto)) {
 						thisRuleMatches = (uint8_t)(rules[rn].v.ipProtocol == (uint8_t)proto);
 					} else {
-						thisRuleMatches = 0;
+						thisRuleMatches = hardNo;
 					}
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_ETHERTYPE:
@@ -281,16 +309,16 @@ static _doZtFilterResult _doZtFilter(
 								if ((rules[rn].v.icmp.flags & 0x01) != 0) {
 									thisRuleMatches = (uint8_t)(frameData[ihl+1] == rules[rn].v.icmp.code);
 								} else {
-									thisRuleMatches = 1;
+									thisRuleMatches = hardYes;
 								}
 							} else {
-								thisRuleMatches = 0;
+								thisRuleMatches = hardNo;
 							}
 						} else {
-							thisRuleMatches = 0;
+							thisRuleMatches = hardNo;
 						}
 					} else {
-						thisRuleMatches = 0;
+						thisRuleMatches = hardNo;
 					}
 				} else if (etherType == ZT_ETHERTYPE_IPV6) {
 					unsigned int pos = 0,proto = 0;
@@ -300,19 +328,19 @@ static _doZtFilterResult _doZtFilter(
 								if ((rules[rn].v.icmp.flags & 0x01) != 0) {
 									thisRuleMatches = (uint8_t)(frameData[pos+1] == rules[rn].v.icmp.code);
 								} else {
-									thisRuleMatches = 1;
+									thisRuleMatches = hardYes;
 								}
 							} else {
-								thisRuleMatches = 0;
+								thisRuleMatches = hardNo;
 							}
 						} else {
-							thisRuleMatches = 0;
+							thisRuleMatches = hardNo;
 						}
 					} else {
-						thisRuleMatches = 0;
+						thisRuleMatches = hardNo;
 					}
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_IP_SOURCE_PORT_RANGE:
@@ -356,10 +384,10 @@ static _doZtFilterResult _doZtFilter(
 						}
 						thisRuleMatches = (p > 0) ? (uint8_t)((p >= (int)rules[rn].v.port[0])&&(p <= (int)rules[rn].v.port[1])) : (uint8_t)0;
 					} else {
-						thisRuleMatches = 0;
+						thisRuleMatches = hardNo;
 					}
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 				break;
 			case ZT_NETWORK_RULE_MATCH_CHARACTERISTICS: {
@@ -444,6 +472,14 @@ static _doZtFilterResult _doZtFilter(
 				const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate());
 				if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) {
 					const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0);
+#ifdef ZT_TRACE
+					/*fprintf(stderr, "\tlocal tag [%u: %u] remote tag [%u: %u] match [%u]",
+							!!localTag ? localTag->id() : 0,
+							!!localTag ? localTag->value() : 0,
+							!!remoteTag ? remoteTag->id() : 0,
+							!!remoteTag ? remoteTag->value() : 0,
+							thisRuleMatches);*/
+#endif
 					if (remoteTag) {
 						const uint32_t ltv = localTag->value();
 						const uint32_t rtv = remoteTag->value();
@@ -459,28 +495,46 @@ static _doZtFilterResult _doZtFilter(
 						} else if (rt == ZT_NETWORK_RULE_MATCH_TAGS_EQUAL) {
 							thisRuleMatches = (uint8_t)((ltv == rules[rn].v.tag.value)&&(rtv == rules[rn].v.tag.value));
 						} else { // sanity check, can't really happen
-							thisRuleMatches = 0;
+							thisRuleMatches = hardNo;
 						}
 					} else {
 						if ((inbound)&&(!superAccept)) {
-							thisRuleMatches = 0;
+							thisRuleMatches = hardNo;
+#ifdef ZT_TRACE
+							//fprintf(stderr, "\tinbound ");
+#endif
 						} else {
 							// Outbound side is not strict since if we have to match both tags and
 							// we are sending a first packet to a recipient, we probably do not know
 							// about their tags yet. They will filter on inbound and we will filter
 							// once we get their tag. If we are a tee/redirect target we are also
 							// not strict since we likely do not have these tags.
-							thisRuleMatches = 1;
+							skipDrop = 1;
+							thisRuleMatches = hardYes;
+#ifdef ZT_TRACE
+							//fprintf(stderr, "\toutbound ");
+#endif
 						}
 					}
 				} else {
-					thisRuleMatches = 0;
+					thisRuleMatches = hardNo;
 				}
 			}	break;
 			case ZT_NETWORK_RULE_MATCH_TAG_SENDER:
 			case ZT_NETWORK_RULE_MATCH_TAG_RECEIVER: {
+					const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate());
+#ifdef ZT_TRACE
+					/*const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0);
+					fprintf(stderr, "\tlocal tag [%u: %u] remote tag [%u: %u] match [%u]",
+							!!localTag ? localTag->id() : 0,
+							!!localTag ? localTag->value() : 0,
+							!!remoteTag ? remoteTag->id() : 0,
+							!!remoteTag ? remoteTag->value() : 0,
+							thisRuleMatches);*/
+#endif
 				if (superAccept) {
-					thisRuleMatches = 1;
+					skipDrop = 1;
+					thisRuleMatches = hardYes;
 				} else if ( ((rt == ZT_NETWORK_RULE_MATCH_TAG_SENDER)&&(inbound)) || ((rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER)&&(!inbound)) ) {
 					const Tag *const remoteTag = ((membership) ? membership->getTag(nconf,rules[rn].v.tag.id) : (const Tag *)0);
 					if (remoteTag) {
@@ -489,17 +543,17 @@ static _doZtFilterResult _doZtFilter(
 						if (rt == ZT_NETWORK_RULE_MATCH_TAG_RECEIVER) {
 							// If we are checking the receiver and this is an outbound packet, we
 							// can't be strict since we may not yet know the receiver's tag.
-							thisRuleMatches = 1;
+							skipDrop = 1;
+							thisRuleMatches = hardYes;
 						} else {
-							thisRuleMatches = 0;
+							thisRuleMatches = hardNo;
 						}
 					}
 				} else { // sender and outbound or receiver and inbound
-					const Tag *const localTag = std::lower_bound(&(nconf.tags[0]),&(nconf.tags[nconf.tagCount]),rules[rn].v.tag.id,Tag::IdComparePredicate());
 					if ((localTag != &(nconf.tags[nconf.tagCount]))&&(localTag->id() == rules[rn].v.tag.id)) {
 						thisRuleMatches = (uint8_t)(localTag->value() == rules[rn].v.tag.value);
 					} else {
-						thisRuleMatches = 0;
+						thisRuleMatches = hardNo;
 					}
 				}
 			}	break;
@@ -1259,6 +1313,7 @@ void Network::requestConfiguration(void *tPtr)
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION);
+	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_OS_ARCH,ZT_TARGET_NAME);
 
 	RR->t->networkConfigRequestSent(tPtr,*this,ctrl);
 

node/Network.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/NetworkConfig.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/NetworkConfig.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -105,6 +105,8 @@ namespace ZeroTier {
 
 // Network config version
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v"
+// Network config version
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_OS_ARCH "o"
 // Protocol version (see Packet.hpp)
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv"
 // Software vendor

node/NetworkController.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Node.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -35,6 +35,7 @@
 #include "Network.hpp"
 #include "Trace.hpp"
 #include "Metrics.hpp"
+#include "PacketMultiplexer.hpp"
 
 // FIXME: remove this suppression and actually fix warnings
 #ifdef __GNUC__
@@ -119,9 +120,10 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		const unsigned long mcs = sizeof(Multicaster) + (((sizeof(Multicaster) & 0xf) != 0) ? (16 - (sizeof(Multicaster) & 0xf)) : 0);
 		const unsigned long topologys = sizeof(Topology) + (((sizeof(Topology) & 0xf) != 0) ? (16 - (sizeof(Topology) & 0xf)) : 0);
 		const unsigned long sas = sizeof(SelfAwareness) + (((sizeof(SelfAwareness) & 0xf) != 0) ? (16 - (sizeof(SelfAwareness) & 0xf)) : 0);
-		const unsigned long bc = sizeof(Bond) + (((sizeof(Bond) & 0xf) != 0) ? (16 - (sizeof(Bond) & 0xf)) : 0);
+		const unsigned long bcs = sizeof(Bond) + (((sizeof(Bond) & 0xf) != 0) ? (16 - (sizeof(Bond) & 0xf)) : 0);
+		const unsigned long pms = sizeof(PacketMultiplexer) + (((sizeof(PacketMultiplexer) & 0xf) != 0) ? (16 - (sizeof(PacketMultiplexer) & 0xf)) : 0);
 
-		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + mcs + topologys + sas + bc));
+		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + mcs + topologys + sas + bcs + pms));
 		if (!m) {
 			throw std::bad_alloc();
 		}
@@ -141,6 +143,8 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		RR->sa = new (m) SelfAwareness(RR);
 		m += sas;
 		RR->bc = new (m) Bond(RR);
+		m += bcs;
+		RR->pm = new (m) PacketMultiplexer(RR);
 	} catch ( ... ) {
 		if (RR->sa) {
 			RR->sa->~SelfAwareness();
@@ -160,6 +164,9 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		if (RR->bc) {
 			RR->bc->~Bond();
 		}
+		if (RR->pm) {
+			RR->pm->~PacketMultiplexer();
+		}
 		::free(m);
 		throw;
 	}
@@ -191,6 +198,9 @@ Node::~Node()
 	if (RR->bc) {
 		RR->bc->~Bond();
 	}
+	if (RR->pm) {
+		RR->pm->~PacketMultiplexer();
+	}
 	::free(RR->rtmem);
 }
 
@@ -230,6 +240,11 @@ ZT_ResultCode Node::processVirtualNetworkFrame(
 	}
 }
 
+void Node::initMultithreading(unsigned int concurrency, bool cpuPinningEnabled)
+{
+	RR->pm->setUpPostDecodeReceiveThreads(concurrency, cpuPinningEnabled);
+}
+
 // Closure used to ping upstream and active/online peers
 class _PingPeersThatNeedPing
 {
@@ -578,6 +593,7 @@ ZT_PeerList *Node::peers() const
 			if((*path)->valid()) {
 				memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage));
 				p->paths[p->pathCount].localSocket = (*path)->localSocket();
+				p->paths[p->pathCount].localPort = (*path)->localPort();
 				p->paths[p->pathCount].lastSend = (*path)->lastOut();
 				p->paths[p->pathCount].lastReceive = (*path)->lastIn();
 				p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address());

node/Node.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -283,7 +283,10 @@ public:
 		return _lowBandwidthMode;
 	}
 
-private:
+	void initMultithreading(unsigned int concurrency, bool cpuPinningEnabled);
+
+
+public:
 	RuntimeEnvironment _RR;
 	RuntimeEnvironment *RR;
 	void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P

node/OutboundMulticast.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/OutboundMulticast.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Packet.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Packet.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/PacketMultiplexer.cpp

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c)2013-2021 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "PacketMultiplexer.hpp"
+
+#include "Node.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Constants.hpp"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+namespace ZeroTier {
+
+PacketMultiplexer::PacketMultiplexer(const RuntimeEnvironment* renv)
+{
+	RR = renv;
+};
+
+void PacketMultiplexer::putFrame(void* tPtr, uint64_t nwid, void** nuptr, const MAC& source, const MAC& dest, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len, unsigned int flowId)
+{
+#if defined(__APPLE__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__WINDOWS__)
+	RR->node->putFrame(tPtr,nwid,nuptr,source,dest,etherType,vlanId,(const void *)data,len);
+	return;
+#endif
+
+	if (!_enabled) {
+		RR->node->putFrame(tPtr,nwid,nuptr,source,dest,etherType,vlanId,(const void *)data,len);
+		return;
+	}
+
+	PacketRecord* packet;
+	_rxPacketVector_m.lock();
+	if (_rxPacketVector.empty()) {
+		packet = new PacketRecord;
+	}
+	else {
+		packet = _rxPacketVector.back();
+		_rxPacketVector.pop_back();
+	}
+	_rxPacketVector_m.unlock();
+
+	packet->tPtr = tPtr;
+	packet->nwid = nwid;
+	packet->nuptr = nuptr;
+	packet->source = source.toInt();
+	packet->dest = dest.toInt();
+	packet->etherType = etherType;
+	packet->vlanId = vlanId;
+	packet->len = len;
+	packet->flowId = flowId;
+	memcpy(packet->data, data, len);
+
+	int bucket = flowId % _concurrency;
+	_rxPacketQueues[bucket]->postLimit(packet, 2048);
+}
+
+void PacketMultiplexer::setUpPostDecodeReceiveThreads(unsigned int concurrency, bool cpuPinningEnabled)
+{
+#if defined(__APPLE__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__WINDOWS__)
+	return;
+#endif
+	_enabled = true;
+	_concurrency = concurrency;
+	bool _enablePinning = cpuPinningEnabled;
+
+	for (unsigned int i = 0; i < _concurrency; ++i) {
+		fprintf(stderr, "Reserved queue for thread %d\n", i);
+		_rxPacketQueues.push_back(new BlockingQueue<PacketRecord*>());
+	}
+
+	// Each thread picks from its own queue to feed into the core
+	for (unsigned int i = 0; i < _concurrency; ++i) {
+		_rxThreads.push_back(std::thread([this, i, _enablePinning]() {
+			fprintf(stderr, "Created post-decode packet ingestion thread %d\n", i);
+
+			PacketRecord* packet = nullptr;
+			for (;;) {
+				if (! _rxPacketQueues[i]->get(packet)) {
+					break;
+				}
+				if (! packet) {
+					break;
+				}
+
+				// fprintf(stderr, "popped packet from queue %d\n", i);
+
+				MAC sourceMac = MAC(packet->source);
+				MAC destMac = MAC(packet->dest);
+
+				RR->node->putFrame(packet->tPtr, packet->nwid, packet->nuptr, sourceMac, destMac, packet->etherType, 0, (const void*)packet->data, packet->len);
+				{
+					Mutex::Lock l(_rxPacketVector_m);
+					_rxPacketVector.push_back(packet);
+				}
+				/*
+				if (ZT_ResultCode_isFatal(err)) {
+					char tmp[256];
+					OSUtils::ztsnprintf(tmp, sizeof(tmp), "error processing packet: %d", (int)err);
+					Mutex::Lock _l(_termReason_m);
+					_termReason = ONE_UNRECOVERABLE_ERROR;
+					_fatalErrorMessage = tmp;
+					this->terminate();
+					break;
+				}
+				*/
+			}
+		}));
+	}
+}
+
+}	// namespace ZeroTier

node/PacketMultiplexer.hpp

@@ -0,0 +1,65 @@
+/*
+ * Copyright (c)2013-2021 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_PACKET_MULTIPLEXER_HPP
+#define ZT_PACKET_MULTIPLEXER_HPP
+
+#include "../osdep/BlockingQueue.hpp"
+#include "MAC.hpp"
+#include "Mutex.hpp"
+#include "RuntimeEnvironment.hpp"
+
+#include <thread>
+#include <vector>
+
+namespace ZeroTier {
+
+struct PacketRecord {
+	void* tPtr;
+	uint64_t nwid;
+	void** nuptr;
+	uint64_t source;
+	uint64_t dest;
+	unsigned int etherType;
+	unsigned int vlanId;
+	uint8_t data[ZT_MAX_MTU];
+	unsigned int len;
+	unsigned int flowId;
+};
+
+class PacketMultiplexer {
+  public:
+	const RuntimeEnvironment* RR;
+
+	PacketMultiplexer(const RuntimeEnvironment* renv);
+
+	void setUpPostDecodeReceiveThreads(unsigned int concurrency, bool cpuPinningEnabled);
+
+	void putFrame(void* tPtr, uint64_t nwid, void** nuptr, const MAC& source, const MAC& dest, unsigned int etherType, unsigned int vlanId, const void* data, unsigned int len, unsigned int flowId);
+
+	std::vector<BlockingQueue<PacketRecord*>*> _rxPacketQueues;
+
+	unsigned int _concurrency;
+	// pool
+	std::vector<PacketRecord*> _rxPacketVector;
+	std::vector<std::thread> _rxPacketThreads;
+	Mutex _rxPacketVector_m, _rxPacketThreads_m;
+
+	std::vector<std::thread> _rxThreads;
+	unsigned int _rxThreadCount;
+	bool _enabled;
+};
+
+}	// namespace ZeroTier
+
+#endif	 // ZT_PACKET_MULTIPLEXER_HPP

node/Path.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Path.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -84,6 +84,7 @@ public:
 		_lastIn(0),
 		_lastTrustEstablishedPacketReceived(0),
 		_lastEchoRequestReceived(0),
+		_localPort(0),
 		_localSocket(-1),
 		_latencyMean(0.0),
 		_latencyVariance(0.0),
@@ -106,6 +107,7 @@ public:
 		_lastIn(0),
 		_lastTrustEstablishedPacketReceived(0),
 		_lastEchoRequestReceived(0),
+		_localPort(0),
 		_localSocket(localSocket),
 		_latencyMean(0.0),
 		_latencyVariance(0.0),
@@ -177,6 +179,11 @@ public:
 	 */
 	inline int64_t localSocket() const { return _localSocket; }
 
+	/**
+	 * @return Local port corresponding to the localSocket
+	 */
+	inline int64_t localPort() const { return _localPort; }
+
 	/**
 	 * @return Physical address
 	 */
@@ -375,6 +382,7 @@ private:
 
 	int64_t _lastEchoRequestReceived;
 
+	uint16_t _localPort;
 	int64_t _localSocket;
 
 	volatile float _latencyMean;

node/Peer.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Peer.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Poly1305.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Revocation.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/Revocation.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/RingBuffer.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/RuntimeEnvironment.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.
@@ -31,6 +31,7 @@ class NetworkController;
 class SelfAwareness;
 class Trace;
 class Bond;
+class PacketMultiplexer;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -77,6 +78,7 @@ public:
 	Topology *topology;
 	SelfAwareness *sa;
 	Bond *bc;
+	PacketMultiplexer *pm;
 
 	// This node's identity and string representations thereof
 	Identity identity;

node/SHA512.hpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.

node/SelfAwareness.cpp

@@ -4,7 +4,7 @@
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
  *
- * Change Date: 2025-01-01
+ * Change Date: 2026-01-01
  *
  * On the date above, in accordance with the Business Source License, use
  * of this software will be governed by version 2.0 of the Apache License.