nic_router: smarter emergency free on exhaustion

Re-implements an emergency freeing of resources on exhaustion of session quota.
In contrast to the past one, the new algorithm is executed directly where the
exhaustion occurs. Instead of interupting the packet handling and restart it
from the beginning after the freeing action, packet handling is now continued
at the point of exhaustion (if enough resources could be freed). Furthermore,
the new algorithm frees only 100 objects (instead of 1024) at a max as we found
this to better match real-life observations. And finally, the router now drops
ICMP first, then UDP, then TCP - as this better reflects priorities - and
refrains from dropping TCP connections in the ESTABLISHED state. If the router
cannot free a sufficient amount of resources, the packet that caused the
exhaustion is dropped with a warning (verbose_packet_drop="yes").

Ref #4729

repos/os/src/server/nic_router/interface.cc

@@ -24,6 +24,7 @@
 #include <configuration.h>
 #include <l3_protocol.h>
 #include <assertion.h>
+#include <retry.h>
 
 using namespace Net;
 using Genode::Deallocator;
@@ -98,10 +99,10 @@ static void _destroy_links(Link_list   &links,
 
 
 template <typename LINK_TYPE>
-static void _destroy_some_links(Link_list     &links,
-                                Link_list     &dissolved_links,
-                                Deallocator   &dealloc,
-                                unsigned long &max)
+static void _early_drop_links(Link_list     &links,
+                              Link_list     &dissolved_links,
+                              Deallocator   &dealloc,
+                              unsigned long &max)
 {
 	if (!max) {
 		return; }
@@ -112,10 +113,15 @@ static void _destroy_some_links(Link_list     &links,
 		if (!--max) {
 			return; }
 	}
-	while (Link *link = links.first()) {
-		_destroy_link<LINK_TYPE>(*link, links, dealloc);
-		if (!--max) {
-			return; }
+	Link *link_ptr = links.first();
+	while (link_ptr) {
+		Link *next_ptr = link_ptr->next();
+		if (static_cast<LINK_TYPE *>(link_ptr)->can_early_drop()) {
+			_destroy_link<LINK_TYPE>(*link_ptr, links, dealloc);
+			if (!--max) {
+				return; }
+		}
+		link_ptr = next_ptr;
 	}
 }
 
@@ -493,6 +499,15 @@ void Interface::_detach_from_domain()
 }
 
 
+void Interface::_try_emergency_free_quota()
+{
+	unsigned long max = MAX_FREE_OPS_PER_EMERGENCY;
+	_early_drop_links<Icmp_link>(_icmp_links, _dissolved_icmp_links, _alloc, max);
+	_early_drop_links<Udp_link> (_udp_links,  _dissolved_udp_links,  _alloc, max);
+	_early_drop_links<Tcp_link> (_tcp_links,  _dissolved_tcp_links,  _alloc, max);
+}
+
+
 Packet_result Interface::_new_link(L3_protocol          const  protocol,
                                    Domain                     &local_domain,
                                    Link_side_id         const &local,
@@ -503,49 +518,37 @@ Packet_result Interface::_new_link(L3_protocol          const  protocol,
 	Packet_result result { };
 	switch (protocol) {
 	case L3_protocol::TCP:
-		try {
-			new (_alloc)
+		retry_once<Out_of_ram, Out_of_caps>(
+			[&] {
+				new (_alloc)
 				Tcp_link { *this, local_domain, local, remote_port_alloc_ptr, remote_domain,
-				           remote, _timer, _config(), protocol, _tcp_stats };
-		}
-		catch (Out_of_ram)  {
-			_tcp_stats.refused_for_ram++;
-			result = packet_drop("out of RAM while creating TCP link");
-		}
-		catch (Out_of_caps) {
-			_tcp_stats.refused_for_ram++;
-			result = packet_drop("out of CAPs while creating TCP link");
-		}
+				           remote, _timer, _config(), protocol, _tcp_stats }; },
+			[&] { _try_emergency_free_quota(); },
+			[&] {
+				_tcp_stats.refused_for_ram++;
+				result = packet_drop("out of quota while creating TCP link"); });
 		break;
 	case L3_protocol::UDP:
-		try {
-			new (_alloc)
-				Udp_link { *this, local_domain, local, remote_port_alloc_ptr, remote_domain,
-				           remote, _timer, _config(), protocol, _udp_stats };
-		}
-		catch (Out_of_ram) {
-			_udp_stats.refused_for_ram++;
-			result = packet_drop("out of RAM while creating UDP link");
-		}
-		catch (Out_of_caps) {
-			_udp_stats.refused_for_ram++;
-			result = packet_drop("out of CAPs while creating UDP link");
-		}
+		retry_once<Out_of_ram, Out_of_caps>(
+			[&] {
+				new (_alloc)
+					Udp_link { *this, local_domain, local, remote_port_alloc_ptr, remote_domain,
+					           remote, _timer, _config(), protocol, _udp_stats }; },
+			[&] { _try_emergency_free_quota(); },
+			[&] {
+				_udp_stats.refused_for_ram++;
+				result = packet_drop("out of quota while creating UDP link"); });
 		break;
 	case L3_protocol::ICMP:
-		try {
-			new (_alloc)
-				Icmp_link { *this, local_domain, local, remote_port_alloc_ptr, remote_domain,
-				            remote, _timer, _config(), protocol, _icmp_stats };
-		}
-		catch (Out_of_ram) {
-			_icmp_stats.refused_for_ram++;
-			result = packet_drop("out of RAM while creating ICMP link");
-		}
-		catch (Out_of_caps) {
-			_icmp_stats.refused_for_ram++;
-			result = packet_drop("out of CAPs while creating ICMP link");
-		}
+		retry_once<Out_of_ram, Out_of_caps>(
+			[&] {
+				new (_alloc)
+					Icmp_link { *this, local_domain, local, remote_port_alloc_ptr, remote_domain,
+					            remote, _timer, _config(), protocol, _icmp_stats }; },
+			[&] { _try_emergency_free_quota(); },
+			[&] {
+				_icmp_stats.refused_for_ram++;
+				result = packet_drop("out of quota while creating ICMP link"); });
 		break;
 	default: ASSERT_NEVER_REACHED; }
 	return result;
@@ -606,12 +609,13 @@ Packet_result Interface::_adapt_eth(Ethernet_frame          &eth,
 					interface._broadcast_arp_request(
 						remote_ip_cfg.interface().address, hop_ip);
 				});
-				try {
-					new (_alloc) Arp_waiter { *this, remote_domain, hop_ip, pkt };
-					result = packet_postponed();
-				}
-				catch (Out_of_ram)  { result = packet_drop("out of RAM while creating ARP waiter"); }
-				catch (Out_of_caps) { result = packet_drop("out of CAPs while creating ARP waiter"); }
+				retry_once<Out_of_ram, Out_of_caps>(
+					[&] {
+						new (_alloc) Arp_waiter { *this, remote_domain, hop_ip, pkt };
+						result = packet_postponed();
+					},
+					[&] { _try_emergency_free_quota(); },
+					[&] { result = packet_drop("out of quota while creating ARP waiter"); });
 			}
 		);
 	};
@@ -774,26 +778,31 @@ Packet_result Interface::_new_dhcp_allocation(Ethernet_frame &eth,
                                               Domain         &local_domain)
 {
 	Packet_result result { };
-	auto ok_fn = [&] (Ipv4_address const &ip) {
-		Dhcp_allocation &allocation = *new (_alloc)
-			Dhcp_allocation { *this, ip, dhcp.client_mac(),
-			                  _timer, _config().dhcp_offer_timeout() };
+	dhcp_srv.alloc_ip().with_result(
+		[&] (Ipv4_address const &ip) {
+			retry_once<Out_of_ram, Out_of_caps>(
+				[&] {
+					Dhcp_allocation &allocation = *new (_alloc)
+						Dhcp_allocation { *this, ip, dhcp.client_mac(),
+						                  _timer, _config().dhcp_offer_timeout() };
 
-		_dhcp_allocations.insert(allocation);
-		if (_config().verbose()) {
-			log("[", local_domain, "] offer DHCP allocation: ", allocation); }
+					_dhcp_allocations.insert(allocation);
+					if (_config().verbose()) {
+						log("[", local_domain, "] offer DHCP allocation: ", allocation); }
 
-		_send_dhcp_reply(dhcp_srv, eth.src(), dhcp.client_mac(),
-		                 allocation.ip(),
-		                 Dhcp_packet::Message_type::OFFER,
-		                 dhcp.xid(),
-		                 local_domain.ip_config().interface());
+					_send_dhcp_reply(dhcp_srv, eth.src(), dhcp.client_mac(),
+					                 allocation.ip(),
+					                 Dhcp_packet::Message_type::OFFER,
+					                 dhcp.xid(),
+					                 local_domain.ip_config().interface());
+
+					result = packet_handled();
+				},
+				[&] { _try_emergency_free_quota(); },
+				[&] { result = packet_drop("out of quota while creating DHCP allocation"); });
+		},
+		[&] (auto) { result = packet_drop("failed to allocate IP for DHCP client"); });
 
-		result = packet_handled();
-	};
-	try { dhcp_srv.alloc_ip().with_result(ok_fn, [&] (auto) { result = packet_drop("failed to allocate IP for DHCP client"); }); }
-	catch (Out_of_ram)  { result = packet_drop("out of RAM while creating DHCP allocation"); }
-	catch (Out_of_caps) { result = packet_drop("out of CAPs while creating DHCP allocation"); }
 	return result;
 }
 

repos/os/src/server/nic_router/interface.h

@@ -128,7 +128,7 @@ class Net::Interface : private Interface_list::Element
 		using Signal_context_capability = Genode::Signal_context_capability;
 
 		enum { IPV4_TIME_TO_LIVE          = 64 };
-		enum { MAX_FREE_OPS_PER_EMERGENCY = 1024 };
+		enum { MAX_FREE_OPS_PER_EMERGENCY = 100 };
 
 		struct Update_domain
 		{
@@ -193,6 +193,8 @@ class Net::Interface : private Interface_list::Element
 		void _release_dhcp_allocation(Dhcp_allocation &allocation,
 		                              Domain          &local_domain);
 
+		void _try_emergency_free_quota();
+
 		[[nodiscard]] Packet_result _new_dhcp_allocation(Ethernet_frame &eth,
 		                                                Dhcp_packet    &dhcp,
 		                                                Dhcp_server    &dhcp_srv,

repos/os/src/server/nic_router/link.cc

@@ -253,7 +253,7 @@ void Tcp_link::_tcp_packet(Tcp_packet &tcp,
 			}
 		}
 	}
-	if (_state == State::OPEN) {
+	if (_state == State::OPENING || _state == State::OPEN) {
 		_packet();
 	} else {
 		_dissolve_timeout.schedule(
@@ -266,6 +266,7 @@ void Tcp_link::server_packet(Tcp_packet &tcp)
 {
 	if (_opening) {
 		_opening = false;
+		_state = State::OPEN;
 		_stats_curr()--;
 		if (&_stats_curr() == &_stats.opening) { _stats_curr = _stats.open; }
 		_stats_curr()++;

repos/os/src/server/nic_router/link.h

@@ -256,7 +256,7 @@ class Net::Tcp_link : public Link
 {
 	private:
 
-		enum class State : Genode::uint8_t { OPEN, CLOSING, CLOSED, };
+		enum class State : Genode::uint8_t { OPENING, OPEN, CLOSING, CLOSED };
 
 		struct Peer
 		{
@@ -264,7 +264,7 @@ class Net::Tcp_link : public Link
 			bool fin_acked { false };
 		};
 
-		State _state  { State::OPEN };
+		State _state  { State::OPENING };
 		Peer  _client { };
 		Peer  _server { };
 
@@ -292,6 +292,8 @@ class Net::Tcp_link : public Link
 		void client_packet(Tcp_packet &tcp) { _tcp_packet(tcp, _client, _server); }
 
 		void server_packet(Tcp_packet &tcp);
+
+		bool can_early_drop() { return _state != State::OPEN; }
 };
 
 
@@ -311,6 +313,8 @@ struct Net::Udp_link : Link
 	void client_packet() { _packet(); }
 
 	void server_packet();
+
+	bool can_early_drop() { return true; }
 };
 
 
@@ -330,6 +334,8 @@ struct Net::Icmp_link : Link
 	void client_packet() { _packet(); }
 
 	void server_packet();
+
+	bool can_early_drop() { return true; }
 };
 
 #endif /* _LINK_H_ */

repos/os/src/server/nic_router/retry.h

@@ -0,0 +1,38 @@
+/*
+ * \brief  Utility to execute a function repeatedly
+ * \author Martin Stein
+ * \date   2015-04-29
+ */
+
+/*
+ * Copyright (C) 2015-2017 Genode Labs GmbH
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef _RETRY_H_
+#define _RETRY_H_
+
+namespace Net {
+
+	template <typename EXCEPTION_1, typename EXCEPTION_2>
+	void retry_once(auto const &attempt_fn, auto const &exception_fn, auto const &failed_fn)
+	{
+		for (unsigned i = 0; ; i++) {
+			try {
+				attempt_fn();
+				return;
+			}
+			catch (EXCEPTION_1) { }
+			catch (EXCEPTION_2) { }
+			if (i == 1) {
+				failed_fn();
+				return;
+			}
+			exception_fn();
+		}
+	}
+}
+
+#endif /* _RETRY_H_ */