platform_drv: make Device_pd a Io_mmu::Domain

By transforming the Device_pd into an Io_mmu::Domain, we implement an
IOMMU device that uses the kernel API for controlling the IOMMU. This
device gets special treatment and is used by default for every device
that has no <io_mmu/> child.

genodelabs/genode#4761

repos/os/src/drivers/platform/common.h

@@ -20,6 +20,7 @@
 #include <root.h>
 #include <device_owner.h>
 #include <io_mmu.h>
+#include <device_pd.h>
 
 namespace Driver { class Common; };
 
@@ -95,6 +96,10 @@ void Driver::Common::acquire_io_mmu_devices()
 		});
 
 	});
+
+	/* if kernel implements iommu, instantiate Kernel_iommu */
+	if (_iommu())
+		new (_heap) Kernel_iommu(_env, _io_mmu_devices, "kernel_iommu");
 }
 
 

repos/os/src/drivers/platform/device_pd.cc

@@ -2,23 +2,25 @@
  * \brief  Pci device protection for platform driver
  * \author Alexander Boettcher
  * \author Stefan Kalkowski
+ * \author Johannes Schlatow
  * \date   2013-02-10
  */
 
 /*
- * Copyright (C) 2013-2022 Genode Labs GmbH
+ * Copyright (C) 2013-2023 Genode Labs GmbH
  *
  * This file is part of the Genode OS framework, which is distributed
  * under the terms of the GNU Affero General Public License version 3.
  */
 
+/* Genode includes */
 #include <base/log.h>
 #include <dataspace/client.h>
 #include <region_map/client.h>
 #include <pd_session/client.h>
-
 #include <util/retry.h>
 
+/* local includes */
 #include <device_pd.h>
 
 using namespace Driver;
@@ -67,77 +69,20 @@ void Device_pd::Region_map_client::upgrade_caps()
 }
 
 
-addr_t Device_pd::_dma_addr(addr_t const phys_addr,
+void Device_pd::add_range(Io_mmu::Range        const & range,
-                            size_t const size,
+                          Dataspace_capability const   cap)
-                            bool   const force_phys_addr)
-{
-	using Alloc_error = Allocator::Alloc_error;
-
-	if (!_iommu) return phys_addr;
-
-	/*
-	 * 1:1 mapping (allocate at specified range from DMA memory allocator)
-	 */
-	if (force_phys_addr) {
-		return _dma_alloc.alloc_addr(size, phys_addr).convert<addr_t>(
-			[&] (void *) -> addr_t { return phys_addr; },
-			[&] (Alloc_error err) -> addr_t {
-				switch (err) {
-				case Alloc_error::OUT_OF_RAM:  throw Out_of_ram();
-				case Alloc_error::OUT_OF_CAPS: throw Out_of_caps();
-				case Alloc_error::DENIED:
-					error("Could not attach DMA range at ",
-					      Hex_range(phys_addr, size), " (error: ", err, ")");
-					break;
-				}
-				return 0UL;
-			});
-	}
-
-	/* natural size align (to some limit) for better IOMMU TLB usage */
-	unsigned size_align_log2 = unsigned(log2(size));
-	if (size_align_log2 < 12) /* 4 kB */
-		size_align_log2 = 12;
-	if (size_align_log2 > 24) /* 16 MB */
-		size_align_log2 = 24;
-
-	return _dma_alloc.alloc_aligned(size, size_align_log2).convert<addr_t>(
-		[&] (void *ptr) { return (addr_t)ptr; },
-		[&] (Alloc_error err) -> addr_t {
-			switch (err) {
-			case Alloc_error::OUT_OF_RAM:  throw Out_of_ram();
-			case Alloc_error::OUT_OF_CAPS: throw Out_of_caps();
-			case Alloc_error::DENIED:
-				error("Could not allocate DMA area of size: ", size,
-				      " alignment: ", size_align_log2,
-				      " total avail: ", _dma_alloc.avail(),
-				     " (error: ", err, ")");
-				break;
-			};
-			return 0;
-		});
-}
-
-
-addr_t Device_pd::attach_dma_mem(Dataspace_capability ds_cap,
-                                 addr_t const         phys_addr,
-                                 bool   const         force_phys_addr)
 {
 	using namespace Genode;
 
 	bool retry = false;
 
-	Dataspace_client ds_client(ds_cap);
+	if (range.start == 0) return;
-	size_t size     = ds_client.size();
-	addr_t dma_addr = _dma_addr(phys_addr, size, force_phys_addr);
-
-	if (dma_addr == 0) return 0;
 
 	do {
-		_pd.attach_dma(ds_cap, dma_addr).with_result(
+		_pd.attach_dma(cap, range.start).with_result(
 			[&] (Pd_session::Attach_dma_ok) {
 				/* trigger eager mapping of memory */
-				_pd.map(dma_addr, ds_client.size());
+				_pd.map(range.start, range.size);
 				retry = false;
 			},
 			[&] (Pd_session::Attach_dma_error e) {
@@ -151,26 +96,23 @@ addr_t Device_pd::attach_dma_mem(Dataspace_capability ds_cap,
 					retry = true;
 					break;
 				case Pd_session::Attach_dma_error::DENIED:
-					_address_space.detach(dma_addr);
+					_address_space.detach(range.start);
 					error("Device PD: attach_dma denied!");
 					break;
 				}
 			}
 		);
 	} while (retry);
-
-	return dma_addr;
 }
 
 
-void Device_pd::free_dma_mem(addr_t dma_addr)
+void Device_pd::remove_range(Io_mmu::Range const & range)
 {
-	if (_iommu)
+	_address_space.detach(range.start);
-		_dma_alloc.free((void *)dma_addr);
 }
 
 
-void Device_pd::assign_pci(Io_mem_dataspace_capability const io_mem_cap,
+void Device_pd::enable_pci_device(Io_mem_dataspace_capability const io_mem_cap,
                                   Pci::Bdf                    const bdf)
 {
 	addr_t addr = _address_space.attach(io_mem_cap, 0x1000);
@@ -191,28 +133,25 @@ void Device_pd::assign_pci(Io_mem_dataspace_capability const io_mem_cap,
 }
 
 
+void Device_pd::disable_pci_device(Io_mem_dataspace_capability const,
+                                   Pci::Bdf                    const)
+{
+	warning("Cannot unassign PCI device from device PD (not implemented by kernel).");
+}
+
+
+
 Device_pd::Device_pd(Env                        & env,
-                     Allocator       & md_alloc,
                      Ram_quota_guard            & ram_guard,
                      Cap_quota_guard            & cap_guard,
-                     bool const iommu)
+                     Kernel_iommu               & io_mmu,
+                     Allocator                  & md_alloc,
+                     Registry<Dma_buffer> const & buffer_registry)
+
 :
+	Io_mmu::Domain(io_mmu, md_alloc, buffer_registry),
 	_pd(env, Pd_connection::Device_pd()),
-	_dma_alloc(&md_alloc), _iommu(iommu),
 	_address_space(env, _pd, ram_guard, cap_guard)
 {
-	/* 0x1000 - 4GB per device PD */
-	enum { DMA_SIZE = 0xffffe000 };
-	_dma_alloc.add_range(0x1000, DMA_SIZE);
-
-	/*
-	 * Interrupt address range is special handled and in general not
-	 * usable for normal DMA translations, see chapter 3.15
-	 * of "Intel Virtualization Technology for Directed I/O"
-	 * (March 2023, Revision 4.1)
-	 */
-	enum { IRQ_RANGE_BASE = 0xfee00000u, IRQ_RANGE_SIZE = 0x100000 };
-	_dma_alloc.remove_range(IRQ_RANGE_BASE, IRQ_RANGE_SIZE);
-
 	_pd.ref_account(env.pd_session_cap());
 }

repos/os/src/drivers/platform/device_pd.h

@@ -1,6 +1,7 @@
 /*
  * \brief  Device PD handling for the platform driver
  * \author Alexander Boettcher
+ * \author Johannes Schlatow
  * \date   2015-11-05
  */
 
@@ -23,18 +24,22 @@
 #include <pd_session/connection.h>
 #include <io_mem_session/capability.h>
 
+/* local inludes */
+#include <io_mmu.h>
+
 namespace Driver {
 	using namespace Genode;
+
 	class Device_pd;
+	class Kernel_iommu;
 }
 
-class Driver::Device_pd
+
+class Driver::Device_pd : public Io_mmu::Domain
 {
 	private:
 
 		Pd_connection _pd;
-		Allocator_avl _dma_alloc;
-		bool const    _iommu;
 
 		/**
 		 * Custom handling of PD-session depletion during attach operations
@@ -73,19 +78,60 @@ class Driver::Device_pd
 			void upgrade_caps();
 		} _address_space;
 
-		addr_t _dma_addr(addr_t phys_addr, size_t size, bool const force_phys_addr);
+	public:
+
+		Device_pd(Env                        & env,
+		          Ram_quota_guard            & ram_guard,
+		          Cap_quota_guard            & cap_guard,
+		          Kernel_iommu               & iommu,
+		          Allocator                  & md_alloc,
+		          Registry<Dma_buffer> const & buffer_registry);
+
+		void add_range(Io_mmu::Range const &, Dataspace_capability const) override;
+		void remove_range(Io_mmu::Range const &) override;
+
+		void enable_pci_device(Io_mem_dataspace_capability const,
+		                       Pci::Bdf const) override;
+		void disable_pci_device(Io_mem_dataspace_capability const,
+		                        Pci::Bdf const) override;
+};
+
+
+class Driver::Kernel_iommu : public Io_mmu
+{
+	private:
+
+		Env & _env;
 
 	public:
 
-		Device_pd(Env &env,
+		/**
-		          Allocator &md_alloc,
+		 * Iommu interface
-		          Ram_quota_guard &ram_guard,
+		 */
-		          Cap_quota_guard &cap_guard,
-		          bool const iommu);
 
-		addr_t attach_dma_mem(Dataspace_capability, addr_t phys_addr, bool force_phys_addr);
+		Driver::Io_mmu::Domain & create_domain(
-		void free_dma_mem(addr_t dma_addr);
+			Allocator                  & md_alloc,
-		void assign_pci(Io_mem_dataspace_capability const, Pci::Bdf const);
+			Registry<Dma_buffer> const & buffer_registry,
+			Ram_quota_guard            & ram_guard,
+			Cap_quota_guard            & cap_guard) override
+		{
+			return *new (md_alloc) Device_pd(_env,
+			                                 ram_guard,
+			                                 cap_guard,
+			                                 *this,
+			                                 md_alloc,
+			                                 buffer_registry);
+		}
+
+
+		Kernel_iommu(Env                      & env,
+		             Io_mmu_devices           & io_mmu_devices,
+		             Device::Name       const & name)
+		: Io_mmu(io_mmu_devices, name),
+		  _env(env)
+		{ };
+
+		~Kernel_iommu() { _destroy_domains(); }
 };
 
 #endif /* _SRC__DRIVERS__PLATFORM__DEVICE_PD_H_ */

repos/os/src/drivers/platform/pci.cc

@@ -11,11 +11,13 @@
  * under the terms of the GNU Affero General Public License version 3.
  */
 
+/* Genode includes */
 #include <base/attached_io_mem_dataspace.h>
 #include <timer_session/connection.h>
 #include <pci/config.h>
 #include <util/reconstructible.h>
 
+/* local includes */
 #include <device.h>
 #include <device_pd.h>
 #include <device_component.h>
@@ -58,10 +60,22 @@ struct Config_helper
 	              Driver::Device::Pci_config const & cfg)
 	: _env(env), _dev(dev), _cfg(cfg) { _config.scan(); }
 
-	void enable(Driver::Device_pd & pd)
+	void enable(Driver::Io_mmu_domain_registry & domain_registry)
 	{
-		pd.assign_pci(_io_mem.cap(),
+		auto enable_fn = [&] (Driver::Io_mmu::Domain & domain) {
+			domain.enable_pci_device(_io_mem.cap(),
 			                         { _cfg.bus_num, _cfg.dev_num, _cfg.func_num });
+		};
+
+		_dev.for_each_io_mmu(
+			/* non-empty list fn */
+			[&] (Driver::Device::Io_mmu const &io_mmu) {
+				domain_registry.with_domain(io_mmu.name, enable_fn, [&] () {}); },
+
+			/* empty list fn */
+			[&] () {
+				domain_registry.with_default_domain(enable_fn); }
+		);
 
 		_config.power_on(delayer(_env));
 
@@ -93,7 +107,7 @@ struct Config_helper
 		_config.write<Config::Command>(cmd);
 	}
 
-	void disable()
+	void disable(Driver::Io_mmu_domain_registry & domain_registry)
 	{
 		Config::Command::access_t cmd =
 			_config.read<Config::Command>();
@@ -104,6 +118,21 @@ struct Config_helper
 		_config.write<Config::Command>(cmd);
 
 		_config.power_off();
+
+		auto disable_fn = [&] (Driver::Io_mmu::Domain & domain) {
+			domain.disable_pci_device(_io_mem.cap(),
+			                          { _cfg.bus_num, _cfg.dev_num, _cfg.func_num });
+		};
+
+		_dev.for_each_io_mmu(
+			/* non-empty list fn */
+			[&] (Driver::Device::Io_mmu const &io_mmu) {
+				domain_registry.with_domain(io_mmu.name, disable_fn, [&] () {}); },
+
+			/* empty list fn */
+			[&] () {
+				domain_registry.with_default_domain(disable_fn); }
+		);
 	}
 
 	void apply_quirks()
@@ -135,17 +164,21 @@ struct Config_helper
 };
 
 
-void Driver::pci_enable(Env & env, Device_pd & pd, Device const & dev)
+void Driver::pci_enable(Env                            & env,
+                        Driver::Io_mmu_domain_registry & domain_registry,
+                        Device const                   & dev)
 {
 	dev.for_pci_config([&] (Device::Pci_config const & pc) {
-		Config_helper(env, dev, pc).enable(pd); });
+		Config_helper(env, dev, pc).enable(domain_registry); });
 }
 
 
-void Driver::pci_disable(Env & env, Device const & dev)
+void Driver::pci_disable(Env                            & env,
+                         Driver::Io_mmu_domain_registry & domain_registry,
+                         Device const                   & dev)
 {
 	dev.for_pci_config([&] (Device::Pci_config const & pc) {
-		Config_helper(env, dev, pc).disable(); });
+		Config_helper(env, dev, pc).disable(domain_registry); });
 }
 
 

repos/os/src/drivers/platform/pci.h

@@ -14,18 +14,25 @@
 #ifndef _SRC__DRIVERS__PLATFORM__PCI_H_
 #define _SRC__DRIVERS__PLATFORM__PCI_H_
 
+/* Genode includes */
 #include <base/env.h>
 #include <irq_session/irq_session.h>
 #include <os/session_policy.h>
 
+/* local includes */
 #include <device.h>
+#include <io_mmu_domain_registry.h>
 
 namespace Driver {
 	class Device_component;
 	class Device_pd;
 
-	void pci_enable(Genode::Env & env, Device_pd & pd, Device const & dev);
+	void pci_enable(Genode::Env            & env,
-	void pci_disable(Genode::Env & env, Device const & dev);
+	                Io_mmu_domain_registry & domain_registry,
+	                Device const           & dev);
+	void pci_disable(Genode::Env            & env,
+	                 Io_mmu_domain_registry & domain_registry,
+	                 Device const           & dev);
 	void pci_apply_quirks(Genode::Env & env, Device const & dev);
 	void pci_msi_enable(Genode::Env & env, Device_component & dc,
 	                    addr_t cfg_space, Genode::Irq_session::Info const info,

repos/os/src/drivers/platform/session_component.cc

@@ -231,7 +231,7 @@ void Session_component::update_devices_rom()
 
 void Session_component::enable_device(Device const & device)
 {
-	pci_enable(_env, device_pd(), device);
+	pci_enable(_env, domain_registry(), device);
 
 	auto fn = [&] (Driver::Io_mmu::Domain & domain) {
 		domain.enable_device();
@@ -251,7 +251,7 @@ void Session_component::enable_device(Device const & device)
 
 void Session_component::disable_device(Device const & device)
 {
-	pci_disable(_env, device);
+	pci_disable(_env, domain_registry(), device);
 
 	auto fn = [&] (Driver::Io_mmu::Domain & domain) {
 		domain.disable_device();

repos/os/src/drivers/platform/session_component.h

@@ -117,11 +117,6 @@ class Driver::Session_component
 		bool                           _info;
 		Policy_version                 _version;
 		bool const                     _iommu;
-		Device_pd                      _device_pd { _env,
-		                                            _md_alloc,
-		                                            _ram_quota_guard(),
-		                                            _cap_quota_guard(),
-		                                            _iommu };
 		Dma_allocator                  _dma_allocator { _md_alloc, _iommu };
 
 		Device_capability _acquire(Device & device);