platfrom_drv: map DMA memory non-natural when iommu is present

Consume '<iommu/>' tag from 'devices' report. In case an IOMMU is
present map physical memory to arbitrary locations within IO page table
range 1K-4G. This way every device PD has access to ~4GB of DMA space.

issue #4665

repos/os/src/drivers/platform/common.h

@@ -21,18 +21,20 @@ class Driver::Common : Device_reporter
 
 		Env                    & _env;
 		String<64>               _rom_name;
-		Attached_rom_dataspace   _devices_rom  { _env, _rom_name.string() };
+		Attached_rom_dataspace   _devices_rom   { _env, _rom_name.string() };
-		Heap                     _heap         { _env.ram(), _env.rm()    };
+		Attached_rom_dataspace   _platform_info { _env, "platform_info"    };
-		Sliced_heap              _sliced_heap  { _env.ram(), _env.rm()    };
+		Heap                     _heap          { _env.ram(), _env.rm()    };
-		Device_model             _devices      { _env, _heap, *this       };
+		Sliced_heap              _sliced_heap   { _env.ram(), _env.rm()    };
-		Signal_handler<Common>   _dev_handler  { _env.ep(), *this,
+		Device_model             _devices       { _env, _heap, *this       };
-		                                         &Common::_handle_devices };
+		Signal_handler<Common>   _dev_handler   { _env.ep(), *this,
+		                                          &Common::_handle_devices };
 		Driver::Root             _root;
 
 		Constructible<Expanding_reporter> _cfg_reporter { };
 		Constructible<Expanding_reporter> _dev_reporter { };
 
 		void _handle_devices();
+		bool _iommu();
 
 	public:
 
@@ -63,6 +65,16 @@ void Driver::Common::_handle_devices()
 }
 
 
+bool Driver::Common::_iommu()
+{
+	bool iommu = false;
+	_platform_info.xml().with_optional_sub_node("kernel", [&] (Xml_node xml) {
+		iommu = xml.attribute_value("iommu", false); });
+
+	return iommu;
+}
+
+
 void Driver::Common::update_report()
 {
 	if (_dev_reporter.constructed())
@@ -103,7 +115,7 @@ Driver::Common::Common(Genode::Env                  & env,
 	_env(env),
 	_rom_name(config_rom.xml().attribute_value("devices_rom",
 	                                           String<64>("devices"))),
-	_root(_env, _sliced_heap, config_rom, _devices)
+	_root(_env, _sliced_heap, config_rom, _devices, _iommu())
 {
 	_devices_rom.sigh(_dev_handler);
 	_handle_devices();

repos/os/src/drivers/platform/device_component.cc

@@ -220,7 +220,7 @@ Device_component::Device_component(Registry<Device_component> & registry,
 			iomem.io_mem.construct(_env, iomem.range.start,
 			                       iomem.range.size, false);
 			session.device_pd().attach_dma_mem(iomem.io_mem->dataspace(),
-			                                   iomem.range.start);
+			                                   iomem.range.start, true);
 		});
 	} catch(...) {
 		_release_resources();

repos/os/src/drivers/platform/device_pd.cc

@@ -67,13 +67,65 @@ void Device_pd::Region_map_client::upgrade_caps()
 }
 
 
-void Device_pd::attach_dma_mem(Dataspace_capability ds_cap,
+addr_t Device_pd::_dma_addr(addr_t const phys_addr,
-                               addr_t const         dma_addr)
+                            size_t const size,
+                            bool   const force_phys_addr)
+{
+	using Range_ok = Range_allocator::Range_ok;
+	using Alloc_error = Allocator::Alloc_error;
+
+	if (!_iommu) return phys_addr;
+
+	/*
+	 * 1:1 mapping (remove from DMA memory allocator)
+	 */
+	if (force_phys_addr) {
+			_dma_alloc.remove_range(phys_addr, size).with_result(
+				[&] (Range_ok) -> addr_t { return phys_addr; },
+				[&] (Alloc_error err) -> addr_t {
+					switch (err) {
+					case Alloc_error::OUT_OF_RAM:  throw Out_of_ram();
+					case Alloc_error::OUT_OF_CAPS: throw Out_of_caps();
+					case Alloc_error::DENIED:
+						error("Could not free DMA range ",
+						      Hex(phys_addr), " - ", Hex(phys_addr + size - 1),
+						     " (error: ", err, ")");
+						break;
+					}
+					return 0;
+			});
+	}
+
+	return _dma_alloc.alloc_aligned(size, 12).convert<addr_t>(
+		[&] (void *ptr) { return (addr_t)ptr; },
+		[&] (Alloc_error err) -> addr_t {
+			switch (err) {
+			case Alloc_error::OUT_OF_RAM:  throw Out_of_ram();
+			case Alloc_error::OUT_OF_CAPS: throw Out_of_caps();
+			case Alloc_error::DENIED:
+				error("Could not allocate DMA area of size: ", size,
+				      " total avail: ", _dma_alloc.avail(),
+				     " (error: ", err, ")");
+				break;
+			};
+			return 0;
+		});
+}
+
+
+addr_t Device_pd::attach_dma_mem(Dataspace_capability ds_cap,
+                                 addr_t const         phys_addr,
+                                 bool   const         force_phys_addr)
 {
 	using namespace Genode;
 
-	bool             retry = false;
+	bool retry = false;
+
 	Dataspace_client ds_client(ds_cap);
+	size_t size     = ds_client.size();
+	addr_t dma_addr = _dma_addr(phys_addr, size, force_phys_addr);
+
+	if (dma_addr == 0) return 0;
 
 	do {
 		_pd.attach_dma(ds_cap, dma_addr).with_result(
@@ -100,6 +152,15 @@ void Device_pd::attach_dma_mem(Dataspace_capability ds_cap,
 			}
 		);
 	} while (retry);
+
+	return dma_addr;
+}
+
+
+void Device_pd::free_dma_mem(addr_t dma_addr)
+{
+	if (_iommu)
+		_dma_alloc.free((void *)dma_addr);
 }
 
 
@@ -125,11 +186,18 @@ void Device_pd::assign_pci(Io_mem_dataspace_capability const io_mem_cap,
 
 
 Device_pd::Device_pd(Env             & env,
+                     Allocator       & md_alloc,
                      Ram_quota_guard & ram_guard,
-                     Cap_quota_guard & cap_guard)
+                     Cap_quota_guard & cap_guard,
+                     bool const iommu)
 :
 	_pd(env, Pd_connection::Device_pd()),
+	_dma_alloc(&md_alloc), _iommu(iommu),
 	_address_space(env, _pd, ram_guard, cap_guard)
 {
+	/* 0x1000 - 4GB per device PD */
+	enum { DMA_SIZE = 0xffffe000 };
+	_dma_alloc.add_range(0x1000, DMA_SIZE);
+
 	_pd.ref_account(env.pd_session_cap());
 }

repos/os/src/drivers/platform/device_pd.h

@@ -15,6 +15,7 @@
 #define _SRC__DRIVERS__PLATFORM__DEVICE_PD_H_
 
 /* base */
+#include <base/allocator_avl.h>
 #include <base/env.h>
 #include <base/quota_guard.h>
 #include <region_map/client.h>
@@ -32,6 +33,8 @@ class Driver::Device_pd
 	private:
 
 		Pd_connection _pd;
+		Allocator_avl _dma_alloc;
+		bool const    _iommu;
 
 		/**
 		 * Custom handling of PD-session depletion during attach operations
@@ -70,13 +73,18 @@ class Driver::Device_pd
 			void upgrade_caps();
 		} _address_space;
 
+		addr_t _dma_addr(addr_t phys_addr, size_t size, bool const force_phys_addr);
+
 	public:
 
 		Device_pd(Env &env,
+		          Allocator &md_alloc,
 		          Ram_quota_guard &ram_guard,
-		          Cap_quota_guard &cap_guard);
+		          Cap_quota_guard &cap_guard,
+		          bool const iommu);
 
-		void attach_dma_mem(Dataspace_capability, addr_t dma_addr);
+		addr_t attach_dma_mem(Dataspace_capability, addr_t phys_addr, bool force_phys_addr);
+		void free_dma_mem(addr_t dma_addr);
 		void assign_pci(Io_mem_dataspace_capability const, Pci::Bdf const);
 };
 

repos/os/src/drivers/platform/root.cc

@@ -46,7 +46,8 @@ Driver::Session_component * Driver::Root::_create_session(const char *args)
 			                  session_resources_from_args(args),
 			                  session_diag_from_args(args),
 			                  policy.attribute_value("info", false),
-			                  policy.attribute_value("version", Version()));
+			                  policy.attribute_value("version", Version()),
+			                  _iommu);
 	} catch (Session_policy::No_policy_defined) {
 		error("Invalid session request, no matching policy for ",
 		      "'", label_from_args(args).string(), "'");
@@ -70,6 +71,7 @@ void Driver::Root::_upgrade_session(Session_component * sc, const char * args)
 Driver::Root::Root(Env                          & env,
                    Sliced_heap                  & sliced_heap,
                    Attached_rom_dataspace const & config,
-                   Device_model                 & devices)
+                   Device_model                 & devices,
+                   bool const                     iommu)
 : Root_component<Session_component>(env.ep(), sliced_heap),
-  _env(env), _config(config), _devices(devices) { }
+  _env(env), _config(config), _devices(devices), _iommu(iommu) { }

repos/os/src/drivers/platform/root.h

@@ -30,7 +30,8 @@ class Driver::Root : public Root_component<Session_component>
 		Root(Env                          & env,
 		     Sliced_heap                  & sliced_heap,
 		     Attached_rom_dataspace const & config,
-		     Device_model                 & devices);
+		     Device_model                 & devices,
+		     bool const                     iommu);
 
 		void update_policy();
 
@@ -43,6 +44,7 @@ class Driver::Root : public Root_component<Session_component>
 		Env                          & _env;
 		Attached_rom_dataspace const & _config;
 		Device_model                 & _devices;
+		bool const                     _iommu;
 		Registry<Session_component>    _sessions {};
 };
 

repos/os/src/drivers/platform/session_component.cc

@@ -44,6 +44,7 @@ void Session_component::_release_device(Device_component & dc)
 void Session_component::_free_dma_buffer(Dma_buffer & buf)
 {
 	Ram_dataspace_capability cap = buf.cap;
+	_device_pd.free_dma_mem(buf.dma_addr);
 	destroy(heap(), &buf);
 	_env_ram.free(cap);
 }
@@ -195,10 +196,9 @@ Session_component::alloc_dma_buffer(size_t const size, Cache cache)
 
 	if (!ram_cap.valid()) return ram_cap;
 
+	Dma_buffer *buf { nullptr };
 	try {
-		Dma_buffer & buf =
+			buf = new (heap()) Dma_buffer(_buffer_registry, ram_cap);
-			*(new (heap()) Dma_buffer(_buffer_registry, ram_cap));
-		_device_pd.attach_dma_mem(ram_cap, _env.pd().dma_addr(buf.cap));
 	} catch (Out_of_ram)  {
 		_env_ram.free(ram_cap);
 		throw;
@@ -207,6 +207,18 @@ Session_component::alloc_dma_buffer(size_t const size, Cache cache)
 		throw;
 	}
 
+	try {
+		buf->dma_addr = _device_pd.attach_dma_mem(ram_cap, _env.pd().dma_addr(buf->cap), false);
+	} catch (Out_of_ram)  {
+		destroy(heap(), buf);
+		_env_ram.free(ram_cap);
+		throw;
+	} catch (Out_of_caps) {
+		destroy(heap(), buf);
+		_env_ram.free(ram_cap);
+		throw;
+	}
+
 	return ram_cap;
 }
 
@@ -230,7 +242,7 @@ Genode::addr_t Session_component::dma_addr(Ram_dataspace_capability ram_cap)
 
 	_buffer_registry.for_each([&] (Dma_buffer const & buf) {
 		if (buf.cap.local_name() == ram_cap.local_name())
-			ret = _env.pd().dma_addr(buf.cap); });
+			ret = buf.dma_addr; });
 
 	return ret;
 }
@@ -244,12 +256,14 @@ Session_component::Session_component(Env                          & env,
                                      Resources      const         & resources,
                                      Diag           const         & diag,
                                      bool           const           info,
-                                     Policy_version const           version)
+                                     Policy_version const           version,
+                                     bool           const           iommu)
 :
 	Session_object<Platform::Session>(env.ep(), resources, label, diag),
 	Session_registry::Element(registry, *this),
 	Dynamic_rom_session::Xml_producer("devices"),
-	_env(env), _config(config), _devices(devices), _info(info), _version(version)
+	_env(env), _config(config), _devices(devices), _info(info), _version(version),
+	_iommu(iommu)
 {
 	/*
 	 * FIXME: As the ROM session does not propagate Out_of_*

repos/os/src/drivers/platform/session_component.h

@@ -52,7 +52,8 @@ class Driver::Session_component
 		                  Resources        const       & resources,
 		                  Diag             const       & diag,
 		                  bool             const         info,
-		                  Policy_version   const         version);
+		                  Policy_version   const         version,
+		                  bool             const         iommu);
 
 		~Session_component();
 
@@ -90,6 +91,7 @@ class Driver::Session_component
 		struct Dma_buffer : Registry<Dma_buffer>::Element
 		{
 			Ram_dataspace_capability const cap;
+			addr_t dma_addr { 0 };
 
 			Dma_buffer(Registry<Dma_buffer> & registry,
 			           Ram_dataspace_capability const cap)
@@ -110,9 +112,12 @@ class Driver::Session_component
 		                                              _env.rm(), *this    };
 		bool                           _info;
 		Policy_version                 _version;
+		bool const                     _iommu;
 		Device_pd                      _device_pd { _env,
+		                                            _md_alloc,
 		                                            _ram_quota_guard(),
-		                                            _cap_quota_guard() };
+		                                            _cap_quota_guard(),
+		                                            _iommu };
 
 		Device_capability _acquire(Device & device);
 		void              _release_device(Device_component & dc);