diff --git a/repos/pc/src/driver/platform/pc/intel/invalidator.cc b/repos/pc/src/driver/platform/pc/intel/invalidator.cc index 72e94f94b0..ed1b4eaf39 100644 --- a/repos/pc/src/driver/platform/pc/intel/invalidator.cc +++ b/repos/pc/src/driver/platform/pc/intel/invalidator.cc @@ -120,6 +120,21 @@ void Intel::Register_invalidator::invalidate_all(Domain_id domain_id, Pci::rid_t } +/* Clear interrupt entry cache */ +void Intel::Queued_invalidator::invalidate_irq(unsigned idx, bool global) +{ + Descriptor::access_t *entry = _tail(); + Iec::Type::set(*entry, Iec::Type::IEC); + Iec::Global::set(*entry, global ? Iec::Global::GLOBAL : Iec::Global::INDEX); + Iec::Index::set(*entry, idx); + + _next(); + + /* wait for completion */ + while (!_empty()); +} + + /** * Clear IOTLB. * diff --git a/repos/pc/src/driver/platform/pc/intel/invalidator.h b/repos/pc/src/driver/platform/pc/intel/invalidator.h index 4619090cc7..6a3f968253 100644 --- a/repos/pc/src/driver/platform/pc/intel/invalidator.h +++ b/repos/pc/src/driver/platform/pc/intel/invalidator.h @@ -38,6 +38,7 @@ class Intel::Invalidator virtual ~Invalidator() { } + virtual void invalidate_irq(unsigned, bool) { }; virtual void invalidate_iotlb(Domain_id) = 0; virtual void invalidate_context(Domain_id domain, Pci::rid_t) = 0; virtual void invalidate_all(Domain_id domain = Domain_id { Domain_id::INVALID }, @@ -171,6 +172,7 @@ class Intel::Queued_invalidator : public Invalidator enum { CONTEXT = 1, IOTLB = 2, + IEC = 4 }; }; @@ -197,6 +199,17 @@ class Intel::Queued_invalidator : public Invalidator struct Dr : Bitfield<7,1> { }; }; + struct Iec : Descriptor + { + struct Global : Bitfield<4,1> { + enum { + GLOBAL = 0, + INDEX = 1 + }; + }; + struct Index : Bitfield<32,16> { }; + }; + bool _empty() { return _queue_mmio.read() == _queue_mmio.read(); } @@ -216,6 +229,7 @@ class Intel::Queued_invalidator : public Invalidator public: + void invalidate_irq(unsigned, bool) override; void invalidate_iotlb(Domain_id) override; void invalidate_context(Domain_id domain, Pci::rid_t) override; void invalidate_all(Domain_id domain = Domain_id { Domain_id::INVALID }, diff --git a/repos/pc/src/driver/platform/pc/intel/io_mmu.cc b/repos/pc/src/driver/platform/pc/intel/io_mmu.cc index 4f8b990e41..3b423c5c96 100644 --- a/repos/pc/src/driver/platform/pc/intel/io_mmu.cc +++ b/repos/pc/src/driver/platform/pc/intel/io_mmu.cc @@ -261,6 +261,9 @@ void Intel::Io_mmu::generate(Xml_generator & xml) xml.attribute("mask", (bool)read()); }); + if (read()) + _irq_table.generate(xml); + if (!read()) return; @@ -371,6 +374,43 @@ void Intel::Io_mmu::resume() } +void Intel::Io_mmu::_enable_irq_remapping() +{ + /* + * If IRQ remapping has already been enabled during boot, the kernel is + * in charge of the remapping. Since there is no way to get the required + * unremapped vector for requested MSI, we cannot take over control. + */ + + if (read()) { + warning("IRQ remapping is controlled by kernel for ", name()); + return; + } + + /* caches must be cleared if Esirtps is not set */ + if (read()) + invalidator().invalidate_irq(0, true); + + /* set interrupt remapping table address */ + write( + Irq_table_address::Size::bits(Irq_table::ENTRIES_LOG2-1) | + Irq_table_address::Address::masked(_irq_table_phys)); + + /* issue set interrupt remapping table pointer command */ + _global_command(1); + + /* disable compatibility format interrupts */ + _global_command(0); + + /* enable interrupt remapping */ + _global_command(1); + + log("enabled interrupt remapping for ", name()); + + _remap_irqs = true; +} + + Intel::Io_mmu::Io_mmu(Env & env, Io_mmu_devices & io_mmu_devices, Device::Name const & name, @@ -380,10 +420,11 @@ Intel::Io_mmu::Io_mmu(Env & env, : Attached_mmio(env, {(char *)range.start, range.size}), Driver::Io_mmu(io_mmu_devices, name), _env(env), - _managed_root_table(_env, table_allocator, *this, !coherent_page_walk()), - _default_mappings(_env, table_allocator, *this, !coherent_page_walk(), - _sagaw_to_levels()), - _domain_allocator(_max_domains()-1) + _table_allocator(table_allocator), + _domain_allocator(_max_domains()-1), + _managed_root_table(_env, _table_allocator, *this, !coherent_page_walk()), + _default_mappings(_env, _table_allocator, *this, !coherent_page_walk(), + _sagaw_to_levels()) { if (_broken_device()) { error(name, " reports invalid capability registers. Please disable VT-d/IOMMU."); @@ -422,6 +463,7 @@ Intel::Io_mmu::Io_mmu(Env & env, _fault_irq->ack_irq(); Irq_session::Info info = _fault_irq->info(); + if (info.type == Irq_session::Info::INVALID) error("Unable to enable fault event interrupts for ", name); else { @@ -430,4 +472,12 @@ Intel::Io_mmu::Io_mmu(Env & env, write(0); } } + + /* + * We always enable IRQ remapping if its supported by the IOMMU. Note, there + * might be the possibility that the ACPI DMAR table says otherwise but + * we've never seen such a case yet. + */ + if (read()) + _enable_irq_remapping(); } diff --git a/repos/pc/src/driver/platform/pc/intel/io_mmu.h b/repos/pc/src/driver/platform/pc/intel/io_mmu.h index a920db504f..34056dd1bc 100644 --- a/repos/pc/src/driver/platform/pc/intel/io_mmu.h +++ b/repos/pc/src/driver/platform/pc/intel/io_mmu.h @@ -32,6 +32,7 @@ #include #include #include +#include #include namespace Intel { @@ -40,6 +41,14 @@ namespace Intel { using Context_table_allocator = Managed_root_table::Allocator; + /** + * We use a 4KB interrupt remap table since kernels (nova, hw) do not + * support more than 256 interrupts anyway. We can thus reuse the + * context-table allocator. + */ + using Irq_table = Irq_remap_table<12>; + using Irq_allocator = Irq_table::Irq_allocator; + class Io_mmu; class Io_mmu_factory; } @@ -71,6 +80,7 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, Domain_allocator & _domain_allocator; Domain_id _domain_id { _domain_allocator.alloc() }; bool _skip_invalidation { false }; + Irq_allocator & _irq_allocator; addr_t _translation_table_phys { _table_allocator.construct() }; @@ -131,7 +141,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, Registry const & buffer_registry, Env & env, Ram_allocator & ram_alloc, - Domain_allocator & domain_allocator) + Domain_allocator & domain_allocator, + Irq_allocator & irq_allocator) : Driver::Io_mmu::Domain(intel_iommu, md_alloc), Registered_translation_table(intel_iommu), _intel_iommu(intel_iommu), @@ -139,7 +150,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, _ram_alloc(ram_alloc), _buffer_registry(buffer_registry), _table_allocator(_env, md_alloc, ram_alloc, 2), - _domain_allocator(domain_allocator) + _domain_allocator(domain_allocator), + _irq_allocator(irq_allocator) { Invalidation_guard guard { *this, _intel_iommu.caching_mode() }; @@ -168,7 +180,39 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, private: - Env & _env; + static + Irq_table & _irq_table_virt(Context_table_allocator & alloc, addr_t phys) + { + addr_t va { 0 }; + + alloc.with_table(phys, + [&] (Irq_table & t) { va = (addr_t)&t; }, + [&] () { /* never reached */ }); + + /** + * Dereferencing is save because _irq_table_phys is never 0 + * (allocator throws exception) and with_table() thus always sets + * a valid virtual address. + */ + return *(Irq_table*)va; + } + + + Env & _env; + bool _verbose { false }; + Context_table_allocator & _table_allocator; + + const addr_t _irq_table_phys { + _table_allocator.construct() }; + + Irq_table & _irq_table { + _irq_table_virt(_table_allocator, _irq_table_phys) }; + + Irq_allocator _irq_allocator { }; + + Report_helper _report_helper { *this }; + Domain_allocator _domain_allocator; + Domain_id _default_domain { _domain_allocator.alloc() }; /** * For a start, we keep a distinct root table for every hardware unit. @@ -184,12 +228,11 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, * The default root table holds default mappings (e.g. reserved memory) * that needs to be accessible even if devices have not been acquired yet. */ - bool _verbose { false }; Managed_root_table _managed_root_table; Default_mappings _default_mappings; - Report_helper _report_helper { *this }; - Domain_allocator _domain_allocator; - Domain_id _default_domain { _domain_allocator.alloc() }; + + bool _remap_irqs { false }; + Constructible _fault_irq { }; Signal_handler _fault_handler { _env.ep(), *this, &Io_mmu::_handle_faults }; @@ -263,8 +306,14 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, /* queued invalidation enable */ struct Qie : Bitfield<26,1> { }; + /* interrupt remapping enable */ + struct Ire : Bitfield<25,1> { }; + /* set interrupt remap table pointer */ struct Sirtp : Bitfield<24,1> { }; + + /* compatibility format interrupts */ + struct Cfi : Bitfield<23,1> { }; }; struct Global_status : Register<0x1c, 32> @@ -296,6 +345,14 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, struct Address : Bitfield<12,52> { }; }; + struct Irq_table_address : Register<0xB8, 64> + { + struct Size : Bitfield< 0, 4> { }; + struct Address : Bitfield<12,52> { }; + + /* not using extended interrupt mode (x2APIC) */ + }; + struct Fault_status : Register<0x34, 32> { /* fault record index */ @@ -437,6 +494,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, void _handle_faults(); + void _enable_irq_remapping(); + /** * Io_mmu interface */ @@ -523,6 +582,33 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, void apply_default_mappings(Pci::Bdf const & bdf) { _default_mappings.copy_stage2(_managed_root_table, bdf); } + /** + * Io_mmu interface for IRQ remapping + */ + void unmap_irq(Pci::Bdf const & bdf, unsigned idx) override + { + if (!_remap_irqs) + return; + + if (_irq_table.unmap(_irq_allocator, bdf, idx)) + invalidator().invalidate_irq(idx, false); + } + + Irq_info map_irq(Pci::Bdf const & bdf, + Irq_info const & info, + Irq_config const & config) override + { + if (!_remap_irqs) + return info; + + return _irq_table.map(_irq_allocator, bdf, info, config, [&] (unsigned idx) { + if (caching_mode()) + invalidator().invalidate_irq(idx, false); + else + flush_write_buffer(); + }); + } + /** * Io_mmu interface */ @@ -540,7 +626,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, buffer_registry, _env, ram_alloc, - _domain_allocator); + _domain_allocator, + _irq_allocator); if (!read() && read()) error("IOMMU requires 5-level translation tables (not implemented)"); @@ -551,7 +638,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, buffer_registry, _env, ram_alloc, - _domain_allocator); + _domain_allocator, + _irq_allocator); } /** @@ -568,6 +656,7 @@ class Intel::Io_mmu : private Attached_mmio<0x800>, ~Io_mmu() { _domain_allocator.free(_default_domain); + _table_allocator.destruct(_irq_table_phys); _destroy_domains(); } }; diff --git a/repos/pc/src/driver/platform/pc/intel/irq_remap_table.cc b/repos/pc/src/driver/platform/pc/intel/irq_remap_table.cc new file mode 100644 index 0000000000..f73c1ed405 --- /dev/null +++ b/repos/pc/src/driver/platform/pc/intel/irq_remap_table.cc @@ -0,0 +1,60 @@ +/* + * \brief Intel IOMMU Interrupt Remapping Table implementation + * \author Johannes Schlatow + * \date 2023-11-09 + * + * The interrupt remapping table is a page-aligned table structure of up to 64K + * 128bit entries (see section 9.9 [1]). Each entries maps a virtual interrupt + * index to a destination ID and vector. + * + * [1] "Intel® Virtualization Technology for Directed I/O" + * Revision 4.1, March 2023 + */ + +/* + * Copyright (C) 2023-2024 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +/* local includes */ +#include + +Intel::Irq_remap::Hi::access_t Intel::Irq_remap::hi_val(Pci::Bdf const & bdf) +{ + return Hi::Svt::bits(Hi::Svt::SOURCE_ID) | + Hi::Sq::bits(Hi::Sq::ALL_BITS) | + Hi::Source_id::bits(Pci::Bdf::rid(bdf)); +} + + +Intel::Irq_remap::Lo::access_t Intel::Irq_remap::lo_val(Irq_session::Info const & info, + Driver::Irq_controller::Irq_config const & config) +{ + using Irq_config = Driver::Irq_controller::Irq_config; + + Irq_address::access_t address = info.address; + Irq_data::access_t data = info.value; + + if (info.type == Irq_session::Info::MSI) + return + Lo::Present::bits(1) | + Lo::Destination_id::bits(Irq_address::Destination_id::get(address)) | + Lo::Destination_mode::bits(Irq_address::Destination_mode::get(address)) | + Lo::Redirection_hint::bits(Irq_address::Redirection_hint::get(address)) | + Lo::Trigger_mode::bits(Irq_data::Trigger_mode::get(data)) | + Lo::Delivery_mode::bits(Irq_data::Delivery_mode::get(data)) | + Lo::Vector::bits(Irq_data::Vector::get(data)); + else if (config.mode != Irq_config::INVALID) + return + Lo::Present::bits(1) | + Lo::Destination_id::bits(config.destination) | + Lo::Destination_mode::bits(config.mode == Irq_config::LOGICAL ? 1 : 0) | + Lo::Trigger_mode::bits(config.trigger == Irq_session::TRIGGER_LEVEL ? 1 : 0) | + Lo::Vector::bits(config.vector); + else + error("Unable to set IRQ remap table entry: missing information"); + + return 0; +} diff --git a/repos/pc/src/driver/platform/pc/intel/irq_remap_table.h b/repos/pc/src/driver/platform/pc/intel/irq_remap_table.h new file mode 100644 index 0000000000..4a1fce1c36 --- /dev/null +++ b/repos/pc/src/driver/platform/pc/intel/irq_remap_table.h @@ -0,0 +1,242 @@ +/* + * \brief Intel IOMMU Interrupt Remapping Table implementation + * \author Johannes Schlatow + * \date 2023-11-09 + * + * The interrupt remapping table is a page-aligned table structure of up to 64K + * 128bit entries (see section 9.9 [1]). Each entries maps a virtual interrupt + * index to a destination ID and vector. + * + * [1] "Intel® Virtualization Technology for Directed I/O" + * Revision 4.1, March 2023 + */ + +/* + * Copyright (C) 2023-2024 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#ifndef _SRC__DRIVERS__PLATFORM__INTEL__IRQ_REMAP_TABLE_H_ +#define _SRC__DRIVERS__PLATFORM__INTEL__IRQ_REMAP_TABLE_H_ + +/* Genode includes */ +#include +#include +#include +#include +#include + +/* platform-driver includes */ +#include + +/* local includes */ +#include + +namespace Intel { + using namespace Genode; + + class Irq_remap; + + template + class Irq_remap_table; +} + +struct Intel::Irq_remap +{ + struct Hi : Genode::Register<64> + { + struct Source_id : Bitfield<0, 16> { }; + + /* Source-id qualifier */ + struct Sq : Bitfield<16, 2> { + enum { + ALL_BITS = 0, + IGNORE_BITS_2 = 1, + IGNORE_BITS_2_1 = 2, + IGNORE_BITS_2_1_0 = 3 + }; + }; + + /* Source validation type */ + struct Svt : Bitfield<18, 2> { + enum { + DISABLE = 0, + SOURCE_ID = 1, + BUS_ID_ONLY = 2 + }; + }; + }; + + struct Lo : Genode::Register<64> + { + struct Present : Bitfield< 0, 1> { }; + struct Ignore_faults : Bitfield< 1, 1> { }; + struct Destination_mode : Bitfield< 2, 1> { }; + struct Redirection_hint : Bitfield< 3, 1> { }; + struct Trigger_mode : Bitfield< 4, 1> { }; + struct Delivery_mode : Bitfield< 5, 3> { }; + struct Vector : Bitfield<16, 8> { }; + struct Destination_id : Bitfield<40, 8> { }; + }; + + struct Irq_address : Register<64> + { + struct Destination_mode : Bitfield<2,1> { }; + struct Redirection_hint : Bitfield<3,1> { }; + + struct Format : Bitfield<4,1> { + enum { + COMPATIBILITY = 0, + REMAPPABLE = 1 + }; + }; + + struct Destination_id : Bitfield<12,8> { }; + struct Handle : Bitfield<5,15> { }; + }; + + struct Irq_data : Register<64> + { + struct Vector : Bitfield< 0,8> { }; + struct Delivery_mode : Bitfield< 8,3> { }; + struct Trigger_mode : Bitfield<15,1> { }; + }; + + static Hi::access_t hi_val(Pci::Bdf const &); + static Lo::access_t lo_val(Irq_session::Info const &, + Driver::Irq_controller::Irq_config const &); +}; + + +template +class Intel::Irq_remap_table +{ + public: + + static constexpr size_t ENTRIES_LOG2 = SIZE_LOG2 - 4; + static constexpr size_t ENTRIES = 1 << ENTRIES_LOG2; + + private: + + Irq_remap::Lo::access_t _entries[ENTRIES*2]; + + static size_t _lo_index(unsigned idx) { return 2*idx; } + static size_t _hi_index(unsigned idx) { return 2*idx + 1; } + + public: + + using Irq_allocator = Bit_allocator; + using Irq_info = Driver::Io_mmu::Irq_info; + using Irq_config = Driver::Irq_controller::Irq_config; + + bool present(unsigned idx) { + return Irq_remap::Lo::Present::get(_entries[_lo_index(idx)]); } + + unsigned destination_id(unsigned idx) { + return Irq_remap::Lo::Destination_id::get(_entries[_lo_index(idx)]); } + + Pci::rid_t source_id(unsigned idx) { + return Irq_remap::Hi::Source_id::get(_entries[_hi_index(idx)]); } + + template + Irq_info map(Irq_allocator & irq_alloc, + Pci::Bdf const & bdf, + Irq_info const & info, + Irq_config const & config, + FN && fn) + { + using Format = Irq_remap::Irq_address::Format; + + Irq_session::Info session_info = info.session_info; + + /* check whether info is already in remapped format */ + if (Format::get(session_info.address) == Format::REMAPPABLE) + return info; + + try { + unsigned idx = (unsigned)irq_alloc.alloc(); + + _entries[_hi_index(idx)] = Irq_remap::hi_val(bdf); + _entries[_lo_index(idx)] = Irq_remap::lo_val(session_info, config); + + clflush(&_entries[_lo_index(idx)]); + clflush(&_entries[_hi_index(idx)]); + + fn(idx); + + if (session_info.type == Irq_session::Info::Type::MSI) { + session_info.address = 0xfee00000U + | Irq_remap::Irq_address::Handle::bits(idx) + | Format::bits(Format::REMAPPABLE); + session_info.value = 0; + } + + /* XXX support multi-vectors MSI (see 5.1.5.2) */ + + /* return remapped Irq_info */ + return { Irq_info::REMAPPED, session_info, idx }; + + } catch (typename Irq_allocator::Out_of_indices) { + error("IRQ remapping table is full"); } + + return info; + } + + bool unmap(Irq_allocator & irq_alloc, Pci::Bdf const & bdf, unsigned idx) + { + Pci::rid_t rid = Pci::Bdf::rid(bdf); + + if (present(idx) && source_id(idx) == rid) { + _entries[_lo_index(idx)] = 0; + clflush(&_entries[_lo_index(idx)]); + irq_alloc.free(idx); + + return true; + } + + return false; + } + + void generate(Xml_generator & xml) + { + auto attribute_hex = [&] (Xml_generator & xml, + char const * name, + unsigned long long value) + { + xml.attribute(name, Genode::String<32>(Genode::Hex(value))); + }; + + for (unsigned idx = 0; idx < ENTRIES; idx++) { + if (!present(idx)) + continue; + + xml.node("irt_entry", [&] () { + attribute_hex(xml, "index", idx); + attribute_hex(xml, "source_id", source_id(idx)); + attribute_hex(xml, "hi", _entries[_hi_index(idx)]); + attribute_hex(xml, "lo", _entries[_lo_index(idx)]); + }); + } + } + + void flush_all() { + for (unsigned i=0; i < ENTRIES*2; i+=8) + clflush(&_entries[i]); + } + + Irq_remap_table() + { + for (unsigned i=0; i < ENTRIES; i++) { + _entries[_lo_index(i)] = 0; + _entries[_hi_index(i)] = 0; + } + + flush_all(); + } + +} __attribute__((aligned(4096))); + + +#endif /* _SRC__DRIVERS__PLATFORM__INTEL__IRQ_REMAP_TABLE_H_ */ diff --git a/repos/pc/src/driver/platform/pc/spec/x86_64/target.mk b/repos/pc/src/driver/platform/pc/spec/x86_64/target.mk index b4ae7ccc2e..4ecdd68c4e 100644 --- a/repos/pc/src/driver/platform/pc/spec/x86_64/target.mk +++ b/repos/pc/src/driver/platform/pc/spec/x86_64/target.mk @@ -11,6 +11,7 @@ SRC_CC += intel/page_table.cc SRC_CC += intel/default_mappings.cc SRC_CC += intel/invalidator.cc SRC_CC += ioapic.cc +SRC_CC += intel/irq_remap_table.cc INC_DIR += $(PRG_DIR)/../../