ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-02-11 13:35:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

platform/pc: implement IRQ remapping for Intel

Browse Source 
genodelabs/genode#5066
This commit is contained in:
Johannes Schlatow 2023-11-24 14:10:31 +01:00 committed by Christian Helmuth
parent 3c5b88111c
commit 4a0ce32faa
7 changed files with 484 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
repos/pc/src/driver/platform/pc/intel/invalidator.cc
View File

@ -120,6 +120,21 @@ void Intel::Register_invalidator::invalidate_all(Domain_id domain_id, Pci::rid_t
}
/* Clear interrupt entry cache */
void Intel::Queued_invalidator::invalidate_irq(unsigned idx, bool global)
{
Descriptor::access_t *entry = _tail();
Iec::Type::set(*entry, Iec::Type::IEC);
Iec::Global::set(*entry, global ? Iec::Global::GLOBAL : Iec::Global::INDEX);
Iec::Index::set(*entry, idx);
_next();
/* wait for completion */
while (!_empty());
}
/**
* Clear IOTLB.
*

14
repos/pc/src/driver/platform/pc/intel/invalidator.h
View File

@ -38,6 +38,7 @@ class Intel::Invalidator
virtual ~Invalidator() { }
virtual void invalidate_irq(unsigned, bool) { };
virtual void invalidate_iotlb(Domain_id) = 0;
virtual void invalidate_context(Domain_id domain, Pci::rid_t) = 0;
virtual void invalidate_all(Domain_id domain = Domain_id { Domain_id::INVALID },
@ -171,6 +172,7 @@ class Intel::Queued_invalidator : public Invalidator
enum {
CONTEXT = 1,
IOTLB = 2,
IEC = 4
};
};
@ -197,6 +199,17 @@ class Intel::Queued_invalidator : public Invalidator
struct Dr : Bitfield<7,1> { };
};
struct Iec : Descriptor
{
struct Global : Bitfield<4,1> {
enum {
GLOBAL = 0,
INDEX = 1
};
};
struct Index : Bitfield<32,16> { };
};
bool _empty() {
return _queue_mmio.read<Queue_mmio::Head>() == _queue_mmio.read<Queue_mmio::Tail>(); }
@ -216,6 +229,7 @@ class Intel::Queued_invalidator : public Invalidator
public:
void invalidate_irq(unsigned, bool) override;
void invalidate_iotlb(Domain_id) override;
void invalidate_context(Domain_id domain, Pci::rid_t) override;
void invalidate_all(Domain_id domain = Domain_id { Domain_id::INVALID },

58
repos/pc/src/driver/platform/pc/intel/io_mmu.cc
View File

@ -261,6 +261,9 @@ void Intel::Io_mmu::generate(Xml_generator & xml)
xml.attribute("mask", (bool)read<Fault_event_control::Mask>());
});
if (read<Global_status::Irtps>())
_irq_table.generate(xml);
if (!read<Global_status::Rtps>())
return;
@ -371,6 +374,43 @@ void Intel::Io_mmu::resume()
}
void Intel::Io_mmu::_enable_irq_remapping()
{
/*
* If IRQ remapping has already been enabled during boot, the kernel is
* in charge of the remapping. Since there is no way to get the required
* unremapped vector for requested MSI, we cannot take over control.
*/
if (read<Global_status::Ires>()) {
warning("IRQ remapping is controlled by kernel for ", name());
return;
}
/* caches must be cleared if Esirtps is not set */
if (read<Capability::Esirtps>())
invalidator().invalidate_irq(0, true);
/* set interrupt remapping table address */
write<Irq_table_address>(
Irq_table_address::Size::bits(Irq_table::ENTRIES_LOG2-1) |
Irq_table_address::Address::masked(_irq_table_phys));
/* issue set interrupt remapping table pointer command */
_global_command<Global_command::Sirtp>(1);
/* disable compatibility format interrupts */
_global_command<Global_command::Cfi>(0);
/* enable interrupt remapping */
_global_command<Global_command::Ire>(1);
log("enabled interrupt remapping for ", name());
_remap_irqs = true;
}
Intel::Io_mmu::Io_mmu(Env & env,
Io_mmu_devices & io_mmu_devices,
Device::Name const & name,
@ -380,10 +420,11 @@ Intel::Io_mmu::Io_mmu(Env & env,
: Attached_mmio(env, {(char *)range.start, range.size}),
Driver::Io_mmu(io_mmu_devices, name),
_env(env),
_managed_root_table(_env, table_allocator, *this, !coherent_page_walk()),
_default_mappings(_env, table_allocator, *this, !coherent_page_walk(),
_sagaw_to_levels()),
_domain_allocator(_max_domains()-1)
_table_allocator(table_allocator),
_domain_allocator(_max_domains()-1),
_managed_root_table(_env, _table_allocator, *this, !coherent_page_walk()),
_default_mappings(_env, _table_allocator, *this, !coherent_page_walk(),
_sagaw_to_levels())
{
if (_broken_device()) {
error(name, " reports invalid capability registers. Please disable VT-d/IOMMU.");
@ -422,6 +463,7 @@ Intel::Io_mmu::Io_mmu(Env & env,
_fault_irq->ack_irq();
Irq_session::Info info = _fault_irq->info();
if (info.type == Irq_session::Info::INVALID)
error("Unable to enable fault event interrupts for ", name);
else {
@ -430,4 +472,12 @@ Intel::Io_mmu::Io_mmu(Env & env,
write<Fault_event_control::Mask>(0);
}
}
/*
* We always enable IRQ remapping if its supported by the IOMMU. Note, there
* might be the possibility that the ACPI DMAR table says otherwise but
* we've never seen such a case yet.
*/
if (read<Extended_capability::Ir>())
_enable_irq_remapping();
}

107
repos/pc/src/driver/platform/pc/intel/io_mmu.h
View File

@ -32,6 +32,7 @@
#include <intel/domain_allocator.h>
#include <intel/default_mappings.h>
#include <intel/invalidator.h>
#include <intel/irq_remap_table.h>
#include <expanding_page_table_allocator.h>
namespace Intel {
@ -40,6 +41,14 @@ namespace Intel {
using Context_table_allocator = Managed_root_table::Allocator;
/**
* We use a 4KB interrupt remap table since kernels (nova, hw) do not
* support more than 256 interrupts anyway. We can thus reuse the
* context-table allocator.
*/
using Irq_table = Irq_remap_table<12>;
using Irq_allocator = Irq_table::Irq_allocator;
class Io_mmu;
class Io_mmu_factory;
}
@ -71,6 +80,7 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
Domain_allocator & _domain_allocator;
Domain_id _domain_id { _domain_allocator.alloc() };
bool _skip_invalidation { false };
Irq_allocator & _irq_allocator;
addr_t _translation_table_phys {
_table_allocator.construct<TABLE>() };
@ -131,7 +141,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
Registry<Dma_buffer> const & buffer_registry,
Env & env,
Ram_allocator & ram_alloc,
Domain_allocator & domain_allocator)
Domain_allocator & domain_allocator,
Irq_allocator & irq_allocator)
: Driver::Io_mmu::Domain(intel_iommu, md_alloc),
Registered_translation_table(intel_iommu),
_intel_iommu(intel_iommu),
@ -139,7 +150,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
_ram_alloc(ram_alloc),
_buffer_registry(buffer_registry),
_table_allocator(_env, md_alloc, ram_alloc, 2),
_domain_allocator(domain_allocator)
_domain_allocator(domain_allocator),
_irq_allocator(irq_allocator)
{
Invalidation_guard guard { *this, _intel_iommu.caching_mode() };
@ -168,7 +180,39 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
private:
Env & _env;
static
Irq_table & _irq_table_virt(Context_table_allocator & alloc, addr_t phys)
{
addr_t va { 0 };
alloc.with_table<Irq_table>(phys,
[&] (Irq_table & t) { va = (addr_t)&t; },
[&] () { /* never reached */ });
/**
* Dereferencing is save because _irq_table_phys is never 0
* (allocator throws exception) and with_table() thus always sets
* a valid virtual address.
*/
return *(Irq_table*)va;
}
Env & _env;
bool _verbose { false };
Context_table_allocator & _table_allocator;
const addr_t _irq_table_phys {
_table_allocator.construct<Irq_table>() };
Irq_table & _irq_table {
_irq_table_virt(_table_allocator, _irq_table_phys) };
Irq_allocator _irq_allocator { };
Report_helper _report_helper { *this };
Domain_allocator _domain_allocator;
Domain_id _default_domain { _domain_allocator.alloc() };
/**
* For a start, we keep a distinct root table for every hardware unit.
@ -184,12 +228,11 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
* The default root table holds default mappings (e.g. reserved memory)
* that needs to be accessible even if devices have not been acquired yet.
*/
bool _verbose { false };
Managed_root_table _managed_root_table;
Default_mappings _default_mappings;
Report_helper _report_helper { *this };
Domain_allocator _domain_allocator;
Domain_id _default_domain { _domain_allocator.alloc() };
bool _remap_irqs { false };
Constructible<Irq_connection> _fault_irq { };
Signal_handler<Io_mmu> _fault_handler {
_env.ep(), *this, &Io_mmu::_handle_faults };
@ -263,8 +306,14 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
/* queued invalidation enable */
struct Qie : Bitfield<26,1> { };
/* interrupt remapping enable */
struct Ire : Bitfield<25,1> { };
/* set interrupt remap table pointer */
struct Sirtp : Bitfield<24,1> { };
/* compatibility format interrupts */
struct Cfi : Bitfield<23,1> { };
};
struct Global_status : Register<0x1c, 32>
@ -296,6 +345,14 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
struct Address : Bitfield<12,52> { };
};
struct Irq_table_address : Register<0xB8, 64>
{
struct Size : Bitfield< 0, 4> { };
struct Address : Bitfield<12,52> { };
/* not using extended interrupt mode (x2APIC) */
};
struct Fault_status : Register<0x34, 32>
{
/* fault record index */
@ -437,6 +494,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
void _handle_faults();
void _enable_irq_remapping();
/**
* Io_mmu interface
*/
@ -523,6 +582,33 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
void apply_default_mappings(Pci::Bdf const & bdf) {
_default_mappings.copy_stage2(_managed_root_table, bdf); }
/**
* Io_mmu interface for IRQ remapping
*/
void unmap_irq(Pci::Bdf const & bdf, unsigned idx) override
{
if (!_remap_irqs)
return;
if (_irq_table.unmap(_irq_allocator, bdf, idx))
invalidator().invalidate_irq(idx, false);
}
Irq_info map_irq(Pci::Bdf const & bdf,
Irq_info const & info,
Irq_config const & config) override
{
if (!_remap_irqs)
return info;
return _irq_table.map(_irq_allocator, bdf, info, config, [&] (unsigned idx) {
if (caching_mode())
invalidator().invalidate_irq(idx, false);
else
flush_write_buffer();
});
}
/**
* Io_mmu interface
*/
@ -540,7 +626,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
buffer_registry,
_env,
ram_alloc,
_domain_allocator);
_domain_allocator,
_irq_allocator);
if (!read<Capability::Sagaw_3_level>() && read<Capability::Sagaw_5_level>())
error("IOMMU requires 5-level translation tables (not implemented)");
@ -551,7 +638,8 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
buffer_registry,
_env,
ram_alloc,
_domain_allocator);
_domain_allocator,
_irq_allocator);
}
/**
@ -568,6 +656,7 @@ class Intel::Io_mmu : private Attached_mmio<0x800>,
~Io_mmu()
{
_domain_allocator.free(_default_domain);
_table_allocator.destruct<Irq_table>(_irq_table_phys);
_destroy_domains();
}
};

60
repos/pc/src/driver/platform/pc/intel/irq_remap_table.cc Normal file
View File

@ -0,0 +1,60 @@
/*
* \brief Intel IOMMU Interrupt Remapping Table implementation
* \author Johannes Schlatow
* \date 2023-11-09
*
* The interrupt remapping table is a page-aligned table structure of up to 64K
* 128bit entries (see section 9.9 [1]). Each entries maps a virtual interrupt
* index to a destination ID and vector.
*
* [1] "Intel® Virtualization Technology for Directed I/O"
* Revision 4.1, March 2023
*/
/*
* Copyright (C) 2023-2024 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
/* local includes */
#include <intel/irq_remap_table.h>
Intel::Irq_remap::Hi::access_t Intel::Irq_remap::hi_val(Pci::Bdf const & bdf)
{
return Hi::Svt::bits(Hi::Svt::SOURCE_ID) |
Hi::Sq::bits(Hi::Sq::ALL_BITS) |
Hi::Source_id::bits(Pci::Bdf::rid(bdf));
}
Intel::Irq_remap::Lo::access_t Intel::Irq_remap::lo_val(Irq_session::Info const & info,
Driver::Irq_controller::Irq_config const & config)
{
using Irq_config = Driver::Irq_controller::Irq_config;
Irq_address::access_t address = info.address;
Irq_data::access_t data = info.value;
if (info.type == Irq_session::Info::MSI)
return
Lo::Present::bits(1) |
Lo::Destination_id::bits(Irq_address::Destination_id::get(address)) |
Lo::Destination_mode::bits(Irq_address::Destination_mode::get(address)) |
Lo::Redirection_hint::bits(Irq_address::Redirection_hint::get(address)) |
Lo::Trigger_mode::bits(Irq_data::Trigger_mode::get(data)) |
Lo::Delivery_mode::bits(Irq_data::Delivery_mode::get(data)) |
Lo::Vector::bits(Irq_data::Vector::get(data));
else if (config.mode != Irq_config::INVALID)
return
Lo::Present::bits(1) |
Lo::Destination_id::bits(config.destination) |
Lo::Destination_mode::bits(config.mode == Irq_config::LOGICAL ? 1 : 0) |
Lo::Trigger_mode::bits(config.trigger == Irq_session::TRIGGER_LEVEL ? 1 : 0) |
Lo::Vector::bits(config.vector);
else
error("Unable to set IRQ remap table entry: missing information");
return 0;
}

242
repos/pc/src/driver/platform/pc/intel/irq_remap_table.h Normal file
View File

@ -0,0 +1,242 @@
/*
* \brief Intel IOMMU Interrupt Remapping Table implementation
* \author Johannes Schlatow
* \date 2023-11-09
*
* The interrupt remapping table is a page-aligned table structure of up to 64K
* 128bit entries (see section 9.9 [1]). Each entries maps a virtual interrupt
* index to a destination ID and vector.
*
* [1] "Intel® Virtualization Technology for Directed I/O"
* Revision 4.1, March 2023
*/
/*
* Copyright (C) 2023-2024 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#ifndef _SRC__DRIVERS__PLATFORM__INTEL__IRQ_REMAP_TABLE_H_
#define _SRC__DRIVERS__PLATFORM__INTEL__IRQ_REMAP_TABLE_H_
/* Genode includes */
#include <util/register.h>
#include <util/bit_allocator.h>
#include <irq_session/irq_session.h>
#include <util/xml_generator.h>
#include <pci/types.h>
/* platform-driver includes */
#include <io_mmu.h>
/* local includes */
#include <cpu/clflush.h>
namespace Intel {
using namespace Genode;
class Irq_remap;
template <unsigned SIZE_LOG2>
class Irq_remap_table;
}
struct Intel::Irq_remap
{
struct Hi : Genode::Register<64>
{
struct Source_id : Bitfield<0, 16> { };
/* Source-id qualifier */
struct Sq : Bitfield<16, 2> {
enum {
ALL_BITS = 0,
IGNORE_BITS_2 = 1,
IGNORE_BITS_2_1 = 2,
IGNORE_BITS_2_1_0 = 3
};
};
/* Source validation type */
struct Svt : Bitfield<18, 2> {
enum {
DISABLE = 0,
SOURCE_ID = 1,
BUS_ID_ONLY = 2
};
};
};
struct Lo : Genode::Register<64>
{
struct Present : Bitfield< 0, 1> { };
struct Ignore_faults : Bitfield< 1, 1> { };
struct Destination_mode : Bitfield< 2, 1> { };
struct Redirection_hint : Bitfield< 3, 1> { };
struct Trigger_mode : Bitfield< 4, 1> { };
struct Delivery_mode : Bitfield< 5, 3> { };
struct Vector : Bitfield<16, 8> { };
struct Destination_id : Bitfield<40, 8> { };
};
struct Irq_address : Register<64>
{
struct Destination_mode : Bitfield<2,1> { };
struct Redirection_hint : Bitfield<3,1> { };
struct Format : Bitfield<4,1> {
enum {
COMPATIBILITY = 0,
REMAPPABLE = 1
};
};
struct Destination_id : Bitfield<12,8> { };
struct Handle : Bitfield<5,15> { };
};
struct Irq_data : Register<64>
{
struct Vector : Bitfield< 0,8> { };
struct Delivery_mode : Bitfield< 8,3> { };
struct Trigger_mode : Bitfield<15,1> { };
};
static Hi::access_t hi_val(Pci::Bdf const &);
static Lo::access_t lo_val(Irq_session::Info const &,
Driver::Irq_controller::Irq_config const &);
};
template <unsigned SIZE_LOG2>
class Intel::Irq_remap_table
{
public:
static constexpr size_t ENTRIES_LOG2 = SIZE_LOG2 - 4;
static constexpr size_t ENTRIES = 1 << ENTRIES_LOG2;
private:
Irq_remap::Lo::access_t _entries[ENTRIES*2];
static size_t _lo_index(unsigned idx) { return 2*idx; }
static size_t _hi_index(unsigned idx) { return 2*idx + 1; }
public:
using Irq_allocator = Bit_allocator<ENTRIES>;
using Irq_info = Driver::Io_mmu::Irq_info;
using Irq_config = Driver::Irq_controller::Irq_config;
bool present(unsigned idx) {
return Irq_remap::Lo::Present::get(_entries[_lo_index(idx)]); }
unsigned destination_id(unsigned idx) {
return Irq_remap::Lo::Destination_id::get(_entries[_lo_index(idx)]); }
Pci::rid_t source_id(unsigned idx) {
return Irq_remap::Hi::Source_id::get(_entries[_hi_index(idx)]); }
template <typename FN>
Irq_info map(Irq_allocator & irq_alloc,
Pci::Bdf const & bdf,
Irq_info const & info,
Irq_config const & config,
FN && fn)
{
using Format = Irq_remap::Irq_address::Format;
Irq_session::Info session_info = info.session_info;
/* check whether info is already in remapped format */
if (Format::get(session_info.address) == Format::REMAPPABLE)
return info;
try {
unsigned idx = (unsigned)irq_alloc.alloc();
_entries[_hi_index(idx)] = Irq_remap::hi_val(bdf);
_entries[_lo_index(idx)] = Irq_remap::lo_val(session_info, config);
clflush(&_entries[_lo_index(idx)]);
clflush(&_entries[_hi_index(idx)]);
fn(idx);
if (session_info.type == Irq_session::Info::Type::MSI) {
session_info.address = 0xfee00000U
| Irq_remap::Irq_address::Handle::bits(idx)
| Format::bits(Format::REMAPPABLE);
session_info.value = 0;
}
/* XXX support multi-vectors MSI (see 5.1.5.2) */
/* return remapped Irq_info */
return { Irq_info::REMAPPED, session_info, idx };
} catch (typename Irq_allocator::Out_of_indices) {
error("IRQ remapping table is full"); }
return info;
}
bool unmap(Irq_allocator & irq_alloc, Pci::Bdf const & bdf, unsigned idx)
{
Pci::rid_t rid = Pci::Bdf::rid(bdf);
if (present(idx) && source_id(idx) == rid) {
_entries[_lo_index(idx)] = 0;
clflush(&_entries[_lo_index(idx)]);
irq_alloc.free(idx);
return true;
}
return false;
}
void generate(Xml_generator & xml)
{
auto attribute_hex = [&] (Xml_generator & xml,
char const * name,
unsigned long long value)
{
xml.attribute(name, Genode::String<32>(Genode::Hex(value)));
};
for (unsigned idx = 0; idx < ENTRIES; idx++) {
if (!present(idx))
continue;
xml.node("irt_entry", [&] () {
attribute_hex(xml, "index", idx);
attribute_hex(xml, "source_id", source_id(idx));
attribute_hex(xml, "hi", _entries[_hi_index(idx)]);
attribute_hex(xml, "lo", _entries[_lo_index(idx)]);
});
}
}
void flush_all() {
for (unsigned i=0; i < ENTRIES*2; i+=8)
clflush(&_entries[i]);
}
Irq_remap_table()
{
for (unsigned i=0; i < ENTRIES; i++) {
_entries[_lo_index(i)] = 0;
_entries[_hi_index(i)] = 0;
}
flush_all();
}
} __attribute__((aligned(4096)));
#endif /* _SRC__DRIVERS__PLATFORM__INTEL__IRQ_REMAP_TABLE_H_ */

1
repos/pc/src/driver/platform/pc/spec/x86_64/target.mk
View File

@ -11,6 +11,7 @@ SRC_CC += intel/page_table.cc
SRC_CC += intel/default_mappings.cc
SRC_CC += intel/invalidator.cc
SRC_CC += ioapic.cc
SRC_CC += intel/irq_remap_table.cc
INC_DIR += $(PRG_DIR)/../../