ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-02-22 10:21:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

gpu/intel: GEN12+

Browse Source 
* clock frequency
* topology
* exec lists
* IRQ handling
* improved resource management

issue #4664
This commit is contained in:
Sebastian Sumpf 2022-07-01 14:35:04 +02:00 committed by Christian Helmuth
parent 852d21db14
commit 2aa01e309c
4 changed files with 527 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
repos/os/include/gpu/info_intel.h
View File

@ -33,6 +33,7 @@ struct Gpu::Info_intel
using Features = Genode::uint32_t; using Features = Genode::uint32_t;
using size_t = Genode::size_t; using size_t = Genode::size_t;
using Context_id = Genode::uint32_t; using Context_id = Genode::uint32_t;
using uint8_t = Genode::uint8_t;
Chip_id chip_id; Chip_id chip_id;
Features features; Features features;
@ -41,16 +42,55 @@ struct Gpu::Info_intel
Sequence_number last_completed; Sequence_number last_completed;
struct Revision { Genode::uint8_t value; } revision; struct Revision { Genode::uint8_t value; } revision;
struct Slice_mask { unsigned value; } slice_mask; struct Slice_mask { unsigned value; } slice_mask;
struct Subslice_mask { unsigned value; } subslice_mask; struct Subslice_mask { unsigned value; } subslice_mask;
struct Eu_total { unsigned value; } eus; struct Eu_total { unsigned value; } eus;
struct Subslices { unsigned value; } subslices; struct Subslices { unsigned value; } subslices;
struct Clock_frequency { unsigned value; } clock_frequency;
struct Topology
{
enum {
MAX_SLICES = 3,
MAX_SUBSLICES = 32,
MAX_EUS = 16,
};
uint8_t slice_mask { 0 };
uint8_t subslice_mask[MAX_SLICES * (MAX_SUBSLICES / 8)] { };
uint8_t eu_mask[MAX_SLICES * MAX_SUBSLICES * (MAX_EUS / 8)] { };
uint8_t max_slices { 0 };
uint8_t max_subslices { 0 };
uint8_t max_eus_per_subslice { 0 };
uint8_t ss_stride { 0 };
uint8_t eu_stride { 0 };
bool valid { false };
bool has_subslice(unsigned slice, unsigned subslice)
{
unsigned ss_idx = subslice / 8;
uint8_t mask = subslice_mask[slice * ss_stride + ss_idx];
return mask & (1u << (subslice % 8));
}
unsigned eu_idx(unsigned slice, unsigned subslice)
{
unsigned slice_stride = max_slices * eu_stride;
return slice * slice_stride + subslice * eu_stride;
}
};
struct Topology topology { };
Info_intel(Chip_id chip_id, Features features, size_t aperture_size, Info_intel(Chip_id chip_id, Features features, size_t aperture_size,
Context_id ctx_id, Sequence_number last, Context_id ctx_id, Sequence_number last,
Revision rev, Slice_mask s_mask, Subslice_mask ss_mask, Revision rev, Slice_mask s_mask, Subslice_mask ss_mask,
Eu_total eu, Subslices subslice) Eu_total eu, Subslices subslice, Clock_frequency clock_frequency,
Topology topology)
: :
chip_id(chip_id), features(features), chip_id(chip_id), features(features),
aperture_size(aperture_size), ctx_id(ctx_id), aperture_size(aperture_size), ctx_id(ctx_id),
@ -59,7 +99,9 @@ struct Gpu::Info_intel
slice_mask(s_mask), slice_mask(s_mask),
subslice_mask(ss_mask), subslice_mask(ss_mask),
eus(eu), eus(eu),
subslices(subslice) subslices(subslice),
clock_frequency(clock_frequency),
topology(topology)
{ } { }
}; };

264
repos/os/src/drivers/gpu/intel/main.cc
View File

@ -56,7 +56,7 @@ namespace Igd {
struct Igd::Device_info struct Igd::Device_info
{ {
enum Platform { UNKNOWN, BROADWELL, SKYLAKE, KABYLAKE, WHISKEYLAKE }; enum Platform { UNKNOWN, BROADWELL, SKYLAKE, KABYLAKE, WHISKEYLAKE, TIGERLAKE };
enum Stepping { A0, B0, C0, D0, D1, E0, F0, G0 }; enum Stepping { A0, B0, C0, D0, D1, E0, F0, G0 };
uint16_t id; uint16_t id;
@ -89,23 +89,52 @@ struct Igd::Device
** PCI ** ** PCI **
*********/ *********/
struct Pci_backend_alloc : Utils::Backend_alloc struct Pci_backend_alloc : Utils::Backend_alloc, Ram_allocator
{ {
Env &_env;
Platform::Connection &_pci; Platform::Connection &_pci;
Pci_backend_alloc(Platform::Connection &pci) : _pci(pci) { } Pci_backend_alloc(Env &env, Platform::Connection &pci)
: _env(env), _pci(pci) { }
Ram_dataspace_capability alloc(size_t size) override Ram_dataspace_capability alloc(size_t size) override
{ {
return _pci.retry_with_upgrade(Genode::Ram_quota{PAGE_SIZE}, enum {
Genode::Cap_quota{8}, [&] () { UPGRADE_RAM = 8 * PAGE_SIZE,
return _pci.alloc_dma_buffer(size, Genode::UNCACHED); }); UPGRADE_CAPS = 2,
UPGRADE_ATTEMPTS = ~0U
};
return retry<Genode::Out_of_ram>(
[&] () {
return retry<Genode::Out_of_caps>(
[&] () { return _pci.Client::alloc_dma_buffer(size, UNCACHED); },
[&] ()
{
if (_env.pd().avail_caps().value < UPGRADE_CAPS) {
warning("alloc dma buffer: out if caps");
throw Gpu::Session::Out_of_caps();
}
_pci.upgrade_caps(UPGRADE_CAPS);
},
UPGRADE_ATTEMPTS);
},
[&] ()
{
if (_env.pd().avail_ram().value < size) {
warning("alloc dma buffer: out of ram");
throw Gpu::Session::Out_of_ram();
}
_pci.upgrade_ram(size);
},
UPGRADE_ATTEMPTS);
} }
void free(Ram_dataspace_capability cap) override void free(Ram_dataspace_capability cap) override
{ {
if (!cap.valid()) { if (!cap.valid()) {
Genode::error("could not free, capability invalid"); error("could not free, capability invalid");
return; return;
} }
@ -117,15 +146,28 @@ struct Igd::Device
return _pci.dma_addr(ds_cap); return _pci.dma_addr(ds_cap);
} }
} _pci_backend_alloc { _platform }; /**
* RAM allocator interface
*/
size_t dataspace_size(Ram_dataspace_capability) const override { return 0; }
Alloc_result try_alloc(size_t size, Cache) override
{
return alloc(size);
}
} _pci_backend_alloc { _env, _platform };
Device_info _info { };
Gpu::Info_intel::Revision _revision { };
Gpu::Info_intel::Slice_mask _slice_mask { };
Gpu::Info_intel::Subslice_mask _subslice_mask { };
Gpu::Info_intel::Eu_total _eus { };
Gpu::Info_intel::Subslices _subslices { };
Gpu::Info_intel::Topology _topology { };
Gpu::Info_intel::Clock_frequency _clock_frequency { };
Device_info _info { };
Gpu::Info_intel::Revision _revision { };
Gpu::Info_intel::Slice_mask _slice_mask { };
Gpu::Info_intel::Subslice_mask _subslice_mask { };
Gpu::Info_intel::Eu_total _eus { };
Gpu::Info_intel::Subslices _subslices { };
bool _supported(Xml_node & supported, bool _supported(Xml_node & supported,
uint16_t dev_id, uint16_t dev_id,
@ -147,10 +189,10 @@ struct Igd::Device
return; return;
struct Igd::Device_info const info { struct Igd::Device_info const info {
.id = device, .id = device,
.generation = generation, .generation = generation,
.platform = platform_type(platform), .platform = platform_type(platform),
.features = 0 .features = 0,
}; };
if (info.platform == Igd::Device_info::Platform::UNKNOWN) if (info.platform == Igd::Device_info::Platform::UNKNOWN)
@ -158,7 +200,8 @@ struct Igd::Device
if (info.id == dev_id) { if (info.id == dev_id) {
_info = info; _info = info;
_revision.value = rev_id; _revision.value = rev_id;
_clock_frequency.value = _mmio.clock_frequency(generation);
found = true; found = true;
return; return;
@ -178,6 +221,8 @@ struct Igd::Device
return Igd::Device_info::Platform::KABYLAKE; return Igd::Device_info::Platform::KABYLAKE;
if (platform == "whiskeylake") if (platform == "whiskeylake")
return Igd::Device_info::Platform::WHISKEYLAKE; return Igd::Device_info::Platform::WHISKEYLAKE;
if (platform == "tigerlake")
return Igd::Device_info::Platform::TIGERLAKE;
return Igd::Device_info::UNKNOWN; return Igd::Device_info::UNKNOWN;
} }
@ -185,7 +230,6 @@ struct Igd::Device
** GGTT ** ** GGTT **
**********/ **********/
size_t _ggtt_size() size_t _ggtt_size()
{ {
/* /*
@ -575,7 +619,9 @@ struct Igd::Device
_device._slice_mask, _device._slice_mask,
_device._subslice_mask, _device._subslice_mask,
_device._eus, _device._eus,
_device._subslices); _device._subslices,
_device._clock_frequency,
_device._topology);
} }
~Vgpu() ~Vgpu()
@ -622,7 +668,9 @@ struct Igd::Device
+ ((_device.generation().value == 9) ? 6 : 0) + ((_device.generation().value == 9) ? 6 : 0)
+ ((_device.generation().value == 8) ? 20 : 22) /* epilog + w/a */ + ((_device.generation().value == 8) ? 20 : 22) /* epilog + w/a */
+ (dc_flush_wa ? 12 : 0); + (dc_flush_wa ? 12 : 0);
if (!el.ring_avail(need)) { el.ring_reset_and_fill_zero(); }
if (!el.ring_avail(need))
el.ring_reset_and_fill_zero();
/* save old tail */ /* save old tail */
Ring_buffer::Index const tail = el.ring_tail(); Ring_buffer::Index const tail = el.ring_tail();
@ -888,6 +936,23 @@ struct Igd::Device
_mmio.write<Igd::Mmio::EXECLIST_SUBMITPORT_RSCUNIT>(desc[0]); _mmio.write<Igd::Mmio::EXECLIST_SUBMITPORT_RSCUNIT>(desc[0]);
} }
void _submit_execlist_gen12(Engine<Rcs_context> &engine)
{
if (_mmio.read<Igd::Mmio::GEN12_EXECLIST_STATUS_RSCUNIT::Execution_queue_invalid>() == 0)
return;
Execlist &el = *engine.execlist;
_mmio.write<Igd::Mmio::GEN12_EXECLIST_SQ_CONTENTS_RSCUNIT>(el.elem0().low(), 0);
_mmio.write<Igd::Mmio::GEN12_EXECLIST_SQ_CONTENTS_RSCUNIT>(el.elem0().high(), 1);
for (unsigned i = 2; i < 16; i++)
_mmio.write<Igd::Mmio::GEN12_EXECLIST_SQ_CONTENTS_RSCUNIT>(0, i);
/* load SQ to EQ */
_mmio.write<Igd::Mmio::GEN12_EXECLIST_CONTROL_RSCUNIT::Load>(1);
}
Vgpu *_unschedule_current_vgpu() Vgpu *_unschedule_current_vgpu()
{ {
Vgpu *result = nullptr; Vgpu *result = nullptr;
@ -916,7 +981,10 @@ struct Igd::Device
_mmio.flush_gfx_tlb(); _mmio.flush_gfx_tlb();
_submit_execlist(rcs); if (_info.generation < 11)
_submit_execlist(rcs);
else
_submit_execlist_gen12(rcs);
_active_vgpu = gpu; _active_vgpu = gpu;
_timer.trigger_once(WATCHDOG_TIMEOUT); _timer.trigger_once(WATCHDOG_TIMEOUT);
@ -926,11 +994,6 @@ struct Igd::Device
** INTR ** ** INTR **
**********/ **********/
void _clear_rcs_iir(Mmio::GT_0_INTERRUPT_IIR::access_t const v)
{
_mmio.write_post<Mmio::GT_0_INTERRUPT_IIR>(v);
}
/** /**
* \return true, if Vgpu is done and has not further work * \return true, if Vgpu is done and has not further work
*/ */
@ -989,7 +1052,8 @@ struct Igd::Device
if (!_active_vgpu) { return; } if (!_active_vgpu) { return; }
Genode::error("watchdog triggered: engine stuck," Genode::error("watchdog triggered: engine stuck,"
" vGPU=", _active_vgpu->id()); " vGPU=", _active_vgpu->id(), " IRQ: ",
Hex(_mmio.read_irq_vector(_info.generation)));
_mmio.dump(); _mmio.dump();
_mmio.error_dump(); _mmio.error_dump();
_mmio.fault_dump(); _mmio.fault_dump();
@ -1021,7 +1085,7 @@ struct Igd::Device
_mmio.reset(_info.generation); _mmio.reset(_info.generation);
_mmio.clear_errors(); _mmio.clear_errors();
_mmio.init(); _mmio.init();
_mmio.enable_intr(); _mmio.enable_intr(_info.generation);
} }
/** /**
@ -1084,6 +1148,9 @@ struct Igd::Device
_subslices.value ++; _subslices.value ++;
_init_eu_total(3, SUBSLICE_MAX, 8); _init_eu_total(3, SUBSLICE_MAX, 8);
} else
if (_info.generation == 12) {
_init_topology_gen12();
} else } else
Genode::error("unsupported platform ", (int)_info.platform); Genode::error("unsupported platform ", (int)_info.platform);
@ -1093,6 +1160,46 @@ struct Igd::Device
_timer.sigh(_watchdog_timeout_sigh); _timer.sigh(_watchdog_timeout_sigh);
} }
void _init_topology_gen12()
{
/* NOTE: This needs to be different for DG2 and Xe_HP */
_topology.max_slices = 1;
_topology.max_subslices = 6;
_topology.max_eus_per_subslice = 16;
_topology.ss_stride = 1; /* roundup(6/8) */
_topology.eu_stride = 2; /* 16/8 */
/* NOTE: 1 for >=12.5 */
_topology.slice_mask = _mmio.read<Igd::Mmio::MIRROR_GT_SLICE_EN::Enabled>();
if (_topology.slice_mask > 1)
error("topology: slices > 1");
uint32_t dss_en = _mmio.read<Igd::Mmio::MIRROR_GT_DSS_ENABLE>();
memcpy(_topology.subslice_mask, &dss_en, sizeof(dss_en));
/* Gen12 uses dual-subslices */
uint8_t eu_en_fuse = ~_mmio.read<Igd::Mmio::MIRROR_EU_DISABLE0::Disabled>();
uint16_t eu_en { 0 };
for (unsigned i = 0; i < _topology.max_eus_per_subslice / 2; i++) {
if (eu_en_fuse & (1u << i)) {
_eus.value += 2;
eu_en |= (3u << (i * 2));
}
}
for (unsigned i = 0; i < _topology.max_subslices; i++) {
if (_topology.has_subslice(0, i)) {
_subslices.value++;
unsigned offset = _topology.eu_idx(0, i);
for (unsigned j = 0; j < _topology.eu_stride; j++) {
_topology.eu_mask[offset + j] = (eu_en >> (8 * j)) & 0xff;
}
}
}
_topology.valid = true;
}
void _clock_gating() void _clock_gating()
{ {
if (_info.platform == Device_info::Platform::KABYLAKE) { if (_info.platform == Device_info::Platform::KABYLAKE) {
@ -1382,22 +1489,24 @@ struct Igd::Device
_clear_fence(id); _clear_fence(id);
} }
unsigned handle_irq() bool handle_irq()
{ {
Mmio::MASTER_INT_CTL::access_t master = _mmio.read<Mmio::MASTER_INT_CTL>(); bool display_irq = _mmio.display_irq(_info.generation);
/* handle render interrupts only */ /* handle render interrupts only */
if (Mmio::MASTER_INT_CTL::Render_interrupts_pending::get(master) == 0) if (_mmio.render_irq(_info.generation) == false)
return master; return display_irq;
_mmio.disable_master_irq(); _mmio.disable_master_irq(_info.generation);
Mmio::GT_0_INTERRUPT_IIR::access_t const v = _mmio.read<Mmio::GT_0_INTERRUPT_IIR>(); Mmio::GEN12_RENDER_INTR_VEC::access_t const v = _mmio.read_irq_vector(_info.generation);
bool const ctx_switch = Mmio::GT_0_INTERRUPT_IIR::Cs_ctx_switch_interrupt::get(v); bool const ctx_switch = Mmio::GEN12_RENDER_INTR_VEC::Cs_ctx_switch_interrupt::get(v);
bool const user_complete = Mmio::GT_0_INTERRUPT_IIR::Cs_mi_user_interrupt::get(v); bool const user_complete = Mmio::GEN12_RENDER_INTR_VEC::Cs_mi_user_interrupt::get(v);
if (v) { _clear_rcs_iir(v); } if (v) {
_mmio.clear_render_irq(_info.generation, v);
}
Vgpu *notify_gpu = nullptr; Vgpu *notify_gpu = nullptr;
if (user_complete) { if (user_complete) {
@ -1428,10 +1537,10 @@ struct Igd::Device
} }
} }
return master; return display_irq;
} }
void enable_master_irq() { _mmio.enable_master_irq(); } void enable_master_irq() { _mmio.enable_master_irq(_info.generation); }
private: private:
@ -1465,10 +1574,10 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
Genode::Env &_env; Genode::Env &_env;
Genode::Region_map &_rm; Genode::Region_map &_rm;
Constrained_ram_allocator _ram; Constrained_ram_allocator _ram;
Heap _heap { _ram, _rm }; Igd::Device &_device;
Heap _heap { _device._pci_backend_alloc, _rm };
Capability<Gpu::Session> _session_cap { cap() }; Capability<Gpu::Session> _session_cap { cap() };
Igd::Device &_device;
Igd::Device::Vgpu _vgpu; Igd::Device::Vgpu _vgpu;
struct Resource_guard struct Resource_guard
@ -1483,24 +1592,38 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
{ } { }
/* worst case */ /* worst case */
bool avail_caps() { return _cap_quota_guard.have_avail(Cap_quota { 5 }); } bool avail_caps() { return _cap_quota_guard.have_avail(Cap_quota { 15 }); }
/* size + possible heap allocations */ /* size + possible heap allocations + possible page table allocation +
* + 16KB for 'ep.manage' + unkown overhead */
bool avail_ram(size_t size = 0) { bool avail_ram(size_t size = 0) {
return _ram_quota_guard.have_avail(Ram_quota { size + 2*1024*1024 }); } return _ram_quota_guard.have_avail(Ram_quota { size + 2*1024*1024+4096 +
1024*1024 + 16*1024 + 1024*1024}); }
void withdraw(size_t caps, size_t ram) void withdraw(size_t caps_old, size_t caps_new,
size_t ram_old, size_t ram_new)
{ {
size_t caps = caps_old > caps_new ? caps_old - caps_new : 0;
size_t ram = ram_old > ram_new ? ram_old - ram_new : 0;
try { try {
_cap_quota_guard.withdraw(Cap_quota { caps }); _cap_quota_guard.withdraw(Cap_quota { caps });
_ram_quota_guard.withdraw(Ram_quota { ram }); _ram_quota_guard.withdraw(Ram_quota { ram });
} catch (... /* intentional catch-all */) { } catch (Genode::Out_of_caps) {
/* /*
* At this point something in the accounting went wrong * At this point something in the accounting went wrong
* and as quick-fix let the client abort rather than the * and as quick-fix let the client abort rather than the
* multiplexer. * multiplexer.
*/ */
throw Service_denied(); Genode::error("Quota guard out of caps! from ", __builtin_return_address(0));
throw Gpu::Session::Out_of_caps();
} catch (Genode::Out_of_ram) {
Genode::error("Quota guard out of ram! from ", __builtin_return_address(0));
Genode::error("guard ram: ", _ram_quota_guard.avail().value, " requested: ", ram);
throw Gpu::Session::Out_of_ram();
} catch (...) {
Genode::error("Unknown exception in 'Resourcd_guard::withdraw'");
throw;
} }
} }
@ -1600,9 +1723,13 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
void _apply_buffer_local(Gpu::Buffer_id id, FN const &fn) void _apply_buffer_local(Gpu::Buffer_id id, FN const &fn)
{ {
Buffer_local::Id_space::Id local_id { .value = id.value }; Buffer_local::Id_space::Id local_id { .value = id.value };
_buffer_space.apply<Buffer_local>(local_id, [&] (Buffer_local &buffer) { try {
fn(buffer); _buffer_space.apply<Buffer_local>(local_id, [&] (Buffer_local &buffer) {
}); fn(buffer);
});
} catch (Buffer_local::Id_space::Unknown_id) {
error("Unknown id: ", id.value);
}
} }
Genode::uint64_t seqno { 0 }; Genode::uint64_t seqno { 0 };
@ -1758,8 +1885,8 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
size_t ram_before = _env.pd().avail_ram().value; size_t ram_before = _env.pd().avail_ram().value;
Ram_dataspace_capability ds_cap = _device.alloc_buffer(_heap, size); Ram_dataspace_capability ds_cap = _device.alloc_buffer(_heap, size);
addr_t phys_addr = _device.dma_addr(ds_cap); addr_t phys_addr = _device.dma_addr(ds_cap);
Buffer *buffer = new (&_heap) Buffer(ds_cap, phys_addr, _session_cap); Buffer *buffer = new (&_heap) Buffer(ds_cap, phys_addr, _session_cap);
_env.ep().manage(*buffer); _env.ep().manage(*buffer);
try { try {
@ -1768,17 +1895,17 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
_env.ep().dissolve(*buffer); _env.ep().dissolve(*buffer);
destroy(&_heap, buffer); destroy(&_heap, buffer);
_device.free_buffer(_heap, ds_cap); _device.free_buffer(_heap, ds_cap);
throw Gpu::Session_component::Conflicting_id(); return Dataspace_capability();
} }
size_t caps_after = _env.pd().avail_caps().value; size_t caps_after = _env.pd().avail_caps().value;
size_t ram_after = _env.pd().avail_ram().value; size_t ram_after = _env.pd().avail_ram().value;
/* limit to buffer size */ /* limit to buffer size for replenish */
buffer->ram_used = min(ram_before - ram_after, size); buffer->ram_used = min(ram_before > ram_after ? ram_before - ram_after : 0, size);
buffer->caps_used = (caps_before - caps_after) > 0; buffer->caps_used = caps_before > caps_after ? true : false;
_resource_guard.withdraw(caps_before - caps_after, buffer->ram_used); _resource_guard.withdraw(caps_before, caps_after, ram_before, ram_after);
return ds_cap; return ds_cap;
} }
@ -1877,8 +2004,8 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
size_t caps_after = _env.pd().avail_caps().value; size_t caps_after = _env.pd().avail_caps().value;
size_t ram_after = _env.pd().avail_ram().value; size_t ram_after = _env.pd().avail_ram().value;
_resource_guard.withdraw(caps_before - caps_after, _resource_guard.withdraw(caps_before, caps_after,
ram_before - ram_after); ram_before , ram_after);
return true; return true;
}; };
@ -1956,8 +2083,8 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
size_t caps_after = _env.pd().avail_caps().value; size_t caps_after = _env.pd().avail_caps().value;
size_t ram_after = _env.pd().avail_ram().value; size_t ram_after = _env.pd().avail_ram().value;
_resource_guard.withdraw(caps_before - caps_after, _resource_guard.withdraw(caps_before, caps_after,
ram_before - ram_after); ram_before, ram_after);
return true; return true;
} }
@ -2081,7 +2208,7 @@ class Gpu::Root : public Gpu::Root_component
using namespace Genode; using namespace Genode;
return new (md_alloc()) return new (md_alloc())
Session_component(_env, _env.ep(), _env.ram(), _env.rm(), Session_component(_env, _env.ep(), _device->_pci_backend_alloc, _env.rm(),
resources, resources,
session_label_from_args(args), session_label_from_args(args),
session_diag_from_args(args), session_diag_from_args(args),
@ -2198,9 +2325,9 @@ struct Main : Irq_ack_handler, Gpu_reset_handler
void handle_irq() void handle_irq()
{ {
unsigned master = 0; bool display_irq = false;
if (_igd_device.constructed()) if (_igd_device.constructed())
master = _igd_device->handle_irq(); display_irq = _igd_device->handle_irq();
/* GPU not present forward all IRQs to platform client */ /* GPU not present forward all IRQs to platform client */
else { else {
_platform_root.handle_irq(); _platform_root.handle_irq();
@ -2211,10 +2338,9 @@ struct Main : Irq_ack_handler, Gpu_reset_handler
* GPU present check for display engine related IRQs before calling platform * GPU present check for display engine related IRQs before calling platform
* client * client
*/ */
using Master = Igd::Mmio::MASTER_INT_CTL; if (display_irq && _platform_root.handle_irq()) {
if (Master::De_interrupts_pending::get(master) &&
(_platform_root.handle_irq()))
return; return;
}
ack_irq(); ack_irq();
} }

298
repos/os/src/drivers/gpu/intel/mmio.h
View File

@ -194,6 +194,66 @@ class Igd::Mmio : public Platform::Device::Mmio
struct GT_3_INTERRUPT_IIR : GT_3_INTERRUPT<0x44338> { }; struct GT_3_INTERRUPT_IIR : GT_3_INTERRUPT<0x44338> { };
struct GT_3_INTERRUPT_IER : GT_3_INTERRUPT<0x4433C> { }; struct GT_3_INTERRUPT_IER : GT_3_INTERRUPT<0x4433C> { };
/**
* GEN11+ interrupt registers
* IHD-OS-TGL-Vol 2c-12.21 part1 p. 1039 ff.
*/
struct GEN12_GFX_MSTR_INTR : Register<0x190010, 32>
{
struct Master_interrupt_enable : Bitfield<31, 1> { };
struct Display : Bitfield<16, 1> { };
struct Gt_dw_1 : Bitfield< 1, 1> { };
struct Gt_dw_0 : Bitfield< 0, 1> { };
};
/* p. 1089 */
struct GEN12_GT_INTR_DW0 : Register<0x190018, 32>
{
struct Rcs0 : Bitfield<0, 1> { };
};
/* p. 1091 */
struct GEN12_INTR_IDENTITY_REG0 : Register<0x190060, 32>
{
struct Valid : Bitfield<31, 1> { };
struct Engine_interrupt : Bitfield<0, 16> { };
};
/*
* p. 1092
*
* Select engine ID to read INTR_IDENTITY from, use only one bit at a time,
* layout is as INTR_DW0 register
*/
struct GEN12_INTR_IIR_SELECTOR0 : Register<0x190070, 32, true>
{
struct Rcs0 : Bitfield<0, 1> { };
};
/* p. 1077 */
struct GEN12_RENDER_COPY_INTR_ENABLE : Register<0x190030, 32>
{
struct Render_enable : Bitfield<16, 16> { };
struct Copy_enable : Bitfield< 0, 16> { };
};
/*
* IHD-OS-TGL-Vol 2d-12.21 p. 818 ff
*/
struct GEN12_RENDER_INTR_VEC : Genode::Register<16>
{
struct Catastrophic_error : Bitfield<15, 1> { };
struct Eu_restart : Bitfield<14, 1> { };
struct Context_stall : Bitfield<13, 1> { };
struct Cs_wait_on_semaphore : Bitfield<11, 1> { };
struct Cs_ctx_switch_interrupt : Bitfield< 8, 1> { };
struct Legacy_page_fault_error : Bitfield< 7, 1> { };
struct Cs_watchdog_counter_expired : Bitfield< 6, 1> { };
struct Cs_pipe_control_notify : Bitfield< 4, 1> { };
struct Cs_error_interrupt : Bitfield< 3, 1> { };
struct Cs_mi_user_interrupt : Bitfield< 0, 1> { };
};
/************************** /**************************
** Page-table registers ** ** Page-table registers **
**************************/ **************************/
@ -696,6 +756,36 @@ class Igd::Mmio : public Platform::Device::Mmio
struct ELEM_DESCRIPTOR1 : Register<0x4400, 32> { }; struct ELEM_DESCRIPTOR1 : Register<0x4400, 32> { };
struct ELEM_DESCRIPTOR2 : Register<0x4404, 32> { }; struct ELEM_DESCRIPTOR2 : Register<0x4404, 32> { };
/**
* GEN9+ GPM unit lx 5.15
*/
struct CTC_MODE : Register<0xa26c, 32>
{
struct Source_divide_logic : Bitfield<0, 1> { };
};
/**
* GEN8+ RPM unit
*/
struct RPM_CONFIG0 : Register<0x0d00, 32>
{
struct Ctc_shift : Bitfield<1, 2> { };
struct Crystal_clock_frequency : Bitfield<3, 3> { };
static uint32_t clock_frequency(Crystal_clock_frequency::access_t freq)
{
switch (freq) {
case 0: return 24'000'000;
case 1: return 19'200'000;
case 2: return 38'400'000;
case 3: return 25'000'000;
default:
Genode::error("Unknown crystal clock frequency: ", freq);
return 0;
}
}
};
/** /**
* Forcewake for GEN9 & GEN10, lx 5.13 * Forcewake for GEN9 & GEN10, lx 5.13
*/ */
@ -823,6 +913,28 @@ class Igd::Mmio : public Platform::Device::Mmio
*/ */
struct EU_DISABLE : Register_array<0x9134, 32, 12, 8> { }; struct EU_DISABLE : Register_array<0x9134, 32, 12, 8> { };
/*
* IHD-OS-TGL-Vol 2c-12.21 part2 p. 81
*/
struct MIRROR_EU_DISABLE0 : Register<0x9134, 32>
{
struct Disabled : Bitfield<0, 8> { };
};
/*
* IHD-OS-TGL-Vol 2c-12.2 part 2 p.98
*/
struct MIRROR_GT_SLICE_EN : Register<0x9138, 32>
{
struct Enabled : Bitfield<0, 8> { };
};
/*
* IHD-OS-TGL-Vol 2c-12.21 part2 p.97
*/
struct MIRROR_GT_DSS_ENABLE : Register<0x913c, 32> { };
/* /*
* IHD-OS-BDW-Vol 2c-11.15 p. 611 ff. * IHD-OS-BDW-Vol 2c-11.15 p. 611 ff.
*/ */
@ -835,6 +947,7 @@ class Igd::Mmio : public Platform::Device::Mmio
struct Fence_valid : Bitfield< 0, 1> { }; struct Fence_valid : Bitfield< 0, 1> { };
}; };
/* /*
* IHD-OS-BDW-Vol 12-11.15 p. 5 * IHD-OS-BDW-Vol 12-11.15 p. 5
*/ */
@ -845,6 +958,7 @@ class Igd::Mmio : public Platform::Device::Mmio
** EXECLIST registers ** ** EXECLIST registers **
************************/ ************************/
/* GEN8-11 */
/* /*
* IHD-OS-BDW-Vol 2c-11.15 p. 435 * IHD-OS-BDW-Vol 2c-11.15 p. 435
*/ */
@ -895,6 +1009,34 @@ class Igd::Mmio : public Platform::Device::Mmio
struct EXECLIST_SUBMITPORT_VCSUNIT1 : EXECLIST_SUBMITPORT_BASE<0x1C000> { }; struct EXECLIST_SUBMITPORT_VCSUNIT1 : EXECLIST_SUBMITPORT_BASE<0x1C000> { };
struct EXECLIST_SUBMITPORT_BSCUNIT : EXECLIST_SUBMITPORT_BASE<0x22000> { }; struct EXECLIST_SUBMITPORT_BSCUNIT : EXECLIST_SUBMITPORT_BASE<0x22000> { };
/* GEN12 */
template <long int BASE>
struct GEN12_EXECLIST_STATUS_BASE : Register<BASE + 0x234, 64>
{
using B = Register<BASE + 0x234, 64>;
struct Execution_queue_invalid : B::template Bitfield<0, 1> { };
};
struct GEN12_EXECLIST_STATUS_RSCUNIT : GEN12_EXECLIST_STATUS_BASE<0x02000> { };
/*
* IHD-OS-TGL-Vol 2c-12.21 p. 896
*/
struct GEN12_EXECLIST_SQ_CONTENTS_RSCUNIT : Register_array<0x2510, 32, 16, 32> { };
/*
* IHD-OS-TGL-Vol 2c-12.21 p.891
*/
template <long int BASE>
struct GEN12_EXECLIST_CONTROL_BASE : Register<BASE + 0x550, 32>
{
using B = Register<BASE + 0x550, 32>;
/* load submission queue into exec queue */
struct Load : B::template Bitfield<0, 1> { };
};
struct GEN12_EXECLIST_CONTROL_RSCUNIT : GEN12_EXECLIST_CONTROL_BASE<0x02000> { };
/* /*
* IHD-OS-BDW-Vol 2c-11.15 p. 266 ff. * IHD-OS-BDW-Vol 2c-11.15 p. 266 ff.
* IHD-OS-BDW-Vol 6-11.15 p. 19 ff. * IHD-OS-BDW-Vol 6-11.15 p. 19 ff.
@ -1023,7 +1165,6 @@ class Igd::Mmio : public Platform::Device::Mmio
using namespace Genode; using namespace Genode;
while (read<typename REG_ACK::Kernel>()) { while (read<typename REG_ACK::Kernel>()) {
log(__func__, " ", __LINE__, " wait ", Hex(read<REG_ACK>()));
_delayer.usleep(500 * 1000); _delayer.usleep(500 * 1000);
_fw_enable_wa<REG, REG_ACK>(); _fw_enable_wa<REG, REG_ACK>();
@ -1048,7 +1189,6 @@ class Igd::Mmio : public Platform::Device::Mmio
using namespace Genode; using namespace Genode;
while (read<typename REG_ACK::Fallback_kernel>()) { while (read<typename REG_ACK::Fallback_kernel>()) {
log(__func__, " ", __LINE__, " wait ", Hex(read<REG_ACK>()));
_delayer.usleep(500 * 1000); _delayer.usleep(500 * 1000);
} }
@ -1059,12 +1199,7 @@ class Igd::Mmio : public Platform::Device::Mmio
_delayer.usleep(100 * 1000); _delayer.usleep(100 * 1000);
log(__func__, " ", __LINE__, " ",
Genode::Hex(read<REG>()), " ",
Genode::Hex(read<REG_ACK>()));
while (!(read<typename REG_ACK::Fallback_kernel>())) { while (!(read<typename REG_ACK::Fallback_kernel>())) {
log(__func__, " ", __LINE__, " wait ", Hex(read<REG_ACK>()));
_delayer.usleep(500 * 1000); _delayer.usleep(500 * 1000);
} }
@ -1092,8 +1227,6 @@ class Igd::Mmio : public Platform::Device::Mmio
write<REG>(v); write<REG>(v);
while (read<typename REG_ACK::Kernel>()) { while (read<typename REG_ACK::Kernel>()) {
Genode::log(__func__, " ", __LINE__, " wait ",
Genode::Hex(read<REG_ACK>()));
_delayer.usleep(500 * 1000); _delayer.usleep(500 * 1000);
} }
} }
@ -1119,6 +1252,12 @@ class Igd::Mmio : public Platform::Device::Mmio
/* TODO DE intr handling (p. 363) */ /* TODO DE intr handling (p. 363) */
} }
void _intr_reset_gen12()
{
write<GEN12_GFX_MSTR_INTR::Master_interrupt_enable>(0);
write_post<HWSTAM>(0xffffffff);
}
/** /**
* Enable interrupts * Enable interrupts
*/ */
@ -1201,6 +1340,16 @@ class Igd::Mmio : public Platform::Device::Mmio
write_post<MASTER_INT_CTL::Master_interrupt_enable>(1); write_post<MASTER_INT_CTL::Master_interrupt_enable>(1);
} }
void _intr_enable_gen12()
{
GEN12_RENDER_INTR_VEC::access_t vec { 0 };
GEN12_RENDER_INTR_VEC::Cs_mi_user_interrupt::set(vec, 1);
GEN12_RENDER_INTR_VEC::Cs_ctx_switch_interrupt::set(vec, 1);
write<GEN12_RENDER_COPY_INTR_ENABLE::Render_enable>(vec);
write_post<HWSTAM>(~vec);
write<GEN12_GFX_MSTR_INTR::Master_interrupt_enable>(1);
}
/** /**
* Disable Render P-states * Disable Render P-states
*/ */
@ -1421,6 +1570,31 @@ class Igd::Mmio : public Platform::Device::Mmio
write<GFX_MODE>(v); write<GFX_MODE>(v);
} }
uint32_t _clock_frequency_gen12()
{
uint32_t freq { 0 };
/* TIMESTAMP_OVERRIDE when source divide logic */
if (read<CTC_MODE::Source_divide_logic>()) {
Genode::error("clock frequency: source divide not implemented");
}
/* RMP_CONFIG when crystal logic */
else {
freq = RPM_CONFIG0::clock_frequency(
read<RPM_CONFIG0::Crystal_clock_frequency>());
Genode::log("clock frequency: ", freq, " Hz from crystal logic");
/*
* Shift by Ctc_shift for lowest timestamp counter because counter may
* not increment at every cycle
*/
freq >>= 3 - read<RPM_CONFIG0::Ctc_shift>();
}
return freq;
}
public: public:
Mmio(Platform::Device & device, Genode::Env & env) Mmio(Platform::Device & device, Genode::Env & env)
@ -1433,6 +1607,15 @@ class Igd::Mmio : public Platform::Device::Mmio
(void)read<T>(); (void)read<T>();
} }
uint32_t clock_frequency(unsigned const generation)
{
if (generation >= 11)
return _clock_frequency_gen12();
return 0;
}
void forcewake_gen8_enable() { _fw_enable(FORCEWAKE_ID_RENDER); } void forcewake_gen8_enable() { _fw_enable(FORCEWAKE_ID_RENDER); }
void forcewake_gen8_disable() { _fw_disable(FORCEWAKE_ID_RENDER); } void forcewake_gen8_disable() { _fw_disable(FORCEWAKE_ID_RENDER); }
@ -1440,12 +1623,12 @@ class Igd::Mmio : public Platform::Device::Mmio
{ {
_fw_enable_gt(); _fw_enable_gt();
_fw_enable_render(); _fw_enable_render();
_fw_enable_media(); //_fw_enable_media();
} }
void forcewake_gen9_disable() void forcewake_gen9_disable()
{ {
_fw_disable_media(); //_fw_disable_media();
_fw_disable_render(); _fw_disable_render();
_fw_disable_gt(); _fw_disable_gt();
} }
@ -1457,6 +1640,7 @@ class Igd::Mmio : public Platform::Device::Mmio
forcewake_gen8_enable(); forcewake_gen8_enable();
return; return;
case 9: case 9:
case 12:
forcewake_gen9_enable(); forcewake_gen9_enable();
return; return;
default: default:
@ -1470,7 +1654,8 @@ class Igd::Mmio : public Platform::Device::Mmio
case 8: case 8:
forcewake_gen8_disable(); forcewake_gen8_disable();
return; return;
case 9: case 9: [[fallthrough]];
case 12:
forcewake_gen9_disable(); forcewake_gen9_disable();
return; return;
default: default:
@ -1486,7 +1671,10 @@ class Igd::Mmio : public Platform::Device::Mmio
return; return;
case 9: case 9:
reset_gen9(); reset_gen9();
return; break;
case 12:
reset_gen12();
break;;
default: default:
Genode::error(__func__, " unsupported generation ", generation); Genode::error(__func__, " unsupported generation ", generation);
} }
@ -1516,29 +1704,101 @@ class Igd::Mmio : public Platform::Device::Mmio
_set_page_attributes(); _set_page_attributes();
} }
void reset_gen12()
{
_intr_reset_gen12();
_fw_reset_gen9();
forcewake_gen9_enable();
_reset_device();
_reset_fences();
_disable_nde_handshake();
_set_page_attributes();
}
void init() void init()
{ {
_disable_rps(); _disable_rps();
_enable_execlist(); _enable_execlist();
} }
void enable_intr() void enable_intr(unsigned const generation)
{ {
write<Igd::Mmio::RCS_EMR>(0xffffff00); write<Igd::Mmio::RCS_EMR>(0xffffff00);
_intr_enable(); if (generation < 11)
_intr_enable();
else
_intr_enable_gen12();
} }
void disable_master_irq() void disable_master_irq(unsigned const generation)
{ {
write_post<Igd::Mmio::MASTER_INT_CTL::Master_interrupt_enable>(0); if (generation < 11)
write_post<Igd::Mmio::MASTER_INT_CTL::Master_interrupt_enable>(0);
else
write<GEN12_GFX_MSTR_INTR::Master_interrupt_enable>(0);
} }
void enable_master_irq() void enable_master_irq(unsigned const generation)
{ {
write_post<Igd::Mmio::MASTER_INT_CTL::Master_interrupt_enable>(1); if (generation < 11)
write_post<Igd::Mmio::MASTER_INT_CTL::Master_interrupt_enable>(1);
else
write<GEN12_GFX_MSTR_INTR::Master_interrupt_enable>(1);
} }
bool render_irq(unsigned const generation)
{
if (generation < 11)
return read<MASTER_INT_CTL::Render_interrupts_pending>() == 1;
else {
if (read<GEN12_GFX_MSTR_INTR::Gt_dw_0>() == 1 &&
read<GEN12_GT_INTR_DW0::Rcs0>() == 1)
return true;
}
return false;
}
GEN12_RENDER_INTR_VEC::access_t read_irq_vector(unsigned const generation)
{
GEN12_RENDER_INTR_VEC::access_t vec = 0;
if (generation < 11) {
vec = read<GT_0_INTERRUPT_IIR>();
} else {
write<GEN12_INTR_IIR_SELECTOR0::Rcs0>(1);
try {
wait_for(Attempts(50), Microseconds(500), _delayer,
GEN12_INTR_IDENTITY_REG0::Valid::Equal(1));
} catch (Polling_timeout) {
Genode::error(__func__, " IRQ vector not valid");
return vec;
}
vec = read<GEN12_INTR_IDENTITY_REG0::Engine_interrupt>();
write<GEN12_INTR_IDENTITY_REG0::Valid>(1);
}
return vec;
};
void clear_render_irq(unsigned const generation, Genode::uint16_t v)
{
if (generation < 11)
write_post<GT_0_INTERRUPT_IIR>(v);
else
write<GEN12_GT_INTR_DW0::Rcs0>(1);
}
bool display_irq(unsigned const generation)
{
if (generation < 11)
return read<MASTER_INT_CTL::De_interrupts_pending>() != 0;
else
return read<GEN12_GFX_MSTR_INTR::Display>() == 1;
}
void flush_gfx_tlb() { _gfx_flush_cntl(); } void flush_gfx_tlb() { _gfx_flush_cntl(); }
void clear_errors() void clear_errors()

6
repos/os/src/drivers/gpu/intel/ppgtt_allocator.h
View File

@ -73,9 +73,9 @@ class Igd::Ppgtt_allocator : public Genode::Translation_table_allocator
try { try {
ds = _backend.alloc(alloc_size); ds = _backend.alloc(alloc_size);
} }
catch (Genode::Out_of_ram) { return Alloc_error::OUT_OF_RAM; } catch (Gpu::Session::Out_of_ram) { throw; }
catch (Genode::Out_of_caps) { return Alloc_error::OUT_OF_CAPS; } catch (Gpu::Session::Out_of_caps) { throw; }
catch (...) { return Alloc_error::DENIED; } catch (...) { return Alloc_error::DENIED; }
Alloc_error alloc_error = Alloc_error::DENIED; Alloc_error alloc_error = Alloc_error::DENIED;