ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2025-03-22 12:06:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

gpu/intel: enable GPU reset for GEN(8)/9/12

Browse Source 
* add semaphore command to ring in order to be able to stop ring
  execution before reset (Wa KabyLake)
* implement reset sequence as done by the Linux driver
* reset ring and cancel job of vgpu causing hang
* lower watchdog timeout 1000ms -> 200ms
* improve scheduling of vgpus so progress after reset is made
* improve the generation chaos a little
* tested on Skylake, Kaby Lake, Tiger Lake

issue #4916
This commit is contained in:
Sebastian Sumpf 2023-05-22 21:57:22 +02:00 committed by Norman Feske
parent b599f4e106
commit e3c2fdf414
8 changed files with 637 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
repos/os/src/drivers/gpu/intel/commands.h
View File

@ -31,6 +31,7 @@ namespace Igd {
struct Mi_arb_on_off;
struct Mi_user_interrupt;
struct Mi_batch_buffer_start;
struct Mi_semaphore_wait;
struct Pipe_control;
void cmd_dump(uint32_t cmd, uint32_t index = 0);
@ -184,6 +185,48 @@ struct Igd::Mi_batch_buffer_start : Mi_cmd
};
/*
* IHD-OS-BDW-Vol 2a-11.15 p. 888 ff.
*
* Note: Legnth 2 on GEN8+ and 3 on GEN12+
*/
struct Igd::Mi_semaphore_wait : Mi_cmd
{
struct Compare_operation : Bitfield<12, 3>
{
enum { SAD_EQUAL_SDD = 0x4 };
};
struct Wait_mode : Bitfield<15, 1>
{
enum { SIGNAL = 0b0, POLL = 0b1};
};
struct Memory_type : Bitfield<22, 1>
{
enum { PPGTT = 0b0, GGTT = 0b1};
};
/* number of words - 2 */
struct Dword_length : Bitfield<0, 8> { };
Mi_semaphore_wait()
:
Mi_cmd(Mi_cmd_opcode::MI_SEMAPHORE_WAIT)
{
Memory_type::set(Cmd_header::value, Memory_type::GGTT);
Wait_mode::set(Cmd_header::value, Wait_mode::POLL);
/* wait for address data equal data word */
Compare_operation::set(Cmd_header::value, Compare_operation::SAD_EQUAL_SDD);
}
void dword_length(unsigned const value)
{
Dword_length::set(Cmd_header::value, value);
}
};
/*
* IHD-OS-BDW-Vol 2a-11.15 p. 983 ff.
*/

26
repos/os/src/drivers/gpu/intel/context.h
View File

@ -21,7 +21,7 @@
/* local includes */
#include <types.h>
#include <commands.h>
#include <utils.h>
namespace Igd {
@ -693,6 +693,12 @@ class Igd::Hardware_status_page : public Igd::Common_context_regs
*/
struct Ring_head_ptr_storage : Common_register<4> { };
/*
* dword 0x30 to 0x3ff is available for driver usage
*/
struct Sequence_number : Register<0x30*4, 64> { };
struct Semaphore : Register<0x32*4, 32> { };
/*
* See CTXT_ST_BUF
*/
@ -704,7 +710,23 @@ class Igd::Hardware_status_page : public Igd::Common_context_regs
struct Last_written_status_offset : Common_register<31> { };
Hardware_status_page(addr_t base)
: Common_context_regs(base) { }
: Common_context_regs(base)
{
semaphore(0);
}
void semaphore(uint32_t value)
{
write<Semaphore>(value);
Igd::wmb();
}
uint64_t sequence_number()
{
/* invalidates cache before reading */
Utils::clflush((void *)(base() + Sequence_number::OFFSET));
return read<Sequence_number>();
}
void dump(bool raw = false)
{

241
repos/os/src/drivers/gpu/intel/main.cc
View File

@ -41,11 +41,12 @@
#include <context.h>
#include <context_descriptor.h>
#include <platform_session.h>
#include <reset.h>
#include <ring_buffer.h>
#include <workarounds.h>
using namespace Genode;
static constexpr bool DEBUG = false;
namespace Igd {
@ -74,13 +75,15 @@ struct Igd::Device
struct Out_of_ram : Genode::Exception { };
struct Could_not_map_vram : Genode::Exception { };
enum { WATCHDOG_TIMEOUT = 1*1000*1000, };
/* 200 ms */
enum { WATCHDOG_TIMEOUT = 200*1000 };
Env & _env;
Allocator & _md_alloc;
Platform::Connection & _platform;
Rm_connection & _rm;
Igd::Mmio & _mmio;
Igd::Reset _reset { _mmio };
Platform::Device::Mmio & _gmadr;
uint8_t _gmch_ctl;
Timer::Connection _timer { _env };
@ -198,10 +201,13 @@ struct Igd::Device
if (info.platform == Igd::Device_info::Platform::UNKNOWN)
return;
/* set generation for device IO as early as possible */
_mmio.generation(generation);
if (info.id == dev_id) {
_info = info;
_revision.value = rev_id;
_clock_frequency.value = _mmio.clock_frequency(generation);
_clock_frequency.value = _mmio.clock_frequency();
found = true;
return;
@ -392,6 +398,7 @@ struct Igd::Device
Ring_buffer::Index ring_max() const { return _ring.max(); }
void ring_reset_and_fill_zero() { _ring.reset_and_fill_zero(); }
void ring_update_head(Ring_buffer::Index head) { _ring.update_head(head); }
void ring_reset_to_head(Ring_buffer::Index head) { _ring.reset_to_head(head); }
void ring_flush(Ring_buffer::Index from, Ring_buffer::Index to)
{
@ -650,10 +657,16 @@ struct Igd::Device
Gpu::Sequence_number { .value = _device.seqno() };
}
void mark_completed() {
_info.last_completed = Gpu::Sequence_number { .value = current_seqno() };
}
bool setup_ring_vram(Gpu::addr_t const vram_addr)
{
_current_seqno++;
unsigned generation = _device.generation().value;
Execlist &el = *rcs.execlist;
Ring_buffer::Index advance = 0;
@ -663,8 +676,8 @@ struct Igd::Device
Device_info::Stepping::B0);
size_t const need = 4 /* batchvram cmd */ + 6 /* prolog */
+ ((_device.generation().value == 9) ? 6 : 0)
+ ((_device.generation().value == 8) ? 20 : 22) /* epilog + w/a */
+ ((generation == 9) ? 6 : 0)
+ ((generation == 8) ? 20 : 22) /* epilog + w/a */
+ (dc_flush_wa ? 12 : 0);
if (!el.ring_avail(need))
@ -683,7 +696,7 @@ struct Igd::Device
* on GEN9: emit empty pipe control before VF_CACHE_INVALIDATE
* - Linux 5.13 gen8_emit_flush_rcs()
*/
if (_device.generation().value == 9) {
if (generation == 9) {
enum { CMD_NUM = 6 };
Genode::uint32_t cmd[CMD_NUM] = {};
Igd::Pipe_control pc(CMD_NUM);
@ -709,7 +722,7 @@ struct Igd::Device
/* prolog */
if (1)
{
enum { CMD_NUM = 6, HWS_DATA = 0xc0, };
enum { CMD_NUM = 6 };
Genode::uint32_t cmd[CMD_NUM] = {};
Igd::Pipe_control pc(CMD_NUM);
cmd[0] = pc.value;
@ -753,7 +766,7 @@ struct Igd::Device
*
* batch-vram commands
*/
if (_device.generation().value == 8)
if (generation == 8)
{
enum { CMD_NUM = 4 };
Genode::uint32_t cmd[CMD_NUM] = {};
@ -774,7 +787,7 @@ struct Igd::Device
*
* batch-vram commands
*/
if (_device.generation().value >= 9)
if (generation >= 9)
{
enum { CMD_NUM = 6 };
Genode::uint32_t cmd[CMD_NUM] = {};
@ -813,7 +826,8 @@ struct Igd::Device
/* epilog 2/3 - gen8_emit_fini_breadcrumb_rcs, gen8_emit_ggtt_write_rcs */
if (1)
{
enum { CMD_NUM = 6, HWS_DATA = 0xc0, };
using HWS_DATA = Hardware_status_page::Sequence_number;
enum { CMD_NUM = 6 };
Genode::uint32_t cmd[CMD_NUM] = {};
Igd::Pipe_control pc(CMD_NUM);
cmd[0] = pc.value;
@ -825,7 +839,7 @@ struct Igd::Device
tmp |= Igd::Pipe_control::STORE_DATA_INDEX;
cmd[1] = tmp;
cmd[2] = HWS_DATA;
cmd[2] = HWS_DATA::OFFSET;
cmd[3] = 0; /* upper addr 0 */
cmd[4] = _current_seqno & 0xffffffff;
cmd[5] = _current_seqno >> 32;
@ -835,6 +849,46 @@ struct Igd::Device
}
}
/*
* emit semaphore we can later block on in order to stop ring
*
* 'emit_preempt_busywait' and 'gen12_emit_preempt_busywait'
*/
if (1)
{
enum { CMD_NUM = 6 };
Genode::uint32_t cmd[CMD_NUM] = { };
Igd::Mi_semaphore_wait sw;
/* number of dwords after [1] */
sw.dword_length(generation < 12 ? 2 : 3);
cmd[0] = Mi_arb_check().value;
cmd[1] = sw.value;
cmd[2] = 0; /* data word zero */
cmd[3] = _device.hw_status_page_semaphore(); /* semaphore address low */
cmd[4] = 0; /* semaphore address high */
cmd[5] = generation < 12 ? Mi_noop().value : 0;
for (size_t i = 0; i < CMD_NUM; i++) {
advance += el.ring_append(cmd[i]);
}
}
if (1)
{
enum { CMD_NUM = 2 };
Genode::uint32_t cmd[2] = {};
Igd::Mi_user_interrupt ui;
cmd[0] = Mi_arb_on_off(true).value;
cmd[1] = ui.value;
for (size_t i = 0; i < CMD_NUM; i++) {
advance += el.ring_append(cmd[i]);
}
}
/*
* epilog 3/3 - gen8_emit_fini_breadcrumb_rcs, gen8_emit_fini_breadcrumb_tail
*
@ -842,20 +896,6 @@ struct Igd::Device
*
* HWS page layout dword 48 - 1023 for driver usage
*/
if (1)
{
enum { CMD_NUM = 2 };
Genode::uint32_t cmd[2] = {};
Igd::Mi_user_interrupt ui;
cmd[0] = ui.value;
cmd[1] = Mi_arb_on_off(true).value;
for (size_t i = 0; i < CMD_NUM; i++) {
advance += el.ring_append(cmd[i]);
}
}
if (1)
{
/* gen8_emit_fini_breadcrumb_tail -> gen8_emit_wa_tail */
@ -1038,7 +1078,6 @@ struct Igd::Device
}
/************
** FENCES **
************/
@ -1069,34 +1108,51 @@ struct Igd::Device
** watchdog timeout **
**********************/
void _handle_vgpu_after_reset(Vgpu &vgpu)
{
/* signal completion of last job to vgpu */
vgpu.mark_completed();
_notify_complete(&vgpu);
/* offset of head in ring context */
size_t head_offset = vgpu.rcs.context->head_offset();
/* set head = tail in ring and ring context */
Execlist &el = *vgpu.rcs.execlist;
el.ring_reset_to_head(head_offset);
/* set tail in context in qwords */
vgpu.rcs.context->tail_offset((head_offset % (vgpu.rcs.ring_size())) / 8);
}
void _handle_watchdog_timeout()
{
if (!_active_vgpu) { return; }
Genode::error("watchdog triggered: engine stuck,"
" vGPU=", _active_vgpu->id(), " IRQ: ",
Hex(_mmio.read_irq_vector(_info.generation)));
_mmio.dump();
_mmio.error_dump();
_mmio.fault_dump();
_mmio.execlist_status_dump();
Hex(_mmio.read_irq_vector()));
_active_vgpu->rcs.context->dump();
_hw_status_page->dump();
Execlist &el = *_active_vgpu->rcs.execlist;
el.ring_update_head(_active_vgpu->rcs.context->head_offset());
el.ring_dump(4096, _active_vgpu->rcs.context->tail_offset() * 2,
_active_vgpu->rcs.context->head_offset());
if (DEBUG) {
_mmio.dump();
_mmio.error_dump();
_mmio.fault_dump();
_mmio.execlist_status_dump();
_device_reset_and_init();
if (_active_vgpu == _current_vgpu()) {
_unschedule_current_vgpu();
_active_vgpu->rcs.context->dump();
_hw_status_page->dump();
Execlist &el = *_active_vgpu->rcs.execlist;
el.ring_update_head(_active_vgpu->rcs.context->head_offset());
el.ring_dump(4096, _active_vgpu->rcs.context->tail_offset() * 2,
_active_vgpu->rcs.context->head_offset());
}
if (_current_vgpu()) {
_schedule_current_vgpu();
}
Vgpu *vgpu = reset();
if (!vgpu)
error("reset vgpu is null");
/* the stuck vgpu */
_handle_vgpu_after_reset(*vgpu);
}
Genode::Signal_handler<Device> _watchdog_timeout_sigh {
@ -1104,10 +1160,10 @@ struct Igd::Device
void _device_reset_and_init()
{
_mmio.reset(_info.generation);
_mmio.reset();
_mmio.clear_errors();
_mmio.init();
_mmio.enable_intr(_info.generation);
_mmio.enable_intr();
}
/**
@ -1132,6 +1188,7 @@ struct Igd::Device
if (!_supported(supported, device_id, revision))
throw Unsupported_device();
_ggtt.dump();
_vgpu_avail = (_gmadr.size() - APERTURE_RESERVED) / Vgpu::APERTURE_SIZE;
@ -1277,7 +1334,52 @@ struct Igd::Device
/**
* Reset the physical device
*/
void reset() { _device_reset_and_init(); }
Vgpu *reset()
{
/* Stop render engine
*
* WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES):
* KabyLake suffers from system hangs when batchbuffer is progressing during
* reset
*/
hw_status_page_pause_ring(true);
Vgpu *vgpu { nullptr };
/* unschedule current vgpu */
if (_active_vgpu) {
vgpu = _active_vgpu;
vgpu_unschedule(*_active_vgpu);
}
/* reset */
_reset.execute();
/* set address of global hardware status page */
if (_hw_status_ctx.constructed()) {
Mmio::HWS_PGA_RCSUNIT::access_t const addr = _hw_status_ctx->gmaddr();
_mmio.write_post<Igd::Mmio::HWS_PGA_RCSUNIT>(addr);
}
_mmio.clear_errors();
/* clear pending irqs */
_mmio.clear_render_irq();
/*
* Restore "Hardware Status Mask Register", this register controls which
* IRQs are even written to the PCI bus (should be same as unmasked in IMR)
*/
_mmio.restore_hwstam();
hw_status_page_pause_ring(false);
if (_current_vgpu()) {
_schedule_current_vgpu();
}
return vgpu;
}
/**
* Get chip id of the physical device
@ -1332,12 +1434,25 @@ struct Igd::Device
** Hardware status page **
**************************/
addr_t hw_status_page() const { return _hw_status_ctx->gmaddr(); }
addr_t hw_status_page_gmaddr() const { return _hw_status_ctx->gmaddr(); }
uint64_t seqno() const
addr_t hw_status_page_semaphore() const
{
Utils::clflush((uint32_t*)(_hw_status_ctx->vaddr() + 0xc0));
return *(uint32_t*)(_hw_status_ctx->vaddr() + 0xc0);
return hw_status_page_gmaddr() + Hardware_status_page::Semaphore::OFFSET;
}
/*
* Pause the physical ring by setting semaphore value programmed by
* 'setup_ring_vram' to 1, causing GPU to spin.
*/
void hw_status_page_pause_ring(bool pause)
{
_hw_status_page->semaphore(pause ? 1 : 0);
}
uint64_t seqno()
{
return _hw_status_page->sequence_number();
}
@ -1397,7 +1512,6 @@ struct Igd::Device
if (vgpu.enqueued())
_vgpu_list.remove(vgpu);
}
/*******************
@ -1521,21 +1635,21 @@ struct Igd::Device
bool handle_irq()
{
bool display_irq = _mmio.display_irq(_info.generation);
bool display_irq = _mmio.display_irq();
/* handle render interrupts only */
if (_mmio.render_irq(_info.generation) == false)
if (_mmio.render_irq() == false)
return display_irq;
_mmio.disable_master_irq(_info.generation);
_mmio.disable_master_irq();
Mmio::GEN12_RENDER_INTR_VEC::access_t const v = _mmio.read_irq_vector(_info.generation);
unsigned const v = _mmio.read_irq_vector();
bool const ctx_switch = Mmio::GEN12_RENDER_INTR_VEC::Cs_ctx_switch_interrupt::get(v);
bool const user_complete = Mmio::GEN12_RENDER_INTR_VEC::Cs_mi_user_interrupt::get(v);
bool ctx_switch = _mmio.context_switch(v);
bool user_complete = _mmio.user_complete(v);
if (v) {
_mmio.clear_render_irq(_info.generation, v);
_mmio.clear_render_irq(v);
}
Vgpu *notify_gpu = nullptr;
@ -1569,7 +1683,7 @@ struct Igd::Device
return display_irq;
}
void enable_master_irq() { _mmio.enable_master_irq(_info.generation); }
void enable_master_irq() { _mmio.enable_master_irq(); }
private:
@ -1886,7 +2000,7 @@ class Gpu::Session_component : public Genode::Session_object<Gpu::Session>
void vgpu_unschedule()
{
return _device.vgpu_unschedule(_vgpu);
_device.vgpu_unschedule(_vgpu);
}
/***************************
@ -2225,10 +2339,11 @@ class Gpu::Root : public Gpu::Root_component
if (s->vgpu_active()) {
Genode::warning("vGPU active, reset device and hope for the best");
_device->reset();
} else {
/* remove from scheduled list */
s->vgpu_unschedule();
}
/* remove from scheduled list */
s->vgpu_unschedule();
Genode::destroy(md_alloc(), s);
}

331
repos/os/src/drivers/gpu/intel/mmio.h
View File

@ -1,11 +1,11 @@
/*
* \brief Broadwell MMIO definitions
* \brief GEN8/9/12 MMIO definitions
* \author Josef Soentgen
* \data 2017-03-15
*/
/*
* Copyright (C) 2017 Genode Labs GmbH
* Copyright (C) 2023 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
@ -30,6 +30,19 @@ class Igd::Mmio : public Platform::Device::Mmio
{
public:
unsigned _generation { 0 };
void generation(unsigned gen) { _generation = gen; }
unsigned generation() const
{
if (!_generation) {
Genode::error("Unsupported generation: ", _generation);
}
return _generation;
}
enum {
/*
* XXX IDs are taken from Linux, still looking
@ -513,19 +526,28 @@ class Igd::Mmio : public Platform::Device::Mmio
*/
struct GFX_MODE : Register<0x0229C, 32>
{
/* to change bits below, accordings mask bit must be set */
struct Mask_bits : Bitfield<16, 16> { };
struct Execlist_enable_mask : Bitfield<31, 1> { };
struct Execlist_enable : Bitfield<15, 1> { };
struct Execlist_enable : Bitfield<15, 1> { };
struct Gen12_prefetch_disable : Bitfield<10, 1> { };
struct Gen11_gfx_disable_legacy_mode : Bitfield< 3, 1> { };
struct Privilege_check_disable : Bitfield< 0, 1> { };
struct Ppgtt_enable_mask : Bitfield<25, 1> { };
struct Ppgtt_enable : Bitfield< 9, 1> { };
/*
* these bits are only valid in ringer buffer mode, we use execlist with
* contexts
*/
struct Ppgtt_enable : Bitfield< 9, 1> { };
struct Virtual_addressing_enable : Bitfield< 7, 1> { };
struct Virtual_addressing_enable_mask : Bitfield<23, 1> { };
struct Virtual_addressing_enable : Bitfield< 7, 1> { };
struct Privilege_check_disable_mask : Bitfield<16, 1> { };
struct Privilege_check_disable : Bitfield< 0, 1> { };
template <typename T>
static access_t set(access_t v, bool bit)
{
T::set(v, bit);
Mask_bits::set(v, 1u << T::SHIFT);
return v;
}
};
/*
@ -633,6 +655,15 @@ class Igd::Mmio : public Platform::Device::Mmio
struct Fault_entry_page_address : Bitfield<12, 20> { };
};
/*
* IHD-OS-BDW-Vol 2c-11.15 p. 535
*/
struct GAMTARBMODE : Register<0x4a08, 32>
{
struct Arbiter_mode_control_1 : Bitfield<1, 1> { };
};
/*
* IHD-OS-BDW-Vol 2c-11.15 p. 1315 ff.
*/
@ -682,6 +713,9 @@ class Igd::Mmio : public Platform::Device::Mmio
/*
* IHD-OS-BDW-Vol 2c-11.15 p. 711
*
* "Hardware Status Mask Register", this register controls which IRQs are
* even written to the PCI bus (should be same as unmasked in IMR)
*/
struct HWSTAM : Register<0x02098, 32> { };
@ -827,6 +861,18 @@ class Igd::Mmio : public Platform::Device::Mmio
struct Kernel : Bitfield< 0, 1> { };
};
/*
* MSG idle register (one per engine) force wake status (undocumented)
*/
struct MSG_IDLE_CS : Register<0x8000, 32>
{
struct Pending_status : Bitfield<9, 5> { };
struct Pending_mask : Bitfield<25, 5> { };
};
/* force wake status used with MSG_IDLE_CS (undocumented) */
struct GEN9_PWRGT_DOMAIN_STATUS : Register<0xa2a0, 32> { };
/*
* IHD-OS-BDW-Vol 2c-11.15 p. 703
*
@ -851,9 +897,10 @@ class Igd::Mmio : public Platform::Device::Mmio
{
using B = Register<BASE + 0xD0, 32>;
struct Mask_bits : B::template Bitfield<16, 16> { };
struct Ready_for_reset : B::template Bitfield< 1, 1> { };
struct Request_reset : B::template Bitfield< 0, 1> { };
struct Mask_bits : B::template Bitfield<16, 16> { };
struct Catastrophic_error : B::template Bitfield< 2, 1> { };
struct Ready_for_reset : B::template Bitfield< 1, 1> { };
struct Request_reset : B::template Bitfield< 0, 1> { };
};
template <unsigned long BASE>
@ -878,6 +925,14 @@ class Igd::Mmio : public Platform::Device::Mmio
struct BCS_RESET_CTRL : RESET_CTRL_BASE <0x22000> { };
struct BCS_MI_MODE_CTRL : MI_MODE_CTRL_BASE<0x22000> { };
/*
* IHD-OS-BDW-Vol 2c-11.15 p. 790 ff.
*/
struct HSW_IDICR : Register<0x9008, 32>
{
struct Idi_hash_mask : Bitfield<16, 6> { };
};
/*
* IHD-OS-BDW-Vol 2c-11.15 p. 609 ff.
*/
@ -947,6 +1002,13 @@ class Igd::Mmio : public Platform::Device::Mmio
struct Fence_valid : Bitfield< 0, 1> { };
};
/*
* IHD-OS-BDW-Vol 12-11.15 p. 4
*/
struct TILECTL : Register<0x101000, 32>
{
struct SWZCTL : Bitfield<0, 2> { };
};
/*
* IHD-OS-BDW-Vol 12-11.15 p. 5
@ -1071,7 +1133,6 @@ class Igd::Mmio : public Platform::Device::Mmio
struct MI_DISP_PWR_DWN : Register<0x20E0, 32> { };
struct MI_ARB_STATE : Register<0x20E4, 32> { };
struct MI_RDRET_STATE : Register<0x20FC, 32> { };
struct MI_MODE : Register<0x209c, 32> { };
struct ECOSKPD : Register<0x21D0, 32> { };
private:
@ -1093,6 +1154,23 @@ class Igd::Mmio : public Platform::Device::Mmio
read<MISC_CTRL0>();
}
void _fw_reset_gen9()
{
write_post<FORCEWAKE_MEDIA_GEN9>(FORCEWAKE_MT::RESET);
write_post<FORCEWAKE_RENDER_GEN9>(FORCEWAKE_MT::RESET);
write_post<FORCEWAKE_GT_GEN9>(FORCEWAKE_MT::RESET);
}
void _forcewake_reset()
{
if (generation() == 8)
_fw_reset_gen8();
else if (generation() <= 12)
_fw_reset_gen9();
else Genode::error(__func__, " unsupported generation", generation());
}
/**
* Set forcewake state, i.e., prevent GT from powering down
*/
@ -1140,13 +1218,6 @@ class Igd::Mmio : public Platform::Device::Mmio
}
}
void _fw_reset_gen9()
{
write_post<FORCEWAKE_MEDIA_GEN9>(FORCEWAKE_MT::RESET);
write_post<FORCEWAKE_RENDER_GEN9>(FORCEWAKE_MT::RESET);
write_post<FORCEWAKE_GT_GEN9>(FORCEWAKE_MT::RESET);
}
/**
* Set forcewake state, i.e., prevent from powering down
*/
@ -1258,6 +1329,16 @@ class Igd::Mmio : public Platform::Device::Mmio
write_post<HWSTAM>(0xffffffff);
}
void _interrupt_reset()
{
if (generation() < 11)
_intr_reset();
else if (generation() <= 12)
_intr_reset_gen12();
else
Genode::error(__func__, " unsupported generation ", generation());
}
/**
* Enable interrupts
*/
@ -1268,7 +1349,7 @@ class Igd::Mmio : public Platform::Device::Mmio
/* GT0 RCS/BCS */
{
uint32_t tmp = read<GT_0_INTERRUPT_IIR>();
if (tmp != 0) { error("GT_0_INTERRUPT_IIR not zero"); }
if (tmp != 0) { error("GT_0_INTERRUPT_IIR not zero: ", Hex(tmp)); }
GT_0_INTERRUPT_IER::access_t ier = 0;
GT_0_INTERRUPT_IER::Cs_mi_user_interrupt::set(ier, 1);
@ -1557,19 +1638,6 @@ class Igd::Mmio : public Platform::Device::Mmio
write<NDE_RSTWRN_OPT>(v);
}
/**
* Enable execlist
*/
void _enable_execlist()
{
GFX_MODE::access_t v = read<GFX_MODE>();
GFX_MODE::Execlist_enable_mask::set(v, 1);
GFX_MODE::Execlist_enable::set(v, 1);
write<GFX_MODE>(v);
}
uint32_t _clock_frequency_gen12()
{
uint32_t freq { 0 };
@ -1600,6 +1668,8 @@ class Igd::Mmio : public Platform::Device::Mmio
Mmio(Platform::Device & device, Genode::Env & env)
: Platform::Device::Mmio(device, {0}), _delayer(env) { }
Delayer &delayer() { return _delayer; }
template <typename T>
void write_post(typename T::access_t const value)
{
@ -1607,10 +1677,32 @@ class Igd::Mmio : public Platform::Device::Mmio
(void)read<T>();
}
uint32_t clock_frequency(unsigned const generation)
/**
* Enable execlist
*/
void enable_execlist()
{
if (generation >= 11)
/*
* This disables all interrupts delivered to the bus, make sure to
* enable IRQs later
*/
write_post<HWSTAM>(~0u);
write<CS_MI_MODE_CTRL::Stop_rings>(0);
using G = GFX_MODE;
G::access_t v = 0;
if (generation() >= 11)
v = G::set<G::Gen11_gfx_disable_legacy_mode>(v, 1);
else
v = G::set<G::Execlist_enable>(v, 1);
write<G>(v);
}
uint32_t clock_frequency()
{
if (generation() >= 11)
return _clock_frequency_gen12();
return 0;
@ -1633,9 +1725,9 @@ class Igd::Mmio : public Platform::Device::Mmio
_fw_disable_gt();
}
void forcewake_enable(unsigned const generation)
void forcewake_enable()
{
switch (generation) {
switch (generation()) {
case 8:
forcewake_gen8_enable();
return;
@ -1644,13 +1736,13 @@ class Igd::Mmio : public Platform::Device::Mmio
forcewake_gen9_enable();
return;
default:
Genode::error(__func__, " unsupported generation ", generation);
Genode::error(__func__, " unsupported generation ", generation());
}
}
void forcewake_disable(unsigned const generation)
void forcewake_disable()
{
switch (generation) {
switch (generation()) {
case 8:
forcewake_gen8_disable();
return;
@ -1659,56 +1751,16 @@ class Igd::Mmio : public Platform::Device::Mmio
forcewake_gen9_disable();
return;
default:
Genode::error(__func__, " unsupported generation ", generation);
Genode::error(__func__, " unsupported generation ", generation());
}
}
void reset(unsigned const generation)
/* reset used during intialization only */
void reset()
{
switch (generation) {
case 8:
reset_gen8();
return;
case 9:
reset_gen9();
break;
case 12:
reset_gen12();
break;;
default:
Genode::error(__func__, " unsupported generation ", generation);
}
}
void reset_gen8()
{
_intr_reset();
_fw_reset_gen8();
forcewake_gen8_enable();
_reset_device();
_reset_fences();
_disable_nde_handshake();
_set_page_attributes();
}
void reset_gen9()
{
_intr_reset();
_fw_reset_gen9();
forcewake_gen9_enable();
_reset_device();
_reset_fences();
_disable_nde_handshake();
_set_page_attributes();
}
void reset_gen12()
{
_intr_reset_gen12();
_fw_reset_gen9();
forcewake_gen9_enable();
_interrupt_reset();
_forcewake_reset();
forcewake_enable();
_reset_device();
_reset_fences();
@ -1719,51 +1771,47 @@ class Igd::Mmio : public Platform::Device::Mmio
void init()
{
_disable_rps();
_enable_execlist();
enable_execlist();
}
void enable_intr(unsigned const generation)
{
void enable_intr()
{
write<Igd::Mmio::RCS_EMR>(0xffffff00);
if (generation < 11)
if (generation() < 11)
_intr_enable();
else
_intr_enable_gen12();
}
void disable_master_irq(unsigned const generation)
void restore_hwstam()
{
if (generation < 11)
if (generation() < 11)
write_post<HWSTAM>(read<GT_0_INTERRUPT_IMR>());
else
write_post<HWSTAM>(~read<GEN12_RENDER_COPY_INTR_ENABLE::Render_enable>());
}
void disable_master_irq()
{
if (generation() < 11)
write_post<Igd::Mmio::MASTER_INT_CTL::Master_interrupt_enable>(0);
else
write<GEN12_GFX_MSTR_INTR::Master_interrupt_enable>(0);
}
void enable_master_irq(unsigned const generation)
void enable_master_irq()
{
if (generation < 11)
if (generation() < 11)
write_post<Igd::Mmio::MASTER_INT_CTL::Master_interrupt_enable>(1);
else
write<GEN12_GFX_MSTR_INTR::Master_interrupt_enable>(1);
}
bool render_irq(unsigned const generation)
unsigned read_irq_vector()
{
if (generation < 11)
return read<MASTER_INT_CTL::Render_interrupts_pending>() == 1;
else {
if (read<GEN12_GFX_MSTR_INTR::Gt_dw_0>() == 1 &&
read<GEN12_GT_INTR_DW0::Rcs0>() == 1)
return true;
}
return false;
}
GEN12_RENDER_INTR_VEC::access_t read_irq_vector(unsigned const generation)
{
GEN12_RENDER_INTR_VEC::access_t vec = 0;
if (generation < 11) {
unsigned vec = 0;
if (generation() < 11) {
vec = read<GT_0_INTERRUPT_IIR>();
} else {
write<GEN12_INTR_IIR_SELECTOR0::Rcs0>(1);
@ -1772,7 +1820,6 @@ class Igd::Mmio : public Platform::Device::Mmio
wait_for(Attempts(50), Microseconds(500), _delayer,
GEN12_INTR_IDENTITY_REG0::Valid::Equal(1));
} catch (Polling_timeout) {
Genode::error(__func__, " IRQ vector not valid");
return vec;
}
vec = read<GEN12_INTR_IDENTITY_REG0::Engine_interrupt>();
@ -1782,22 +1829,61 @@ class Igd::Mmio : public Platform::Device::Mmio
return vec;
};
void clear_render_irq(unsigned const generation, Genode::uint16_t v)
void clear_render_irq(unsigned v)
{
if (generation < 11)
if (generation() < 11)
write_post<GT_0_INTERRUPT_IIR>(v);
else
write<GEN12_GT_INTR_DW0::Rcs0>(1);
}
bool display_irq(unsigned const generation)
void clear_render_irq()
{
if (generation < 11)
unsigned v = 0;
if (generation() < 11)
v = read_irq_vector();
clear_render_irq(v);
}
/**
* IRQ causes
*/
bool render_irq()
{
if (generation() < 11)
return read<MASTER_INT_CTL::Render_interrupts_pending>() == 1;
else {
if (read<GEN12_GFX_MSTR_INTR::Gt_dw_0>() == 1 &&
read<GEN12_GT_INTR_DW0::Rcs0>() == 1)
return true;
}
return false;
}
bool display_irq()
{
if (generation() < 11)
return read<MASTER_INT_CTL::De_interrupts_pending>() != 0;
else
return read<GEN12_GFX_MSTR_INTR::Display>() == 1;
}
bool context_switch(unsigned const vector)
{
if (generation() < 11)
return GT_0_INTERRUPT_IIR::Cs_ctx_switch_interrupt::get(vector);
else
return GEN12_RENDER_INTR_VEC::Cs_ctx_switch_interrupt::get(vector);
}
bool user_complete(unsigned const vector)
{
if (generation() < 11)
return GT_0_INTERRUPT_IIR::Cs_mi_user_interrupt::get(vector);
else
return GEN12_RENDER_INTR_VEC::Cs_mi_user_interrupt::get(vector);
}
void flush_gfx_tlb() { _gfx_flush_cntl(); }
@ -1808,8 +1894,10 @@ class Igd::Mmio : public Platform::Device::Mmio
void update_context_status_pointer()
{
size_t const context_status_size = generation() < 11 ? 6 : 12;
RCS_RING_CONTEXT_STATUS_PTR::access_t const wp = read<RCS_RING_CONTEXT_STATUS_PTR::Write_pointer>();
if (wp > 0x05) {
if (wp > (context_status_size - 1)) {
Genode::warning("ring context status write-pointer invalid", Genode::Hex(wp));
return;
}
@ -1886,8 +1974,9 @@ class Igd::Mmio : public Platform::Device::Mmio
struct MASK_WAKEMEM : Bitfield<13, 1> {};
};
struct DISP_ARB_CTL : Register<0x45000, 32> {
struct DISP_FBC_MEMORY_WAKE : Bitfield<31, 1> {};
struct DISP_FBC_WM_DIS : Bitfield<15, 1> {};
struct DISP_FBC_MEMORY_WAKE : Bitfield<31, 1> {};
struct DISP_TILE_SURFACE_SWIZZLING : Bitfield<13, 1> { };
struct DISP_FBC_WM_DIS : Bitfield<15, 1> {};
};
void gen9_clock_gating()

2
repos/os/src/drivers/gpu/intel/mmio_dump.cc
View File

@ -52,7 +52,7 @@ void Igd::Mmio::dump()
log("0x20E0 - MI_DISP_PWR_DWN ", Hex(read<MI_DISP_PWR_DWN>()));
log("0x20E4 - MI_ARB_STATE ", Hex(read<MI_ARB_STATE>()));
log("0x20FC - MI_RDRET_STATE ", Hex(read<MI_RDRET_STATE>()));
log("0x209C - MI_MODE ", Hex(read<MI_MODE>()));
log("0x209C - MI_MODE ", Hex(read<CS_MI_MODE_CTRL>()));
log("0x21D0 - ECOSKPD ", Hex(read<ECOSKPD>()));
}

171
repos/os/src/drivers/gpu/intel/reset.h Normal file
View File

@ -0,0 +1,171 @@
/*
* \brief Render engine reset based on the Linux driver
* \author Sebastian Sumpf
* \date 2023-06-05
*/
/*
* Copyright (C) 2023 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#ifndef _RESET_H_
#define _RESET_H_
#include "mmio.h"
#include "workarounds.h"
namespace Igd {
class Reset;
}
class Igd::Reset
{
private:
Igd::Mmio &_mmio;
void _stop_engine_cs()
{
/* write stop bit to render mode */
_mmio.write<Mmio::CS_MI_MODE_CTRL::Stop_rings>(1);
/*
* Wa_22011802037 : GEN11, GNE12, Prior to doing a reset, ensure CS is
* stopped, set ring stop bit and prefetch disable bit to halt CS
*/
if (_mmio.generation() == 11 || _mmio.generation() == 12) {
Mmio::GFX_MODE::access_t v = 0;
using G = Mmio::GFX_MODE;
v = G::set<G::Gen12_prefetch_disable>(v, 1);
_mmio.write<Mmio::GFX_MODE>(v);
}
try {
_mmio.wait_for(Mmio::Attempts(10), Mmio::Microseconds(100'000), _mmio.delayer(),
Mmio::CS_MI_MODE_CTRL::Rings_idle::Equal(1));
} catch(Mmio::Polling_timeout) {
Genode::warning("stop engine cs timeout");
}
/* read to let GPU writes be flushed to memory */
_mmio.read<Mmio::CS_MI_MODE_CTRL>();
}
/* not documented
* Wa_22011802037: gen11/gen12: In addition to stopping the cs, we need
* to wait for any pending mi force wakeups
* MSG_IDLE_CS 0x8000 force wake
*/
void _wait_for_pending_force_wakeups()
{
if (_mmio.generation() < 11 && _mmio.generation() > 12) return;
unsigned fw_status = _mmio.read<Mmio::MSG_IDLE_CS::Pending_status>();
unsigned fw_mask = _mmio.read<Mmio::MSG_IDLE_CS::Pending_mask>();
_mmio.delayer().usleep(1);
for (unsigned i = 0; i < 10; i++) {
unsigned status = _mmio.read<Mmio::GEN9_PWRGT_DOMAIN_STATUS>() & fw_mask;
_mmio.delayer().usleep(1);
if (status == fw_status) return;
_mmio.delayer().usleep(50000);
}
_mmio.delayer().usleep(1);
Genode::warning("wait pending force wakeup timeout");
}
void _ready_for_reset()
{
if (_mmio.read<Mmio::CS_RESET_CTRL::Catastrophic_error>()) {
/* For catastrophic errors, ready-for-reset sequence
* needs to be bypassed: HAS#396813
*/
try {
_mmio.wait_for(Mmio::Attempts(7), Mmio::Microseconds(100'000), _mmio.delayer(),
Mmio::CS_RESET_CTRL::Catastrophic_error::Equal(0));
} catch (Mmio::Polling_timeout) {
Genode::warning("catastrophic error reset not cleared");
}
return;
}
if (_mmio.read<Mmio::CS_RESET_CTRL::Ready_for_reset>()) return;
Mmio::CS_RESET_CTRL::access_t request = 0;
Mmio::CS_RESET_CTRL::Mask_bits::set(request, 1);
Mmio::CS_RESET_CTRL::Request_reset::set(request, 1);
_mmio.write_post<Mmio::CS_RESET_CTRL>(request);
try {
_mmio.wait_for(Mmio::Attempts(7), Mmio::Microseconds(100'000), _mmio.delayer(),
Mmio::CS_RESET_CTRL::Ready_for_reset::Equal(1));
} catch (Mmio::Polling_timeout) {
Genode::warning("not ready for reset");
}
}
void _unready_for_reset()
{
Mmio::CS_RESET_CTRL::access_t request = 0;
Mmio::CS_RESET_CTRL::Mask_bits::set(request, 1);
Mmio::CS_RESET_CTRL::Request_reset::set(request, 0);
_mmio.write_post<Mmio::CS_RESET_CTRL>(request);
}
void _reset_hw()
{
/* full sw reset */
_mmio.write<Mmio::GDRST::Graphics_full_soft_reset_ctl>(1);
try {
/* do NOT attempt more than 2 times */
_mmio.wait_for(Mmio::Attempts(2), Mmio::Microseconds(200'000), _mmio.delayer(),
Mmio::GDRST::Graphics_full_soft_reset_ctl::Equal(0));
} catch (Mmio::Polling_timeout) {
Genode::error("resetting device failed");
}
/* some devices still show volatile state */
_mmio.delayer().usleep(50);
}
void _init_swizzling()
{
_mmio.write<Mmio::DISP_ARB_CTL::DISP_TILE_SURFACE_SWIZZLING>(1);
_mmio.write<Mmio::TILECTL::SWZCTL>(1);
if (_mmio.generation() == 8)
_mmio.write<Mmio::GAMTARBMODE::Arbiter_mode_control_1>(1);
}
public:
Reset(Igd::Mmio &mmio) : _mmio(mmio) { }
void execute()
{
_stop_engine_cs();
_wait_for_pending_force_wakeups();
_ready_for_reset();
_reset_hw();
_unready_for_reset();
if (_mmio.generation() < 9)
_mmio.write<Mmio::HSW_IDICR::Idi_hash_mask>(0xf);
apply_workarounds(_mmio, _mmio.generation());
_init_swizzling();
_mmio.enable_execlist();
}
};
#endif /* _RESET_H_ */

9
repos/os/src/drivers/gpu/intel/ring_buffer.h
View File

@ -98,6 +98,15 @@ class Igd::Ring_buffer
_head = head;
}
/**
* Update head and set tail to head
*/
void reset_to_head(Index head)
{
update_head(head);
_tail = _head;
}
/**
* Insert new command at given index
*

2
repos/os/src/drivers/gpu/intel/workarounds.h
View File

@ -13,7 +13,7 @@
#include <mmio.h>
namespace Igd {
void apply_workarounds(Mmio &mmio, unsigned generation);
static void apply_workarounds(Mmio &mmio, unsigned generation);
}
void Igd::apply_workarounds(Mmio &mmio, unsigned generation)