From e3c2fdf414135aa9e8f84b7767ca0c3d7a0c7098 Mon Sep 17 00:00:00 2001 From: Sebastian Sumpf Date: Mon, 22 May 2023 21:57:22 +0200 Subject: [PATCH] gpu/intel: enable GPU reset for GEN(8)/9/12 * add semaphore command to ring in order to be able to stop ring execution before reset (Wa KabyLake) * implement reset sequence as done by the Linux driver * reset ring and cancel job of vgpu causing hang * lower watchdog timeout 1000ms -> 200ms * improve scheduling of vgpus so progress after reset is made * improve the generation chaos a little * tested on Skylake, Kaby Lake, Tiger Lake issue #4916 --- repos/os/src/drivers/gpu/intel/commands.h | 43 +++ repos/os/src/drivers/gpu/intel/context.h | 26 +- repos/os/src/drivers/gpu/intel/main.cc | 241 ++++++++++---- repos/os/src/drivers/gpu/intel/mmio.h | 331 ++++++++++++------- repos/os/src/drivers/gpu/intel/mmio_dump.cc | 2 +- repos/os/src/drivers/gpu/intel/reset.h | 171 ++++++++++ repos/os/src/drivers/gpu/intel/ring_buffer.h | 9 + repos/os/src/drivers/gpu/intel/workarounds.h | 2 +- 8 files changed, 637 insertions(+), 188 deletions(-) create mode 100644 repos/os/src/drivers/gpu/intel/reset.h diff --git a/repos/os/src/drivers/gpu/intel/commands.h b/repos/os/src/drivers/gpu/intel/commands.h index ecc4118972..299de19e71 100644 --- a/repos/os/src/drivers/gpu/intel/commands.h +++ b/repos/os/src/drivers/gpu/intel/commands.h @@ -31,6 +31,7 @@ namespace Igd { struct Mi_arb_on_off; struct Mi_user_interrupt; struct Mi_batch_buffer_start; + struct Mi_semaphore_wait; struct Pipe_control; void cmd_dump(uint32_t cmd, uint32_t index = 0); @@ -184,6 +185,48 @@ struct Igd::Mi_batch_buffer_start : Mi_cmd }; +/* + * IHD-OS-BDW-Vol 2a-11.15 p. 888 ff. + * + * Note: Legnth 2 on GEN8+ and 3 on GEN12+ + */ +struct Igd::Mi_semaphore_wait : Mi_cmd +{ + + struct Compare_operation : Bitfield<12, 3> + { + enum { SAD_EQUAL_SDD = 0x4 }; + }; + + struct Wait_mode : Bitfield<15, 1> + { + enum { SIGNAL = 0b0, POLL = 0b1}; + }; + + struct Memory_type : Bitfield<22, 1> + { + enum { PPGTT = 0b0, GGTT = 0b1}; + }; + + /* number of words - 2 */ + struct Dword_length : Bitfield<0, 8> { }; + + Mi_semaphore_wait() + : + Mi_cmd(Mi_cmd_opcode::MI_SEMAPHORE_WAIT) + { + Memory_type::set(Cmd_header::value, Memory_type::GGTT); + Wait_mode::set(Cmd_header::value, Wait_mode::POLL); + /* wait for address data equal data word */ + Compare_operation::set(Cmd_header::value, Compare_operation::SAD_EQUAL_SDD); + } + + void dword_length(unsigned const value) + { + Dword_length::set(Cmd_header::value, value); + } +}; + /* * IHD-OS-BDW-Vol 2a-11.15 p. 983 ff. */ diff --git a/repos/os/src/drivers/gpu/intel/context.h b/repos/os/src/drivers/gpu/intel/context.h index 7805b5f56b..70c4a0b35c 100644 --- a/repos/os/src/drivers/gpu/intel/context.h +++ b/repos/os/src/drivers/gpu/intel/context.h @@ -21,7 +21,7 @@ /* local includes */ #include #include - +#include namespace Igd { @@ -693,6 +693,12 @@ class Igd::Hardware_status_page : public Igd::Common_context_regs */ struct Ring_head_ptr_storage : Common_register<4> { }; + /* + * dword 0x30 to 0x3ff is available for driver usage + */ + struct Sequence_number : Register<0x30*4, 64> { }; + struct Semaphore : Register<0x32*4, 32> { }; + /* * See CTXT_ST_BUF */ @@ -704,7 +710,23 @@ class Igd::Hardware_status_page : public Igd::Common_context_regs struct Last_written_status_offset : Common_register<31> { }; Hardware_status_page(addr_t base) - : Common_context_regs(base) { } + : Common_context_regs(base) + { + semaphore(0); + } + + void semaphore(uint32_t value) + { + write(value); + Igd::wmb(); + } + + uint64_t sequence_number() + { + /* invalidates cache before reading */ + Utils::clflush((void *)(base() + Sequence_number::OFFSET)); + return read(); + } void dump(bool raw = false) { diff --git a/repos/os/src/drivers/gpu/intel/main.cc b/repos/os/src/drivers/gpu/intel/main.cc index 8075fa2440..e9ebceb5fb 100644 --- a/repos/os/src/drivers/gpu/intel/main.cc +++ b/repos/os/src/drivers/gpu/intel/main.cc @@ -41,11 +41,12 @@ #include #include #include +#include #include -#include using namespace Genode; +static constexpr bool DEBUG = false; namespace Igd { @@ -74,13 +75,15 @@ struct Igd::Device struct Out_of_ram : Genode::Exception { }; struct Could_not_map_vram : Genode::Exception { }; - enum { WATCHDOG_TIMEOUT = 1*1000*1000, }; + /* 200 ms */ + enum { WATCHDOG_TIMEOUT = 200*1000 }; Env & _env; Allocator & _md_alloc; Platform::Connection & _platform; Rm_connection & _rm; Igd::Mmio & _mmio; + Igd::Reset _reset { _mmio }; Platform::Device::Mmio & _gmadr; uint8_t _gmch_ctl; Timer::Connection _timer { _env }; @@ -198,10 +201,13 @@ struct Igd::Device if (info.platform == Igd::Device_info::Platform::UNKNOWN) return; + /* set generation for device IO as early as possible */ + _mmio.generation(generation); + if (info.id == dev_id) { _info = info; _revision.value = rev_id; - _clock_frequency.value = _mmio.clock_frequency(generation); + _clock_frequency.value = _mmio.clock_frequency(); found = true; return; @@ -392,6 +398,7 @@ struct Igd::Device Ring_buffer::Index ring_max() const { return _ring.max(); } void ring_reset_and_fill_zero() { _ring.reset_and_fill_zero(); } void ring_update_head(Ring_buffer::Index head) { _ring.update_head(head); } + void ring_reset_to_head(Ring_buffer::Index head) { _ring.reset_to_head(head); } void ring_flush(Ring_buffer::Index from, Ring_buffer::Index to) { @@ -650,10 +657,16 @@ struct Igd::Device Gpu::Sequence_number { .value = _device.seqno() }; } + void mark_completed() { + _info.last_completed = Gpu::Sequence_number { .value = current_seqno() }; + } + bool setup_ring_vram(Gpu::addr_t const vram_addr) { _current_seqno++; + unsigned generation = _device.generation().value; + Execlist &el = *rcs.execlist; Ring_buffer::Index advance = 0; @@ -663,8 +676,8 @@ struct Igd::Device Device_info::Stepping::B0); size_t const need = 4 /* batchvram cmd */ + 6 /* prolog */ - + ((_device.generation().value == 9) ? 6 : 0) - + ((_device.generation().value == 8) ? 20 : 22) /* epilog + w/a */ + + ((generation == 9) ? 6 : 0) + + ((generation == 8) ? 20 : 22) /* epilog + w/a */ + (dc_flush_wa ? 12 : 0); if (!el.ring_avail(need)) @@ -683,7 +696,7 @@ struct Igd::Device * on GEN9: emit empty pipe control before VF_CACHE_INVALIDATE * - Linux 5.13 gen8_emit_flush_rcs() */ - if (_device.generation().value == 9) { + if (generation == 9) { enum { CMD_NUM = 6 }; Genode::uint32_t cmd[CMD_NUM] = {}; Igd::Pipe_control pc(CMD_NUM); @@ -709,7 +722,7 @@ struct Igd::Device /* prolog */ if (1) { - enum { CMD_NUM = 6, HWS_DATA = 0xc0, }; + enum { CMD_NUM = 6 }; Genode::uint32_t cmd[CMD_NUM] = {}; Igd::Pipe_control pc(CMD_NUM); cmd[0] = pc.value; @@ -753,7 +766,7 @@ struct Igd::Device * * batch-vram commands */ - if (_device.generation().value == 8) + if (generation == 8) { enum { CMD_NUM = 4 }; Genode::uint32_t cmd[CMD_NUM] = {}; @@ -774,7 +787,7 @@ struct Igd::Device * * batch-vram commands */ - if (_device.generation().value >= 9) + if (generation >= 9) { enum { CMD_NUM = 6 }; Genode::uint32_t cmd[CMD_NUM] = {}; @@ -813,7 +826,8 @@ struct Igd::Device /* epilog 2/3 - gen8_emit_fini_breadcrumb_rcs, gen8_emit_ggtt_write_rcs */ if (1) { - enum { CMD_NUM = 6, HWS_DATA = 0xc0, }; + using HWS_DATA = Hardware_status_page::Sequence_number; + enum { CMD_NUM = 6 }; Genode::uint32_t cmd[CMD_NUM] = {}; Igd::Pipe_control pc(CMD_NUM); cmd[0] = pc.value; @@ -825,7 +839,7 @@ struct Igd::Device tmp |= Igd::Pipe_control::STORE_DATA_INDEX; cmd[1] = tmp; - cmd[2] = HWS_DATA; + cmd[2] = HWS_DATA::OFFSET; cmd[3] = 0; /* upper addr 0 */ cmd[4] = _current_seqno & 0xffffffff; cmd[5] = _current_seqno >> 32; @@ -835,6 +849,46 @@ struct Igd::Device } } + + /* + * emit semaphore we can later block on in order to stop ring + * + * 'emit_preempt_busywait' and 'gen12_emit_preempt_busywait' + */ + if (1) + { + enum { CMD_NUM = 6 }; + Genode::uint32_t cmd[CMD_NUM] = { }; + + Igd::Mi_semaphore_wait sw; + /* number of dwords after [1] */ + sw.dword_length(generation < 12 ? 2 : 3); + + cmd[0] = Mi_arb_check().value; + cmd[1] = sw.value; + cmd[2] = 0; /* data word zero */ + cmd[3] = _device.hw_status_page_semaphore(); /* semaphore address low */ + cmd[4] = 0; /* semaphore address high */ + cmd[5] = generation < 12 ? Mi_noop().value : 0; + + for (size_t i = 0; i < CMD_NUM; i++) { + advance += el.ring_append(cmd[i]); + } + } + + if (1) + { + enum { CMD_NUM = 2 }; + Genode::uint32_t cmd[2] = {}; + Igd::Mi_user_interrupt ui; + cmd[0] = Mi_arb_on_off(true).value; + cmd[1] = ui.value; + + for (size_t i = 0; i < CMD_NUM; i++) { + advance += el.ring_append(cmd[i]); + } + } + /* * epilog 3/3 - gen8_emit_fini_breadcrumb_rcs, gen8_emit_fini_breadcrumb_tail * @@ -842,20 +896,6 @@ struct Igd::Device * * HWS page layout dword 48 - 1023 for driver usage */ - - if (1) - { - enum { CMD_NUM = 2 }; - Genode::uint32_t cmd[2] = {}; - Igd::Mi_user_interrupt ui; - cmd[0] = ui.value; - cmd[1] = Mi_arb_on_off(true).value; - - for (size_t i = 0; i < CMD_NUM; i++) { - advance += el.ring_append(cmd[i]); - } - } - if (1) { /* gen8_emit_fini_breadcrumb_tail -> gen8_emit_wa_tail */ @@ -1038,7 +1078,6 @@ struct Igd::Device } - /************ ** FENCES ** ************/ @@ -1069,34 +1108,51 @@ struct Igd::Device ** watchdog timeout ** **********************/ + void _handle_vgpu_after_reset(Vgpu &vgpu) + { + /* signal completion of last job to vgpu */ + vgpu.mark_completed(); + _notify_complete(&vgpu); + + /* offset of head in ring context */ + size_t head_offset = vgpu.rcs.context->head_offset(); + /* set head = tail in ring and ring context */ + Execlist &el = *vgpu.rcs.execlist; + el.ring_reset_to_head(head_offset); + /* set tail in context in qwords */ + vgpu.rcs.context->tail_offset((head_offset % (vgpu.rcs.ring_size())) / 8); + } + void _handle_watchdog_timeout() { if (!_active_vgpu) { return; } Genode::error("watchdog triggered: engine stuck," " vGPU=", _active_vgpu->id(), " IRQ: ", - Hex(_mmio.read_irq_vector(_info.generation))); - _mmio.dump(); - _mmio.error_dump(); - _mmio.fault_dump(); - _mmio.execlist_status_dump(); + Hex(_mmio.read_irq_vector())); - _active_vgpu->rcs.context->dump(); - _hw_status_page->dump(); - Execlist &el = *_active_vgpu->rcs.execlist; - el.ring_update_head(_active_vgpu->rcs.context->head_offset()); - el.ring_dump(4096, _active_vgpu->rcs.context->tail_offset() * 2, - _active_vgpu->rcs.context->head_offset()); + if (DEBUG) { + _mmio.dump(); + _mmio.error_dump(); + _mmio.fault_dump(); + _mmio.execlist_status_dump(); - _device_reset_and_init(); - - if (_active_vgpu == _current_vgpu()) { - _unschedule_current_vgpu(); + _active_vgpu->rcs.context->dump(); + _hw_status_page->dump(); + Execlist &el = *_active_vgpu->rcs.execlist; + el.ring_update_head(_active_vgpu->rcs.context->head_offset()); + el.ring_dump(4096, _active_vgpu->rcs.context->tail_offset() * 2, + _active_vgpu->rcs.context->head_offset()); } - if (_current_vgpu()) { - _schedule_current_vgpu(); - } + + Vgpu *vgpu = reset(); + + if (!vgpu) + error("reset vgpu is null"); + + /* the stuck vgpu */ + _handle_vgpu_after_reset(*vgpu); } Genode::Signal_handler _watchdog_timeout_sigh { @@ -1104,10 +1160,10 @@ struct Igd::Device void _device_reset_and_init() { - _mmio.reset(_info.generation); + _mmio.reset(); _mmio.clear_errors(); _mmio.init(); - _mmio.enable_intr(_info.generation); + _mmio.enable_intr(); } /** @@ -1132,6 +1188,7 @@ struct Igd::Device if (!_supported(supported, device_id, revision)) throw Unsupported_device(); + _ggtt.dump(); _vgpu_avail = (_gmadr.size() - APERTURE_RESERVED) / Vgpu::APERTURE_SIZE; @@ -1277,7 +1334,52 @@ struct Igd::Device /** * Reset the physical device */ - void reset() { _device_reset_and_init(); } + Vgpu *reset() + { + /* Stop render engine + * + * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES): + * KabyLake suffers from system hangs when batchbuffer is progressing during + * reset + */ + hw_status_page_pause_ring(true); + + Vgpu *vgpu { nullptr }; + + /* unschedule current vgpu */ + if (_active_vgpu) { + vgpu = _active_vgpu; + vgpu_unschedule(*_active_vgpu); + } + + /* reset */ + _reset.execute(); + + /* set address of global hardware status page */ + if (_hw_status_ctx.constructed()) { + Mmio::HWS_PGA_RCSUNIT::access_t const addr = _hw_status_ctx->gmaddr(); + _mmio.write_post(addr); + } + + _mmio.clear_errors(); + + /* clear pending irqs */ + _mmio.clear_render_irq(); + + /* + * Restore "Hardware Status Mask Register", this register controls which + * IRQs are even written to the PCI bus (should be same as unmasked in IMR) + */ + _mmio.restore_hwstam(); + + hw_status_page_pause_ring(false); + + if (_current_vgpu()) { + _schedule_current_vgpu(); + } + + return vgpu; + } /** * Get chip id of the physical device @@ -1332,12 +1434,25 @@ struct Igd::Device ** Hardware status page ** **************************/ - addr_t hw_status_page() const { return _hw_status_ctx->gmaddr(); } + addr_t hw_status_page_gmaddr() const { return _hw_status_ctx->gmaddr(); } - uint64_t seqno() const + addr_t hw_status_page_semaphore() const { - Utils::clflush((uint32_t*)(_hw_status_ctx->vaddr() + 0xc0)); - return *(uint32_t*)(_hw_status_ctx->vaddr() + 0xc0); + return hw_status_page_gmaddr() + Hardware_status_page::Semaphore::OFFSET; + } + + /* + * Pause the physical ring by setting semaphore value programmed by + * 'setup_ring_vram' to 1, causing GPU to spin. + */ + void hw_status_page_pause_ring(bool pause) + { + _hw_status_page->semaphore(pause ? 1 : 0); + } + + uint64_t seqno() + { + return _hw_status_page->sequence_number(); } @@ -1397,7 +1512,6 @@ struct Igd::Device if (vgpu.enqueued()) _vgpu_list.remove(vgpu); - } /******************* @@ -1521,21 +1635,21 @@ struct Igd::Device bool handle_irq() { - bool display_irq = _mmio.display_irq(_info.generation); + bool display_irq = _mmio.display_irq(); /* handle render interrupts only */ - if (_mmio.render_irq(_info.generation) == false) + if (_mmio.render_irq() == false) return display_irq; - _mmio.disable_master_irq(_info.generation); + _mmio.disable_master_irq(); - Mmio::GEN12_RENDER_INTR_VEC::access_t const v = _mmio.read_irq_vector(_info.generation); + unsigned const v = _mmio.read_irq_vector(); - bool const ctx_switch = Mmio::GEN12_RENDER_INTR_VEC::Cs_ctx_switch_interrupt::get(v); - bool const user_complete = Mmio::GEN12_RENDER_INTR_VEC::Cs_mi_user_interrupt::get(v); + bool ctx_switch = _mmio.context_switch(v); + bool user_complete = _mmio.user_complete(v); if (v) { - _mmio.clear_render_irq(_info.generation, v); + _mmio.clear_render_irq(v); } Vgpu *notify_gpu = nullptr; @@ -1569,7 +1683,7 @@ struct Igd::Device return display_irq; } - void enable_master_irq() { _mmio.enable_master_irq(_info.generation); } + void enable_master_irq() { _mmio.enable_master_irq(); } private: @@ -1886,7 +2000,7 @@ class Gpu::Session_component : public Genode::Session_object void vgpu_unschedule() { - return _device.vgpu_unschedule(_vgpu); + _device.vgpu_unschedule(_vgpu); } /*************************** @@ -2225,10 +2339,11 @@ class Gpu::Root : public Gpu::Root_component if (s->vgpu_active()) { Genode::warning("vGPU active, reset device and hope for the best"); _device->reset(); + } else { + /* remove from scheduled list */ + s->vgpu_unschedule(); } - /* remove from scheduled list */ - s->vgpu_unschedule(); Genode::destroy(md_alloc(), s); } diff --git a/repos/os/src/drivers/gpu/intel/mmio.h b/repos/os/src/drivers/gpu/intel/mmio.h index c5a4358486..c7a68ac1c9 100644 --- a/repos/os/src/drivers/gpu/intel/mmio.h +++ b/repos/os/src/drivers/gpu/intel/mmio.h @@ -1,11 +1,11 @@ /* - * \brief Broadwell MMIO definitions + * \brief GEN8/9/12 MMIO definitions * \author Josef Soentgen * \data 2017-03-15 */ /* - * Copyright (C) 2017 Genode Labs GmbH + * Copyright (C) 2023 Genode Labs GmbH * * This file is part of the Genode OS framework, which is distributed * under the terms of the GNU Affero General Public License version 3. @@ -30,6 +30,19 @@ class Igd::Mmio : public Platform::Device::Mmio { public: + unsigned _generation { 0 }; + + void generation(unsigned gen) { _generation = gen; } + + unsigned generation() const + { + if (!_generation) { + Genode::error("Unsupported generation: ", _generation); + } + + return _generation; + } + enum { /* * XXX IDs are taken from Linux, still looking @@ -513,19 +526,28 @@ class Igd::Mmio : public Platform::Device::Mmio */ struct GFX_MODE : Register<0x0229C, 32> { + /* to change bits below, accordings mask bit must be set */ struct Mask_bits : Bitfield<16, 16> { }; - struct Execlist_enable_mask : Bitfield<31, 1> { }; - struct Execlist_enable : Bitfield<15, 1> { }; + struct Execlist_enable : Bitfield<15, 1> { }; + struct Gen12_prefetch_disable : Bitfield<10, 1> { }; + struct Gen11_gfx_disable_legacy_mode : Bitfield< 3, 1> { }; + struct Privilege_check_disable : Bitfield< 0, 1> { }; - struct Ppgtt_enable_mask : Bitfield<25, 1> { }; - struct Ppgtt_enable : Bitfield< 9, 1> { }; + /* + * these bits are only valid in ringer buffer mode, we use execlist with + * contexts + */ + struct Ppgtt_enable : Bitfield< 9, 1> { }; + struct Virtual_addressing_enable : Bitfield< 7, 1> { }; - struct Virtual_addressing_enable_mask : Bitfield<23, 1> { }; - struct Virtual_addressing_enable : Bitfield< 7, 1> { }; - - struct Privilege_check_disable_mask : Bitfield<16, 1> { }; - struct Privilege_check_disable : Bitfield< 0, 1> { }; + template + static access_t set(access_t v, bool bit) + { + T::set(v, bit); + Mask_bits::set(v, 1u << T::SHIFT); + return v; + } }; /* @@ -633,6 +655,15 @@ class Igd::Mmio : public Platform::Device::Mmio struct Fault_entry_page_address : Bitfield<12, 20> { }; }; + /* + * IHD-OS-BDW-Vol 2c-11.15 p. 535 + */ + struct GAMTARBMODE : Register<0x4a08, 32> + { + struct Arbiter_mode_control_1 : Bitfield<1, 1> { }; + }; + + /* * IHD-OS-BDW-Vol 2c-11.15 p. 1315 ff. */ @@ -682,6 +713,9 @@ class Igd::Mmio : public Platform::Device::Mmio /* * IHD-OS-BDW-Vol 2c-11.15 p. 711 + * + * "Hardware Status Mask Register", this register controls which IRQs are + * even written to the PCI bus (should be same as unmasked in IMR) */ struct HWSTAM : Register<0x02098, 32> { }; @@ -827,6 +861,18 @@ class Igd::Mmio : public Platform::Device::Mmio struct Kernel : Bitfield< 0, 1> { }; }; + /* + * MSG idle register (one per engine) force wake status (undocumented) + */ + struct MSG_IDLE_CS : Register<0x8000, 32> + { + struct Pending_status : Bitfield<9, 5> { }; + struct Pending_mask : Bitfield<25, 5> { }; + }; + + /* force wake status used with MSG_IDLE_CS (undocumented) */ + struct GEN9_PWRGT_DOMAIN_STATUS : Register<0xa2a0, 32> { }; + /* * IHD-OS-BDW-Vol 2c-11.15 p. 703 * @@ -851,9 +897,10 @@ class Igd::Mmio : public Platform::Device::Mmio { using B = Register; - struct Mask_bits : B::template Bitfield<16, 16> { }; - struct Ready_for_reset : B::template Bitfield< 1, 1> { }; - struct Request_reset : B::template Bitfield< 0, 1> { }; + struct Mask_bits : B::template Bitfield<16, 16> { }; + struct Catastrophic_error : B::template Bitfield< 2, 1> { }; + struct Ready_for_reset : B::template Bitfield< 1, 1> { }; + struct Request_reset : B::template Bitfield< 0, 1> { }; }; template @@ -878,6 +925,14 @@ class Igd::Mmio : public Platform::Device::Mmio struct BCS_RESET_CTRL : RESET_CTRL_BASE <0x22000> { }; struct BCS_MI_MODE_CTRL : MI_MODE_CTRL_BASE<0x22000> { }; + /* + * IHD-OS-BDW-Vol 2c-11.15 p. 790 ff. + */ + struct HSW_IDICR : Register<0x9008, 32> + { + struct Idi_hash_mask : Bitfield<16, 6> { }; + }; + /* * IHD-OS-BDW-Vol 2c-11.15 p. 609 ff. */ @@ -947,6 +1002,13 @@ class Igd::Mmio : public Platform::Device::Mmio struct Fence_valid : Bitfield< 0, 1> { }; }; + /* + * IHD-OS-BDW-Vol 12-11.15 p. 4 + */ + struct TILECTL : Register<0x101000, 32> + { + struct SWZCTL : Bitfield<0, 2> { }; + }; /* * IHD-OS-BDW-Vol 12-11.15 p. 5 @@ -1071,7 +1133,6 @@ class Igd::Mmio : public Platform::Device::Mmio struct MI_DISP_PWR_DWN : Register<0x20E0, 32> { }; struct MI_ARB_STATE : Register<0x20E4, 32> { }; struct MI_RDRET_STATE : Register<0x20FC, 32> { }; - struct MI_MODE : Register<0x209c, 32> { }; struct ECOSKPD : Register<0x21D0, 32> { }; private: @@ -1093,6 +1154,23 @@ class Igd::Mmio : public Platform::Device::Mmio read(); } + void _fw_reset_gen9() + { + write_post(FORCEWAKE_MT::RESET); + write_post(FORCEWAKE_MT::RESET); + write_post(FORCEWAKE_MT::RESET); + } + + void _forcewake_reset() + { + if (generation() == 8) + _fw_reset_gen8(); + else if (generation() <= 12) + _fw_reset_gen9(); + else Genode::error(__func__, " unsupported generation", generation()); + } + + /** * Set forcewake state, i.e., prevent GT from powering down */ @@ -1140,13 +1218,6 @@ class Igd::Mmio : public Platform::Device::Mmio } } - void _fw_reset_gen9() - { - write_post(FORCEWAKE_MT::RESET); - write_post(FORCEWAKE_MT::RESET); - write_post(FORCEWAKE_MT::RESET); - } - /** * Set forcewake state, i.e., prevent from powering down */ @@ -1258,6 +1329,16 @@ class Igd::Mmio : public Platform::Device::Mmio write_post(0xffffffff); } + void _interrupt_reset() + { + if (generation() < 11) + _intr_reset(); + else if (generation() <= 12) + _intr_reset_gen12(); + else + Genode::error(__func__, " unsupported generation ", generation()); + } + /** * Enable interrupts */ @@ -1268,7 +1349,7 @@ class Igd::Mmio : public Platform::Device::Mmio /* GT0 RCS/BCS */ { uint32_t tmp = read(); - if (tmp != 0) { error("GT_0_INTERRUPT_IIR not zero"); } + if (tmp != 0) { error("GT_0_INTERRUPT_IIR not zero: ", Hex(tmp)); } GT_0_INTERRUPT_IER::access_t ier = 0; GT_0_INTERRUPT_IER::Cs_mi_user_interrupt::set(ier, 1); @@ -1557,19 +1638,6 @@ class Igd::Mmio : public Platform::Device::Mmio write(v); } - /** - * Enable execlist - */ - void _enable_execlist() - { - GFX_MODE::access_t v = read(); - - GFX_MODE::Execlist_enable_mask::set(v, 1); - GFX_MODE::Execlist_enable::set(v, 1); - - write(v); - } - uint32_t _clock_frequency_gen12() { uint32_t freq { 0 }; @@ -1600,6 +1668,8 @@ class Igd::Mmio : public Platform::Device::Mmio Mmio(Platform::Device & device, Genode::Env & env) : Platform::Device::Mmio(device, {0}), _delayer(env) { } + Delayer &delayer() { return _delayer; } + template void write_post(typename T::access_t const value) { @@ -1607,10 +1677,32 @@ class Igd::Mmio : public Platform::Device::Mmio (void)read(); } - - uint32_t clock_frequency(unsigned const generation) + /** + * Enable execlist + */ + void enable_execlist() { - if (generation >= 11) + /* + * This disables all interrupts delivered to the bus, make sure to + * enable IRQs later + */ + write_post(~0u); + write(0); + + using G = GFX_MODE; + G::access_t v = 0; + + if (generation() >= 11) + v = G::set(v, 1); + else + v = G::set(v, 1); + + write(v); + } + + uint32_t clock_frequency() + { + if (generation() >= 11) return _clock_frequency_gen12(); return 0; @@ -1633,9 +1725,9 @@ class Igd::Mmio : public Platform::Device::Mmio _fw_disable_gt(); } - void forcewake_enable(unsigned const generation) + void forcewake_enable() { - switch (generation) { + switch (generation()) { case 8: forcewake_gen8_enable(); return; @@ -1644,13 +1736,13 @@ class Igd::Mmio : public Platform::Device::Mmio forcewake_gen9_enable(); return; default: - Genode::error(__func__, " unsupported generation ", generation); + Genode::error(__func__, " unsupported generation ", generation()); } } - void forcewake_disable(unsigned const generation) + void forcewake_disable() { - switch (generation) { + switch (generation()) { case 8: forcewake_gen8_disable(); return; @@ -1659,56 +1751,16 @@ class Igd::Mmio : public Platform::Device::Mmio forcewake_gen9_disable(); return; default: - Genode::error(__func__, " unsupported generation ", generation); + Genode::error(__func__, " unsupported generation ", generation()); } } - void reset(unsigned const generation) + /* reset used during intialization only */ + void reset() { - switch (generation) { - case 8: - reset_gen8(); - return; - case 9: - reset_gen9(); - break; - case 12: - reset_gen12(); - break;; - default: - Genode::error(__func__, " unsupported generation ", generation); - } - } - - void reset_gen8() - { - _intr_reset(); - _fw_reset_gen8(); - forcewake_gen8_enable(); - _reset_device(); - _reset_fences(); - - _disable_nde_handshake(); - _set_page_attributes(); - } - - void reset_gen9() - { - _intr_reset(); - _fw_reset_gen9(); - forcewake_gen9_enable(); - _reset_device(); - _reset_fences(); - - _disable_nde_handshake(); - _set_page_attributes(); - } - - void reset_gen12() - { - _intr_reset_gen12(); - _fw_reset_gen9(); - forcewake_gen9_enable(); + _interrupt_reset(); + _forcewake_reset(); + forcewake_enable(); _reset_device(); _reset_fences(); @@ -1719,51 +1771,47 @@ class Igd::Mmio : public Platform::Device::Mmio void init() { _disable_rps(); - _enable_execlist(); + enable_execlist(); } - void enable_intr(unsigned const generation) - { + void enable_intr() + { write(0xffffff00); - if (generation < 11) + if (generation() < 11) _intr_enable(); else _intr_enable_gen12(); } - void disable_master_irq(unsigned const generation) + void restore_hwstam() { - if (generation < 11) + if (generation() < 11) + write_post(read()); + else + write_post(~read()); + } + + void disable_master_irq() + { + if (generation() < 11) write_post(0); else write(0); } - void enable_master_irq(unsigned const generation) + void enable_master_irq() { - if (generation < 11) + if (generation() < 11) write_post(1); else write(1); } - bool render_irq(unsigned const generation) + unsigned read_irq_vector() { - if (generation < 11) - return read() == 1; - else { - if (read() == 1 && - read() == 1) - return true; - } - return false; - } - - GEN12_RENDER_INTR_VEC::access_t read_irq_vector(unsigned const generation) - { - GEN12_RENDER_INTR_VEC::access_t vec = 0; - if (generation < 11) { + unsigned vec = 0; + if (generation() < 11) { vec = read(); } else { write(1); @@ -1772,7 +1820,6 @@ class Igd::Mmio : public Platform::Device::Mmio wait_for(Attempts(50), Microseconds(500), _delayer, GEN12_INTR_IDENTITY_REG0::Valid::Equal(1)); } catch (Polling_timeout) { - Genode::error(__func__, " IRQ vector not valid"); return vec; } vec = read(); @@ -1782,22 +1829,61 @@ class Igd::Mmio : public Platform::Device::Mmio return vec; }; - void clear_render_irq(unsigned const generation, Genode::uint16_t v) + void clear_render_irq(unsigned v) { - if (generation < 11) + if (generation() < 11) write_post(v); else write(1); } - bool display_irq(unsigned const generation) + void clear_render_irq() { - if (generation < 11) + unsigned v = 0; + if (generation() < 11) + v = read_irq_vector(); + clear_render_irq(v); + } + + /** + * IRQ causes + */ + bool render_irq() + { + if (generation() < 11) + return read() == 1; + else { + if (read() == 1 && + read() == 1) + return true; + } + return false; + } + + bool display_irq() + { + if (generation() < 11) return read() != 0; else return read() == 1; } + bool context_switch(unsigned const vector) + { + if (generation() < 11) + return GT_0_INTERRUPT_IIR::Cs_ctx_switch_interrupt::get(vector); + else + return GEN12_RENDER_INTR_VEC::Cs_ctx_switch_interrupt::get(vector); + } + + bool user_complete(unsigned const vector) + { + if (generation() < 11) + return GT_0_INTERRUPT_IIR::Cs_mi_user_interrupt::get(vector); + else + return GEN12_RENDER_INTR_VEC::Cs_mi_user_interrupt::get(vector); + } + void flush_gfx_tlb() { _gfx_flush_cntl(); } @@ -1808,8 +1894,10 @@ class Igd::Mmio : public Platform::Device::Mmio void update_context_status_pointer() { + size_t const context_status_size = generation() < 11 ? 6 : 12; + RCS_RING_CONTEXT_STATUS_PTR::access_t const wp = read(); - if (wp > 0x05) { + if (wp > (context_status_size - 1)) { Genode::warning("ring context status write-pointer invalid", Genode::Hex(wp)); return; } @@ -1886,8 +1974,9 @@ class Igd::Mmio : public Platform::Device::Mmio struct MASK_WAKEMEM : Bitfield<13, 1> {}; }; struct DISP_ARB_CTL : Register<0x45000, 32> { - struct DISP_FBC_MEMORY_WAKE : Bitfield<31, 1> {}; - struct DISP_FBC_WM_DIS : Bitfield<15, 1> {}; + struct DISP_FBC_MEMORY_WAKE : Bitfield<31, 1> {}; + struct DISP_TILE_SURFACE_SWIZZLING : Bitfield<13, 1> { }; + struct DISP_FBC_WM_DIS : Bitfield<15, 1> {}; }; void gen9_clock_gating() diff --git a/repos/os/src/drivers/gpu/intel/mmio_dump.cc b/repos/os/src/drivers/gpu/intel/mmio_dump.cc index f27825d3db..f20942cfeb 100644 --- a/repos/os/src/drivers/gpu/intel/mmio_dump.cc +++ b/repos/os/src/drivers/gpu/intel/mmio_dump.cc @@ -52,7 +52,7 @@ void Igd::Mmio::dump() log("0x20E0 - MI_DISP_PWR_DWN ", Hex(read())); log("0x20E4 - MI_ARB_STATE ", Hex(read())); log("0x20FC - MI_RDRET_STATE ", Hex(read())); - log("0x209C - MI_MODE ", Hex(read())); + log("0x209C - MI_MODE ", Hex(read())); log("0x21D0 - ECOSKPD ", Hex(read())); } diff --git a/repos/os/src/drivers/gpu/intel/reset.h b/repos/os/src/drivers/gpu/intel/reset.h new file mode 100644 index 0000000000..5a4bade46f --- /dev/null +++ b/repos/os/src/drivers/gpu/intel/reset.h @@ -0,0 +1,171 @@ +/* + * \brief Render engine reset based on the Linux driver + * \author Sebastian Sumpf + * \date 2023-06-05 + */ + +/* + * Copyright (C) 2023 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#ifndef _RESET_H_ +#define _RESET_H_ + +#include "mmio.h" +#include "workarounds.h" + +namespace Igd { + class Reset; +} + + +class Igd::Reset +{ + private: + + Igd::Mmio &_mmio; + + void _stop_engine_cs() + { + /* write stop bit to render mode */ + _mmio.write(1); + + /* + * Wa_22011802037 : GEN11, GNE12, Prior to doing a reset, ensure CS is + * stopped, set ring stop bit and prefetch disable bit to halt CS + */ + if (_mmio.generation() == 11 || _mmio.generation() == 12) { + Mmio::GFX_MODE::access_t v = 0; + using G = Mmio::GFX_MODE; + v = G::set(v, 1); + _mmio.write(v); + } + + try { + _mmio.wait_for(Mmio::Attempts(10), Mmio::Microseconds(100'000), _mmio.delayer(), + Mmio::CS_MI_MODE_CTRL::Rings_idle::Equal(1)); + } catch(Mmio::Polling_timeout) { + Genode::warning("stop engine cs timeout"); + } + + /* read to let GPU writes be flushed to memory */ + _mmio.read(); + } + + /* not documented + * Wa_22011802037: gen11/gen12: In addition to stopping the cs, we need + * to wait for any pending mi force wakeups + * MSG_IDLE_CS 0x8000 force wake + */ + void _wait_for_pending_force_wakeups() + { + if (_mmio.generation() < 11 && _mmio.generation() > 12) return; + + unsigned fw_status = _mmio.read(); + unsigned fw_mask = _mmio.read(); + + _mmio.delayer().usleep(1); + + for (unsigned i = 0; i < 10; i++) { + unsigned status = _mmio.read() & fw_mask; + + _mmio.delayer().usleep(1); + + if (status == fw_status) return; + + _mmio.delayer().usleep(50000); + } + + _mmio.delayer().usleep(1); + Genode::warning("wait pending force wakeup timeout"); + } + + void _ready_for_reset() + { + if (_mmio.read()) { + /* For catastrophic errors, ready-for-reset sequence + * needs to be bypassed: HAS#396813 + */ + try { + _mmio.wait_for(Mmio::Attempts(7), Mmio::Microseconds(100'000), _mmio.delayer(), + Mmio::CS_RESET_CTRL::Catastrophic_error::Equal(0)); + } catch (Mmio::Polling_timeout) { + Genode::warning("catastrophic error reset not cleared"); + } + return; + } + + if (_mmio.read()) return; + + Mmio::CS_RESET_CTRL::access_t request = 0; + Mmio::CS_RESET_CTRL::Mask_bits::set(request, 1); + Mmio::CS_RESET_CTRL::Request_reset::set(request, 1); + _mmio.write_post(request); + try { + _mmio.wait_for(Mmio::Attempts(7), Mmio::Microseconds(100'000), _mmio.delayer(), + Mmio::CS_RESET_CTRL::Ready_for_reset::Equal(1)); + } catch (Mmio::Polling_timeout) { + Genode::warning("not ready for reset"); + } + } + + void _unready_for_reset() + { + Mmio::CS_RESET_CTRL::access_t request = 0; + Mmio::CS_RESET_CTRL::Mask_bits::set(request, 1); + Mmio::CS_RESET_CTRL::Request_reset::set(request, 0); + _mmio.write_post(request); + } + + void _reset_hw() + { + /* full sw reset */ + _mmio.write(1); + try { + /* do NOT attempt more than 2 times */ + _mmio.wait_for(Mmio::Attempts(2), Mmio::Microseconds(200'000), _mmio.delayer(), + Mmio::GDRST::Graphics_full_soft_reset_ctl::Equal(0)); + } catch (Mmio::Polling_timeout) { + Genode::error("resetting device failed"); + } + + /* some devices still show volatile state */ + _mmio.delayer().usleep(50); + } + + void _init_swizzling() + { + _mmio.write(1); + _mmio.write(1); + + if (_mmio.generation() == 8) + _mmio.write(1); + } + + public: + + Reset(Igd::Mmio &mmio) : _mmio(mmio) { } + + void execute() + { + _stop_engine_cs(); + _wait_for_pending_force_wakeups(); + _ready_for_reset(); + + _reset_hw(); + + _unready_for_reset(); + + if (_mmio.generation() < 9) + _mmio.write(0xf); + + apply_workarounds(_mmio, _mmio.generation()); + _init_swizzling(); + _mmio.enable_execlist(); + } +}; + +#endif /* _RESET_H_ */ diff --git a/repos/os/src/drivers/gpu/intel/ring_buffer.h b/repos/os/src/drivers/gpu/intel/ring_buffer.h index 2b526911bf..344018ebbc 100644 --- a/repos/os/src/drivers/gpu/intel/ring_buffer.h +++ b/repos/os/src/drivers/gpu/intel/ring_buffer.h @@ -98,6 +98,15 @@ class Igd::Ring_buffer _head = head; } + /** + * Update head and set tail to head + */ + void reset_to_head(Index head) + { + update_head(head); + _tail = _head; + } + /** * Insert new command at given index * diff --git a/repos/os/src/drivers/gpu/intel/workarounds.h b/repos/os/src/drivers/gpu/intel/workarounds.h index fbb0165f05..e84ae36edd 100644 --- a/repos/os/src/drivers/gpu/intel/workarounds.h +++ b/repos/os/src/drivers/gpu/intel/workarounds.h @@ -13,7 +13,7 @@ #include namespace Igd { - void apply_workarounds(Mmio &mmio, unsigned generation); + static void apply_workarounds(Mmio &mmio, unsigned generation); } void Igd::apply_workarounds(Mmio &mmio, unsigned generation)