ExternalVendorCode/genode
1
0
Fork 0
mirror of https://github.com/genodelabs/genode.git synced 2024-12-21 06:33:31 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

base: extend cache maintainance functionality

Browse Source 
Introduce two new cache maintainance functions:
* cache_clean_invalidate_data
* cache_invalidate_data
used to flush or invalidate data-cache lines.

Both functions are typically empty, accept for the ARM architecture.
The commit provides implementations for the base-hw kernel, and Fiasco.OC.

Fixes #4207
This commit is contained in:
Stefan Kalkowski 2021-06-24 14:27:37 +02:00 committed by Christian Helmuth
parent ed0cc5330e
commit e7067050be
26 changed files with 301 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
repos/base-foc/src/lib/base/cache.cc
View File

@ -21,3 +21,15 @@ void Genode::cache_coherent(addr_t addr, size_t size)
{
Foc::l4_cache_coherent(addr, addr + size);
}
void Genode::cache_clean_invalidate_data(Genode::addr_t addr, Genode::size_t size)
{
Foc::l4_cache_flush_data(addr, addr + size);
}
void Genode::cache_invalidate_data(Genode::addr_t addr, Genode::size_t size)
{
Foc::l4_cache_inv_data(addr, addr + size);
}

42
repos/base-hw/include/kernel/interface.h
View File

@ -37,13 +37,15 @@ namespace Kernel {
constexpr Call_arg call_id_ack_signal() { return 11; }
constexpr Call_arg call_id_print_char() { return 12; }
constexpr Call_arg call_id_cache_coherent_region() { return 13; }
constexpr Call_arg call_id_ack_cap() { return 14; }
constexpr Call_arg call_id_delete_cap() { return 15; }
constexpr Call_arg call_id_timeout() { return 16; }
constexpr Call_arg call_id_timeout_max_us() { return 17; }
constexpr Call_arg call_id_time() { return 18; }
constexpr Call_arg call_id_run_vm() { return 19; }
constexpr Call_arg call_id_pause_vm() { return 20; }
constexpr Call_arg call_id_cache_clean_inv_region() { return 14; }
constexpr Call_arg call_id_cache_inv_region() { return 15; }
constexpr Call_arg call_id_ack_cap() { return 16; }
constexpr Call_arg call_id_delete_cap() { return 17; }
constexpr Call_arg call_id_timeout() { return 18; }
constexpr Call_arg call_id_timeout_max_us() { return 19; }
constexpr Call_arg call_id_time() { return 20; }
constexpr Call_arg call_id_run_vm() { return 21; }
constexpr Call_arg call_id_pause_vm() { return 22; }
/*****************************************************************
@ -188,6 +190,32 @@ namespace Kernel {
}
/**
* Clean and invalidate D-Cache lines of the given memory region
*
* \param base base of the region within the current domain
* \param size size of the region
*/
inline void cache_clean_invalidate_data_region(addr_t const base,
size_t const size)
{
call(call_id_cache_clean_inv_region(), (Call_arg)base, (Call_arg)size);
}
/**
* Invalidate D-Cache lines of the given memory region
*
* \param base base of the region within the current domain
* \param size size of the region
*/
inline void cache_invalidate_data_region(addr_t const base,
size_t const size)
{
call(call_id_cache_inv_region(), (Call_arg)base, (Call_arg)size);
}
/**
* Send request message and await receipt of corresponding reply message
*

1
repos/base-hw/lib/mk/spec/arm/core-hw.inc
View File

@ -15,6 +15,7 @@ SRC_CC += spec/arm/kernel/cpu.cc
SRC_CC += spec/arm/kernel/pd.cc
SRC_CC += spec/arm/cpu.cc
SRC_CC += spec/arm/kernel/thread.cc
SRC_CC += spec/arm/kernel/thread_caches.cc
SRC_CC += spec/arm/platform_support.cc
# add assembly sources

1
repos/base-hw/lib/mk/spec/arm_v8/core-hw.inc
View File

@ -7,6 +7,7 @@ SRC_CC += kernel/cpu_mp.cc
SRC_CC += spec/64bit/memory_map.cc
SRC_CC += spec/arm/generic_timer.cc
SRC_CC += spec/arm/kernel/lock.cc
SRC_CC += spec/arm/kernel/thread_caches.cc
SRC_CC += spec/arm/platform_support.cc
SRC_CC += spec/arm_v8/cpu.cc
SRC_CC += spec/arm_v8/kernel/cpu.cc

2
repos/base-hw/src/core/kernel/thread.cc
View File

@ -726,6 +726,8 @@ void Thread::_call()
unsigned const call_id = user_arg_0();
switch (call_id) {
case call_id_cache_coherent_region(): _call_cache_coherent_region(); return;
case call_id_cache_clean_inv_region(): _call_cache_clean_invalidate_data_region(); return;
case call_id_cache_inv_region(): _call_cache_invalidate_data_region(); return;
case call_id_stop_thread(): _call_stop_thread(); return;
case call_id_restart_thread(): _call_restart_thread(); return;
case call_id_yield_thread(): _call_yield_thread(); return;

2
repos/base-hw/src/core/kernel/thread.h
View File

@ -225,6 +225,8 @@ class Kernel::Thread : private Kernel::Object, public Cpu_job, private Timeout
void _call_send_reply_msg();
void _call_invalidate_tlb();
void _call_cache_coherent_region();
void _call_cache_clean_invalidate_data_region();
void _call_cache_invalidate_data_region();
void _call_print_char();
void _call_await_signal();
void _call_pending_signal();

18
repos/base-hw/src/core/spec/arm/cpu.cc
View File

@ -152,16 +152,24 @@ void Arm_cpu::cache_coherent_region(addr_t const base,
}
void Arm_cpu::clean_data_cache_by_virt_region(addr_t const base,
size_t const size)
void Arm_cpu::cache_invalidate_data_region(addr_t const base,
size_t const size)
{
auto lambda = [] (addr_t const base) { Dcimvac::write(base); };
cache_maintainance(base, size, Cpu::data_cache_line_size(), lambda);
}
void Arm_cpu::cache_clean_data_region(addr_t const base,
size_t const size)
{
auto lambda = [] (addr_t const base) { Dccmvac::write(base); };
cache_maintainance(base, size, Cpu::data_cache_line_size(), lambda);
}
void Arm_cpu::clean_invalidate_data_cache_by_virt_region(addr_t const base,
size_t const size)
void Arm_cpu::cache_clean_invalidate_data_region(addr_t const base,
size_t const size)
{
auto lambda = [] (addr_t const base) { Dccimvac::write(base); };
cache_maintainance(base, size, Cpu::data_cache_line_size(), lambda);
@ -197,7 +205,7 @@ void Arm_cpu::clear_memory_region(addr_t const addr,
* DMA memory, which needs to be evicted from the D-cache
*/
if (changed_cache_properties) {
Cpu::clean_invalidate_data_cache_by_virt_region(addr, size);
Cpu::cache_clean_invalidate_data_region(addr, size);
}
/**

22
repos/base-hw/src/core/spec/arm/cpu_support.h
View File

@ -70,19 +70,6 @@ struct Genode::Arm_cpu : public Hw::Arm_cpu
static void invalidate_instr_cache() {
asm volatile ("mcr p15, 0, %0, c7, c5, 0" :: "r" (0) : ); }
/**
* Clean data-cache for virtual region 'base' - 'base + size'
*/
static void clean_data_cache_by_virt_region(addr_t const base,
size_t const size);
/**
* Clean and invalidate data-cache for virtual region
* 'base' - 'base + size'
*/
static void clean_invalidate_data_cache_by_virt_region(addr_t const base,
size_t const size);
static void clear_memory_region(addr_t const addr,
size_t const size,
bool changed_cache_properties);
@ -90,6 +77,15 @@ struct Genode::Arm_cpu : public Hw::Arm_cpu
static void cache_coherent_region(addr_t const addr,
size_t const size);
static void cache_clean_data_region(addr_t const base,
size_t const size);
static void cache_clean_invalidate_data_region(addr_t const addr,
size_t const size);
static void cache_invalidate_data_region(addr_t const addr,
size_t const size);
/**
* Invalidate TLB regarding the given address space id
*/

30
repos/base-hw/src/core/spec/arm/kernel/thread.cc
View File

@ -55,36 +55,6 @@ void Thread::exception(Cpu & cpu)
}
void Kernel::Thread::_call_cache_coherent_region()
{
addr_t base = (addr_t) user_arg_1();
size_t const size = (size_t) user_arg_2();
/**
* sanity check that only one small page is affected,
* because we only want to lookup one page in the page tables
* to limit execution time within the kernel
*/
if (Hw::trunc_page(base) != Hw::trunc_page(base+size-1)) {
Genode::raw(*this, " tried to make cross-page region cache coherent ",
(void*)base, " ", size);
return;
}
/**
* Lookup whether the page is backed, and if so make the memory coherent
* in between I-, and D-cache
*/
addr_t phys = 0;
if (pd().platform_pd().lookup_translation(base, phys)) {
Cpu::cache_coherent_region(base, size);
} else {
Genode::raw(*this, " tried to make invalid address ",
base, " cache coherent");
}
}
/**
* on ARM with multiprocessing extensions, maintainance operations on TLB,
* and caches typically work coherently across CPUs when using the correct

74
repos/base-hw/src/core/spec/arm/kernel/thread_caches.cc Normal file
View File

@ -0,0 +1,74 @@
/*
* \brief Kernel backend for threads - cache maintainance
* \author Stefan Kalkowski
* \date 2021-06-24
*/
/*
* Copyright (C) 2021 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#include <cpu.h>
#include <platform_pd.h>
#include <kernel/pd.h>
#include <kernel/thread.h>
using namespace Kernel;
template <typename FN>
static void for_cachelines(addr_t base,
size_t const size,
Kernel::Thread & thread,
FN const & fn)
{
/**
* sanity check that only one small page is affected,
* because we only want to lookup one page in the page tables
* to limit execution time within the kernel
*/
if (Hw::trunc_page(base) != Hw::trunc_page(base+size-1)) {
Genode::raw(thread, " tried to make cross-page region cache coherent ",
(void*)base, " ", size);
return;
}
/**
* Lookup whether the page is backed, and if so make the memory coherent
* in between I-, and D-cache
*/
addr_t phys = 0;
if (thread.pd().platform_pd().lookup_translation(base, phys)) {
fn(base, size);
} else {
Genode::raw(thread, " tried to make invalid address ",
base, " cache coherent");
}
}
void Kernel::Thread::_call_cache_coherent_region()
{
for_cachelines((addr_t)user_arg_1(), (size_t)user_arg_2(), *this,
[] (addr_t addr, size_t size) {
Genode::Cpu::cache_coherent_region(addr, size); });
}
void Kernel::Thread::_call_cache_clean_invalidate_data_region()
{
for_cachelines((addr_t)user_arg_1(), (size_t)user_arg_2(), *this,
[] (addr_t addr, size_t size) {
Genode::Cpu::cache_clean_invalidate_data_region(addr, size); });
}
void Kernel::Thread::_call_cache_invalidate_data_region()
{
for_cachelines((addr_t)user_arg_1(), (size_t)user_arg_2(), *this,
[] (addr_t addr, size_t size) {
Genode::Cpu::cache_invalidate_data_region(addr, size); });
}

2
repos/base-hw/src/core/spec/arm_v6/translation_table.h
View File

@ -29,7 +29,7 @@ constexpr bool Hw::Page_table::Descriptor_base::_smp() { return false; }
void Hw::Page_table::_table_changed(unsigned long addr, unsigned long size)
{
Genode::Arm_cpu::clean_data_cache_by_virt_region(addr, size);
Genode::Arm_cpu::cache_clean_data_region(addr, size);
}
#endif /* _CORE__SPEC__ARM_V6__TRANSLATION_TABLE_H_ */

32
repos/base-hw/src/core/spec/arm_v8/cpu.cc
View File

@ -101,7 +101,7 @@ static inline void cache_maintainance(Genode::addr_t const base,
void Genode::Cpu::cache_coherent_region(addr_t const base,
size_t const size)
size_t const size)
{
Genode::memory_barrier();
@ -117,6 +117,36 @@ void Genode::Cpu::cache_coherent_region(addr_t const base,
}
void Genode::Cpu::cache_clean_invalidate_data_region(addr_t const base,
size_t const size)
{
Genode::memory_barrier();
auto lambda = [] (addr_t const base) {
asm volatile("dc civac, %0" :: "r" (base));
asm volatile("dsb ish");
asm volatile("isb");
};
cache_maintainance(base, size, lambda);
}
void Genode::Cpu::cache_invalidate_data_region(addr_t const base,
size_t const size)
{
Genode::memory_barrier();
auto lambda = [] (addr_t const base) {
asm volatile("dc ivac, %0" :: "r" (base));
asm volatile("dsb ish");
asm volatile("isb");
};
cache_maintainance(base, size, lambda);
}
void Genode::Cpu::clear_memory_region(addr_t const addr,
size_t const size,
bool changed_cache_properties)

4
repos/base-hw/src/core/spec/arm_v8/cpu.h
View File

@ -95,6 +95,10 @@ struct Genode::Cpu : Hw::Arm_64_cpu
static void cache_coherent_region(addr_t const addr,
size_t const size);
static void cache_clean_invalidate_data_region(addr_t const addr,
size_t const size);
static void cache_invalidate_data_region(addr_t const addr,
size_t const size);
};
#endif /* _CORE__SPEC__ARM_V8__CPU_H_ */

30
repos/base-hw/src/core/spec/arm_v8/kernel/thread.cc
View File

@ -107,36 +107,6 @@ bool Kernel::Pd::invalidate_tlb(Cpu &, addr_t addr, size_t size)
}
void Kernel::Thread::_call_cache_coherent_region()
{
addr_t base = (addr_t) user_arg_1();
size_t const size = (size_t) user_arg_2();
/**
* sanity check that only one small page is affected,
* because we only want to lookup one page in the page tables
* to limit execution time within the kernel
*/
if (Hw::trunc_page(base) != Hw::trunc_page(base+size-1)) {
Genode::raw(*this, " tried to make cross-page region cache coherent ",
(void*)base, " ", size);
return;
}
/**
* Lookup whether the page is backed, and if so make the memory coherent
* in between I-, and D-cache
*/
addr_t phys = 0;
if (pd().platform_pd().lookup_translation(base, phys)) {
Cpu::cache_coherent_region(base, size);
} else {
Genode::raw(*this, " tried to make invalid address ",
base, " cache coherent");
}
}
void Thread::proceed(Cpu & cpu)
{
cpu.switch_to(*regs, pd().mmu_regs);

2
repos/base-hw/src/core/spec/cortex_a8/translation_table.h
View File

@ -37,7 +37,7 @@ void Hw::Page_table::_table_changed(unsigned long addr, unsigned long size)
* page table entry is added. We only do this as core as the kernel
* adds translations solely before MMU and caches are enabled.
*/
Genode::Cpu::clean_data_cache_by_virt_region(addr, size);
Genode::Cpu::cache_clean_data_region(addr, size);
}
#endif /* _CORE__SPEC__CORTEX_A8__TRANSLATION_TABLE_H_ */

6
repos/base-hw/src/core/spec/cortex_a9/cpu.h
View File

@ -28,10 +28,10 @@ struct Genode::Cpu : Arm_v7_cpu
* Clean and invalidate data-cache for virtual region
* 'base' - 'base + size'
*/
static void clean_invalidate_data_cache_by_virt_region(addr_t const base,
size_t const size)
static void cache_clean_invalidate_data_region(addr_t const base,
size_t const size)
{
Arm_cpu::clean_invalidate_data_cache_by_virt_region(base, size);
Arm_cpu::cache_clean_invalidate_data_region(base, size);
Board::l2_cache().clean_invalidate();
}

6
repos/base-hw/src/core/spec/riscv/kernel/thread.cc
View File

@ -92,6 +92,12 @@ void Thread::exception(Cpu & cpu)
void Thread::_call_cache_coherent_region() { }
void Kernel::Thread::_call_cache_clean_invalidate_data_region() { }
void Kernel::Thread::_call_cache_invalidate_data_region() { }
void Kernel::Thread::proceed(Cpu & cpu)
{
cpu.switch_to(_pd->mmu_regs);

6
repos/base-hw/src/core/spec/x86_64/kernel/thread.cc
View File

@ -35,6 +35,12 @@ void Kernel::Thread::Tlb_invalidation::execute()
void Kernel::Thread::_call_cache_coherent_region() { }
void Kernel::Thread::_call_cache_clean_invalidate_data_region() { }
void Kernel::Thread::_call_cache_invalidate_data_region() { }
void Kernel::Thread::proceed(Cpu & cpu)
{
cpu.switch_to(*regs, pd().mmu_regs);

3
repos/base-hw/src/include/hw/spec/arm/cpu.h
View File

@ -206,6 +206,9 @@ struct Hw::Arm_cpu
/* Branch predictor invalidate all */
ARM_CP15_REGISTER_32BIT(Bpimva, c7, c5, 0, 7);
/* Data Cache Invalidate by MVA to PoC */
ARM_CP15_REGISTER_32BIT(Dcimvac, c7, c6, 0, 1);
/* Data Cache Clean by MVA to PoC */
ARM_CP15_REGISTER_32BIT(Dccmvac, c7, c10, 0, 1);

34
repos/base-hw/src/lib/base/cache.cc
View File

@ -17,22 +17,44 @@
#include <cpu/cache.h>
#include <util/misc_math.h>
using namespace Genode;
void Genode::cache_coherent(Genode::addr_t addr, Genode::size_t size)
template <typename FN>
static void for_cachelines(addr_t addr, size_t size, FN const & fn)
{
using namespace Genode;
/**
* The kernel accepts the 'cache_coherent_region' call for one designated
* The kernel accepts the cache maintainance calls for one designated
* page only. Otherwise, it just ignores the call to limit the time being
* uninteruppptible in the kernel. Therefor, we have to loop if more than
* uninteruppptible in the kernel. Therefore, we have to loop if more than
* one page is affected by the given region.
*/
while (size) {
addr_t next_page = align_addr(addr+1, get_page_size_log2());
size_t s = min(size, next_page - addr);
Kernel::cache_coherent_region(addr, s);
fn(addr, s);
addr += s;
size -= s;
}
}
void Genode::cache_coherent(addr_t addr, size_t size)
{
for_cachelines(addr, size, [] (addr_t addr, size_t size) {
Kernel::cache_coherent_region(addr, size); });
}
void Genode::cache_clean_invalidate_data(addr_t addr, size_t size)
{
for_cachelines(addr, size, [] (addr_t addr, size_t size) {
Kernel::cache_clean_invalidate_data_region(addr, size); });
}
void Genode::cache_invalidate_data(addr_t addr, size_t size)
{
for_cachelines(addr, size, [] (addr_t addr, size_t size) {
Kernel::cache_invalidate_data_region(addr, size); });
}

13
repos/base-linux/src/lib/base/cpu/arm/cache.cc
View File

@ -13,9 +13,22 @@
#include <linux_syscalls.h>
#include <base/log.h>
#include <cpu/cache.h>
void Genode::cache_coherent(Genode::addr_t addr, Genode::size_t size)
{
lx_syscall(__ARM_NR_cacheflush, addr, addr + size, 0);
}
void Genode::cache_clean_invalidate_data(Genode::addr_t, Genode::size_t)
{
error(__func__, " not implemented for this kernel!");
}
void Genode::cache_invalidate_data(Genode::addr_t, Genode::size_t)
{
error(__func__, " not implemented for this kernel!");
}

2
repos/base-sel4/lib/mk/spec/arm/base-sel4.mk
View File

@ -1,3 +1,5 @@
LIBS += timeout-arm
vpath cache.cc $(REP_DIR)/src/lib/base/arm
include $(REP_DIR)/lib/mk/base-sel4.inc

32
repos/base-sel4/src/lib/base/arm/cache.cc Normal file
View File

@ -0,0 +1,32 @@
/*
* \brief Implementation of the cache operations
* \author Stefan Kalkowski
* \date 2021-06-24
*/
/*
* Copyright (C) 2021 Genode Labs GmbH
*
* This file is part of the Genode OS framework, which is distributed
* under the terms of the GNU Affero General Public License version 3.
*/
#include <base/log.h>
#include <cpu/cache.h>
void Genode::cache_coherent(Genode::addr_t, Genode::size_t)
{
error(__func__, " not implemented for this kernel!");
}
void Genode::cache_clean_invalidate_data(Genode::addr_t, Genode::size_t)
{
error(__func__, " not implemented for this kernel!");
}
void Genode::cache_invalidate_data(Genode::addr_t, Genode::size_t)
{
error(__func__, " not implemented for this kernel!");
}

13
repos/base/include/cpu/cache.h
View File

@ -1,6 +1,7 @@
/*
* \brief Cache operations
* \author Christian Prochaska
* \author Stefan Kalkowski
* \date 2014-05-13
*/
@ -20,8 +21,20 @@ namespace Genode {
/*
* Make D-Cache and I-Cache coherent
*
* That means write back the D-Cache lines, and invalidate the I-Cache lines
*/
void cache_coherent(Genode::addr_t addr, Genode::size_t size);
/*
* Write back and delete D-Cache (commonly known as flush)
*/
void cache_clean_invalidate_data(Genode::addr_t addr, Genode::size_t size);
/*
* Delete D-Cache lines only
*/
void cache_invalidate_data(Genode::addr_t addr, Genode::size_t size);
}
#endif /* _INCLUDE__CPU__CACHE_H_ */

2
repos/base/lib/symbols/ld
View File

@ -125,6 +125,8 @@ _ZN6Genode14Signal_contextD0Ev T
_ZN6Genode14Signal_contextD1Ev T
_ZN6Genode14Signal_contextD2Ev T
_ZN6Genode14cache_coherentEmm T
_ZN6Genode21cache_invalidate_dataEmm T
_ZN6Genode27cache_clean_invalidate_dataEmm T
_ZN6Genode14env_deprecatedEv T
_ZN6Genode14ipc_reply_waitERKNS_17Native_capabilityENS_18Rpc_exception_codeERNS_11Msgbuf_baseES5_ T
_ZN6Genode15Connection_baseC1Ev T

8
repos/base/src/lib/base/cache.cc
View File

@ -14,8 +14,14 @@
#include <cpu/cache.h>
/*
* This function needs to be implemented only for base platforms with ARM
* These functions need to be implemented only for base platforms with ARM
* support right now, so the default implementation does nothing.
*/
void Genode::cache_coherent(Genode::addr_t, Genode::size_t) { }
void Genode::cache_clean_invalidate_data(Genode::addr_t, Genode::size_t) { }
void Genode::cache_invalidate_data(Genode::addr_t, Genode::size_t) { }