From 08c56e61e16e1b5f6dd12f3469140027e3c83e59 Mon Sep 17 00:00:00 2001 From: Stefan Kalkowski Date: Mon, 19 Dec 2022 11:17:43 +0100 Subject: [PATCH] hw: avoid overhead for cache maintainance When running on x86, and riscv never enter the kernel for cache maintainance, but use the dummy implementation of the generic base library instead. On ARMv8 it is not necessary to enter privileged mode for cache cleaning, and unification of instruction/data cache, but only for invalidating cache lines at all levels, which is necessary for the use cases, where this function it needed (coherency of DMA memory). Fix genodelabs/genode#4339 --- .../lib/mk/{base-hw.mk => base-hw.inc} | 0 repos/base-hw/lib/mk/spec/arm/base-hw.mk | 3 + repos/base-hw/lib/mk/spec/arm_64/base-hw.mk | 3 + repos/base-hw/lib/mk/spec/riscv/base-hw.mk | 1 + repos/base-hw/lib/mk/spec/x86_64/base-hw.mk | 1 + repos/base-hw/recipes/src/base-hw_content.inc | 2 +- repos/base-hw/src/core/spec/arm_v6/cpu.h | 4 ++ .../src/core/spec/arm_v7/cpu_support.h | 5 ++ .../base-hw/src/include/base/internal/cache.h | 54 ++++++++++++++++ repos/base-hw/src/lib/base/arm/cache.cc | 41 ++++++++++++ repos/base-hw/src/lib/base/arm_64/cache.cc | 57 +++++++++++++++++ repos/base-hw/src/lib/base/cache.cc | 62 ------------------- 12 files changed, 170 insertions(+), 63 deletions(-) rename repos/base-hw/lib/mk/{base-hw.mk => base-hw.inc} (100%) create mode 100644 repos/base-hw/lib/mk/spec/arm/base-hw.mk create mode 100644 repos/base-hw/lib/mk/spec/arm_64/base-hw.mk create mode 100644 repos/base-hw/lib/mk/spec/riscv/base-hw.mk create mode 100644 repos/base-hw/lib/mk/spec/x86_64/base-hw.mk create mode 100644 repos/base-hw/src/include/base/internal/cache.h create mode 100644 repos/base-hw/src/lib/base/arm/cache.cc create mode 100644 repos/base-hw/src/lib/base/arm_64/cache.cc delete mode 100644 repos/base-hw/src/lib/base/cache.cc diff --git a/repos/base-hw/lib/mk/base-hw.mk b/repos/base-hw/lib/mk/base-hw.inc similarity index 100% rename from repos/base-hw/lib/mk/base-hw.mk rename to repos/base-hw/lib/mk/base-hw.inc diff --git a/repos/base-hw/lib/mk/spec/arm/base-hw.mk b/repos/base-hw/lib/mk/spec/arm/base-hw.mk new file mode 100644 index 0000000000..2cc9ea1bbe --- /dev/null +++ b/repos/base-hw/lib/mk/spec/arm/base-hw.mk @@ -0,0 +1,3 @@ +vpath cache.cc $(REP_DIR)/src/lib/base/arm + +include $(REP_DIR)/lib/mk/base-hw.inc diff --git a/repos/base-hw/lib/mk/spec/arm_64/base-hw.mk b/repos/base-hw/lib/mk/spec/arm_64/base-hw.mk new file mode 100644 index 0000000000..51e578f490 --- /dev/null +++ b/repos/base-hw/lib/mk/spec/arm_64/base-hw.mk @@ -0,0 +1,3 @@ +vpath cache.cc $(REP_DIR)/src/lib/base/arm_64 + +include $(REP_DIR)/lib/mk/base-hw.inc diff --git a/repos/base-hw/lib/mk/spec/riscv/base-hw.mk b/repos/base-hw/lib/mk/spec/riscv/base-hw.mk new file mode 100644 index 0000000000..38ea05026d --- /dev/null +++ b/repos/base-hw/lib/mk/spec/riscv/base-hw.mk @@ -0,0 +1 @@ +include $(REP_DIR)/lib/mk/base-hw.inc diff --git a/repos/base-hw/lib/mk/spec/x86_64/base-hw.mk b/repos/base-hw/lib/mk/spec/x86_64/base-hw.mk new file mode 100644 index 0000000000..38ea05026d --- /dev/null +++ b/repos/base-hw/lib/mk/spec/x86_64/base-hw.mk @@ -0,0 +1 @@ +include $(REP_DIR)/lib/mk/base-hw.inc diff --git a/repos/base-hw/recipes/src/base-hw_content.inc b/repos/base-hw/recipes/src/base-hw_content.inc index ee1b9ae2f8..38ab511db0 100644 --- a/repos/base-hw/recipes/src/base-hw_content.inc +++ b/repos/base-hw/recipes/src/base-hw_content.inc @@ -133,7 +133,7 @@ SRC_CORE += $(notdir $(wildcard $(BASE_HW_DIR)/src/core/*.cc)) \ # names of the lib/mk/ files to consider for inclusion in the src archive LIB_MK_FILES := base-common.inc base-hw-common.mk \ - base.inc base-hw.mk \ + base.inc base-hw.inc base-hw.mk \ bootstrap-hw.inc bootstrap-hw-$(BOARD).inc bootstrap-hw-$(BOARD).mk \ core-hw.inc core-hw-$(BOARD).inc core-hw-$(BOARD).mk \ startup.inc startup-hw.mk \ diff --git a/repos/base-hw/src/core/spec/arm_v6/cpu.h b/repos/base-hw/src/core/spec/arm_v6/cpu.h index 6a9c6dfeb9..fae5dcc532 100644 --- a/repos/base-hw/src/core/spec/arm_v6/cpu.h +++ b/repos/base-hw/src/core/spec/arm_v6/cpu.h @@ -57,6 +57,10 @@ struct Genode::Cpu : Arm_cpu return instruction_cache_line_size; } + + static inline size_t cache_line_size() { + return Genode::min(data_cache_line_size(), + instruction_cache_line_size()); } }; #endif /* _CORE__SPEC__ARM_V6__CPU_H_ */ diff --git a/repos/base-hw/src/core/spec/arm_v7/cpu_support.h b/repos/base-hw/src/core/spec/arm_v7/cpu_support.h index 4085c7506e..8b0ddf515b 100644 --- a/repos/base-hw/src/core/spec/arm_v7/cpu_support.h +++ b/repos/base-hw/src/core/spec/arm_v7/cpu_support.h @@ -79,6 +79,11 @@ struct Genode::Arm_v7_cpu : Arm_cpu return instruction_cache_line_size; } + + + static inline size_t cache_line_size() { + return Genode::min(data_cache_line_size(), + instruction_cache_line_size()); } }; #endif /* _CORE__SPEC__ARM_V7__CPU_SUPPORT_H_ */ diff --git a/repos/base-hw/src/include/base/internal/cache.h b/repos/base-hw/src/include/base/internal/cache.h new file mode 100644 index 0000000000..327db162d7 --- /dev/null +++ b/repos/base-hw/src/include/base/internal/cache.h @@ -0,0 +1,54 @@ +/* + * \brief Cache maintainance utilities + * \author Stefan Kalkowski + * \date 2022-12-16 + */ + +/* + * Copyright (C) 2022 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#ifndef _INCLUDE__BASE__INTERNAL__CACHE_H_ +#define _INCLUDE__BASE__INTERNAL__CACHE_H_ + +#include +#include +#include + +template +static inline void for_each_page(Genode::addr_t addr, + Genode::size_t size, + FN const & fn) +{ + using namespace Genode; + + while (size) { + addr_t next_page = align_addr(addr+1, get_page_size_log2()); + size_t s = min(size, next_page - addr); + touch_read(reinterpret_cast(addr)); + fn(addr, s); + addr += s; + size -= s; + } +} + + +template +static inline void for_each_cache_line(Genode::addr_t addr, + Genode::size_t size, + FN const & fn) +{ + using namespace Genode; + + static size_t cache_line_size = Kernel::cache_line_size(); + + addr_t start = addr; + addr_t const end = addr + size; + for (; start < end; start += cache_line_size) + fn(start); +} + +#endif /* _INCLUDE__BASE__INTERNAL__CACHE_H_ */ diff --git a/repos/base-hw/src/lib/base/arm/cache.cc b/repos/base-hw/src/lib/base/arm/cache.cc new file mode 100644 index 0000000000..40a25c419e --- /dev/null +++ b/repos/base-hw/src/lib/base/arm/cache.cc @@ -0,0 +1,41 @@ +/* + * \brief Implementation of cache operations for ARM + * \author Stefan Kalkowski + * \date 2022-12-16 + */ + +/* + * Copyright (C) 2022 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include + +#include +#include + +using namespace Genode; + + +void Genode::cache_coherent(addr_t addr, size_t size) +{ + for_each_page(addr, size, [] (addr_t addr, size_t size) { + Kernel::cache_coherent_region(addr, size); }); +} + + +void Genode::cache_clean_invalidate_data(addr_t addr, size_t size) +{ + for_each_page(addr, size, [] (addr_t addr, size_t size) { + Kernel::cache_clean_invalidate_data_region(addr, size); }); +} + + +void Genode::cache_invalidate_data(addr_t addr, size_t size) +{ + for_each_page(addr, size, [] (addr_t addr, size_t size) { + Kernel::cache_invalidate_data_region(addr, size); }); +} + diff --git a/repos/base-hw/src/lib/base/arm_64/cache.cc b/repos/base-hw/src/lib/base/arm_64/cache.cc new file mode 100644 index 0000000000..337bd00019 --- /dev/null +++ b/repos/base-hw/src/lib/base/arm_64/cache.cc @@ -0,0 +1,57 @@ +/* + * \brief Implementation of the cache maintainance on ARMv8 + * \author Stefan Kalkowski + * \date 2022-12-16 + */ + +/* + * Copyright (C) 2022 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include + +#include +#include +#include + +using namespace Genode; + +void Genode::cache_coherent(addr_t addr, size_t size) +{ + Genode::memory_barrier(); + + for_each_page(addr, size, [] (addr_t addr, size_t size) { + for_each_cache_line(addr, size, [&] (addr_t addr) { + asm volatile("dc cvau, %0" :: "r" (addr)); + asm volatile("dsb ish"); + asm volatile("ic ivau, %0" :: "r" (addr)); + asm volatile("dsb ish"); + asm volatile("isb"); + }); + }); +} + + +void Genode::cache_clean_invalidate_data(addr_t addr, size_t size) +{ + Genode::memory_barrier(); + + for_each_page(addr, size, [&] (addr_t addr, size_t size) { + + for_each_cache_line(addr, size, [&] (addr_t addr) { + asm volatile("dc civac, %0" :: "r" (addr)); }); + }); + + asm volatile("dsb sy"); + asm volatile("isb"); +} + + +void Genode::cache_invalidate_data(addr_t addr, size_t size) +{ + for_each_page(addr, size, [] (addr_t addr, size_t size) { + Kernel::cache_invalidate_data_region(addr, size); }); +} diff --git a/repos/base-hw/src/lib/base/cache.cc b/repos/base-hw/src/lib/base/cache.cc deleted file mode 100644 index b2b1b61160..0000000000 --- a/repos/base-hw/src/lib/base/cache.cc +++ /dev/null @@ -1,62 +0,0 @@ -/* - * \brief Implementation of the cache operations - * \author Christian Prochaska - * \date 2014-05-13 - */ - -/* - * Copyright (C) 2014-2017 Genode Labs GmbH - * - * This file is part of the Genode OS framework, which is distributed - * under the terms of the GNU Affero General Public License version 3. - */ - -#include - -#include -#include -#include -#include - -using namespace Genode; - - -template -static void for_cachelines(addr_t addr, size_t size, FN const & fn) -{ - /** - * The kernel accepts the cache maintainance calls for one designated - * page only. Otherwise, it just ignores the call to limit the time being - * uninteruppptible in the kernel. Therefore, we have to loop if more than - * one page is affected by the given region. - */ - while (size) { - addr_t next_page = align_addr(addr+1, get_page_size_log2()); - size_t s = min(size, next_page - addr); - touch_read(reinterpret_cast(addr)); - fn(addr, s); - addr += s; - size -= s; - } -} - - -void Genode::cache_coherent(addr_t addr, size_t size) -{ - for_cachelines(addr, size, [] (addr_t addr, size_t size) { - Kernel::cache_coherent_region(addr, size); }); -} - - -void Genode::cache_clean_invalidate_data(addr_t addr, size_t size) -{ - for_cachelines(addr, size, [] (addr_t addr, size_t size) { - Kernel::cache_clean_invalidate_data_region(addr, size); }); -} - - -void Genode::cache_invalidate_data(addr_t addr, size_t size) -{ - for_cachelines(addr, size, [] (addr_t addr, size_t size) { - Kernel::cache_invalidate_data_region(addr, size); }); -}