base-hw: invalidate VM TLB entries in deletion

This commit introduces a hypervisor calling interface. The host kernel can
use it to either switch to a guest VM, or to invalidate the TLB with
regard to a specified VM id.
The VM-specific TLB invalidation is called whenever a VM (Vcpu)
gets destroyed.

Fix genodelabs/genode#4528

repos/base-hw/src/core/kernel/vm.h

@@ -77,6 +77,8 @@ class Kernel::Vm : private Kernel::Object, public Cpu_job
 		   Kernel::Signal_context & context,
 		   Identity               & id);
 
+		~Vm();
+
 		/**
 		 * Inject an interrupt to this VM
 		 *

repos/base-hw/src/core/spec/arm_v7/trustzone/kernel/vm.cc

@@ -38,6 +38,9 @@ Vm::Vm(Irq::Pool              & user_irq_pool,
 }
 
 
+Vm::~Vm() {}
+
+
 void Vm::exception(Cpu & cpu)
 {
 	switch(_state.cpu_exception) {

repos/base-hw/src/core/spec/arm_v7/virtualization/exception_vector.s

@@ -15,7 +15,7 @@
 	push  { r0 }
 	mrc   p15, 4, r0, c1, c1, 0 /* read HCR register */
 	tst   r0, #1                /* check VM bit      */
-	beq   _host_to_vm
+	beq   _from_host
 	mov   r0, #\exception_type
 	b     _vm_to_host
 .endm /* _vm_exit */
@@ -47,10 +47,27 @@ _vt_dab_entry: _vm_exit 5
 _vt_irq_entry: _vm_exit 6
 _vt_trp_entry: _vm_exit 8
 
-_host_to_vm:
+_from_host:
+	pop   { r0 }
+	cmp   r0, #0
+	beq   _to_vm
+	cmp   r0, #1
+	beq   _invalidate_tlb
+	eret
+
+
+_invalidate_tlb:
+	push  { r3, r4 }
+	mrrc  p15, 6, r3, r4, c2    /* save  VTTBR   */
+	mcrr  p15, 6, r1, r2, c2    /* write VTTBR   */
+	mcr   p15, 0, r0, c8, c3, 0 /* TLBIALLIS     */
+	mcrr  p15, 6, r3, r4, c2    /* restore VTTBR */
+	eret
+
+_to_vm:
 	push  { r1 }
-	ldr   r0, [sp, #1*4]
-	add   r0, r0, #13*4
+	push  { r2 }
+	add   r0, r1, #13*4
 	ldmia r0!, { r1-r5 }
 	msr   sp_usr,    r1
 	mov   lr,        r2
@@ -115,6 +132,7 @@ _host_to_vm:
 	ldmia r0, {r0-r12}            /* load vm's r0-r12   */
 	eret
 
+
 _vm_to_host:
 	push  { r0 }                  /* push cpu excep.    */
 	ldr   r0,  [sp, #3*4]         /* load vm state ptr  */
@@ -218,6 +236,7 @@ _vm_to_host:
 
 
 /* host kernel must jump to this point to switch to a vm */
-.global hypervisor_enter_vm
-hypervisor_enter_vm:
+.global hypervisor_call
+hypervisor_call:
 	hvc   #0
+	bx    lr

repos/base-hw/src/core/spec/arm_v7/virtualization/hypervisor.h

@@ -0,0 +1,55 @@
+/*
+ * \brief  Interface between kernel and hypervisor
+ * \author Stefan Kalkowski
+ * \date   2022-06-13
+ */
+
+/*
+ * Copyright (C) 2022 Genode Labs GmbH
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef _SPEC__ARM_V7__VIRTUALIZATION_HYPERVISOR_H_
+#define _SPEC__ARM_V7__VIRTUALIZATION_HYPERVISOR_H_
+
+#include <base/stdint.h>
+#include <cpu/vm_state_virtualization.h>
+
+namespace Hypervisor {
+
+	struct Host_context;
+
+	enum Call_number {
+		WORLD_SWITCH   = 0,
+		TLB_INVALIDATE = 1,
+	};
+
+	using Call_arg = Genode::umword_t;
+	using Call_ret = Genode::umword_t;
+
+	extern "C"
+	Call_ret hypervisor_call(Call_arg call_id,
+	                         Call_arg arg0,
+	                         Call_arg arg1);
+
+
+	inline void invalidate_tlb(Genode::uint64_t vttbr)
+	{
+		hypervisor_call(TLB_INVALIDATE,
+		                (vttbr & 0xffffffff),
+		                ((vttbr >> 32U) & 0xffffffff));
+	}
+
+
+	inline void switch_world(Genode::Vm_state & vm_state,
+	                         Host_context     & host_state)
+	{
+		hypervisor_call(WORLD_SWITCH,
+		                (Call_arg)&vm_state,
+		                (Call_arg)&host_state);
+	}
+}
+
+#endif /* _SPEC__ARM_V7__VIRTUALIZATION_HYPERVISOR_H_ */

repos/base-hw/src/core/spec/arm_v7/virtualization/kernel/vm.cc

@@ -19,6 +19,7 @@
 #include <kernel/cpu.h>
 #include <kernel/vm.h>
 #include <kernel/main.h>
+#include <spec/arm_v7/virtualization/hypervisor.h>
 
 namespace Kernel {
 
@@ -41,7 +42,7 @@ namespace Kernel {
 using namespace Kernel;
 
 
-struct Host_context
+struct Hypervisor::Host_context
 {
 	Cpu::Ttbr_64bit::access_t vttbr;
 	Cpu::Hcr::access_t        hcr;
@@ -61,15 +62,13 @@ struct Host_context
 } vt_host_context;
 
 
-extern "C" void hypervisor_enter_vm(Genode::Vm_state&, Host_context&);
-
-
-static Host_context & host_context(Cpu & cpu)
+static Hypervisor::Host_context & host_context(Cpu & cpu)
 {
-	static Genode::Constructible<Host_context> host_context[NR_OF_CPUS];
+	static Genode::Constructible<Hypervisor::Host_context>
+		host_context[NR_OF_CPUS];
 	if (!host_context[cpu.id()].constructed()) {
 		host_context[cpu.id()].construct();
-		Host_context & c = *host_context[cpu.id()];
+		Hypervisor::Host_context & c = *host_context[cpu.id()];
 		c.sp     = cpu.stack_start();
 		c.ttbr0  = Cpu::Ttbr0_64bit::read();
 		c.ttbr1  = Cpu::Ttbr1_64bit::read();
@@ -152,6 +151,15 @@ Kernel::Vm::Vm(Irq::Pool              & user_irq_pool,
 }
 
 
+Kernel::Vm::~Vm()
+{
+	Cpu::Ttbr_64bit::access_t vttbr =
+		Cpu::Ttbr_64bit::Ba::masked((Cpu::Ttbr_64bit::access_t)_id.table);
+	Cpu::Ttbr_64bit::Asid::set(vttbr, _id.id);
+	Hypervisor::invalidate_tlb(vttbr);
+}
+
+
 void Kernel::Vm::exception(Cpu & cpu)
 {
 	switch(_state.cpu_exception) {
@@ -190,7 +198,7 @@ void Kernel::Vm::proceed(Cpu & cpu)
 	_state.esr_el2   = Cpu::Hstr::init();
 	_state.hpfar_el2 = Cpu::Hcr::init();
 
-	hypervisor_enter_vm(_state, host_context(cpu));
+	Hypervisor::switch_world(_state, host_context(cpu));
 }
 
 

repos/base-hw/src/core/spec/arm_v8/virtualization/exception_vector.s

@@ -22,24 +22,34 @@
 .global hypervisor_exception_vector
 hypervisor_exception_vector:
 	.rept 16
-	add sp, sp, #-16   /* push x0, x1 to stack               */
-	stp x0, x1, [sp]
-	mrs x1, hcr_el2    /* read HCR register                  */
-	tst x1, #1         /* check VM bit                       */
-	beq _host_to_vm    /* if VM bit is not set, switch to VM */
-	ldr x0, [sp, #32]  /* otherwise, load vm_state pointer   */
-	adr x1, .          /* hold exception vector offset in x1 */
-	and x1, x1, #0xf80
-	b   _vm_to_host
+	add sp, sp, #-16   /* push x29, x30 to stack */
+	stp x29, x30, [sp]
+	mrs x30, hcr_el2   /* read HCR register */
+	tst x30, #1        /* check VM bit */
+	beq _from_host     /* if VM bit is not set, its a host call */
+	ldr x29, [sp, #32] /* otherwise, load vm_state pointer */
+	adr x30, .         /* hold exception vector offset in x30 */
+	and x30, x30, #0xf80
+	b   _from_vm
 	.balign 128
 	.endr
 
-_host_to_vm:
+_from_host:
+	ldp x29, x30, [sp], #2*8  /* pop x29, x30 from stack */
+	cmp x0, #0
+	beq _to_vm
+	cmp x0, #1
+	beq _invalidate_tlb
+	eret
 
-	add sp, sp, #-16    /* push arg2 (vm pic state) to stack */
-	str x2, [sp]
+_to_vm:
+	add sp, sp, #-16    /* push arg1/2 (vm/host state to stack */
+	stp x1, x2, [sp]
+	add sp, sp, #-16    /* push arg3 (vm pic state) to stack */
+	str x3, [sp]
 
-	msr vttbr_el2, x3   /* stage2 table pointer was arg3 */
+	msr vttbr_el2, x4   /* stage2 table pointer was arg4 */
+	mov x0, x1
 
 	add  x0, x0, #31*8  /* skip x0...x30, loaded later */
 
@@ -179,27 +189,38 @@ _host_to_vm:
 
 	eret
 
-_vm_to_host:
+
+_invalidate_tlb:
+	msr  vttbr_el2, x1
+	tlbi vmalle1is
+	msr  vttbr_el2, xzr
+	eret
+
+
+_from_vm:
 
 	/*********************
 	 ** Save vm context **
 	 *********************/
 
 	/** general-purpose register **/
-	add        x0,   x0, #2*8    /* skip x0 and x1 for now */
-	stp   x2,  x3, [x0], #2*8
-	stp   x4,  x5, [x0], #2*8
-	stp   x6,  x7, [x0], #2*8
-	stp   x8,  x9, [x0], #2*8
-	stp  x10, x11, [x0], #2*8
-	stp  x12, x13, [x0], #2*8
-	stp  x14, x15, [x0], #2*8
-	stp  x16, x17, [x0], #2*8
-	stp  x18, x19, [x0], #2*8
-	stp  x20, x21, [x0], #2*8
-	stp  x22, x23, [x0], #2*8
-	stp  x24, x25, [x0], #2*8
-	stp  x26, x27, [x0], #2*8
+	stp   x0,  x1, [x29], #2*8
+	stp   x2,  x3, [x29], #2*8
+	stp   x4,  x5, [x29], #2*8
+	stp   x6,  x7, [x29], #2*8
+	stp   x8,  x9, [x29], #2*8
+	stp  x10, x11, [x29], #2*8
+	stp  x12, x13, [x29], #2*8
+	stp  x14, x15, [x29], #2*8
+	stp  x16, x17, [x29], #2*8
+	stp  x18, x19, [x29], #2*8
+	stp  x20, x21, [x29], #2*8
+	stp  x22, x23, [x29], #2*8
+	stp  x24, x25, [x29], #2*8
+	stp  x26, x27, [x29], #2*8
+	mov  x0,  x29
+	mov  x1,  x30
+	ldp  x29, x30, [sp], #2*8  /* pop x29, x30 from stack */
 	stp  x28, x29, [x0], #2*8
 	str       x30, [x0], #1*8
 
@@ -284,11 +305,8 @@ _vm_to_host:
 	mov  x0, #0b111
 	msr  cnthctl_el2, x0
 
-
-	ldp x0,  x1, [sp], #2*8    /* pop x0, x1 from stack             */
-	ldr x29,     [sp], #2*8    /* pop vm pic state from stack       */
-	ldp x2, x30, [sp], #2*8    /* pop vm, and host state from stack */
-	stp x0,  x1, [x2]          /* save x0, x1 to vm state           */
+	ldr x29,     [sp], #2*8  /* pop vm pic state from stack       */
+	ldp x2, x30, [sp], #2*8  /* pop vm, and host state from stack */
 
 
 	/**********************
@@ -364,6 +382,7 @@ _vm_to_host:
 	eret
 
 /* host kernel must jump to this point to switch to a vm */
-.global hypervisor_enter_vm
-hypervisor_enter_vm:
+.global hypervisor_call
+hypervisor_call:
 	hvc #0
+	ret

repos/base-hw/src/core/spec/arm_v8/virtualization/hypervisor.h

@@ -0,0 +1,52 @@
+/*
+ * \brief  Interface between kernel and hypervisor
+ * \author Stefan Kalkowski
+ * \date   2022-06-13
+ */
+
+/*
+ * Copyright (C) 2022 Genode Labs GmbH
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef _SPEC__ARM_V8__VIRTUALIZATION_HYPERVISOR_H_
+#define _SPEC__ARM_V8__VIRTUALIZATION_HYPERVISOR_H_
+
+#include <base/stdint.h>
+
+namespace Hypervisor {
+
+	enum Call_number {
+		WORLD_SWITCH   = 0,
+		TLB_INVALIDATE = 1,
+	};
+
+	using Call_arg = Genode::umword_t;
+	using Call_ret = Genode::umword_t;
+
+	extern "C"
+	Call_ret hypervisor_call(Call_arg call_id,
+	                         Call_arg arg0,
+	                         Call_arg arg1,
+	                         Call_arg arg2,
+	                         Call_arg arg3);
+
+
+	inline void invalidate_tlb(Call_arg ttbr)
+	{
+		hypervisor_call(TLB_INVALIDATE, ttbr, 0, 0, 0);
+	}
+
+
+	inline void switch_world(Call_arg guest_state,
+	                         Call_arg host_state,
+	                         Call_arg pic_state,
+	                         Call_arg ttbr)
+	{
+		hypervisor_call(WORLD_SWITCH, guest_state, host_state, pic_state, ttbr);
+	}
+}
+
+#endif /* _SPEC__ARM_V8__VIRTUALIZATION_HYPERVISOR_H_ */

repos/base-hw/src/core/spec/arm_v8/virtualization/kernel/vm.cc

@@ -23,14 +23,12 @@
 #include <kernel/vm.h>
 #include <kernel/main.h>
 
+#include <spec/arm_v8/virtualization/hypervisor.h>
+
 using Genode::addr_t;
 using Kernel::Cpu;
 using Kernel::Vm;
 
-extern     void * kernel_stack;
-extern "C" void   hypervisor_enter_vm(addr_t vm, addr_t host,
-                                      addr_t pic, addr_t guest_table);
-
 
 static Genode::Vm_state & host_context(Cpu & cpu)
 {
@@ -154,6 +152,15 @@ Vm::Vm(Irq::Pool              & user_irq_pool,
 }
 
 
+Vm::~Vm()
+{
+	Cpu::Vttbr_el2::access_t vttbr_el2 =
+		Cpu::Vttbr_el2::Ba::masked((Cpu::Vttbr_el2::access_t)_id.table);
+	Cpu::Vttbr_el2::Asid::set(vttbr_el2, _id.id);
+	Hypervisor::invalidate_tlb(vttbr_el2);
+}
+
+
 void Vm::exception(Cpu & cpu)
 {
 	switch (_state.exception_type) {
@@ -197,7 +204,7 @@ void Vm::proceed(Cpu & cpu)
 	addr_t pic   = Hw::Mm::el2_addr(&_vcpu_context.pic);
 	addr_t host  = Hw::Mm::el2_addr(&host_context(cpu));
 
-	hypervisor_enter_vm(guest, host, pic, vttbr_el2);
+	Hypervisor::switch_world(guest, host, pic, vttbr_el2);
 }