vbox: use multiple host CPUs

Fixes #1553

repos/ports/ports/virtualbox.hash

@@ -1 +1 @@
-166105c79c2ebd0d8378af72bb4c392595c7bd67
+59e99301300f881be06c12cd50fbe8b1164e1845

repos/ports/run/vm_win7.vbox

@@ -21,7 +21,7 @@
       <ExtraDataItem name="GUI/LastNormalWindowPosition" value="513,100,1024,790"/>
     </ExtraData>
     <Hardware version="2">
-      <CPU count="1" hotplug="false">
+      <CPU count="2" hotplug="false">
         <HardwareVirtEx enabled="true"/>
         <HardwareVirtExNestedPaging enabled="true"/>
         <HardwareVirtExVPID enabled="true"/>
@@ -46,7 +46,7 @@
       <RemoteDisplay enabled="false" authType="Null"/>
       <BIOS>
         <ACPI enabled="true"/>
-        <IOAPIC enabled="false"/>
+        <IOAPIC enabled="true"/>
         <Logo fadeIn="true" fadeOut="true" displayTime="0"/>
         <BootMenu mode="MessageAndMenu"/>
         <TimeOffset value="0"/>

repos/ports/run/vm_win8.vbox

@@ -21,7 +21,7 @@
       <ExtraDataItem name="GUI/LastNormalWindowPosition" value="0,12,1439,863"/>
     </ExtraData>
     <Hardware version="2">
-      <CPU count="1" hotplug="false">
+      <CPU count="2" hotplug="false">
         <HardwareVirtEx enabled="true"/>
         <HardwareVirtExNestedPaging enabled="true"/>
         <HardwareVirtExVPID enabled="true"/>

repos/ports/src/virtualbox/accloff/sup.cc

@@ -84,7 +84,8 @@ bool create_emt_vcpu(pthread_t * thread, size_t stack_size,
                      const pthread_attr_t *attr,
                      void *(*start_routine)(void *), void *arg,
                      Genode::Cpu_session * cpu_session,
-                     Genode::Affinity::Location location)
+                     Genode::Affinity::Location location,
+                     unsigned int cpu_id)
 {
 	/* no hardware acceleration support */
 	return false;

repos/ports/src/virtualbox/nova/sup.cc

@@ -38,14 +38,26 @@
 #include "vcpu_svm.h"
 #include "vcpu_vmx.h"
 
-
+/* libc memory allocator */
-static Vcpu_handler *vcpu_handler = 0;
+#include <libc_mem_alloc.h>
 
 
-static Genode::Semaphore *r0_halt_sem()
+static Genode::List<Vcpu_handler> &vcpu_handler_list()
 {
-	static Genode::Semaphore sem;
+	static Genode::List<Vcpu_handler> _inst;
-	return &sem;
+	return _inst;
+}
+
+
+static Vcpu_handler *lookup_vcpu_handler(unsigned int cpu_id)
+{
+	for (Vcpu_handler *vcpu_handler = vcpu_handler_list().first();
+	     vcpu_handler;
+	     vcpu_handler = vcpu_handler->next())
+		if (vcpu_handler->cpu_id() == cpu_id)
+			return vcpu_handler;
+
+	return 0;
 }
 
 
@@ -80,7 +92,9 @@ int SUPR3CallVMMR0Fast(PVMR0 pVMR0, unsigned uOperation, VMCPUID idCpu)
 {
 	switch (uOperation) {
 	case SUP_VMMR0_DO_HM_RUN:
-		return vcpu_handler->run_hw(pVMR0, idCpu);
+		Vcpu_handler *vcpu_handler = lookup_vcpu_handler(idCpu);
+		Assert(vcpu_handler);
+		return vcpu_handler->run_hw(pVMR0);
 	}
 	return VERR_INTERNAL_ERROR;
 }
@@ -95,13 +109,25 @@ int SUPR3CallVMMR0Ex(PVMR0 pVMR0, VMCPUID idCpu, unsigned
 		genode_VMMR0_DO_GVMM_CREATE_VM(pReqHdr);
 		return VINF_SUCCESS;
 
-	case VMMR0_DO_GVMM_SCHED_HALT:
+	case VMMR0_DO_GVMM_REGISTER_VMCPU:
-		r0_halt_sem()->down();
+		genode_VMMR0_DO_GVMM_REGISTER_VMCPU(pVMR0, idCpu);
 		return VINF_SUCCESS;
 
-	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
+	case VMMR0_DO_GVMM_SCHED_HALT:
-		r0_halt_sem()->up();
+	{
+		Vcpu_handler *vcpu_handler = lookup_vcpu_handler(idCpu);
+		Assert(vcpu_handler);
+		vcpu_handler->halt();
 		return VINF_SUCCESS;
+	}
+
+	case VMMR0_DO_GVMM_SCHED_WAKE_UP:
+	{
+		Vcpu_handler *vcpu_handler = lookup_vcpu_handler(idCpu);
+		Assert(vcpu_handler);
+		vcpu_handler->wake_up();
+		return VINF_SUCCESS;
+	}
 
 	/* called by 'vmR3HaltGlobal1Halt' */
 	case VMMR0_DO_GVMM_SCHED_POLL:
@@ -120,9 +146,13 @@ int SUPR3CallVMMR0Ex(PVMR0 pVMR0, VMCPUID idCpu, unsigned
 		return VINF_SUCCESS;
 
 	case VMMR0_DO_GVMM_SCHED_POKE:
+	{
+		Vcpu_handler *vcpu_handler = lookup_vcpu_handler(idCpu);
+		Assert(vcpu_handler);
 		if (vcpu_handler)
 			vcpu_handler->recall();
 		return VINF_SUCCESS;
+	}
 
 	default:
 		PERR("SUPR3CallVMMR0Ex: unhandled uOperation %d", uOperation);
@@ -213,23 +243,30 @@ bool create_emt_vcpu(pthread_t * pthread, size_t stack,
                      const pthread_attr_t *attr,
                      void *(*start_routine)(void *), void *arg,
                      Genode::Cpu_session * cpu_session,
-                     Genode::Affinity::Location location)
+                     Genode::Affinity::Location location,
+                     unsigned int cpu_id)
 {
 	Nova::Hip * hip = hip_rom.local_addr<Nova::Hip>();
 
 	if (!hip->has_feature_vmx() && !hip->has_feature_svm())
 		return false;
 
+	Vcpu_handler *vcpu_handler = 0;
+
 	if (hip->has_feature_vmx())
 		vcpu_handler = new (0x10) Vcpu_handler_vmx(stack, attr, start_routine,
-		                                           arg, cpu_session, location);
+		                                           arg, cpu_session, location,
+		                                           cpu_id);
 
 	if (hip->has_feature_svm())
 		vcpu_handler = new (0x10) Vcpu_handler_svm(stack, attr, start_routine,
-		                                           arg, cpu_session, location);
+		                                           arg, cpu_session, location,
+		                                           cpu_id);
 
 	Assert(!(reinterpret_cast<unsigned long>(vcpu_handler) & 0xf));
 
+	vcpu_handler_list().insert(vcpu_handler);
+
 	*pthread = vcpu_handler;
 	return true;
 }

repos/ports/src/virtualbox/nova/vcpu.h

@@ -74,7 +74,8 @@ extern "C" int MMIO2_MAPPED_SYNC(PVM pVM, RTGCPHYS GCPhys, size_t cbWrite,
                                  bool &writeable);
 
 
-class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>
+class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>,
+                     public Genode::List<Vcpu_handler>::Element
 {
 	private:
 
@@ -87,6 +88,9 @@ class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>
 		Genode::addr_t _ec_sel; 
 		bool _irq_win;
 
+		unsigned int      _cpu_id;
+		Genode::Semaphore _halt_sem;
+
 		void fpu_save(char * data) {
 			Assert(!(reinterpret_cast<Genode::addr_t>(data) & 0xF));
  			asm volatile ("fxsave %0" : "=m" (*data));
@@ -219,7 +223,6 @@ class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>
 			using namespace Genode;
 
 			Assert(utcb->actv_state == ACTIVITY_STATE_ACTIVE);
-			Assert(utcb->intr_state == INTERRUPT_STATE_NONE);
 
 			if (utcb->inj_info & IRQ_INJ_VALID_MASK)
 				Vmm::printf("inj_info %x\n", utcb->inj_info);
@@ -635,16 +638,20 @@ class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>
 		Vcpu_handler(size_t stack_size, const pthread_attr_t *attr,
 		             void *(*start_routine) (void *), void *arg,
 		             Genode::Cpu_session * cpu_session,
-		             Genode::Affinity::Location location)
+		             Genode::Affinity::Location location,
+		             unsigned int cpu_id)
 		:
 			Vmm::Vcpu_dispatcher<pthread>(stack_size, _cap_connection,
 			                              cpu_session, location, 
 			                              attr ? *attr : 0, start_routine, arg),
 			_vcpu(cpu_session, location),
 			_ec_sel(Genode::cap_map()->insert()),
-			_irq_win(false)
+			_irq_win(false),
+			_cpu_id(cpu_id)
 		{ }
 
+		unsigned int cpu_id() { return _cpu_id; }
+
 		void start() {
 			_vcpu.start(_ec_sel);
 		}
@@ -660,6 +667,16 @@ class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>
 			}
 		}
 
+		void halt()
+		{
+			_halt_sem.down();
+		}
+
+		void wake_up()
+		{
+			_halt_sem.up();
+		}
+
 		inline void dump_register_state(PCPUMCTX pCtx)
 		{
 			PINF("pCtx");
@@ -742,10 +759,10 @@ class Vcpu_handler : public Vmm::Vcpu_dispatcher<pthread>
 			PLOG("%x %x %x", utcb->intr_state, utcb->actv_state, utcb->mtd);
 		}
 
-		int run_hw(PVMR0 pVMR0, VMCPUID idCpu)
+		int run_hw(PVMR0 pVMR0)
 		{
 			VM     * pVM   = reinterpret_cast<VM *>(pVMR0);
-			PVMCPU   pVCpu = &pVM->aCpus[idCpu];
+			PVMCPU   pVCpu = &pVM->aCpus[_cpu_id];
 			PCPUMCTX pCtx  = CPUMQueryGuestCtxPtr(pVCpu);
 
 			Nova::Utcb *utcb = reinterpret_cast<Nova::Utcb *>(Thread_base::utcb());

repos/ports/src/virtualbox/nova/vcpu_svm.h

@@ -84,10 +84,11 @@ class Vcpu_handler_svm : public Vcpu_handler
 		Vcpu_handler_svm(size_t stack_size, const pthread_attr_t *attr,
 		                 void *(*start_routine) (void *), void *arg,
 		                 Genode::Cpu_session * cpu_session,
-		                 Genode::Affinity::Location location)
+		                 Genode::Affinity::Location location,
+		                 unsigned int cpu_id)
 		:
 			Vcpu_handler(stack_size, attr, start_routine, arg, cpu_session,
-			             location) 
+			             location, cpu_id)
 		{
 			using namespace Nova;
 

repos/ports/src/virtualbox/nova/vcpu_vmx.h

@@ -149,10 +149,11 @@ class Vcpu_handler_vmx : public Vcpu_handler
 		Vcpu_handler_vmx(size_t stack_size, const pthread_attr_t *attr,
 		                 void *(*start_routine) (void *), void *arg,
 		                 Genode::Cpu_session * cpu_session,
-		                 Genode::Affinity::Location location)
+		                 Genode::Affinity::Location location,
+		                 unsigned int cpu_id)
 		:
 			 Vcpu_handler(stack_size, attr, start_routine, arg, cpu_session, 
-			              location)
+			              location, cpu_id)
 		{
 			using namespace Nova;
 

repos/ports/src/virtualbox/patches/series

@@ -19,3 +19,4 @@ avoid_yield.patch
 serial.patch
 rem_irq.patch
 usb.patch
+tm_smp.patch

repos/ports/src/virtualbox/patches/tm_smp.patch

@@ -0,0 +1,27 @@
+tm_smp.patch
+
+diff --git a/src/app/virtualbox/src/VBox/VMM/VMMR3/TM.cpp b/src/app/virtualbox/src/VBox/VMM/VMMR3/TM.cpp
+index a5dc16e..df851a1 100644
+--- a/src/app/virtualbox/src/VBox/VMM/VMMR3/TM.cpp
++++ b/src/app/virtualbox/src/VBox/VMM/VMMR3/TM.cpp
+@@ -1901,10 +1901,19 @@ static DECLCALLBACK(void) tmR3TimerCallback(PRTTIMER pTimer, void *pvUser, uint6
+     NOREF(pTimer);
+ 
+     AssertCompile(TMCLOCK_MAX == 4);
++
++    if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)) {
+ #ifdef DEBUG_Sander /* very annoying, keep it private. */
+-    if (VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER))
+         Log(("tmR3TimerCallback: timer event still pending!!\n"));
+ #endif
++        /*
++         * The VMCPU_FF_TIMER flag could have been set by a non-destination
++         * EMT thread without waking the destination EMT thread.
++         */
++        VMR3NotifyCpuFFU(pVCpuDst->pUVCpu, VMNOTIFYFF_FLAGS_DONE_REM);
++        return;
++	}
++
+     if (    !VMCPU_FF_IS_SET(pVCpuDst, VMCPU_FF_TIMER)
+         &&  (   pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL_SYNC].offSchedule /** @todo FIXME - reconsider offSchedule as a reason for running the timer queues. */
+             ||  pVM->tm.s.paTimerQueuesR3[TMCLOCK_VIRTUAL].offSchedule

repos/ports/src/virtualbox/pgm.cc

@@ -672,6 +672,31 @@ extern "C" int MMIO2_MAPPED_SYNC(PVM pVM, RTGCPHYS GCPhys, size_t cbWrite,
 }
 
 
+/**
+ * Resets a virtual CPU when unplugged.
+ *
+ * @param   pVM                 Pointer to the VM.
+ * @param   pVCpu               Pointer to the VMCPU.
+ */
+VMMR3DECL(void) PGMR3ResetCpu(PVM pVM, PVMCPU pVCpu)
+{
+    int rc = PGMR3ChangeMode(pVM, pVCpu, PGMMODE_REAL);
+    AssertRC(rc);
+
+    /*
+     * Re-init other members.
+     */
+    pVCpu->pgm.s.fA20Enabled = true;
+    pVCpu->pgm.s.GCPhysA20Mask = ~((RTGCPHYS)!pVCpu->pgm.s.fA20Enabled << 20);
+
+    /*
+     * Clear the FFs PGM owns.
+     */
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3);
+    VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL);
+}
+
+
 void PGMR3Reset(PVM pVM)
 {
 	VM_ASSERT_EMT(pVM);

repos/ports/src/virtualbox/sup.cc

@@ -26,6 +26,9 @@
 #include <iprt/uint128.h>
 #include <VBox/err.h>
 
+/* libc memory allocator */
+#include <libc_mem_alloc.h>
+
 
 struct Attached_gip : Genode::Attached_ram_dataspace
 {
@@ -271,3 +274,58 @@ int SUPR3CallVMMR0(PVMR0 pVMR0, VMCPUID idCpu, unsigned uOperation,
 	          ("SUPR3CallVMMR0Ex: unhandled uOperation %d", uOperation));
 	return VERR_GENERAL_FAILURE;
 }
+
+
+void genode_VMMR0_DO_GVMM_CREATE_VM(PSUPVMMR0REQHDR pReqHdr)
+{
+	GVMMCREATEVMREQ &req = reinterpret_cast<GVMMCREATEVMREQ &>(*pReqHdr);
+
+	size_t const cCpus = req.cCpus;
+
+	/*
+	 * Allocate and initialize VM struct
+	 *
+	 * The VM struct is followed by the variable-sizedA array of VMCPU
+	 * objects. 'RT_UOFFSETOF' is used to determine the size including
+	 * the VMCPU array.
+	 *
+	 * VM struct must be page-aligned, which is checked at least in
+	 * PDMR3CritSectGetNop().
+	 */
+	size_t const cbVM = RT_UOFFSETOF(VM, aCpus[cCpus]);
+	VM *pVM = (VM *)Libc::mem_alloc()->alloc(cbVM, Genode::log2(PAGE_SIZE));
+	Genode::memset(pVM, 0, cbVM);
+
+	/*
+	 * On Genode, VMMR0 and VMMR3 share a single address space. Hence, the
+	 * same pVM pointer is valid as pVMR0 and pVMR3.
+	 */
+	pVM->enmVMState       = VMSTATE_CREATING;
+	pVM->pVMR0            = (RTHCUINTPTR)pVM;
+	pVM->pVMRC            = (RTGCUINTPTR)pVM;
+	pVM->pSession         = req.pSession;
+	pVM->cbSelf           = cbVM;
+	pVM->cCpus            = cCpus;
+	pVM->uCpuExecutionCap = 100;  /* expected by 'vmR3CreateU()' */
+	pVM->offVMCPU         = RT_UOFFSETOF(VM, aCpus);
+
+	for (uint32_t i = 0; i < cCpus; i++) {
+		pVM->aCpus[i].pVMR0           = pVM->pVMR0;
+		pVM->aCpus[i].pVMR3           = pVM;
+		pVM->aCpus[i].idHostCpu       = NIL_RTCPUID;
+		pVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
+	}
+
+	pVM->aCpus[0].hNativeThreadR0 = RTThreadNativeSelf();
+
+	/* out parameters of the request */
+	req.pVMR0 = pVM->pVMR0;
+	req.pVMR3 = pVM;
+}
+
+
+void genode_VMMR0_DO_GVMM_REGISTER_VMCPU(PVMR0 pVMR0, VMCPUID idCpu)
+{
+	PVM pVM = reinterpret_cast<PVM>(pVMR0);
+	pVM->aCpus[idCpu].hNativeThreadR0 = RTThreadNativeSelf();
+}

repos/ports/src/virtualbox/sup.h

@@ -16,17 +16,12 @@
 
 /* Genode includes */
 #include <cpu_session/cpu_session.h>
-#include "util/misc_math.h"
-#include "util/string.h"
 
 /* VirtualBox includes */
 #include <VBox/vmm/vm.h>
 #include <VBox/vmm/gvmm.h>
 #include <iprt/param.h>
 
-/* libc memory allocator */
-#include <libc_mem_alloc.h>
-
 /**
  * Returns true if a vCPU could be started. If false we run without
  * hardware acceleration support.
@@ -35,57 +30,15 @@ bool create_emt_vcpu(pthread_t * pthread, size_t stack,
                      const pthread_attr_t *attr,
                      void *(*start_routine)(void *), void *arg,
                      Genode::Cpu_session * cpu_session,
-                     Genode::Affinity::Location location);
+                     Genode::Affinity::Location location,
+                     unsigned int cpu_id);
 
 
 uint64_t genode_cpu_hz();
 
-
-void inline genode_VMMR0_DO_GVMM_CREATE_VM(PSUPVMMR0REQHDR pReqHdr)
-{
-	GVMMCREATEVMREQ &req = reinterpret_cast<GVMMCREATEVMREQ &>(*pReqHdr);
-
-	size_t const cCpus = req.cCpus;
-
-	/*
-	 * Allocate and initialize VM struct
-	 *
-	 * The VM struct is followed by the variable-sizedA array of VMCPU
-	 * objects. 'RT_UOFFSETOF' is used to determine the size including
-	 * the VMCPU array.
-	 *
-	 * VM struct must be page-aligned, which is checked at least in
-	 * PDMR3CritSectGetNop().
-	 */
-	size_t const cbVM = RT_UOFFSETOF(VM, aCpus[cCpus]);
-	VM *pVM = (VM *)Libc::mem_alloc()->alloc(cbVM, Genode::log2(PAGE_SIZE));
-	Genode::memset(pVM, 0, cbVM);
-
-	/*
-	 * On Genode, VMMR0 and VMMR3 share a single address space. Hence, the
-	 * same pVM pointer is valid as pVMR0 and pVMR3.
-	 */
-	pVM->enmVMState       = VMSTATE_CREATING;
-	pVM->pVMR0            = (RTHCUINTPTR)pVM;
-	pVM->pVMRC            = (RTGCUINTPTR)pVM;
-	pVM->pSession         = req.pSession;
-	pVM->cbSelf           = cbVM;
-	pVM->cCpus            = cCpus;
-	pVM->uCpuExecutionCap = 100;  /* expected by 'vmR3CreateU()' */
-	pVM->offVMCPU         = RT_UOFFSETOF(VM, aCpus);
-
-	for (uint32_t i = 0; i < cCpus; i++) {
-		pVM->aCpus[i].pVMR0           = pVM->pVMR0;
-		pVM->aCpus[i].pVMR3           = pVM;
-		pVM->aCpus[i].idHostCpu       = NIL_RTCPUID;
-		pVM->aCpus[i].hNativeThreadR0 = NIL_RTNATIVETHREAD;
-	}
-
-	/* out parameters of the request */
-	req.pVMR0 = pVM->pVMR0;
-	req.pVMR3 = pVM;
-}
-
 Genode::Cpu_session * get_vcpu_cpu_session();
 
+void genode_VMMR0_DO_GVMM_CREATE_VM(PSUPVMMR0REQHDR pReqHdr);
+void genode_VMMR0_DO_GVMM_REGISTER_VMCPU(PVMR0 pVMR0, VMCPUID idCpu);
+
 #endif /* _SUP_H_ */

repos/ports/src/virtualbox/thread.cc

@@ -25,6 +25,8 @@
 /* libc */
 #include <pthread.h>
 #include <errno.h>
+#include <stdio.h>
+#include <string.h>
 
 /* vbox */
 #include <internal/thread.h>
@@ -51,7 +53,7 @@ static Genode::Cpu_connection * cpu_connection(RTTHREADTYPE type) {
 
 	char * data = new (env()->heap()) char[16];
 
-	snprintf(data, 16, "vbox %u", type);
+	Genode::snprintf(data, 16, "vbox %u", type);
 
 	con[type - 1] = new (env()->heap()) Cpu_connection(data, prio);
 
@@ -77,14 +79,20 @@ static int create_thread(pthread_t *thread, const pthread_attr_t *attr,
 		     stack_size);
 
 	/* sanity check - emt and vcpu thread have to have same prio class */
-	if (!Genode::strcmp(rtthread->szName, "EMT"))
+	if (strstr(rtthread->szName, "EMT") == rtthread->szName)
 		Assert(rtthread->enmType == RTTHREADTYPE_EMULATION);
 
 	if (rtthread->enmType == RTTHREADTYPE_EMULATION) {
+
+		unsigned int cpu_id = 0;
+		sscanf(rtthread->szName, "EMT-%u", &cpu_id);
+
 		Genode::Cpu_session * cpu_session = cpu_connection(RTTHREADTYPE_EMULATION);
-		Genode::Affinity::Location location;
+		Genode::Affinity::Space space = cpu_session->affinity_space();
+		Genode::Affinity::Location location(space.location_of_index(cpu_id));
+
 		if (create_emt_vcpu(thread, stack_size, attr, start_routine, arg,
-		                    cpu_session, location))
+		                    cpu_session, location, cpu_id))
 			return 0;
 		/*
 		 * The virtualization layer had no need to setup the EMT

repos/ports/src/virtualbox/unimpl.cc

@@ -79,7 +79,6 @@ DUMMY(PGMR3LockCall)
 DUMMY(PGMR3PoolGrow)
 DUMMY(PGMR3QueryGlobalMemoryStats)
 DUMMY(PGMR3QueryMemoryStats)
-DUMMY(PGMR3ResetCpu)
 
 DUMMY(PGMR3PhysAllocateHandyPages)
 DUMMY(PGMR3PhysAllocateLargeHandyPage)