nova: add migration support for global threads

Fixes #3842

repos/base-nova/include/nova/syscall-generic.h

@@ -236,7 +236,13 @@ namespace Nova {
 	/**
 	 * Ec operations
 	 */
-	enum Ec_op { EC_RECALL = 0U, EC_YIELD = 1U, EC_DONATE_SC = 2U, EC_RESCHEDULE = 3U };
+	enum Ec_op {
+		EC_RECALL = 0U,
+		EC_YIELD  = 1U,
+		EC_DONATE_SC = 2U,
+		EC_RESCHEDULE = 3U,
+		EC_MIGRATE = 4U,
+	};
 
 	/**
 	 * Pd operations
@@ -754,7 +760,7 @@ namespace Nova {
 	enum {
 		PT_SEL_PAGE_FAULT = 0xe,
 		PT_SEL_PARENT     = 0x1a,  /* convention on Genode */
-		PT_SEL_MAIN_EC    = 0x1c,  /* convention on Genode */
+		EC_SEL_THREAD     = 0x1c,  /* convention on Genode */
 		PT_SEL_STARTUP    = 0x1e,
 		SM_SEL_SIGNAL     = 0x1e,  /* alias of PT_SEL_STARTUP */
 		PT_SEL_RECALL     = 0x1f,

repos/base-nova/include/spec/32bit/nova/syscalls.h

@@ -240,9 +240,10 @@ namespace Nova {
 
 
 	ALWAYS_INLINE
-	inline uint8_t ec_ctrl(Ec_op op, mword_t ec = ~0UL, mword_t para = ~0UL)
+	inline uint8_t ec_ctrl(Ec_op op, mword_t ec = ~0UL, mword_t para = ~0UL,
+	                       Crd crd = 0)
 	{
-		return syscall_1(NOVA_EC_CTRL, op, ec, para);
+		return syscall_2(NOVA_EC_CTRL, op, ec, para, crd.value());
 	}
 
 

repos/base-nova/include/spec/64bit/nova/syscalls.h

@@ -192,9 +192,10 @@ namespace Nova {
 
 
 	ALWAYS_INLINE
-	inline uint8_t ec_ctrl(Ec_op op, mword_t ec = ~0UL, mword_t para = ~0UL)
+	inline uint8_t ec_ctrl(Ec_op op, mword_t ec = ~0UL, mword_t para = ~0UL,
+	                       Crd crd = 0)
 	{
-		return syscall_1(NOVA_EC_CTRL, op, ec, para);
+		return syscall_2(NOVA_EC_CTRL, op, ec, para, crd.value());
 	}
 
 

repos/base-nova/ports/nova.hash

@@ -1 +1 @@
-e1ea2d79b890ec24b65effb5b9718019db97ec56
+b05b469a38b6e686d60f6646de31eb1bdfef329b

repos/base-nova/ports/nova.port

@@ -4,7 +4,7 @@ DOWNLOADS := nova.git
 
 # r10 branch
 URL(nova) := https://github.com/alex-ab/NOVA.git
-REV(nova) := 68c2fb1671e75d811a4787e35b0d0c6cc85815c0
+REV(nova) := b48e8661120226bcd9b6f261a29f864c2f06e315
 DIR(nova) := src/kernel/nova
 
 PATCHES   := $(sort $(wildcard $(REP_DIR)/patches/*.patch))

repos/base-nova/src/core/include/pager.h

@@ -92,6 +92,7 @@ namespace Genode {
 					DISSOLVED        = 0x10U,
 					SUBMIT_SIGNAL    = 0x20U,
 					BLOCKED_PAUSE_SM = 0x40U,
+					MIGRATE          = 0x80U
 				};
 				uint8_t _status;
 				bool modified;
@@ -119,11 +120,15 @@ namespace Genode {
 				inline void submit_signal() { _status |= SUBMIT_SIGNAL; }
 				inline void reset_submit() { _status &= ~SUBMIT_SIGNAL; }
 
+				bool migrate() const { return _status & MIGRATE; }
+				void reset_migrate() { _status &= ~MIGRATE; }
+				void request_migrate() { _status |= MIGRATE; }
 			} _state { };
 
 			Cpu_session_capability   _cpu_session_cap;
 			Thread_capability        _thread_cap;
-			Affinity::Location const _location;
+			Affinity::Location       _location;
+			Affinity::Location       _next_location { };
 			Exception_handlers       _exceptions;
 
 			addr_t _pd_target;
@@ -153,6 +158,9 @@ namespace Genode {
 			__attribute__((regparm(3)))
 			static void _oom_handler(addr_t, addr_t, addr_t);
 
+			void _construct_pager();
+			bool _migrate_thread();
+
 		public:
 
 			Pager_object(Cpu_session_capability cpu_session_cap,
@@ -164,7 +172,11 @@ namespace Genode {
 			virtual ~Pager_object();
 
 			unsigned long badge() const { return _badge; }
-			void reset_badge() { _badge = 0; }
+			void reset_badge()
+			{
+				Genode::Mutex::Guard guard(_state_lock);
+				_badge = 0;
+			}
 
 			const char * client_thread() const;
 			const char * client_pd() const;
@@ -181,6 +193,8 @@ namespace Genode {
 
 			Affinity::Location location() const { return _location; }
 
+			void migrate(Affinity::Location);
+
 			/**
 			 * Assign PD selector to PD
 			 */
@@ -315,7 +329,7 @@ namespace Genode {
 			/**
 			 * Portal called by thread that causes a out of memory in kernel.
 			 */
-			addr_t get_oom_portal();
+			addr_t create_oom_portal();
 
 			enum Policy {
 				STOP = 1,

repos/base-nova/src/core/include/platform_thread.h

@@ -70,6 +70,11 @@ namespace Genode {
 			Platform_thread(Platform_thread const &);
 			Platform_thread &operator = (Platform_thread const &);
 
+			/**
+			 * Create OOM portal and delegate it
+			 */
+			bool _create_and_map_oom_portal(Nova::Utcb &);
+
 		public:
 
 			/* mark as vcpu in remote pd if it is a vcpu */
@@ -81,16 +86,13 @@ namespace Genode {
 				return _sel_exc_base;
 			}
 
-			/* invalid thread number */
-			enum { THREAD_INVALID = -1 };
-
 			/**
 			 * Constructor
 			 */
-			Platform_thread(size_t, const char *name = 0,
+			Platform_thread(size_t quota, char const *name,
-			                unsigned priority = 0,
+			                unsigned priority,
-			                Affinity::Location affinity = Affinity::Location(),
+			                Affinity::Location affinity,
-			                int thread_id = THREAD_INVALID);
+			                addr_t utcb);
 
 			/**
 			 * Destructor
@@ -173,10 +175,20 @@ namespace Genode {
 			 */
 			void affinity(Affinity::Location location);
 
+			/**
+			 * Pager_object starts migration preparation and calls for
+			 * finalization of the migration.
+			 * The method delegates the new exception portals to
+			 * the protection domain and set the new acknowledged location.
+			 */
+			void prepare_migration();
+			void finalize_migration(Affinity::Location const location) {
+				_location = location; }
+
 			/**
 			 * Get the executing CPU for this thread
 			 */
-			Affinity::Location affinity() const;
+			Affinity::Location affinity() const { return _location; }
 
 			/**
 			 * Get thread name

repos/base-nova/src/core/pager.cc

@@ -263,13 +263,69 @@ void Pager_object::exception(uint8_t exit_id)
 }
 
 
+bool Pager_object::_migrate_thread()
+{
+	bool const valid_migrate = (_state.migrate() && _badge);
+	if (!valid_migrate)
+		return false;
+
+	_state.reset_migrate();
+
+	try {
+		/* revoke all exception portals pointing to current pager */
+		Platform_thread &thread = *reinterpret_cast<Platform_thread *>(_badge);
+
+		Nova::revoke(Obj_crd(_selectors, 2));
+
+		/* revoke all exception portals selectors */
+		Nova::revoke(Obj_crd(exc_pt_sel_client()+0x00, 4));
+		Nova::revoke(Obj_crd(exc_pt_sel_client()+0x10, 3));
+		Nova::revoke(Obj_crd(exc_pt_sel_client()+0x18, 1));
+		Nova::revoke(Obj_crd(exc_pt_sel_client()+0x1f, 0));
+
+		/* re-create portals bound to pager on new target CPU */
+		_location   = _next_location;
+		_exceptions = Exception_handlers(*this);
+		_construct_pager();
+
+		/* map all exception portals to thread pd */
+		thread.prepare_migration();
+
+		/* syscall to migrate */
+		unsigned const migrate_to = platform_specific().kernel_cpu_id(_location);
+		uint8_t res = syscall_retry(*this, [&]() {
+			return ec_ctrl(EC_MIGRATE, _state.sel_client_ec, migrate_to,
+			               Obj_crd(EC_SEL_THREAD, 0, Obj_crd::RIGHT_EC_RECALL));
+		});
+
+		if (res == Nova::NOVA_OK)
+			thread.finalize_migration(_location);
+
+		return true;
+	} catch (...) {
+		return false;
+	}
+}
+
+
 void Pager_object::_recall_handler(Pager_object &obj)
 {
 	Thread &myself = *Thread::myself();
 	Utcb   &utcb   = *reinterpret_cast<Utcb *>(myself.utcb());
 
+	/* acquire mutex */
 	obj._state_lock.acquire();
 
+	/* check for migration */
+	if (obj._migrate_thread()) {
+		/* release mutex */
+		obj._state_lock.release();
+
+		utcb.set_msg_word(0);
+		utcb.mtd = 0;
+		reply(myself.stack_top());
+	}
+
 	if (obj._state.modified) {
 		obj._copy_state_to_utcb(utcb);
 		obj._state.modified = false;
@@ -589,61 +645,25 @@ Exception_handlers::Exception_handlers(Pager_object &obj)
  ** Pager object **
  ******************/
 
-
+void Pager_object::_construct_pager()
-Pager_object::Pager_object(Cpu_session_capability cpu_session_cap,
-                           Thread_capability thread_cap, unsigned long badge,
-                           Affinity::Location location, Session_label const &,
-                           Cpu_session::Name const &)
-:
-	_badge(badge),
-	_selectors(cap_map().insert(2)),
-	_client_exc_pt_sel(cap_map().insert(NUM_INITIAL_PT_LOG2)),
-	_cpu_session_cap(cpu_session_cap), _thread_cap(thread_cap),
-	_location(location),
-	_exceptions(*this),
-	_pd_target(Native_thread::INVALID_INDEX)
 {
-	uint8_t res;
+	addr_t const pd_sel = platform_specific().core_pd_sel();
-
+	addr_t const ec_sel = pager_thread(_location, platform_specific()).native_thread().ec_sel;
-	addr_t const pd_sel  = platform_specific().core_pd_sel();
-	_state._status       = 0;
-	_state.modified      = false;
-	_state.sel_client_ec = Native_thread::INVALID_INDEX;
-	_state.block();
-
-	if (Native_thread::INVALID_INDEX == _selectors ||
-	    Native_thread::INVALID_INDEX == _client_exc_pt_sel)
-		throw Invalid_thread();
-
-	addr_t const ec_sel = pager_thread(location, platform_specific()).native_thread().ec_sel;
 
 	/* create portal for page-fault handler - 14 */
 	_exceptions.register_handler<14>(*this, Mtd::QUAL | Mtd::ESP | Mtd::EIP,
 	                                 _page_fault_handler);
 
-	/* create portal for startup handler - 26 */
+	/* create portal for recall handler */
-	Mtd const mtd_startup(Mtd::ESP | Mtd::EIP);
-	_exceptions.register_handler<PT_SEL_STARTUP>(*this, mtd_startup,
-	                                             _startup_handler);
-
-	/* create portal for recall handler - 31 */
 	Mtd const mtd_recall(Mtd::ESP | Mtd::EIP | Mtd::ACDB | Mtd::EFL |
 	                     Mtd::EBSD | Mtd::FSGS);
 	_exceptions.register_handler<PT_SEL_RECALL>(*this, mtd_recall,
 	                                            _recall_handler);
 
-	/*
-	 * Create semaphore required for Genode locking. It can be later on
-	 * requested by the thread the same way as all exception portals.
-	 */
-	res = Nova::create_sm(exc_pt_sel_client() + SM_SEL_EC, pd_sel, 0);
-	if (res != Nova::NOVA_OK) {
-		throw Invalid_thread();
-	}
-
 	/* create portal for final cleanup call used during destruction */
-	res = create_portal(sel_pt_cleanup(), pd_sel, ec_sel, Mtd(0),
+	uint8_t res = create_portal(sel_pt_cleanup(), pd_sel, ec_sel, Mtd(0),
-	                    reinterpret_cast<addr_t>(_invoke_handler), this);
+	                            reinterpret_cast<addr_t>(_invoke_handler),
+	                            this);
 	if (res != Nova::NOVA_OK) {
 		error("could not create pager cleanup portal, error=", res);
 		throw Invalid_thread();
@@ -663,6 +683,69 @@ Pager_object::Pager_object(Cpu_session_capability cpu_session_cap,
 }
 
 
+Pager_object::Pager_object(Cpu_session_capability cpu_session_cap,
+                           Thread_capability thread_cap, unsigned long badge,
+                           Affinity::Location location, Session_label const &,
+                           Cpu_session::Name const &)
+:
+	_badge(badge),
+	_selectors(cap_map().insert(2)),
+	_client_exc_pt_sel(cap_map().insert(NUM_INITIAL_PT_LOG2)),
+	_cpu_session_cap(cpu_session_cap), _thread_cap(thread_cap),
+	_location(location),
+	_exceptions(*this),
+	_pd_target(Native_thread::INVALID_INDEX)
+{
+	_state._status       = 0;
+	_state.modified      = false;
+	_state.sel_client_ec = Native_thread::INVALID_INDEX;
+	_state.block();
+
+	if (Native_thread::INVALID_INDEX == _selectors ||
+	    Native_thread::INVALID_INDEX == _client_exc_pt_sel)
+		throw Invalid_thread();
+
+	_construct_pager();
+
+	/* create portal for startup handler */
+	Mtd const mtd_startup(Mtd::ESP | Mtd::EIP);
+	_exceptions.register_handler<PT_SEL_STARTUP>(*this, mtd_startup,
+	                                             _startup_handler);
+
+	/*
+	 * Create semaphore required for Genode locking. It can be later on
+	 * requested by the thread the same way as all exception portals.
+	 */
+	addr_t const pd_sel = platform_specific().core_pd_sel();
+	uint8_t const res = Nova::create_sm(exc_pt_sel_client() + SM_SEL_EC,
+	                                    pd_sel, 0);
+	if (res != Nova::NOVA_OK)
+		throw Invalid_thread();
+}
+
+
+void Pager_object::migrate(Affinity::Location location)
+{
+	Mutex::Guard _state_lock_guard(_state_lock);
+
+	if (_state.blocked())
+		return;
+
+	if (location.xpos() == _location.xpos() &&
+	    location.ypos() == _location.ypos())
+		return;
+
+	/* initiate migration by recall handler */
+	bool const just_recall = false;
+	uint8_t const res = _unsynchronized_client_recall(just_recall);
+	if (res == Nova::NOVA_OK) {
+		_next_location = location;
+
+		_state.request_migrate();
+	}
+}
+
+
 Pager_object::~Pager_object()
 {
 	/* sanity check that object got dissolved already - otherwise bug */
@@ -868,7 +951,7 @@ void Pager_object::_oom_handler(addr_t pager_dst, addr_t pager_src,
 }
 
 
-addr_t Pager_object::get_oom_portal()
+addr_t Pager_object::create_oom_portal()
 {
 	try {
 		addr_t const pt_oom      = sel_oom_portal();

repos/base-nova/src/core/platform_thread.cc

@@ -34,7 +34,6 @@
 using namespace Genode;
 
 
-
 static uint8_t map_thread_portals(Pager_object &pager,
                                   addr_t const target_exc_base,
                                   Nova::Utcb &utcb)
@@ -65,13 +64,41 @@ static uint8_t map_thread_portals(Pager_object &pager,
  ** Platform thread **
  *********************/
 
-void Platform_thread::affinity(Affinity::Location)
+
+void Platform_thread::affinity(Affinity::Location location)
 {
-	error("dynamic affinity change not supported on NOVA");
+	if (!_pager)
+		return;
+
+	if (worker() || vcpu() || !sc_created())
+		return;
+
+	_pager->migrate(platform_specific().sanitize(location));
 }
 
 
-Affinity::Location Platform_thread::affinity() const { return _location; }
+bool Platform_thread::_create_and_map_oom_portal(Nova::Utcb &utcb)
+{
+	addr_t const pt_oom = pager().create_oom_portal();
+	if (!pt_oom)
+		return false;
+
+	addr_t const source_pd = platform_specific().core_pd_sel();
+	return !map_local(source_pd, utcb, Nova::Obj_crd(pt_oom, 0),
+	                  Nova::Obj_crd(_sel_pt_oom(), 0));
+}
+
+
+void Platform_thread::prepare_migration()
+{
+	using Nova::Utcb;
+	Utcb &utcb = *reinterpret_cast<Utcb *>(Thread::myself()->utcb());
+
+	/* map exception portals to target pd */
+	map_thread_portals(pager(), _sel_exc_base, utcb);
+	/* re-create pt_oom portal */
+	_create_and_map_oom_portal(utcb);
+}
 
 
 int Platform_thread::start(void *ip, void *sp)
@@ -83,6 +110,8 @@ int Platform_thread::start(void *ip, void *sp)
 		return -1;
 	}
 
+	Pager_object &pager = *_pager;
+
 	if (!_pd || (main_thread() && !vcpu() &&
 	             _pd->parent_pt_sel() == Native_thread::INVALID_INDEX)) {
 		error("protection domain undefined");
@@ -93,9 +122,7 @@ int Platform_thread::start(void *ip, void *sp)
 	unsigned const kernel_cpu_id = platform_specific().kernel_cpu_id(_location);
 	addr_t const source_pd = platform_specific().core_pd_sel();
 
-	addr_t const pt_oom = _pager->get_oom_portal();
+	if (!_create_and_map_oom_portal(utcb)) {
-	if (!pt_oom || map_local(source_pd, utcb,
-	                         Obj_crd(pt_oom, 0), Obj_crd(_sel_pt_oom(), 0))) {
 		error("setup of out-of-memory notification portal - failed");
 		return -8;
 	}
@@ -109,7 +136,7 @@ int Platform_thread::start(void *ip, void *sp)
 			return -3;
 		}
 
-		uint8_t res = syscall_retry(*_pager,
+		uint8_t res = syscall_retry(pager,
 			[&]() {
 				return create_ec(_sel_ec(), _pd->pd_sel(), kernel_cpu_id,
 				                 utcb_addr, initial_sp, _sel_exc_base,
@@ -122,7 +149,7 @@ int Platform_thread::start(void *ip, void *sp)
 		}
 
 		if (!vcpu())
-			res = map_thread_portals(*_pager, _sel_exc_base, utcb);
+			res = map_thread_portals(pager, _sel_exc_base, utcb);
 
 		if (res != NOVA_OK) {
 			revoke(Obj_crd(_sel_ec(), 0));
@@ -132,12 +159,12 @@ int Platform_thread::start(void *ip, void *sp)
 
 		if (worker()) {
 			/* local/worker threads do not require a startup portal */
-			revoke(Obj_crd(_pager->exc_pt_sel_client() + PT_SEL_STARTUP, 0));
+			revoke(Obj_crd(pager.exc_pt_sel_client() + PT_SEL_STARTUP, 0));
 		}
 
-		_pager->initial_eip((addr_t)ip);
+		pager.initial_eip((addr_t)ip);
-		_pager->initial_esp(initial_sp);
+		pager.initial_esp(initial_sp);
-		_pager->client_set_ec(_sel_ec());
+		pager.client_set_ec(_sel_ec());
 
 		return 0;
 	}
@@ -150,7 +177,7 @@ int Platform_thread::start(void *ip, void *sp)
 	addr_t pd_utcb = 0;
 
 	if (!vcpu()) {
-		_sel_exc_base  = _pager->exc_pt_sel_client();
+		_sel_exc_base = 0;
 
 		pd_utcb = stack_area_virtual_base() + stack_virtual_size() - get_page_size();
 
@@ -161,7 +188,7 @@ int Platform_thread::start(void *ip, void *sp)
 		for (unsigned i = 0; i < sizeof(remap_dst)/sizeof(remap_dst[0]); i++) {
 			if (map_local(source_pd, utcb,
 			              Obj_crd(remap_src[i], 0),
-			              Obj_crd(_sel_exc_base + remap_dst[i], 0)))
+			              Obj_crd(pager.exc_pt_sel_client() + remap_dst[i], 0)))
 				return -6;
 		}
 	}
@@ -169,24 +196,24 @@ int Platform_thread::start(void *ip, void *sp)
 	/* create first thread in task */
 	enum { THREAD_GLOBAL = true };
 	uint8_t res = create_ec(_sel_ec(), _pd->pd_sel(), kernel_cpu_id,
-	                        pd_utcb, 0, vcpu() ? _sel_exc_base : 0,
+	                        pd_utcb, 0, _sel_exc_base,
 	                        THREAD_GLOBAL);
 	if (res != NOVA_OK) {
 		error("create_ec returned ", res);
 		return -7;
 	}
 
-	_pager->client_set_ec(_sel_ec());
+	pager.client_set_ec(_sel_ec());
-	_pager->initial_eip((addr_t)ip);
+	pager.initial_eip((addr_t)ip);
-	_pager->initial_esp((addr_t)sp);
+	pager.initial_esp((addr_t)sp);
 
 	if (vcpu())
 		_features |= REMOTE_PD;
 	else
-		res = map_thread_portals(*_pager, 0, utcb);
+		res = map_thread_portals(pager, 0, utcb);
 
 	if (res == NOVA_OK) {
-		res = syscall_retry(*_pager,
+		res = syscall_retry(pager,
 			[&]() {
 				/* let the thread run */
 				return create_sc(_sel_sc(), _pd->pd_sel(), _sel_ec(),
@@ -195,9 +222,9 @@ int Platform_thread::start(void *ip, void *sp)
 	}
 
 	if (res != NOVA_OK) {
-		_pager->client_set_ec(Native_thread::INVALID_INDEX);
+		pager.client_set_ec(Native_thread::INVALID_INDEX);
-		_pager->initial_eip(0);
+		pager.initial_eip(0);
-		_pager->initial_esp(0);
+		pager.initial_esp(0);
 
 		error("create_sc returned ", res);
 
@@ -326,7 +353,7 @@ void Platform_thread::thread_type(Nova_native_cpu::Thread_type thread_type,
 
 
 Platform_thread::Platform_thread(size_t, const char *name, unsigned prio,
-                                 Affinity::Location affinity, int)
+                                 Affinity::Location affinity, addr_t)
 :
 	_pd(0), _pager(0), _id_base(cap_map().insert(2)),
 	_sel_exc_base(Native_thread::INVALID_INDEX),

repos/base-nova/src/include/base/internal/spin_lock.h

@@ -36,7 +36,7 @@ static inline void spinlock_lock(volatile T *lock_variable)
 
 	Genode::Thread * myself = Genode::Thread::myself();
 	T const tid = myself ? myself->native_thread().ec_sel
-	                     : (Genode::addr_t)Nova::PT_SEL_MAIN_EC;
+	                     : (Genode::addr_t)Nova::EC_SEL_THREAD;
 
 	unsigned help_counter = 0;
 

repos/base-nova/src/lib/base/thread_start.cc

@@ -85,7 +85,7 @@ void Thread::_init_platform_thread(size_t weight, Type type)
 		_thread_cap = main_thread_cap();
 
 		native_thread().exc_pt_sel = 0;
-		native_thread().ec_sel     = Nova::PT_SEL_MAIN_EC;
+		native_thread().ec_sel     = Nova::EC_SEL_THREAD;
 
 		request_native_ec_cap(PT_SEL_PAGE_FAULT, native_thread().ec_sel);
 		return;
@@ -125,7 +125,6 @@ void Thread::_deinit_platform_thread()
 
 	if (native_thread().ec_sel != Native_thread::INVALID_INDEX) {
 		revoke(Obj_crd(native_thread().ec_sel, 0));
-		cap_map().remove(native_thread().ec_sel, 0, false);
 	}
 
 	/* de-announce thread */
@@ -172,9 +171,7 @@ void Thread::start()
 	cpu_thread.start(thread_ip, _stack->top());
 
 	/* request native EC thread cap */ 
-	native_thread().ec_sel = cap_map().insert();
+	native_thread().ec_sel = native_thread().exc_pt_sel + Nova::EC_SEL_THREAD;
-	if (native_thread().ec_sel == Native_thread::INVALID_INDEX)
-		throw Cpu_session::Thread_creation_failed();
 
 	/*
 	 * Requested ec cap that is used for recall and