noux: reflect -no memory- case during process setup

Instead of just red messages in the log and a hanging caller, the issuer
may respond to it - e.g. a noux bash shell will show an error and is
afterwards still usable.

Fixes #1778

repos/ports/include/noux_session/sysio.h

@@ -266,7 +266,8 @@ namespace Noux {
 		enum Clock_Id        { CLOCK_ID_SECOND };
 
 		enum Fcntl_error     { FCNTL_ERR_CMD_INVALID = Vfs::Directory_service::NUM_GENERAL_ERRORS };
-		enum Execve_error    { EXECVE_NONEXISTENT    = Vfs::Directory_service::NUM_GENERAL_ERRORS };
+		enum Execve_error    { EXECVE_NONEXISTENT    = Vfs::Directory_service::NUM_GENERAL_ERRORS, EXECVE_NOMEM };
+		enum Fork_error      { FORK_NOMEM = Vfs::Directory_service::NUM_GENERAL_ERRORS };
 		enum Select_error    { SELECT_ERR_INTERRUPT };
 
 		/**
@@ -339,6 +340,7 @@ namespace Noux {
 			Utimes_error   utimes;
 			Wait4_error    wait4;
 			Kill_error     kill;
+			Fork_error     fork;
 
 		} error;
 

repos/ports/src/lib/libc_noux/plugin.cc

@@ -549,6 +549,11 @@ extern "C" pid_t fork(void)
 
 		if (!noux_syscall(Noux::Session::SYSCALL_FORK)) {
 			PERR("fork error %d", sysio()->error.general);
+			switch (sysio()->error.fork) {
+			case Noux::Sysio::FORK_NOMEM:       errno = ENOMEM; break;
+			default: errno = EAGAIN;
+			}
+			return -1;
 		}
 
 		return sysio()->fork_out.pid;
@@ -1008,6 +1013,7 @@ namespace {
 			PWRN("exec syscall failed for path \"%s\"", filename);
 			switch (sysio()->error.execve) {
 			case Noux::Sysio::EXECVE_NONEXISTENT: errno = ENOENT; break;
+			case Noux::Sysio::EXECVE_NOMEM:       errno = ENOMEM; break;
 			}
 			return -1;
 		}

repos/ports/src/noux/child.h

@@ -296,9 +296,20 @@ namespace Noux {
 
 			bool _syscall_net(Syscall sc);
 
+			void _destruct() {
+
+				_sig_rec->dissolve(&_destruct_dispatcher);
+
+				_entrypoint.dissolve(this);
+
+				if (is_init_process(this))
+					init_process_exited(_child_policy.exit_value());
+			}
+
 		public:
 
 			struct Binary_does_not_exist : Exception { };
+			struct Insufficient_memory : Exception { };
 
 			/**
 			 * Constructor
@@ -312,6 +323,8 @@ namespace Noux {
 			 *                               specified name could not be
 			 *                               looked up at the virtual file
 			 *                               system
+			 * \throw Insufficent_memory if the child could not be started by
+			 *                           the parent
 			 */
 			Child(char const           *binary_name,
 			      Parent_exit          *parent_exit,
@@ -379,19 +392,17 @@ namespace Noux {
 
 				if (!forked && !_elf._binary_ds.valid()) {
 					PERR("Lookup of executable \"%s\" failed", binary_name);
+
+					_destruct();
 					throw Binary_does_not_exist();
 				}
+				if (!_child.main_thread_cap().valid()) {
+					_destruct();
+					throw Insufficient_memory();
+				}
 			}
 
-			~Child()
-			{
-				_sig_rec->dissolve(&_destruct_dispatcher);
-
-				_entrypoint.dissolve(this);
-
-				if (is_init_process(this))
-					init_process_exited(_child_policy.exit_value());
-			}
+			~Child() { _destruct(); }
 
 			void start() { _entrypoint.activate(); }
 

repos/ports/src/noux/main.cc

@@ -387,6 +387,8 @@ bool Noux::Child::syscall(Noux::Session::Syscall sc)
 				}
 				catch (Child::Binary_does_not_exist) {
 					_sysio->error.execve = Sysio::EXECVE_NONEXISTENT; }
+				catch (Child::Insufficient_memory) {
+					_sysio->error.execve = Sysio::EXECVE_NOMEM; }
 
 				break;
 			}
@@ -583,28 +585,34 @@ bool Noux::Child::syscall(Noux::Session::Syscall sc)
 				Genode::addr_t parent_cap_addr = _sysio->fork_in.parent_cap_addr;
 
 				int const new_pid = pid_allocator()->alloc();
+				Child * child = nullptr;
 
-				/*
-				 * XXX To ease debugging, it would be useful to generate a
-				 *     unique name that includes the PID instead of just
-				 *     reusing the name of the parent.
-				 */
-				Child *child = new Child(_child_policy.name(),
-				                         this,
-				                         _kill_broadcaster,
-				                         *this,
-				                         new_pid,
-				                         _sig_rec,
-				                         root_dir(),
-				                         _args,
-				                         _env.env(),
-				                         _cap_session,
-				                         _parent_services,
-				                         _resources.ep,
-				                         true,
-				                         env()->heap(),
-				                         _destruct_queue,
-				                         verbose);
+				try {
+					/*
+					 * XXX To ease debugging, it would be useful to generate a
+					 *     unique name that includes the PID instead of just
+					 *     reusing the name of the parent.
+					 */
+					child = new Child(_child_policy.name(),
+					                  this,
+					                  _kill_broadcaster,
+					                  *this,
+					                  new_pid,
+					                  _sig_rec,
+					                  root_dir(),
+					                  _args,
+					                  _env.env(),
+					                  _cap_session,
+					                  _parent_services,
+					                  _resources.ep,
+					                  true,
+					                  env()->heap(),
+					                  _destruct_queue,
+					                  verbose);
+				} catch (Child::Insufficient_memory) {
+					_sysio->error.fork = Sysio::FORK_NOMEM;
+					break;
+				}
 
 				Family_member::insert(child);
 
@@ -1078,7 +1086,24 @@ Genode::Lock &Noux::signal_lock()
 
 
 void *operator new (Genode::size_t size) {
-	return Genode::env()->heap()->alloc(size); }
+	void * ptr = Genode::env()->heap()->alloc(size);
+	if (!ptr)
+		return ptr;
+
+	Genode::memset(ptr, 0, size);
+	return ptr;
+}
+
+
+void operator delete (void * ptr)
+{
+	if (Genode::env()->heap()->need_size_for_free()) {
+		PWRN("leaking memory");
+		return;
+	}
+
+	Genode::env()->heap()->free(ptr, 0);
+}
 
 
 template <typename FILE_SYSTEM>