libc: translate file-descriptor exhaustion into EMFILE

Fixes #3841

repos/libports/src/lib/libc/fd_alloc.cc

@@ -63,13 +63,15 @@ File_descriptor *File_descriptor_allocator::alloc(Plugin *plugin,
 	bool const any_fd = (libc_fd < 0);
 	Id_space::Id id {(unsigned)libc_fd};
 
-	if (any_fd) {
-		id.value = _id_allocator.alloc();
-	} else {
-		_id_allocator.alloc_addr(addr_t(libc_fd));
-	}
+	try {
+		if (any_fd) {
+			id.value = _id_allocator.alloc();
+		} else {
+			_id_allocator.alloc_addr(addr_t(libc_fd));
+		}
 
-	return new (_alloc) File_descriptor(_id_space, *plugin, *context, id);
+		return new (_alloc) File_descriptor(_id_space, *plugin, *context, id);
+	} catch (...) { return nullptr; }
 }
 
 

repos/libports/src/lib/libc/file_operations.cc

@@ -273,6 +273,7 @@ extern "C" int dup2(int libc_fd, int new_libc_fd)
 		close(new_libc_fd);
 
 	new_fd = file_descriptor_allocator()->alloc(fd->plugin, 0, new_libc_fd);
+	if (!new_fd) return Errno(EMFILE);
 
 	/* new_fd->context must be assigned by the plugin implementing 'dup2' */
 	return fd->plugin->dup2(fd, new_fd);
@@ -513,10 +514,8 @@ __SYS_(int, open, (const char *pathname, int flags, ...),
 	}
 
 	new_fdo = plugin->open(resolved_path.base(), flags);
-	if (!new_fdo) {
-		error("plugin()->open(\"", pathname, "\") failed");
+	if (!new_fdo)
 		return -1;
-	}
 	new_fdo->path(resolved_path.base());
 
 	return new_fdo->libc_fd;

repos/libports/src/lib/libc/internal/unconfirmed.h

@@ -0,0 +1,41 @@
+/*
+ * \brief  Utility to automatically unroll unconfirmed operations
+ * \author Christian Helmuth
+ * \date   2020-07-31
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef _LIBC__INTERNAL__UNCONFIRMED_H_
+#define _LIBC__INTERNAL__UNCONFIRMED_H_
+
+namespace Libc {
+	template <typename> struct Unconfirmed;
+
+	template <typename FUNC>
+	Unconfirmed<FUNC> make_unconfirmed(FUNC const &cleanup)
+	{
+		return { cleanup };
+	}
+};
+
+
+template <typename FUNC>
+struct Libc::Unconfirmed
+{
+	bool confirmed { false };
+
+	FUNC const &cleanup;
+
+	Unconfirmed(FUNC const &cleanup) : cleanup(cleanup) { }
+	~Unconfirmed() { if (!confirmed) cleanup(); }
+
+	void confirm() { confirmed = true; }
+};
+
+#endif /* _LIBC__INTERNAL__UNCONFIRMED_H_ */

repos/libports/src/lib/libc/socket_fs_plugin.cc

@@ -41,6 +41,7 @@
 #include <internal/errno.h>
 #include <internal/init.h>
 #include <internal/suspend.h>
+#include <internal/unconfirmed.h>
 
 
 namespace Libc {
@@ -568,6 +569,16 @@ extern "C" int socket_fs_accept(int libc_fd, sockaddr *addr, socklen_t *addrlen)
 		return Errno(EACCES);
 	}
 
+	File_descriptor *accept_fd =
+		file_descriptor_allocator()->alloc(&plugin(), accept_context);
+	if (!accept_fd) {
+		Libc::Allocator alloc { };
+		destroy(alloc, accept_context);
+		return Errno(EMFILE);
+	}
+
+	auto _accept_fd = make_unconfirmed([&] { file_descriptor_allocator()->free(accept_fd); });
+
 	if (addr && addrlen) {
 		Socket_fs::Remote_functor func(*accept_context, false);
 		int ret = read_sockaddr_in(func, (sockaddr_in *)addr, addrlen);
@@ -578,12 +589,10 @@ extern "C" int socket_fs_accept(int libc_fd, sockaddr *addr, socklen_t *addrlen)
 		}
 	}
 
-	File_descriptor *accept_fd =
-		file_descriptor_allocator()->alloc(&plugin(), accept_context);
-
 	/* inherit the O_NONBLOCK flag if set */
 	accept_context->fd_flags(listen_context->fd_flags());
 
+	_accept_fd.confirm();
 	return accept_fd->libc_fd;
 }
 
@@ -1028,6 +1037,11 @@ extern "C" int socket_fs_socket(int domain, int type, int protocol)
 	} catch (New_socket_failed) { return Errno(EACCES); }
 
 	File_descriptor *fd = file_descriptor_allocator()->alloc(&plugin(), context);
+	if (!fd) {
+		Libc::Allocator alloc { };
+		destroy(alloc, context);
+		return Errno(EMFILE);
+	}
 
 	return fd->libc_fd;
 }

repos/libports/src/lib/libc/vfs_plugin.cc

@@ -559,6 +559,12 @@ Libc::File_descriptor *Libc::Vfs_plugin::dup(File_descriptor *fd)
 	File_descriptor * const new_fd =
 		file_descriptor_allocator()->alloc(this, vfs_context(handle));
 
+	if (!new_fd) {
+		VFS_THREAD_SAFE(handle->close());
+		errno = EMFILE;
+		return nullptr;
+	}
+
 	new_fd->flags = fd->flags;
 	new_fd->path(fd->fd_path);
 

repos/libports/src/test/libc/main.cc

@@ -19,17 +19,19 @@
 
 /* libC includes */
 extern "C" {
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <time.h>
-#include <sys/random.h>
-#include <sys/syscall.h>
 #include <sys/limits.h>
+#include <sys/random.h>
+#include <sys/resource.h>
+#include <sys/syscall.h>
 #include <time.h>
-#include <inttypes.h>
+#include <time.h>
+#include <unistd.h>
 }
 
 int main(int argc, char **argv)
@@ -176,6 +178,33 @@ int main(int argc, char **argv)
 
 	perror("perror");
 
+	{
+		/* test EMFILE (issue #3841) */
+		rlimit limit { 0, 0 };
+		if (getrlimit(RLIMIT_NOFILE, &limit) == -1) {
+			perror("getrlimit");
+			++error_count;
+		} else {
+			int fd[limit.rlim_cur]; memset(fd, -1, sizeof(fd));
+
+			while (limit.rlim_cur--) {
+				int i = open("/dev/log", O_WRONLY);
+				if (i == -1) break;
+				fd[i] = i;
+			}
+
+			if (errno != EMFILE) {
+				printf("open returned '%s' expected EMFILE\n", strerror(errno));
+				++error_count;
+			}
+
+			for (int i : fd) {
+				if (i != -1)
+					close(i);
+			}
+		}
+	}
+
 	struct timespec ts;
 	for (int i = 0; i < 3; ++i) {
 		sleep(1);