libc: duplicate the file descriptor on shared 'mmap()' mappings

Fixes #3909

repos/libports/src/lib/libc/file_operations.cc

@@ -451,7 +451,10 @@ __SYS_(void *, mmap, (void *addr, ::size_t length,
 	}
 
 	void *start = fd->plugin->mmap(addr, length, prot, flags, fd, offset);
-	mmap_registry()->insert(start, length, fd->plugin);
+
+	if (start != MAP_FAILED)
+		mmap_registry()->insert(start, length, fd->plugin);
+
 	return start;
 })
 
@@ -471,6 +474,12 @@ extern "C" int munmap(void *start, ::size_t length)
 	 */
 	Plugin *plugin = mmap_registry()->lookup_plugin_by_addr(start);
 
+	/*
+	 * Remove registry entry before unmapping to avoid double insertion error
+	 * if another thread gets the same start address immediately after unmapping.
+	 */
+	mmap_registry()->remove(start);
+
 	int ret = 0;
 	if (plugin)
 		ret = plugin->munmap(start, length);
@@ -481,7 +490,6 @@ extern "C" int munmap(void *start, ::size_t length)
 		mem_alloc(executable)->free(start);
 	}
 
-	mmap_registry()->remove(start);
 	return ret;
 }
 

repos/libports/src/lib/libc/internal/vfs_plugin.h

@@ -73,6 +73,17 @@ class Libc::Vfs_plugin : public Plugin
 
 	private:
 
+		struct Mmap_entry : Registry<Mmap_entry>::Element
+		{
+			void                  * const start;
+			Libc::File_descriptor * const fd;
+
+			Mmap_entry(Registry<Mmap_entry> &registry, void *start,
+			           Libc::File_descriptor *fd)
+			: Registry<Mmap_entry>::Element(registry, *this), start(start),
+			  fd(fd) { }
+		};
+
 		Genode::Allocator               &_alloc;
 		Vfs::File_system                &_root_fs;
 		Constructible<Genode::Directory> _root_dir { };
@@ -80,6 +91,7 @@ class Libc::Vfs_plugin : public Plugin
 		Update_mtime               const _update_mtime;
 		Current_real_time               &_current_real_time;
 		bool                       const _pipe_configured;
+		Registry<Mmap_entry>             _mmap_registry;
 
 		/**
 		 * Sync a handle

repos/libports/src/lib/libc/vfs_plugin.cc

@@ -1900,13 +1900,13 @@ void *Libc::Vfs_plugin::mmap(void *addr_in, ::size_t length, int prot, int flags
 	if ((prot != PROT_READ) && (prot != (PROT_READ | PROT_WRITE))) {
 		error("mmap for prot=", Hex(prot), " not supported");
 		errno = EACCES;
-		return (void *)-1;
+		return MAP_FAILED;
 	}
 
 	if (flags & MAP_FIXED) {
 		error("mmap for fixed predefined address not supported yet");
 		errno = EINVAL;
-		return (void *)-1;
+		return MAP_FAILED;
 	}
 
 	void *addr = nullptr;
@@ -1922,7 +1922,7 @@ void *Libc::Vfs_plugin::mmap(void *addr_in, ::size_t length, int prot, int flags
 		if (addr == (void *)-1) {
 			error("mmap out of memory");
 			errno = ENOMEM;
-			return (void *)-1;
+			return MAP_FAILED;
 		}
 
 		/* copy variables for complete read */
@@ -1936,15 +1936,25 @@ void *Libc::Vfs_plugin::mmap(void *addr_in, ::size_t length, int prot, int flags
 				error("mmap could not obtain file content");
 				::munmap(addr, length);
 				errno = EACCES;
-				return (void *)-1;
+				return MAP_FAILED;
 			} else if (length_read == 0) /* EOF */
 				break; /* done (length can legally be greater than the file length) */
 			read_remain -= length_read;
 			read_offset += length_read;
 			read_addr += length_read;
 		}
+
 	} else if (flags & MAP_SHARED) {
 
+		/* duplicate the file descriptor to keep the file open as long as the mapping exists */
+
+		Libc::File_descriptor *dup_fd = dup(fd);
+
+		if (!dup_fd) {
+			errno = ENFILE;
+			return MAP_FAILED;
+		}
+
 		Genode::Dataspace_capability ds_cap;
 
 		monitor().monitor([&] {
@@ -1954,11 +1964,20 @@ void *Libc::Vfs_plugin::mmap(void *addr_in, ::size_t length, int prot, int flags
 
 		if (!ds_cap.valid()) {
 			Genode::error("mmap got invalid dataspace capability");
+			close(dup_fd);
 			errno = ENODEV;
-			return (void*)-1;
+			return MAP_FAILED;
 		}
 
-		addr = region_map().attach(ds_cap, length, offset);
+		try {
+			addr = region_map().attach(ds_cap, length, offset);
+		} catch (...) {
+			close(dup_fd);
+			errno = ENOMEM;
+			return MAP_FAILED;
+		}
+
+		new (_alloc) Mmap_entry(_mmap_registry, addr, dup_fd);
 	}
 
 	return addr;
@@ -1967,10 +1986,28 @@ void *Libc::Vfs_plugin::mmap(void *addr_in, ::size_t length, int prot, int flags
 
 int Libc::Vfs_plugin::munmap(void *addr, ::size_t)
 {
-	if (mem_alloc()->size_at(addr) > 0)
+	if (mem_alloc()->size_at(addr) > 0) {
+		/* private mapping */
 		mem_alloc()->free(addr);
-	else
-		region_map().detach(addr);
+		return 0;
+	}
+
+	/* shared mapping */
+
+	Libc::File_descriptor *fd = nullptr;
+
+	_mmap_registry.for_each([&] (Mmap_entry &entry) {
+		if (entry.start == addr) {
+			fd = entry.fd;
+			destroy(_alloc, &entry);
+			region_map().detach(addr);
+		}
+	});
+
+	if (!fd)
+		return Errno(EINVAL);
+
+	close(fd);
 
 	return 0;
 }