New libc_integration test

I created a test program which stresses the interplay between libc,
pthreads and vfs_pipe and may detect regressions.

The program starts a thread that spawns a bunch of workers, sends and
receives data to them through a pipe. When all data of a worker is
collected, the worker is finished (join()) and a new worker is started.
Both the number of bytes sent to the worker and the size of its
answer are random.

Issue #3967

repos/libports/run/libc_integration.run

@@ -0,0 +1,117 @@
+# -wo   number of worker to run default value: 200000000
+# -pw   number of parallel workers to run    default value: 23
+# -ws   maximal buffer to transfer           default value: 16384
+# -ds   data size to write at once           default value: 1024
+
+set config_wo 10000
+set config_pw 23
+set config_ws 16384
+set config_ds 1024
+
+set run_script_timeout 500000
+
+if { [get_cmd_switch --autopilot] } {
+	set current_date [clock format [clock seconds] -format %a]
+	if {[get_cmd_switch --autopilot] && $current_date != "Sat" && $current_date != "Sun" } {
+		puts "\n Run script is not supported on this platform today. \n";
+		exit 0
+	}
+
+	if {[have_include "power_on/qemu"]} {
+		puts "Run script does not support autopilot mode on Qemu"
+		exit 0
+	}
+
+	set run 0
+	set run_script_timeout 1200
+
+	if {[have_board pc]} {
+		set config_wo 10000
+		set run [expr [have_spec nova] || [have_spec hw]]
+	}
+	if {[have_board imx8q_evk]} {
+		set config_wo 10000
+		set run 1
+	}
+	if {[have_board linux]} {
+		set run 1
+	}
+
+	if {!$run} {
+		puts "\n Run script is not supported on this platform. \n";
+		exit 0
+	}
+}
+
+create_boot_directory
+
+set build_components {
+	test/libc_integration
+}
+
+set boot_modules {
+	test-libc_integration
+}
+
+import_from_depot [depot_user]/src/[base_src]
+import_from_depot [depot_user]/src/init
+import_from_depot [depot_user]/src/libc
+import_from_depot [depot_user]/src/posix
+import_from_depot [depot_user]/src/stdcxx
+import_from_depot [depot_user]/src/vfs
+import_from_depot [depot_user]/src/vfs_pipe
+
+build $build_components
+
+set config {}
+append config {
+<config prio_levels="2">
+	<parent-provides>
+		<service name="ROM"/>
+		<service name="IRQ"/>
+		<service name="IO_MEM"/>
+		<service name="IO_PORT"/>
+		<service name="PD"/>
+		<service name="RM"/>
+		<service name="CPU"/>
+		<service name="LOG"/>
+	</parent-provides>
+	<default-route>
+		<any-service> <parent/> <any-child/> </any-service>
+	</default-route>
+	<default caps="200"/>
+	<start name="timer">
+		<resource name="CPU" quantum="20"/>
+		<resource name="RAM" quantum="1M"/>
+		<provides> <service name="Timer"/> </provides>
+	</start>
+	<start name="test-libc_integration" caps="400" priority="-1">
+		<resource name="RAM" quantum="256M"/>
+		<config>
+			<libc stdout="/dev/log" stderr="/dev/log" rtc="/dev/rtc" pipe="/dev/pipe"/>
+			<arg value="test-libc_integration"/>
+			<arg value="-wo"/> <arg value="} $config_wo {"/>
+			<arg value="-pw"/> <arg value="} $config_pw {"/>
+			<arg value="-ws"/> <arg value="} $config_ws {"/>
+			<arg value="-ds"/> <arg value="} $config_ds {"/>
+			<vfs>
+				<dir name="dev">
+					<log/>
+					<inline name="rtc">2019-08-20 15:01</inline>
+					<dir name="pipe">
+						<pipe/>
+					</dir>
+				</dir>
+			</vfs>
+		</config>
+	</start>
+</config>
+}
+
+install_config $config
+
+build_boot_image $boot_modules
+
+append qemu_args " -nographic -smp 10 "
+
+run_genode_until {.*--- test finished ---.*} $run_script_timeout

repos/libports/src/test/libc_integration/README

@@ -0,0 +1,28 @@
+Libc test for heavy multi threading and vfs usage.
+
+
+To build this example for Linux use the following command:
+
+``` bash
+cd `mktemp -d`
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/fd_set.h
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/input_sender.h
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/main.cc
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/pipe.h
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/stdcxx_log.cc
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/stdcxx_log.h
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/thread.cc
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/thread.h
+ln -s <genode_dir>/repos/libports/src/test/libc_integration/definitions.h
+
+g++ -pthread -Wall -Werror -std=c++17 \
+	main.cc thread.cc stdcxx_log.cc \
+	-o integration_test
+./integration_test
+```
+
+Parameters:
+  -wo     number of worker to run              default value: 200000000
+  -pw     number of parallel workers to run    default value: 23
+  -ws     maximal buffer to transfer           default value: 16384
+  -ds     data size to write at once           default value: 1024

repos/libports/src/test/libc_integration/definitions.h

@@ -0,0 +1,38 @@
+/*
+ * \brief  common definitions
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef INTEGRATION_TEST_TYPES_H
+#define INTEGRATION_TEST_TYPES_H
+
+
+/* libc includes */
+#include <algorithm>
+#include <utility>
+
+
+namespace Integration_test
+{
+	using namespace std;
+
+	enum {
+		BUFFER_SIZE       = 256 * 1024,
+		IN_DATA_SIZE      = 8192 * 2,
+		NUMBER_OF_WORKERS = 200000000,
+		PARALLEL_WORKERS  = 23,
+		WRITE_SIZE        = 1024,
+	};
+}
+
+
+#endif  /* INTEGRATION_TEST_TYPES_H */

repos/libports/src/test/libc_integration/fd_set.h

@@ -0,0 +1,78 @@
+/*
+ * \brief  wrapper for the libc fd_set.
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef INTEGRATION_TEST_FD_SET_H
+#define INTEGRATION_TEST_FD_SET_H
+
+/* libc includes */
+#include <sys/select.h>
+
+/* stdcxx includes */
+#include <vector>
+#include <algorithm>
+
+/* local includes */
+#include "stdcxx_log.h"
+
+
+namespace Integration_test
+{
+	using namespace std;
+
+	struct File_descriptor_set
+	{
+		private:
+
+			vector<int> _fds    { };
+			int         _max_fd { 0 };
+
+		public:
+
+			using Fds_iter = vector<int>::const_iterator;
+
+			void add_fd(int const fd)
+			{
+				_fds.push_back(fd);
+			}
+
+			void remove_fd(int const fd)
+			{
+				_fds.erase(find(_fds.begin(), _fds.end(), fd));
+			}
+
+			int max_fd()    const { return _max_fd; }
+			size_t count()  const { return _fds.size(); }
+
+			Fds_iter begin() const { return _fds.cbegin(); }
+			Fds_iter end()   const { return _fds.cend(); }
+
+			fd_set fds()
+			{
+				fd_set set { };
+
+				_max_fd = 0;
+				FD_ZERO(&set);
+
+				for (auto fd : _fds) {
+					FD_SET(fd, &set);
+					_max_fd = max(_max_fd, fd);
+				}
+
+				return set;
+			}
+
+	};
+}
+
+#endif  /* INTEGRATION_TEST_FD_SET_H */

repos/libports/src/test/libc_integration/input_sender.h

@@ -0,0 +1,175 @@
+/*
+ * \brief  thread sends input date to workers.
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef INTEGRATION_TEST_INPUT_SENDER_H
+#define INTEGRATION_TEST_INPUT_SENDER_H
+
+
+/* stdcxx includes */
+#include <algorithm>
+#include <chrono>
+#include <mutex>
+#include <random>
+#include <thread>
+#include <vector>
+
+/* libc includes */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+/* local includes */
+#include "stdcxx_log.h"
+#include "definitions.h"
+
+
+namespace Integration_test
+{
+	using namespace std;
+
+	struct Input_info
+	{
+		size_t   worker_no;
+		ssize_t  num_bytes;
+		size_t   bytes_sent;
+		int      pipe_fd;
+
+		Input_info(size_t no, int fd)
+		:
+			worker_no  { no },
+			num_bytes  { IN_DATA_SIZE },
+			bytes_sent { 0 },
+			pipe_fd    { fd }
+		{ }
+	};
+
+	class Runner;
+	class Input_sender;
+};
+
+
+class Integration_test::Runner
+{
+	private:
+
+		using Work_iter  = std::vector<Input_info>::iterator;
+
+		mutex              _lock           { };
+		vector<Input_info> _workers        { };
+		vector<char>       _send_data;
+
+		Work_iter _find_worker(size_t no)
+		{
+			return find_if(_workers.begin(),
+			               _workers.end(),
+			               [no] (Input_info const &e) { return no == e.worker_no; });
+		}
+
+	public:
+
+		Runner()
+		:
+			_send_data ( static_cast<vector<char>::size_type>(IN_DATA_SIZE) )
+		{
+			random_device                  rd;
+			mt19937                        gen     { rd() };
+			uniform_int_distribution<char> distrib { ' ', '}' };
+
+			for (auto &e : _send_data) {
+				e = distrib(gen);
+			}
+		}
+
+		void add_worker(size_t no, int fd)
+		{
+			lock_guard<mutex> guard { _lock };
+			_workers.emplace_back(no, fd);
+		}
+
+		void remove_worker(size_t w)
+		{
+			lock_guard<mutex> guard { _lock };
+
+			auto it { _find_worker(w) };
+			if (it == _workers.end()) {
+				return;
+			}
+
+			_workers.erase(it);
+		}
+
+		void run()
+		{
+			while (true)
+			{
+				{
+					lock_guard<mutex> guard { _lock };
+					for (auto &worker : _workers) {
+
+						if (worker.bytes_sent < _send_data.size()) {
+
+							size_t cnt    { min(static_cast<size_t>(WRITE_SIZE),
+							                    _send_data.size()-worker.bytes_sent) };
+							ssize_t w_res { write(worker.pipe_fd,
+							                      _send_data.data()+worker.bytes_sent, cnt) };
+
+							if(w_res < 0) {
+								error("error: send data to worker ", worker.worker_no, " write failed");
+								exit(-3);
+							}
+
+							worker.bytes_sent += w_res;
+						}
+					}
+				}
+
+				/* allow access to _workers from the outside */
+				this_thread::sleep_for(chrono::milliseconds(50));
+			}
+		}
+};
+
+
+class Integration_test::Input_sender
+{
+	private:
+
+		Runner     _runner { };
+		thread     _thread;
+
+		Input_sender(Input_sender  &) = delete;
+		Input_sender(Input_sender &&) = delete;
+
+	public:
+
+		Input_sender()
+		:
+			_thread { &Runner::run, &_runner }
+		{
+		}
+
+		void add_worker(size_t no, int fd)
+		{
+			_runner.add_worker(no, fd);
+		}
+
+		void remove_finished_workers(vector<size_t> &workers)
+		{
+			for (auto e : workers) {
+				_runner.remove_worker(e);
+			}
+		}
+};
+
+#endif  /* INTEGRATION_TEST_INPUT_SENDER_H */

repos/libports/src/test/libc_integration/main.cc

@@ -0,0 +1,174 @@
+/*
+ * \brief  Integration test for interplay of the following
+ *         features:
+ *           - libc
+ *           - pthread
+ *           - vfs_pipe
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+
+/* libc includes */
+#include <string.h>
+#include <errno.h>
+#include <sys/select.h>
+
+/* stdcxx includes */
+#include <vector>
+
+/* local includes */
+#include "definitions.h"
+#include "fd_set.h"
+#include "input_sender.h"
+#include "stdcxx_log.h"
+#include "thread.h"
+
+
+namespace Integration_test
+{
+	using namespace std;
+
+	class Main;
+
+	struct Thread_data;
+}
+
+
+struct Integration_test::Thread_data
+{
+	size_t max_workers;
+	size_t parallel_workers;
+	size_t buffer_size;
+	size_t write_size;
+};
+
+
+/*
+ * Thread-function that runs the test.
+ */
+void *test_runner(void *arg)
+{
+	using namespace Integration_test;
+	using namespace std;
+
+	enum { OUTPUT_REDUCTION_FACTOR = 100 };
+
+	Thread_data  data             { *reinterpret_cast<Thread_data*>(arg) };
+	Input_sender sender           { };
+	Thread_list  threads          { data.buffer_size };
+	size_t       threads_started  { 0 };
+
+	for (size_t thread=0; thread<data.parallel_workers; ++thread) {
+		auto handle { threads.add_worker() };
+		sender.add_worker(handle.first, handle.second);
+		++threads_started;
+	}
+
+	while (threads_started < data.max_workers) {
+
+		static size_t cnt { 0 };
+		if ((cnt++ % OUTPUT_REDUCTION_FACTOR) == 0) {
+			log((threads_started - threads.count()), "  workers finished, ",
+			    threads.count(), " currently running");
+		}
+
+		auto fds { threads.fds() };
+		auto num_ready = select(threads.max_fd()+1, &fds, nullptr, nullptr, nullptr);
+
+		if (num_ready == -1) {
+			error("select() failed with '", strerror(errno), "'");
+			exit(-1);
+		}
+
+		vector<size_t> finished         { };
+		for (auto fd : threads.descriptor_set()) {
+
+			if (!FD_ISSET(fd, &fds))
+					continue;
+
+			auto it = threads.find_worker_by_fd(fd);
+
+			if (it == threads.end()) {
+				error("worker not found");
+				exit(-2);
+			}
+
+			uint8_t buf[16*1024] { };
+			auto res { read(fd, buf, sizeof(buf)) };
+			if (res < 0) {
+				error("read error: fd=", fd);
+			} else {
+				it->append_result_data(buf, res);
+			}
+
+			if (it->done()) {
+				finished.push_back(it->worker_no());
+			}
+		}
+
+		sender.remove_finished_workers(finished);
+		threads.remove_finished_workers(finished);
+
+		/* restart more threads when some threads are finished */
+		while (threads.count() < data.parallel_workers && threads_started < data.max_workers) {
+			auto handle { threads.add_worker() };
+			sender.add_worker(handle.first, handle.second);
+			++threads_started;
+		}
+	}
+
+	log("--- test finished ---");
+	exit(0);
+
+	return 0;
+}
+
+
+size_t get_param_by_name(const char *name, int argc, const char *argv[], size_t not_found_value)
+{
+	using namespace Integration_test;
+
+	for (int i=1; i<argc; ++i) {
+
+		if (strcmp(name, argv[i]) == 0) {
+
+			size_t        res { not_found_value };
+			stringstream  str { argv[i+1] };
+			str >> res;
+			return res;
+		}
+	}
+
+	return not_found_value;
+}
+
+
+int main(int argc, char *argv[])
+{
+	using namespace Integration_test;
+
+	Thread_data data {
+	                   get_param_by_name("-wo", argc, const_cast<const char**>(argv), NUMBER_OF_WORKERS),
+	                   get_param_by_name("-pw", argc, const_cast<const char**>(argv), PARALLEL_WORKERS),
+	                   get_param_by_name("-ws", argc, const_cast<const char**>(argv), WRITE_SIZE),
+	                   get_param_by_name("-ds", argc, const_cast<const char**>(argv), IN_DATA_SIZE)
+	                 };
+	log("number of workers  (-wo)  : ", data.max_workers);
+	log("parallel workers   (-pw)  : ", data.parallel_workers);
+	log("write size         (-ws)  : ", data.write_size);
+	log("data size          (-ds)  : ", data.buffer_size);
+	pthread_t   thr;
+
+	pthread_create(&thr, nullptr, test_runner, &data);
+
+	pthread_join(thr, nullptr);
+}

repos/libports/src/test/libc_integration/pipe.h

@@ -0,0 +1,62 @@
+/*
+ * \brief  wrapper for libc pipe.
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef INTEGRATION_TEST_PIPE_H
+#define INTEGRATION_TEST_PIPE_H
+
+/* libc includes */
+#include <fcntl.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+/* local includes */
+#include "stdcxx_log.h"
+
+
+namespace Integration_test
+{
+	class Pipe_creation_failed { };
+	class Pipe;
+}
+
+
+class Integration_test::Pipe
+{
+	private:
+
+		int  _pipe_fds[2] { -1, -1 };
+
+	public:
+
+		Pipe()
+		{
+			if (pipe2(_pipe_fds, O_NONBLOCK) != 0) {
+				throw Pipe_creation_failed { };
+			}
+		}
+
+		~Pipe()
+		{
+			/* close write fd first */
+			close(_pipe_fds[1]);
+			close(_pipe_fds[0]);
+		}
+
+		int read_fd()  const { return _pipe_fds[0]; }
+		int write_fd() const { return _pipe_fds[1]; }
+};
+
+#endif /* INTEGRATION_TEST_PIPE_H */

repos/libports/src/test/libc_integration/stdcxx_log.cc

@@ -0,0 +1,19 @@
+/*
+ * \brief  Wrapper for stdcxx output handling.
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+
+/* local includes */
+#include "stdcxx_log.h"
+
+std::mutex Integration_test::lock { };

repos/libports/src/test/libc_integration/stdcxx_log.h

@@ -0,0 +1,51 @@
+/*
+ * \brief  Wrapper for stdcxx output handling.
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef INTEGRATION_TEST_STDCXX_LOG_H
+#define INTEGRATION_TEST_STDCXX_LOG_H
+
+
+/* stdcxx includes */
+#include <iostream>
+#include <mutex>
+
+
+namespace Integration_test
+{
+	using namespace std;
+
+	extern mutex lock;
+
+	template <typename... ARGS>
+	void log(ARGS &&... args)
+	{
+		lock_guard<mutex> guard { lock };
+
+		((cout << args), ...);
+		cout << endl;
+	}
+
+	template <typename... ARGS>
+	void error(ARGS &&... args)
+	{
+		lock_guard<mutex> guard { lock };
+
+		cerr << "\033[31m";
+		((cerr << args), ...);
+		cerr << "\033[0m";
+		cerr << endl;
+	}
+}
+
+#endif /* INTEGRATION_TEST_STDCXX_LOG_H */

repos/libports/src/test/libc_integration/target.mk

@@ -0,0 +1,11 @@
+TARGET := test-libc_integration
+
+LIBS   += libc
+LIBS   += posix
+LIBS   += stdcxx
+LIBS   += vfs
+LIBS   += vfs_pipe
+
+SRC_CC := main.cc
+SRC_CC += stdcxx_log.cc
+SRC_CC += thread.cc

repos/libports/src/test/libc_integration/thread.cc

@@ -0,0 +1,131 @@
+/*
+ * \brief  worker thread
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+/* libc includes */
+#include <string.h>
+#include <errno.h>
+
+/* stdcxx includes */
+#include <vector>
+
+/* local includes */
+#include "thread.h"
+#include "fd_set.h"
+
+
+/*
+ * Thread-function for the Test_worker.
+ */
+void *worker_func(void *ptr)
+{
+	using namespace Integration_test;
+	using namespace std;
+
+	Work_info       work_info     { *reinterpret_cast<Work_info*>(ptr) };
+
+	int const       max_fd        { work_info.pipe_in_fd };
+	auto const      count         { max(1ul, work_info.num_bytes * 1024 / IN_DATA_SIZE) };
+	size_t          bytes_read    { 0 };
+	size_t          bytes_written { 0 };
+	vector<uint8_t> data_out      { };
+
+	while (bytes_read < IN_DATA_SIZE) {
+
+		/* read data from input pipe */
+		fd_set fds { };
+		FD_ZERO(&fds);
+		FD_SET(work_info.pipe_in_fd, &fds);
+		auto num_ready { select(max_fd+1, &fds, nullptr, nullptr, nullptr) };
+
+		if (num_ready < 0) {
+			error("error: worker ", work_info.worker_no, " select failed");
+			exit(-6);
+		}
+
+		char buf[1024] { };
+		auto r_res     { read(work_info.pipe_in_fd, buf, sizeof(buf)) };
+		if (r_res < 0) {
+			error("error: worker ", work_info.worker_no, " read failed");
+			exit(-7);
+		}
+
+		bytes_read += r_res;
+
+		for (size_t i=0; i<count && data_out.size() < work_info.num_bytes; ++i) {
+			data_out.push_back(buf[random()%r_res]);
+		}
+
+		/* write part of response */
+		size_t  cnt   { min(static_cast<size_t>(work_info.buffer_size), work_info.num_bytes-bytes_written) };
+		ssize_t w_res { write(work_info.pipe_out_fd,
+		                      data_out.data()+bytes_written,
+		                      cnt) };
+
+		if (w_res < 0) {
+			error("error: worker ", work_info.worker_no, " write failed ----", work_info.num_bytes-bytes_written,"--",work_info.pipe_out_fd);
+			exit(-8);
+		}
+
+		bytes_written += w_res;
+
+		/*
+		 * Get out early when the expected num_bytes got written already.
+		 * The receiver will pthread_join this thread and wait for.
+		 */
+		if (bytes_written >= work_info.num_bytes) {
+			return nullptr;
+		}
+
+		if (bytes_written >= data_out.size()) {
+			break;
+		}
+	}
+
+	/* ensure enough data is present */
+	while (data_out.size() < static_cast<size_t>(work_info.num_bytes)) {
+		data_out.push_back(data_out[random()%data_out.size()]);
+	}
+
+	/* simulate output creation requiring some time */
+	usleep(1000ull*random()%300);
+
+	/* write remaining output bytes */
+	while (bytes_written < data_out.size()) {
+
+		size_t  cnt   { min(static_cast<size_t>(work_info.buffer_size), work_info.num_bytes-bytes_written) };
+		ssize_t w_res { write(work_info.pipe_out_fd,
+		                      data_out.data()+bytes_written, cnt) };
+
+		if(w_res < 0) {
+			error("error: worker ", work_info.worker_no, " write failed");
+			exit(-9);
+		}
+
+		bytes_written += w_res;
+	}
+
+	return nullptr;
+}
+
+
+Integration_test::Test_worker::Test_worker(size_t num_bytes, size_t worker_no, size_t buffer_size)
+:
+	_work_info { .num_bytes    = num_bytes,
+	             .worker_no    = worker_no,
+	             .buffer_size  = buffer_size,
+	             .pipe_in_fd   = _pipe_in.read_fd(),
+	             .pipe_out_fd  = _pipe_out.write_fd() }
+{
+	pthread_create(&_thread, nullptr, worker_func, &_work_info);
+}

repos/libports/src/test/libc_integration/thread.h

@@ -0,0 +1,202 @@
+/*
+ * \brief  worker thread
+ * \author Pirmin Duss
+ * \date   2020-11-11
+ */
+
+/*
+ * Copyright (C) 2020 Genode Labs GmbH
+ * Copyright (C) 2020 gapfruit AG
+ *
+ * This file is part of the Genode OS framework, which is distributed
+ * under the terms of the GNU Affero General Public License version 3.
+ */
+
+#ifndef INTEGRATION_TEST_THREAD_H
+#define INTEGRATION_TEST_THREAD_H
+
+
+/* stdcxx includes */
+#include <algorithm>
+#include <list>
+#include <memory>
+#include <random>
+#include <sstream>
+#include <vector>
+
+/* local includes */
+#include "fd_set.h"
+#include "pipe.h"
+#include "stdcxx_log.h"
+#include "definitions.h"
+
+
+namespace Integration_test
+{
+	using namespace std;
+
+	using Worker_handle = pair<size_t, int>;
+
+	struct Work_info
+	{
+		size_t  num_bytes;
+		size_t  worker_no;
+		size_t  buffer_size;
+		int     pipe_in_fd;
+		int     pipe_out_fd;
+	};
+
+	class Thread;
+	class Test_worker;
+	class Thread_list;
+
+	using Work_iter  = std::list<Test_worker>::iterator;
+
+	/*
+	 * abstract away difference in libc pthread_t definition
+	 * between Genode (BSD libc) and Linux.
+	 */
+	#if defined (__GENODE__)
+		#define THREAD_INIT  nullptr
+	#else
+		#define THREAD_INIT  0
+	#endif
+}
+
+
+class Integration_test::Test_worker
+{
+	private:
+
+		Pipe             _pipe_in        { };
+		Pipe             _pipe_out       { };
+		Work_info        _work_info;
+		pthread_t        _thread         { THREAD_INIT };
+		vector<uint8_t>  _result_data    { };
+
+		void _print_data() const
+		{
+			ostringstream str { };
+
+			str << std::hex << (int)_result_data[0] << " " <<
+			                   (int)_result_data[1] << " " <<
+			                   (int)_result_data[2] << " " <<
+			       std::dec << " ... (" << _result_data.size()-6 << " bytes) ... " <<
+			       std::hex << (int)_result_data[_result_data.size()-3] << " " <<
+			                   (int)_result_data[_result_data.size()-2] << " " <<
+			                   (int)_result_data[_result_data.size()-1];
+
+			log("Worker ", worker_no(), " data : ", str.str());
+		}
+
+		Test_worker(const Integration_test::Test_worker&) = delete;
+		Test_worker &operator=(const Integration_test::Test_worker&) = delete;
+
+	public:
+
+		Test_worker(size_t num_bytes, size_t worker_no, size_t buffer_size);
+
+		~Test_worker()
+		{
+			join();
+			_print_data();
+		}
+
+		void    join()            const { pthread_join(_thread, nullptr); }
+		size_t  worker_no()       const { return _work_info.worker_no; }
+		ssize_t number_of_bytes() const { return _work_info.num_bytes; }
+		int     read_fd()         const { return _pipe_out.read_fd(); }
+		int     write_fd()        const { return _pipe_in.write_fd(); }
+
+		bool    done()            const { return _result_data.size() >=
+		                                         _work_info.num_bytes; }
+
+		size_t missing()          const { return _work_info.num_bytes - _result_data.size(); }
+
+		void append_result_data(uint8_t const *buf, size_t count)
+		{
+			_result_data.insert(_result_data.end(), buf, buf+count);
+		}
+};
+
+
+class Integration_test::Thread_list
+{
+	private:
+
+		size_t                           _buffer_size { };
+		list<Test_worker>                _threads     { };
+		File_descriptor_set              _fd_set      { };
+		random_device                    _rd          { };
+		mt19937                          _gen         { _rd() };
+		uniform_int_distribution<size_t> _distrib     { 1, BUFFER_SIZE };
+
+		Work_iter _find_worker_by_worker_no(size_t worker_no)
+		{
+			return find_if(_threads.begin(),
+			               _threads.end(),
+			               [worker_no] (Test_worker const &e) {
+			                  return worker_no == e.worker_no(); });
+		}
+
+	public:
+
+		Thread_list(size_t buffer_size)
+		:
+			_buffer_size { buffer_size }
+		{ }
+
+		fd_set fds()           { return _fd_set.fds(); }
+		int     max_fd() const { return _fd_set.max_fd(); }
+		size_t  count()  const { return _threads.size(); }
+
+		Work_iter end()
+		{
+			return _threads.end();
+		}
+
+		Work_iter find_worker_by_fd(int fd)
+		{
+			return find_if(_threads.begin(),
+			               _threads.end(),
+			               [fd] (Test_worker const &e) {
+			                  return fd == e.read_fd(); });
+		}
+
+		File_descriptor_set &descriptor_set()
+		{
+			return _fd_set;
+		}
+
+		Worker_handle add_worker()
+		{
+
+			static size_t worker_no { 0 };
+			size_t const num_bytes { _distrib(_gen) };
+
+			_threads.emplace_back(num_bytes, worker_no, _buffer_size);
+
+			/* manage fd_set for receiver thread */
+			_fd_set.add_fd(_threads.back().read_fd());
+
+			return { worker_no++, _threads.back().write_fd() };
+		}
+
+		void remove_finished_workers(vector<size_t> &workers)
+		{
+			for (auto no : workers) {
+				auto it { _find_worker_by_worker_no(no) };
+				if (it == _threads.end()) {
+					continue;
+				}
+
+				/* manage fd_set for receiver thread */
+				_fd_set.remove_fd(it->read_fd());
+
+				_threads.erase(it);
+			}
+		}
+};
+
+
+#endif /* INTEGRATION_TEST_THREAD_H */

tool/autopilot.list

@@ -18,6 +18,7 @@ gdb_monitor
 ieee754
 init_smp
 event_filter
+libc_integration
 libc_vfs_fs_ext2
 log_core
 lwip