block_session: server-defined payload alignment

This patch replaces the formerly fixed 2 KiB data alignment within the
packet-stream buffer by a server-defined alignment. This has two
benefits.

First, when using block servers that provide small block sizes like 512
bytes, we avoid fragmenting the packet-stream buffer, which occurs when
aligning 512-byte requests at 2 KiB boundaries. This reduces meta data
costs for the packet-stream allocator and also allows fitting more
requests into the buffer.

Second, block drivers with alignment constraints dictated by the
hardware can now pass those constraints to the client, thereby easing
the use of zero-copy DMA directly into the packet stream.

The alignment is determined by the Block::Session_client at construction
time and applied by the Block::Session_client::alloc_packet method.
Block-session clients should always use this method, not the 'alloc_packet'
method of the packet stream (tx source) directly. The latter merely
applies a default alignment of 2 KiB.

At the server side, the alignment is automatically checked by
block/component.h (old API) and block/request_stream.h (new API).

Issue #3274

repos/dde_linux/src/lib/usb/storage/storage.cc

@@ -148,6 +148,7 @@ class Storage_device : public Genode::List<Storage_device>::Element,
 		{
 			return { .block_size  = _block_size,
 			         .block_count = _block_count,
+			         .align_log2  = Genode::log2(_block_size),
 			         .writeable   = true };
 		}
 

repos/dde_rump/src/lib/rump/io.cc

@@ -57,7 +57,7 @@ class Backend
 			                                   Packet_descriptor::READ;
 			/* allocate packet */
 			try {
-				Packet_descriptor packet( _session.dma_alloc_packet(length),
+				Packet_descriptor packet( _session.alloc_packet(length),
 				                         opcode, offset / _info.block_size,
 				                         length / _info.block_size);
 

repos/gems/src/server/http_block/main.cc

@@ -46,6 +46,7 @@ class Driver : public Block::Driver
 		{
 			return { .block_size  = _block_size,
 			         .block_count = _http.file_size() / _block_size,
+			         .align_log2  = log2(_block_size),
 			         .writeable   = false };
 		}
 

repos/libports/src/lib/fatfs/diskio_block.cc

@@ -62,6 +62,7 @@ extern "C" {
 
 		using Block::Connection::tx;
 		using Block::Connection::sync;
+		using Block::Connection::alloc_packet;
 
 		Drive(Platform &platform, char const *label)
 		: Block::Connection(platform.env, &platform.tx_alloc, 128*1024, label)
@@ -125,7 +126,7 @@ extern "C" DRESULT disk_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count)
 	Genode::size_t const op_len = drive.info.block_size*count;
 
 	/* allocate packet-descriptor for reading */
-	Block::Packet_descriptor p(drive.tx()->alloc_packet(op_len),
+	Block::Packet_descriptor p(drive.alloc_packet(op_len),
 	                           Block::Packet_descriptor::READ, sector, count);
 	drive.tx()->submit_packet(p);
 	p = drive.tx()->get_acked_packet();
@@ -155,7 +156,7 @@ extern "C" DRESULT disk_write (BYTE pdrv, const BYTE* buff, DWORD sector, UINT c
 	Genode::size_t const op_len = drive.info.block_size*count;
 
 	/* allocate packet-descriptor for writing */
-	Block::Packet_descriptor p(drive.tx()->alloc_packet(op_len),
+	Block::Packet_descriptor p(drive.alloc_packet(op_len),
 	                           Block::Packet_descriptor::WRITE, sector, count);
 
 	Genode::memcpy(drive.tx()->packet_content(p), buff, op_len);

repos/os/include/block/component.h

@@ -108,7 +108,8 @@ class Block::Session_component : public Block::Session_component_base,
 
 			/* ignore invalid packets */
 			bool const valid = packet.size() && _range_check(_p_to_handle)
-			                && tx_sink()->packet_valid(packet);
+			                && tx_sink()->packet_valid(packet)
+			                && aligned(packet.offset(), _info.align_log2);
 			if (!valid) {
 				_ack_packet(_p_to_handle);
 				return;

repos/os/include/block/request_stream.h

@@ -26,6 +26,9 @@ class Block::Request_stream : Genode::Noncopyable
 {
 	public:
 
+		struct Block_size { Genode::uint32_t value; };
+		struct Align_log2 { Genode::size_t   value; };
+
 		/**
 		 * Interface for accessing the content of a 'Request'
 		 *
@@ -59,7 +62,7 @@ class Block::Request_stream : Genode::Noncopyable
 					return request.count * _info.block_size;
 				}
 
-				bool _valid_range(Block::Request const &request) const
+				bool _valid_range_and_alignment(Block::Request const &request) const
 				{
 					/* local address of the last byte of the request */
 					Genode::addr_t const request_end = _base + request.offset
@@ -73,6 +76,10 @@ class Block::Request_stream : Genode::Noncopyable
 					if (request_end > _base + _size - 1)
 						return false;
 
+					/* check for proper alignment */
+					if (!Genode::aligned(request.offset, _info.align_log2))
+						return false;
+
 					return true;
 				}
 
@@ -94,7 +101,7 @@ class Block::Request_stream : Genode::Noncopyable
 				template <typename FN>
 				void with_content(Block::Request request, FN const &fn) const
 				{
-					if (_valid_range(request))
+					if (_valid_range_and_alignment(request))
 						fn(_request_ptr(request), _request_size(request));
 				}
 		};

repos/os/include/block_session/block_session.h

@@ -31,7 +31,7 @@ namespace Block {
 
 
 /**
- * Represents an block-operation request
+ * Representation of an block-operation request
  *
  * The data associated with the 'Packet_descriptor' is either
  * the data read from or written to the block indicated by
@@ -42,6 +42,14 @@ class Block::Packet_descriptor : public Genode::Packet_descriptor
 	public:
 
 		enum Opcode    { READ, WRITE, END };
+
+		/*
+		 * Alignment used when allocating a packet directly via the 'tx'
+		 * packet stream. This is not recommended because it does not
+		 * apply the server's alignment constraints. Instead, the
+		 * 'Block::Session_client::alloc_packet' should be used for
+		 * allocating properly aligned block-request packets.
+		 */
 		enum Alignment { PACKET_ALIGNMENT = 11 };
 
 	private:
@@ -82,7 +90,7 @@ class Block::Packet_descriptor : public Genode::Packet_descriptor
 };
 
 
-/*
+/**
  * Block session interface
  *
  * A block session corresponds to a block device that can be used to read
@@ -109,6 +117,7 @@ struct Block::Session : public Genode::Session
 	{
 		Genode::size_t block_size;   /* size of one block in bytes */
 		sector_t       block_count;  /* number of blocks */
+		Genode::size_t align_log2;   /* packet alignment within payload buffer */
 		bool           writeable;
 	};
 

repos/os/include/block_session/client.h

@@ -27,6 +27,8 @@ class Block::Session_client : public Genode::Rpc_client<Session>
 
 		Packet_stream_tx::Client<Tx> _tx;
 
+		Info const _info = info();
+
 	public:
 
 		/**
@@ -59,12 +61,12 @@ class Block::Session_client : public Genode::Rpc_client<Session>
 
 		Genode::Capability<Tx> tx_cap() override { return call<Rpc_tx_cap>(); }
 
-		/*
+		/**
-		 * Wrapper for alloc_packet, allocates 2KB aligned packets
+		 * Allocate packet respecting the server's alignment constraints
 		 */
-		Packet_descriptor dma_alloc_packet(Genode::size_t size)
+		Packet_descriptor alloc_packet(Genode::size_t size)
 		{
-			return tx()->alloc_packet(size, 11);
+			return tx()->alloc_packet(size, _info.align_log2);
 		}
 };
 

repos/os/src/app/block_tester/test_ping_pong.h

@@ -63,7 +63,7 @@ struct Test::Ping_pong : Test_base
 			while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit()) {
 
 				Block::Packet_descriptor tmp =
-					_block->tx()->alloc_packet(_size_in_blocks * _info.block_size);
+					_block->alloc_packet(_size_in_blocks * _info.block_size);
 
 				Block::sector_t const lba = _ping ? _start + _blocks
 				                                  : _end   - _blocks;

repos/os/src/app/block_tester/test_random.h

@@ -127,8 +127,7 @@ struct Test::Random : Test_base
 			bool next = true;
 			while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit() && next) {
 
-				Block::Packet_descriptor tmp =
+				Block::Packet_descriptor tmp = _block->alloc_packet(_size);
-					_block->tx()->alloc_packet(_size);
 
 				Block::sector_t lba = _next_block();
 

repos/os/src/app/block_tester/test_replay.h

@@ -69,7 +69,7 @@ struct Test::Replay : Test_base
 				more = false;
 				requests.dequeue([&] (Request &req) {
 					Block::Packet_descriptor p(
-						_block->tx()->alloc_packet(req.count * _info.block_size),
+						_block->alloc_packet(req.count * _info.block_size),
 						req.op, req.nr, req.count);
 
 					bool const write = req.op == Block::Packet_descriptor::WRITE;

repos/os/src/app/block_tester/test_sequential.h

@@ -66,8 +66,7 @@ struct Test::Sequential : Test_base
 			bool next = true;
 			while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit() && next) {
 
-				Block::Packet_descriptor tmp =
+				Block::Packet_descriptor tmp = _block->alloc_packet(_size);
-					_block->tx()->alloc_packet(_size);
 
 				Block::Packet_descriptor p(tmp,
 					_op, _start, _size_in_blocks);

repos/os/src/drivers/ahci/ata_driver.h

@@ -377,6 +377,7 @@ struct Ata_driver : Port_driver
 	{
 		return { .block_size  = block_size(),
 		         .block_count = block_count(),
+		         .align_log2  = log2(block_size()),
 		         .writeable   = true };
 	}
 

repos/os/src/drivers/ahci/atapi_driver.h

@@ -162,6 +162,7 @@ struct Atapi_driver : Port_driver
 	{
 		return { .block_size  = block_size(),
 		         .block_count = block_count(),
+		         .align_log2  = 11,
 		         .writeable   = false };
 	}
 

repos/os/src/drivers/nvme/main.cc

@@ -1541,6 +1541,7 @@ class Driver : public Block::Driver
 
 			_info = { .block_size  = nsinfo.size,
 			          .block_count = nsinfo.count,
+			          .align_log2  = Genode::log2(nsinfo.size),
 			          .writeable   = true };
 
 			Nvme::Controller::Info const &info = _nvme_ctrlr->info();

repos/os/src/drivers/sd_card/driver_base.h

@@ -48,6 +48,7 @@ class Sd_card::Driver_base : public    Block::Driver,
 		{
 			return { .block_size  = _block_size(),
 			         .block_count = _block_count(),
+			         .align_log2  = Genode::log2(_block_size()),
 			         .writeable   = true };
 		}
 };

repos/os/src/drivers/sd_card/spec/pbxa9/driver.h

@@ -129,6 +129,7 @@ class Sd_card::Driver : public  Block::Driver, private Attached_mmio
 		{
 			return { .block_size  = _block_size,
 			         .block_count = _block_count,
+			         .align_log2  = log2(_block_size),
 			         .writeable   = true };
 		}
 

repos/os/src/drivers/usb_block/main.cc

@@ -808,6 +808,7 @@ struct Usb::Block_driver : Usb::Completion,
 	{
 		return { .block_size  = _block_size,
 		         .block_count = _block_count,
+		         .align_log2  = Genode::log2(_block_size),
 		         .writeable   = _writeable };
 	}
 

repos/os/src/lib/vfs/block_file_system.h

@@ -106,7 +106,7 @@ class Vfs::Block_file_system : public Single_file_system
 						try {
 							Lock::Guard guard(_lock);
 
-							packet = _tx_source->alloc_packet(packet_size);
+							packet = _block.alloc_packet(packet_size);
 							break;
 						} catch (Block::Session::Tx::Source::Packet_alloc_failed) {
 							if (!_tx_source->ready_to_submit())

repos/os/src/server/block_cache/driver.h

@@ -244,7 +244,7 @@ class Driver : public Block::Driver
 
 				/* construct and send the packet */
 				p_to_dev =
-					Block::Packet_descriptor(_blk.dma_alloc_packet(_info.block_size*cnt),
+					Block::Packet_descriptor(_blk.alloc_packet(_info.block_size*cnt),
 					                         Block::Packet_descriptor::READ,
 					                         nr, cnt);
 				_r_list.insert(new (&_r_slab) Request(p_to_dev, packet, buffer));

repos/os/src/server/block_cache/main.cc

@@ -35,7 +35,7 @@ void Driver<POLICY>::Policy::sync(const typename POLICY::Element *e, char *dst)
 		throw Write_failed(off);
 	try {
 		Block::Packet_descriptor
-			p(driver->blk()->dma_alloc_packet(Driver::CACHE_BLK_SIZE),
+			p(driver->blk()->alloc_packet(Driver::CACHE_BLK_SIZE),
 		      Block::Packet_descriptor::WRITE, off / driver->blk_sz(),
 		      Driver::CACHE_BLK_SIZE / driver->blk_sz());
 		driver->blk()->tx()->submit_packet(p);

repos/os/src/server/iso9660/iso9660.cc

@@ -56,7 +56,7 @@ class Iso::Sector {
 		{
 			try {
 				_p = Block::Packet_descriptor(
-					block.dma_alloc_packet(blk_size() * count),
+					block.alloc_packet(blk_size() * count),
 					Block::Packet_descriptor::READ,
 					blk_nr * ((float)blk_size() / BLOCK_SIZE),
 					count * ((float)blk_size() / BLOCK_SIZE));

repos/os/src/server/lx_block/main.cc

@@ -73,6 +73,7 @@ class Lx_block_driver : public Block::Driver
 			return {
 				.block_size  = block_size,
 				.block_count = st.st_size / block_size,
+				.align_log2  = Genode::log2(block_size),
 				.writeable   = xml_attr_ok(config, "writeable")
 			};
 		}

repos/os/src/server/part_block/component.h

@@ -220,6 +220,7 @@ class Block::Session_component : public  Block::Session_rpc_object,
 		{
 			return Info { .block_size  = _driver.blk_size(),
 			              .block_count = _partition->sectors,
+			              .align_log2  = Genode::log2(_driver.blk_size()),
 			              .writeable   = _writeable && _driver.writeable() };
 		}
 

repos/os/src/server/part_block/driver.h

@@ -139,7 +139,7 @@ class Block::Driver
 			    ? Block::Packet_descriptor::WRITE
 			    : Block::Packet_descriptor::READ;
 			Genode::size_t const size = _info.block_size * cnt;
-			Packet_descriptor p(_session.dma_alloc_packet(size),
+			Packet_descriptor p(_session.alloc_packet(size),
 			                    op,  nr, cnt);
 			Request *r = new (&_r_slab) Request(dispatcher, cli, p);
 			_r_list.insert(r);

repos/os/src/server/part_block/partition_table.h

@@ -54,7 +54,7 @@ struct Block::Partition_table : Genode::Interface
 				       unsigned long count,
 				       bool          write = false)
 				: _session(driver.session()),
-				  _p(_session.dma_alloc_packet(driver.blk_size() * count),
+				  _p(_session.alloc_packet(driver.blk_size() * count),
 				     write ? Packet_descriptor::WRITE : Packet_descriptor::READ,
 				     blk_nr, count)
 				{

repos/os/src/server/ram_block/main.cc

@@ -116,6 +116,7 @@ class Ram_block : public Block::Driver
 		{
 			return { .block_size  = _block_size,
 			         .block_count = _block_count,
+			         .align_log2  = log2(_block_size),
 			         .writeable   = true };
 		}
 

repos/os/src/server/rom_block/main.cc

@@ -48,6 +48,7 @@ class Rom_block : public Block::Driver
 		{
 			return { .block_size  = _blk_sz,
 			         .block_count = _blk_cnt,
+			         .align_log2  = log2(_blk_sz),
 			         .writeable   = false };
 		}
 

repos/os/src/server/tz_vmm/block_driver.cc

@@ -175,7 +175,7 @@ void Block_driver::_new_request(Vm_base &vm)
 			size_t  const size = vm.smc_arg_3();
 			void   *const req  = (void*)vm.smc_arg_4();
 
-			Packet_descriptor pkt  = dev.session().tx()->alloc_packet(size);
+			Packet_descriptor pkt  = dev.session().alloc_packet(size);
 			void             *addr = dev.session().tx()->packet_content(pkt);
 			dev.cache().insert(addr, req);
 			vm.smc_ret((long)addr, pkt.offset());

repos/os/src/test/block/bench/main.cc

@@ -67,7 +67,7 @@ class Throughput
 			try {
 				while (_session.tx()->ready_to_submit()) {
 					Block::Packet_descriptor p(
-						_session.tx()->alloc_packet(REQUEST_SIZE),
+						_session.alloc_packet(REQUEST_SIZE),
 						!_read_done ? Block::Packet_descriptor::READ : Block::Packet_descriptor::WRITE,
 						_current, count);
 

repos/os/src/test/block/client/main.cc

@@ -140,7 +140,7 @@ struct Read_test : Test
 
 			try {
 				Block::Packet_descriptor p(
-					_session.dma_alloc_packet(cnt*blk_sz),
+					_session.alloc_packet(cnt*blk_sz),
 					Block::Packet_descriptor::READ, nr, cnt);
 				_session.tx()->submit_packet(p);
 			} catch(Block::Session::Tx::Source::Packet_alloc_failed) {
@@ -241,8 +241,7 @@ struct Write_test : Test
 	{
 		while (!read_packets.empty()) {
 			Block::Packet_descriptor r = read_packets.get();
-			Block::Packet_descriptor w(_session.dma_alloc_packet(r.block_count()
+			Block::Packet_descriptor w(_session.alloc_packet(r.block_count()*blk_sz),
-			                                                     *blk_sz),
 			                    Block::Packet_descriptor::WRITE,
 			                    r.block_number(), r.block_count());
 			signed char *dst = (signed char*)_session.tx()->packet_content(w),
@@ -263,7 +262,7 @@ struct Write_test : Test
 		for (sector_t nr = start, cnt = Genode::min(NR_PER_REQ, end - start); nr < end;
 		     nr += cnt,
 		     cnt = Genode::min<sector_t>(NR_PER_REQ, end-nr)) {
-			Block::Packet_descriptor p(_session.dma_alloc_packet(cnt*blk_sz),
+			Block::Packet_descriptor p(_session.alloc_packet(cnt*blk_sz),
 			                           Block::Packet_descriptor::READ, nr, cnt);
 			_session.tx()->submit_packet(p);
 		}
@@ -338,7 +337,7 @@ struct Violation_test : Test
 
 	void req(Block::sector_t nr, Genode::size_t cnt, bool write)
 	{
-		Block::Packet_descriptor p(_session.dma_alloc_packet(blk_sz),
+		Block::Packet_descriptor p(_session.alloc_packet(blk_sz),
 		                           write ? Block::Packet_descriptor::WRITE
 		                                 : Block::Packet_descriptor::READ,
 		                           nr, cnt);

repos/os/src/test/block/server/main.cc

@@ -67,6 +67,7 @@ class Driver : public Block::Driver
 		{
 			return { .block_size  = _size,
 			         .block_count = _number,
+			         .align_log2  = Genode::log2(_size),
 			         .writeable   = true };
 		}
 

repos/os/src/test/block_request_stream/main.cc

@@ -44,6 +44,7 @@ struct Test::Block_session_component : Rpc_object<Block::Session>,
 		Request_stream(rm, ds, ep, sigh,
 		               Info { .block_size  = BLOCK_SIZE,
 		                      .block_count = NUM_BLOCKS,
+		                      .align_log2  = log2(BLOCK_SIZE),
 		                      .writeable   = true }),
 		_ep(ep)
 	{

repos/os/src/test/rom_block/main.cc

@@ -56,7 +56,7 @@ struct Main
 			size_t const cnt = (info.block_count - i > REQ_PARALLEL)
 			                 ? REQ_PARALLEL : info.block_count - i;
 
-			Packet_descriptor pkt(src.alloc_packet(cnt * info.block_size),
+			Packet_descriptor pkt(block.alloc_packet(cnt * info.block_size),
 			                      Packet_descriptor::READ, i, cnt);
 
 			log("Check blocks ", i, "..", i + cnt - 1);

repos/ports/src/app/seoul/disk.cc

@@ -260,7 +260,7 @@ bool Seoul::Disk::restart(struct disk_session const &disk,
 		Genode::Lock::Guard lock_guard(_alloc_lock);
 
 		packet = Block::Packet_descriptor(
-			source->alloc_packet(blocks * blk_size),
+			disk.blk_con->alloc_packet(blocks * blk_size),
 			(write) ? Block::Packet_descriptor::WRITE
 			        : Block::Packet_descriptor::READ,
 			msg->sector, blocks);
@@ -317,7 +317,7 @@ bool Seoul::Disk::execute(bool const write, struct disk_session const &disk,
 		Genode::Lock::Guard lock_guard(_alloc_lock);
 
 		packet = Block::Packet_descriptor(
-			source->alloc_packet(blocks * blk_size),
+			disk.blk_con->alloc_packet(blocks * blk_size),
 			(write) ? Block::Packet_descriptor::WRITE
 			        : Block::Packet_descriptor::READ,
 			sector, blocks);