genode/repos/os/src/drivers/nvme/main.cc

/*
 * \brief  NVMe Block session component
 * \author Josef Soentgen
 * \date   2018-03-05
 *
 * Spec used: NVM-Express-1_3a-20171024_ratified.pdf
 */

/*
 * Copyright (C) 2018 Genode Labs GmbH
 *
 * This file is part of the Genode OS framework, which is distributed
 * under the terms of the GNU Affero General Public License version 3.
 */

/* Genode includes */
#include <base/allocator_avl.h>
#include <base/attached_rom_dataspace.h>
#include <base/component.h>
#include <base/heap.h>
#include <base/log.h>
#include <block/component.h>
#include <dataspace/client.h>
#include <os/attached_mmio.h>
#include <os/reporter.h>
#include <timer_session/connection.h>
#include <util/bit_array.h>
#include <util/interface.h>
#include <util/misc_math.h>

/* local includes */
#include <util.h>
#include <pci.h>


namespace {

using uint16_t          = Genode::uint16_t;
using uint32_t          = Genode::uint32_t;
using uint64_t          = Genode::uint64_t;
using size_t            = Genode::size_t;
using addr_t            = Genode::addr_t;
using Packet_descriptor = Block::Packet_descriptor;

} /* anonymous namespace */


/**********
 ** NVMe **
 **********/

namespace Nvme {
	using namespace Genode;

	struct Identify_data;
	struct Identify_ns_data;
	struct Doorbell;

	struct Cqe;

	struct Sqe;
	struct Sqe_create_cq;
	struct Sqe_create_sq;
	struct Sqe_identify;
	struct Sqe_io;

	struct Queue;
	struct Sq;
	struct Cq;

	struct Controller;

	enum {
		CQE_LEN                = 16,
		SQE_LEN                = 64,
		MAX_IO_QUEUES          = 1,
		MAX_IO_ENTRIES         = 128,
		MAX_IO_ENTRIES_MASK    = MAX_IO_ENTRIES - 1,
		MAX_IO_PENDING         = MAX_IO_ENTRIES - 1, /* tail + 1 == head -> full */
		MAX_ADMIN_ENTRIES      = 128,
		MAX_ADMIN_ENTRIES_MASK = MAX_ADMIN_ENTRIES - 1,
	};

	enum {
		/*
		 * Limit max I/O requests size; we can map up to 2MiB with one list
		 * page (4K/8 = 512 * 4K) but 1MiB is plenty
		 */
		MAX_IO_LEN       =   1u << 20,
		DMA_DS_SIZE      =   4u << 20,
		DMA_LIST_DS_SIZE = 256u << 10,
		MPS              = 4096u,
	};

	enum {
		IO_NSID    = 1u,
		MAX_NS     = 1u,
		NUM_QUEUES = 1 + MAX_NS,
	};

	enum Opcode {
		/* Admin command set */
		DELETE_IO_SQ = 0x00,
		CREATE_IO_SQ = 0x01,
		DELETE_IO_CQ = 0x04,
		CREATE_IO_CQ = 0x05,
		IDENTIFY     = 0x06,
		SET_FEATURES = 0x09,
		GET_FEATURES = 0x0A,
		/* NVM command set */
		FLUSH        = 0x00,
		WRITE        = 0x01,
		READ         = 0x02,
	};
};


/*
 * Identify command data
 */
struct Nvme::Identify_data : Genode::Mmio
{
	enum {
		SN_OFFSET = 0x04, SN_LEN = 20,
		MN_OFFSET = 0x18, MN_LEN = 40,
		FR_OFFSET = 0x40, FR_LEN = 12,
	};

	using Sn = Genode::String<SN_LEN + 1>;
	using Mn = Genode::String<MN_LEN + 1>;
	using Fr = Genode::String<FR_LEN + 1>;

	Sn sn { }; /* serial number */
	Mn mn { }; /* model number */
	Fr fr { }; /* firmware revision */

	struct Vid   : Register<0x000, 16> { }; /* vendor id */
	struct Ssvid : Register<0x002, 16> { }; /* sub system vendor id */
	/* optional admin command support */
	struct Oacs  : Register<0x100, 32>
	{
		struct Ssr  : Bitfield< 0, 1> { }; /* security send/receive */
		struct Nvmf : Bitfield< 1, 1> { }; /* NVM format */
		struct Fwcd : Bitfield< 2, 1> { }; /* firmware commit/download image */
		struct Nsm  : Bitfield< 3, 1> { }; /* namespace management */
		struct Vm   : Bitfield< 7, 1> { }; /* virtualization management */
	};
	struct Nn    : Register<0x204, 32> { }; /* number of namespaces */
	struct Vwc   : Register<0x204,  8> { }; /* volatile write cache */

	Identify_data(addr_t const base)
	: Genode::Mmio(base)
	{
		char const *p = (char const*)base;

		sn = Sn(Util::extract_string(p, SN_OFFSET, SN_LEN+1));
		mn = Mn(Util::extract_string(p, MN_OFFSET, MN_LEN+1));
		fr = Fr(Util::extract_string(p, FR_OFFSET, FR_LEN+1));
	}
};


/*
 * Identify name space command data
 */
struct Nvme::Identify_ns_data : public Genode::Mmio
{
	struct Nsze   : Register<0x00, 64> { }; /* name space size */
	struct Ncap   : Register<0x08, 64> { }; /* name space capacity */
	struct Nuse   : Register<0x10, 64> { }; /* name space utilization */
	struct Nsfeat : Register<0x18,  8> { }; /* name space features */
	struct Nlbaf  : Register<0x19,  8> { }; /* number of LBA formats */
	/* formatted LBA size */
	struct Flbas  : Register<0x1a,  8>
	{
		struct Formats : Bitfield< 0,  3> { };
	};
	struct Mc     : Register<0x1b,  8> { }; /* metadata capabilities */
	struct Dpc    : Register<0x1c,  8> { }; /* end-to-end data protection capabilities */
	struct Dps    : Register<0x1d,  8> { }; /* end-to-end data protection settings */

	enum { MAX_LBAF = 16, };
	/* LBA format support */
	struct Lbaf   : Register_array<0x80, 32, MAX_LBAF, 32>
	{
		struct Ms    : Bitfield< 0, 16> { }; /* metadata size */
		struct Lbads : Bitfield<16,  8> { }; /* LBA data size (2^n) */
		struct Rp    : Bitfield<24,  2> { }; /* relative performance */
	};

	Identify_ns_data(addr_t const base)
	: Genode::Mmio(base)
	{ }
};


/*
 * Queue doorbell register
 */
struct Nvme::Doorbell : public Genode::Mmio
{
	struct Sqtdbl : Register<0x00, 32>
	{
		struct Sqt : Bitfield< 0, 16> { }; /* submission queue tail */
	};

	struct Cqhdbl : Register<0x04, 32>
	{
		struct Cqh : Bitfield< 0, 16> { }; /* submission queue tail */
	};

	Doorbell(addr_t const base)
	: Genode::Mmio(base) { }
};


/*
 * Completion queue entry
 */
struct Nvme::Cqe : Genode::Mmio
{
	struct Dw0  : Register<0x00, 32> { }; /* command specific */
	struct Dw1  : Register<0x04, 32> { }; /* reserved */

	struct Sqhd : Register<0x08, 16> { };
	struct Sqid : Register<0x0a, 16> { };
	struct Cid  : Register<0x0c, 16> { };
	struct Sf   : Register<0x0e, 16>
	{
		struct P   : Bitfield< 0, 1> { };
		struct Sc  : Bitfield< 1, 8> { }; /* status code */
		struct Sct : Bitfield< 9, 3> { }; /* status code type */
		struct M   : Bitfield<14, 1> { }; /* more (get log) */
		struct Dnr : Bitfield<15, 1> { }; /* do not retry */
	};

	Cqe(addr_t const base) : Genode::Mmio(base) { }

	static uint32_t request_id(Nvme::Cqe const &b)
	{
		return (b.read<Sqid>() << 16)|b.read<Cid>();
	}

	static bool succeeded(Nvme::Cqe const &b)
	{
		return !b.read<Sf::Sc>();
	}

	static void dump(Nvme::Cqe const &b)
	{
		using namespace Genode;
		log("sqhd:",        b.read<Sqhd>(),     " "
		    "sqid:",        b.read<Sqid>(),     " "
		    "cid:",         b.read<Cid>(),      " "
		    "p:",           b.read<Sf::P>(),    " "
		    "status: ", Hex(b.read<Sf>()),      " "
		    "sc:",      Hex(b.read<Sf::Sc>()),  " "
		    "sct:",     Hex(b.read<Sf::Sct>()), " "
		    "m:",           b.read<Sf::M>(),    " "
		    "dnr:",         b.read<Sf::Dnr>());
	}
};


/*
 * Submission queue entry base
 */
struct Nvme::Sqe : Genode::Mmio
{
	struct Cdw0 : Register<0x00, 32>
	{
		struct Opc  : Bitfield< 0,  8> { }; /* opcode */
		struct Fuse : Bitfield< 9,  2> { }; /* fused operation */
		struct Psdt : Bitfield<14,  2> { }; /* PRP or SGL for data transfer */
		struct Cid  : Bitfield<16, 16> { }; /* command identifier */
	};
	struct Nsid : Register<0x04, 32> { };
	struct Mptr : Register<0x10, 64> { };
	struct Prp1 : Register<0x18, 64> { };
	struct Prp2 : Register<0x20, 64> { };

	/* SGL not supported */

	Sqe(addr_t const base) : Genode::Mmio(base) { }

	bool valid() const { return base() != 0ul; }
};


/*
 * Identify command
 */
struct Nvme::Sqe_identify : Nvme::Sqe
{
	struct Cdw10 : Register<0x28, 32>
	{
		struct Cns : Bitfield< 0, 8> { }; /* controller or namespace structure */
	};

	Sqe_identify(addr_t const base) : Sqe(base) { }
};


/*
 *  Create completion queue command
 */
struct Nvme::Sqe_create_cq : Nvme::Sqe
{
	struct Cdw10 : Register<0x28, 32>
	{
		struct Qid   : Bitfield< 0, 16> { }; /* queue identifier */
		struct Qsize : Bitfield<16, 16> { }; /* queue size 0-based vale */
	};

	struct Cdw11 : Register<0x2c, 32>
	{
		struct Pc : Bitfield< 0,  1> { }; /* physically contiguous */
		struct En : Bitfield< 1,  1> { }; /* interrupts enabled */
		struct Iv : Bitfield<16, 16> { }; /* interrupt vector */
	};

	Sqe_create_cq(addr_t const base) : Sqe(base) { }
};


/*
 * Create submission queue command
 */
struct Nvme::Sqe_create_sq : Nvme::Sqe
{
	struct Cdw10 : Register<0x28, 32>
	{
		struct Qid   : Bitfield< 0, 16> { }; /* queue identifier */
		struct Qsize : Bitfield<16, 16> { }; /* queue size 0-based vale */
	};

	struct Cdw11 : Register<0x2c, 32>
	{
		struct Pc    : Bitfield< 0,  1> { }; /* physically contiguous */
		struct Qprio : Bitfield< 1,  2> { }; /* queue priority */
		struct Cqid  : Bitfield<16, 16> { }; /* completion queue identifier */
	};

	Sqe_create_sq(addr_t const base) : Sqe(base) { }
};


/*
 * I/O command
 */
struct Nvme::Sqe_io : Nvme::Sqe
{
	struct Slba_lower : Register<0x28, 32> { };
	struct Slba_upper : Register<0x2c, 32> { };
	struct Slba       : Genode::Bitset_2<Slba_lower, Slba_upper> { };

	struct Cdw12 : Register<0x30, 32>
	{
		struct Nlb : Bitfield<0, 16> { };
	};

	Sqe_io(addr_t const base) : Sqe(base) { }
};


/*
 * Queue base structure
 */
struct Nvme::Queue
{
	Genode::Ram_dataspace_capability ds { };
	addr_t    pa { 0 };
	addr_t    va { 0 };
	uint32_t  max_entries { 0 };

	bool valid() const { return pa != 0ul; }
};


/*
 * Submission queue
 */
struct Nvme::Sq : Nvme::Queue
{
	uint32_t tail { 0 };
	uint16_t id   { 0 };

	addr_t next()
	{
		addr_t a = va + (tail * SQE_LEN);
		Genode::memset((void*)a, 0, SQE_LEN);
		tail = (tail + 1) % max_entries;
		return a;
	}
};


/*
 * Completion queue
 */
struct Nvme::Cq : Nvme::Queue
{
	uint32_t head  { 0 };
	uint32_t phase { 1 };

	addr_t next() { return va + (head * CQE_LEN); }

	void advance_head()
	{
		if (++head >= max_entries) {
			head   = 0;
			phase ^= 1;
		}
	}
};


/*
 * Controller
 */
struct Nvme::Controller : public Genode::Attached_mmio
{
	/**********
	 ** MMIO **
	 **********/

	/*
	 * Controller capabilities (p. 40 ff.)
	 */
	struct Cap : Register<0x0, 64>
	{
		struct Mqes   : Bitfield< 0, 15> { }; /* maximum queue entries supported 0-based */
		struct Cqr    : Bitfield<16,  1> { }; /* contiguous queues required */
		struct Ams    : Bitfield<17,  2> { }; /* arbitration mechanism supported */
		struct To     : Bitfield<24,  8> { }; /* timeout (csts.rdy) */
		struct Dstrd  : Bitfield<32,  4> { }; /* doorbell stride */
		struct Nssrs  : Bitfield<36,  1> { }; /* NVM subsystem reset supported */
		struct Css    : Bitfield<37,  8> { }; /* command sets supported */
		struct Bps    : Bitfield<45,  1> { }; /* boot partition support */
		struct Mpsmin : Bitfield<48,  4> { }; /* memory page size minimum */
		struct Mpsmax : Bitfield<52,  4> { }; /* memory page size maximum */
	};

	/*
	 * Version
	 */
	struct Vs : Register<0x8, 32>
	{
		struct Ter : Bitfield< 0,  8> { }; /* tertiary */
		struct Mnr : Bitfield< 8,  8> { }; /* minor */
		struct Mjr : Bitfield<16, 16> { }; /* major */
	};

	/*
	 * Interrupt mask set (for !MSI-X)
	 */
	struct Intms : Register<0x0c, 32>
	{
		struct Ivms : Bitfield<0, 32> { }; /* interrupt vector mask set */
	};

	/*
	 * Interrupt mask clear
	 */
	struct Intmc : Register<0x10, 32>
	{
		struct Ivmc : Bitfield<0, 32> { }; /* interrupt vector mask clear */
	};

	/*
	 * Controller configuration
	 */
	struct Cc : Register<0x14, 32>
	{
		struct En     : Bitfield< 0,  1> { }; /* enable */
		struct Css    : Bitfield< 4,  3> { }; /* I/O command set selected */
		struct Mps    : Bitfield< 7,  4> { }; /* memory page size */
		struct Ams    : Bitfield<11,  3> { }; /* arbitration mechanism selected */
		struct Shn    : Bitfield<14,  2> { }; /* shutdown notification */
		struct Iosqes : Bitfield<16,  4> { }; /* I/O submission queue entry size */
		struct Iocqes : Bitfield<20,  4> { }; /* I/O completion queue entry size */
	};

	/*
	 * Controller status
	 */
	struct Csts : Register<0x1c, 32>
	{
		struct Rdy   : Bitfield< 0,  1> { }; /* ready */
		struct Cfs   : Bitfield< 1,  1> { }; /* controller fatal status */
		struct Shst  : Bitfield< 2,  1> { }; /* shutdown status */
		struct Nssro : Bitfield< 4,  1> { }; /* NVM subsystem reset occurred */
		struct Pp    : Bitfield< 5,  1> { }; /* processing paused */
	};

	/*
	 * NVM subsystem reset
	 */
	struct Nssr : Register<0x20, 32>
	{
		struct Nssrc : Bitfield< 0, 32> { }; /* NVM subsystem reset control */
	};

	/*
	 * Admin queue attributes
	 */
	struct Aqa : Register<0x24, 32>
	{
		struct Asqs : Bitfield< 0, 12> { }; /* admin submission queue size 0-based */
		struct Acqs : Bitfield<16, 12> { }; /* admin completion queue size 0-based */
	};

	/*
	 * Admin submission queue base address
	 */
	struct Asq : Register<0x28, 64>
	{
		struct Asqb : Bitfield<12, 52> { }; /* admin submission queue base */
	};

	/*
	 * Admin completion queue base address
	 */
	struct Acq : Register<0x30, 64>
	{
		struct Acqb : Bitfield<12, 52> { }; /* admin completion queue base */
	};

	/*
	 * Controller memory buffer location
	 */
	struct Cmbloc : Register<0x38, 32>
	{
		struct Bir  : Bitfield< 0,  2> { }; /* base indicator register */
		struct Ofst : Bitfield<12, 24> { }; /* offset */
	};

	/*
	 * Controller memory buffer size
	 */
	struct Cmbsz : Register<0x3c, 32>
	{
		struct Sqs   : Bitfield< 0,  1> { }; /* submission queue support */
		struct Cqs   : Bitfield< 1,  1> { }; /* completion queue support */
		struct Lists : Bitfield< 2,  1> { }; /* PRP SGL list support */
		struct Rds   : Bitfield< 3,  1> { }; /* read data support */
		struct Wds   : Bitfield< 4,  1> { }; /* write data support */
		struct Szu   : Bitfield< 8,  4> { }; /* size units */
		struct Sz    : Bitfield<12, 24> { }; /* size */
	};

	/*
	 * Boot partition information
	 */
	struct Bpinfo : Register<0x40, 32>
	{
		struct Bpsz  : Bitfield< 0, 14> { }; /* boot partition size (in 128KiB) */
		struct Brs   : Bitfield<24,  2> { }; /* boot read status */
		struct Abpid : Bitfield<31,  1> { }; /* active boot partition id */
	};

	/*
	 * Boot partition read select
	 */
	struct Bprsel : Register<0x44, 32>
	{
		struct Bprsz : Bitfield< 0, 10> { }; /* boot partition read size (in 4KiB) */
		struct Bprof : Bitfield<10, 30> { }; /* boot partition read offset (in 4KiB) */
		struct Bpid  : Bitfield<31,  1> { }; /* boot partition identifier */
	};

	/*
	 * Boot partition memory buffer location
	 */
	struct Bpmbl : Register<0x48, 64>
	{
		struct Bmbba : Bitfield<12, 52> { }; /* boot partition memory buffer base address */
	};

	/*
	 * Admin submission doorbell
	 */
	struct Admin_sdb : Register<0x1000, 32>
	{
		struct Sqt : Bitfield< 0, 16> { }; /* submission queue tail */
	};

	/*
	 * Admin completion doorbell
	 */
	struct Admin_cdb : Register<0x1004, 32>
	{
		struct Cqh : Bitfield< 0, 16> { }; /* completion queue tail */
	};

	/*
	 * I/O submission doorbell
	 */
	struct Io_sdb : Register<0x1008, 32>
	{
		struct Sqt : Bitfield< 0, 16> { }; /* submission queue tail */
	};

	/*
	 * I/O completion doorbell
	 */
	struct Io_cdb : Register<0x100C, 32>
	{
		struct Cqh : Bitfield< 0, 16> { }; /* completion queue tail */
	};

	/**********
	 ** CODE **
	 **********/

	struct Mem_address
	{
		addr_t va { 0 };
		addr_t pa { 0 };
	};

	struct Initialization_failed : Genode::Exception { };

	Genode::Env &_env;

	Util::Dma_allocator &_dma_alloc;
	Mmio::Delayer       &_delayer;

	size_t _mps { 0 };

	Nvme::Cq _cq[NUM_QUEUES] { };
	Nvme::Sq _sq[NUM_QUEUES] { };

	Nvme::Cq &_admin_cq = _cq[0];
	Nvme::Sq &_admin_sq = _sq[0];

	Mem_address _nvme_identify { };

	Genode::Constructible<Identify_data> _identify_data { };

	Mem_address _nvme_nslist { };
	uint32_t    _nvme_nslist_count { 0 };

	enum Cns {
		IDENTIFY_NS = 0x00,
		IDENTIFY    = 0x01,
		NSLIST      = 0x02,
	};

	enum {
		IDENTIFY_LEN = 4096,

		IDENTIFY_CID = 0x666,
		NSLIST_CID,
		QUERYNS_CID,
		CREATE_IO_CQ_CID,
		CREATE_IO_SQ_CID,
	};

	Mem_address _nvme_query_ns[MAX_NS] { };

	struct Info
	{
		Genode::String<8> version { };
		Identify_data::Sn sn { };
		Identify_data::Mn mn { };
		Identify_data::Fr fr { };
	} _info { };

	struct Nsinfo
	{
		Block::sector_t count { 0 };
		size_t          size  { 0 };
		bool valid() const { return count && size; }
	} _nsinfo[MAX_NS] { };

	/**
	 * Wait for ready bit to change
	 *
	 * \param val  value of ready bit
	 *
	 * \throw Mmio::Polling_timeout
	 */
	void _wait_for_rdy(unsigned val)
	{
		enum { MAX = 50u, TO_UNIT = 500u, };
		Attempts     const a(MAX);
		Microseconds const t(((uint64_t)read<Cap::To>() * TO_UNIT) * (1000 / MAX));
		try {
			wait_for(a, t, _delayer, Csts::Rdy::Equal(val));
		} catch (Mmio::Polling_timeout) {
			Genode::error("Csts::Rdy(", val, ") failed");
			throw;
		}
	}

	/**
	 * Reset controller
	 *
	 * \throw Initialization_failed
	 */
	void _reset()
	{
		/* disable intr and ctrlr */
		write<Intms>(~0u);
		write<Cc>(0);

		try { _wait_for_rdy(0); }
		catch (...) { throw Initialization_failed(); }

		/*
		 * For now we limit the memory page size to 4K because besides Qemu
		 * there are not that many consumer NVMe device that support larger
		 * page sizes and we do not want to align the DMA buffers to larger
		 * sizes.
		 */
		Cap::access_t const mpsmax = read<Cap::Mpsmax>();
		if (mpsmax > 0) { Genode::warning("ignore mpsmax:", mpsmax); }

		/* the value written to the register amounts to 2^(12 + v) bytes */
		Cap::access_t const v = Genode::log2((unsigned)Nvme::MPS) - 12;
		_mps = 1u << (12 + v);
		write<Cc::Mps>(v);

		write<Cc::Iocqes>(log2((unsigned)CQE_LEN));
		write<Cc::Iosqes>(log2((unsigned)SQE_LEN));
	}

	/**
	 * Setup queue, i.e., fill out fields
	 *
	 * \param q    reference to queue
	 * \param num  number of entries
	 * \param len  size of one entry
	 */
	void _setup_queue(Queue &q, size_t const num, size_t const len)
	{
		size_t const size = num * len;
		q.ds          = _dma_alloc.alloc(size);
		q.pa          = Dataspace_client(q.ds).phys_addr();
		q.va          = (addr_t)_env.rm().attach(q.ds);
		q.max_entries = num;
	}

	/**
	 * Check if given queue tuple is full
	 *
	 * \param sq  reference to submission queue
	 * \param cq  reference to completion queue
	 *
	 * \return  returns true if queue is full and false otherwise
	 */
	bool _queue_full(Nvme::Sq const &sq, Nvme::Cq const &cq) const
	{
		return ((sq.tail + 1) & (MAX_IO_ENTRIES_MASK)) == cq.head;
	}

	/**
	 * Setup admin queues
	 */
	void _setup_admin()
	{
		_setup_queue(_admin_cq, MAX_ADMIN_ENTRIES, CQE_LEN);
		write<Aqa::Acqs>(MAX_ADMIN_ENTRIES_MASK);
		write<Acq>(_admin_cq.pa);

		_setup_queue(_admin_sq, MAX_ADMIN_ENTRIES, SQE_LEN);
		write<Aqa::Asqs>(MAX_ADMIN_ENTRIES_MASK);
		write<Asq>(_admin_sq.pa);
	}

	/**
	 * Get address of the next free entry in the admin submission queue
	 *
	 * \param opc   entry opcode
	 * \param nsid  namespace identifier
	 * \param cid   command identifier
	 *
	 * \return  returns address of the next free entry or 0 if there is
	 *          no free entry
	 */
	addr_t _admin_command(Opcode opc, uint32_t nsid, uint32_t cid)
	{
		if (_queue_full(_admin_sq, _admin_cq)) { return 0ul; }

		Sqe b(_admin_sq.next());
		b.write<Nvme::Sqe::Cdw0::Opc>(opc);
		b.write<Nvme::Sqe::Cdw0::Cid>(cid);
		b.write<Nvme::Sqe::Nsid>(nsid);
		return b.base();
	}

	/**
	 * Wait until admin command has finished
	 *
	 * \param num  number of attempts
	 * \param cid  command identifier
	 *
	 * \return  returns true if attempt to wait was successfull, otherwise
	 *          false is returned
	 */
	bool _wait_for_admin_cq(uint32_t num, uint16_t cid)
	{
		bool success = false;

		for (uint32_t i = 0; i < num; i++) {
			_delayer.usleep(100 * 1000);

			Cqe b(_admin_cq.next());

			if (b.read<Nvme::Cqe::Cid>() != cid) {
				continue;
			}

			_admin_cq.advance_head();

			success = true;

			write<Admin_cdb::Cqh>(_admin_cq.head);
			break;
		}

		return success;
	}

	/**
	 * Get list of namespaces
	 */
	void _query_nslist()
	{
		if (!_nvme_nslist.va) {
			Ram_dataspace_capability ds = _dma_alloc.alloc(IDENTIFY_LEN);
			_nvme_nslist.va = (addr_t)_env.rm().attach(ds);
			_nvme_nslist.pa = Dataspace_client(ds).phys_addr();
		}

		uint32_t *nslist = (uint32_t*)_nvme_nslist.va;

		bool const nsm = _identify_data->read<Identify_data::Oacs::Nsm>();
		if (!nsm) {
			nslist[0] = 1;
			_nvme_nslist_count = 1;
			return;
		}

		Sqe_identify b(_admin_command(Opcode::IDENTIFY, 0, NSLIST_CID));

		b.write<Nvme::Sqe::Prp1>(_nvme_nslist.pa);
		b.write<Nvme::Sqe_identify::Cdw10::Cns>(Cns::NSLIST);

		write<Admin_sdb::Sqt>(_admin_sq.tail);

		if (!_wait_for_admin_cq(10, NSLIST_CID)) {
			Genode::error("identify name space list failed");
			throw Initialization_failed();
		}

		for (size_t i = 0; i < 1024; i++) {
			if (nslist[i] == 0) { break; }
			++_nvme_nslist_count;
		}
	}

	/**
	 * Get information of namespaces
	 */
	void _query_ns()
	{
		uint32_t const max = _nvme_nslist_count > (uint32_t)MAX_NS ?
		                     (uint32_t)MAX_NS : _nvme_nslist_count;

		if (!max) {
			error("no name spaces found");
			throw Initialization_failed();
		}

		if (max > 1) { warning("only the first name space is used"); }

		uint32_t const *ns = (uint32_t const*)_nvme_nslist.va;
		uint32_t const  id = 0;

		if (!_nvme_query_ns[id].va) {
			Ram_dataspace_capability ds = _dma_alloc.alloc(IDENTIFY_LEN);
			_nvme_query_ns[id].va = (addr_t)_env.rm().attach(ds);
			_nvme_query_ns[id].pa = Dataspace_client(ds).phys_addr();
		}

		Sqe_identify b(_admin_command(Opcode::IDENTIFY, ns[id], QUERYNS_CID));
		b.write<Nvme::Sqe::Prp1>(_nvme_query_ns[id].pa);
		b.write<Nvme::Sqe_identify::Cdw10::Cns>(Cns::IDENTIFY_NS);

		write<Admin_sdb::Sqt>(_admin_sq.tail);

		if (!_wait_for_admin_cq(10, QUERYNS_CID)) {
			Genode::error("identify name space failed");
			throw Initialization_failed();
		}

		Identify_ns_data nsdata(_nvme_query_ns[id].va);
		uint32_t const flbas = nsdata.read<Nvme::Identify_ns_data::Flbas>();

		_nsinfo[id].count = nsdata.read<Nvme::Identify_ns_data::Nsze>();
		_nsinfo[id].size  = 1u << nsdata.read<Nvme::Identify_ns_data::Lbaf::Lbads>(flbas);
	}

	/**
	 * Query the controller information
	 */
	void _identify()
	{
		if (!_nvme_identify.va) {
			Ram_dataspace_capability ds = _dma_alloc.alloc(IDENTIFY_LEN);
			_nvme_identify.va = (addr_t)_env.rm().attach(ds);
			_nvme_identify.pa = Dataspace_client(ds).phys_addr();
		}

		Sqe_identify b(_admin_command(Opcode::IDENTIFY, 0, IDENTIFY_CID));
		b.write<Nvme::Sqe::Prp1>(_nvme_identify.pa);
		b.write<Nvme::Sqe_identify::Cdw10::Cns>(Cns::IDENTIFY);

		write<Admin_sdb::Sqt>(_admin_sq.tail);

		if (!_wait_for_admin_cq(10, IDENTIFY_CID)) {
			Genode::error("identify failed");
			throw Initialization_failed();
		}

		_identify_data.construct(_nvme_identify.va);

		/* store information */
		_info.version = Genode::String<8>(read<Vs::Mjr>(), ".",
		                                  read<Vs::Mnr>(), ".",
		                                  read<Vs::Ter>());
		_info.sn = _identify_data->sn;
		_info.mn = _identify_data->mn;
		_info.fr = _identify_data->fr;
	}

	/**
	 * Setup I/O completion queue
	 *
	 * \param id  identifier of the completion queue
	 *
	 * \throw Initialization_failed() in case the queue could not be created
	 */
	void _setup_io_cq(uint16_t id)
	{
		Nvme::Cq &cq = _cq[id];
		if (!cq.valid()) { _setup_queue(cq, MAX_IO_ENTRIES, CQE_LEN); }

		Sqe_create_cq b(_admin_command(Opcode::CREATE_IO_CQ, 0, CREATE_IO_CQ_CID));
		b.write<Nvme::Sqe::Prp1>(cq.pa);
		b.write<Nvme::Sqe_create_cq::Cdw10::Qid>(id);
		b.write<Nvme::Sqe_create_cq::Cdw10::Qsize>(MAX_IO_ENTRIES_MASK);
		b.write<Nvme::Sqe_create_cq::Cdw11::Pc>(1);
		b.write<Nvme::Sqe_create_cq::Cdw11::En>(1);

		write<Admin_sdb::Sqt>(_admin_sq.tail);

		if (!_wait_for_admin_cq(10, CREATE_IO_CQ_CID)) {
			Genode::error("create I/O cq failed");
			throw Initialization_failed();
		}
	}

	/**
	 * Setup I/O submission queue
	 *
	 * \param id    identifier of the submission queue
	 * \param cqid  identifier of the completion queue
	 *
	 * \throw Initialization_failed() in case the queue could not be created
	 */
	void _setup_io_sq(uint16_t id, uint16_t cqid)
	{
		Nvme::Sq &sq = _sq[id];
		if (!sq.valid()) { _setup_queue(sq, MAX_IO_ENTRIES, SQE_LEN); }

		Sqe_create_sq b(_admin_command(Opcode::CREATE_IO_SQ, 0, CREATE_IO_SQ_CID));
		b.write<Nvme::Sqe::Prp1>(sq.pa);
		b.write<Nvme::Sqe_create_sq::Cdw10::Qid>(id);
		b.write<Nvme::Sqe_create_sq::Cdw10::Qsize>(MAX_IO_ENTRIES_MASK);
		b.write<Nvme::Sqe_create_sq::Cdw11::Pc>(1);
		b.write<Nvme::Sqe_create_sq::Cdw11::Qprio>(0b00); /* urgent for now */
		b.write<Nvme::Sqe_create_sq::Cdw11::Cqid>(cqid);

		write<Admin_sdb::Sqt>(_admin_sq.tail);

		if (!_wait_for_admin_cq(10, CREATE_IO_SQ_CID)) {
			Genode::error("create I/O sq failed");
			throw Initialization_failed();
		}
	}

	/**
	 * Constructor
	 */
	Controller(Genode::Env &env, Util::Dma_allocator &dma_alloc,
	           addr_t const base, size_t const size,
	           Mmio::Delayer &delayer)
	:
		Genode::Attached_mmio(env, base, size),
		_env(env), _dma_alloc(dma_alloc), _delayer(delayer)
	{ }

	/**
	 * Initialize controller
	 *
	 * \throw Initialization_failed
	 */
	void init()
	{
		_reset();
		_setup_admin();

		write<Cc::En>(1);

		try { _wait_for_rdy(1); }
		catch (...) {
			if (read<Csts::Cfs>()) {
				Genode::error("fatal controller status");
			}
			throw Initialization_failed();
		}
	}

	/**
	 * Mask interrupts
	 */
	void mask_intr() { write<Intms>(1); }

	/**
	 * Clean interrupts
	 */
	void clear_intr() { write<Intmc>(1); }

	/*
	 * Identify NVM system
	 */
	void identify()
	{
		_identify();
		_query_nslist();
		_query_ns();
	}

	/**
	 * Setup I/O queue
	 */
	void setup_io(uint16_t cid, uint16_t sid)
	{
		_setup_io_cq(cid);
		_setup_io_sq(sid, cid);
	}

	/**
	 * Get next free IO submission queue slot
	 */
	addr_t io_command(uint16_t id)
	{
		Nvme::Sq &sq = _sq[id];
		Nvme::Cq &cq = _cq[id];

		if (_queue_full(sq, cq)) { return 0ul; }

		Sqe e(sq.next());
		e.write<Nvme::Sqe::Cdw0::Cid>(sq.id++);
		e.write<Nvme::Sqe::Nsid>(id);
		return e.base();
	}

	/**
	 * Write current I/O submission queue tail
	 */
	void commit_io(uint16_t id)
	{
		Nvme::Sq &sq = _sq[id];
		write<Io_sdb::Sqt>(sq.tail);
	}

	/**
	 * Flush cache
	 */
	void flush_cache(uint16_t id)
	{
		(void)id;
	}

	/**
	 * Process every pending I/O completion
	 *
	 * \param func  function that is called on each completion
	 */
	template <typename FUNC>
	void handle_io_completions(uint16_t id, FUNC const &func)
	{
		Nvme::Cq &cq = _cq[id];

		if (!cq.valid()) { return; }

		for (;;) {
			Cqe e(cq.next());

			/* process until old phase */
			if (e.read<Nvme::Cqe::Sf::P>() != cq.phase) { break; }

			func(e);

			cq.advance_head();

			/*
			 * Instead of acknowledging the completions here,
			 * we could handle them batch-wise after the loop.
			 */
			write<Io_cdb::Cqh>(cq.head);
		}
	}

	/**
	 * Get memory page size in bytes
	 */
	size_t mps() const { return _mps; }

	/**
	 * Get block metrics of namespace
	 *
	 * \param nsid  namespace identifier
	 *
	 * \return  returns information of the namespace
	 */
	Nsinfo nsinfo(uint32_t id)
	{
		id = id - 1;
		if (id >= MAX_NS) { return Nsinfo(); }
		return _nsinfo[id];
	}

	/**
	 * Get controller information
	 */
	Info const &info() const { return _info; }

	/***********
	 ** Debug **
	 ***********/

	void dump_cap()
	{
		Genode::log("CAP:", "  ",
		            "Mqes:",   read<Cap::Mqes>()+1, " ",
		            "Cqr:",    read<Cap::Cqr>(),    " ",
		            "Ams:",    read<Cap::Ams>(),    " ",
		            "To:",     read<Cap::To>(),     " ",
		            "Dstrd:",  read<Cap::Dstrd>(),  " ",
		            "Nssrs:",  read<Cap::Nssrs>(),  " ",
		            "Css:",    read<Cap::Css>(),    " ",
		            "Bps:",    read<Cap::Bps>(),    " ",
		            "Mpsmin:", read<Cap::Mpsmin>(), " ",
		            "Mpsmax:", read<Cap::Mpsmax>());

		Genode::log("VS: ", "  ", read<Vs::Mjr>(), ".",
		            read<Vs::Mnr>(), ".", read<Vs::Ter>());
	}

	void dump_identify()
	{
		log("vid:", Hex(_identify_data->read<Identify_data::Vid>()));
		log("ssvid:", Hex(_identify_data->read<Identify_data::Ssvid>()));
		log("oacs:", Hex(_identify_data->read<Identify_data::Oacs>()));
		log("  nsm:", Hex(_identify_data->read<Identify_data::Oacs::Nsm>()));
		log("sn:'", _identify_data->sn.string(), "'");
		log("mn:'", _identify_data->mn.string(), "'");
		log("fr:'", _identify_data->fr.string(), "'");
		log("nn:",   _identify_data->read<Identify_data::Nn>());
		log("vwc:",  _identify_data->read<Identify_data::Vwc>());
	}

	void dump_nslist()
	{
		uint32_t const *p = (uint32_t const*)_nvme_nslist.va;
		if (!p) { return; }

		for (size_t i = 0; i < 1024; i++) {
			if (p[i] == 0) { break; }
			Genode::log("ns:#", p[i], " found");
		}
	}
};


/******************
 ** Block driver **
 ******************/

class Driver : public Block::Driver
{
	public:

		bool _verbose_checks   { false };
		bool _verbose_identify { false };
		bool _verbose_io       { false };
		bool _verbose_mem      { false };
		bool _verbose_regs     { false };

	private:

		Genode::Env       &_env;
		Genode::Allocator &_alloc;

		Genode::Signal_context_capability _announce_sigh;

		Genode::Attached_rom_dataspace _config_rom { _env, "config" };

		void _handle_config_update()
		{
			_config_rom.update();

			if (!_config_rom.valid()) { return; }

			Genode::Xml_node config = _config_rom.xml();
			_verbose_checks   = config.attribute_value("verbose_checks",   _verbose_checks);
			_verbose_identify = config.attribute_value("verbose_identify", _verbose_identify);
			_verbose_io       = config.attribute_value("verbose_io",       _verbose_io);
			_verbose_mem      = config.attribute_value("verbose_mem",      _verbose_mem);
			_verbose_regs     = config.attribute_value("verbose_regs",     _verbose_regs);
		}

		Genode::Signal_handler<Driver> _config_sigh {
			_env.ep(), *this, &Driver::_handle_config_update };

		/**************
		 ** Reporter **
		 **************/

		Genode::Reporter _namespace_reporter { _env, "controller" };

		void _report_namespaces()
		{
			try {
				Genode::Reporter::Xml_generator xml(_namespace_reporter, [&]() {
					Nvme::Controller::Info const &info = _nvme_ctrlr->info();

					xml.attribute("serial", info.sn);
					xml.attribute("model",  info.mn);

					for (int i = 1; i <= Nvme::MAX_NS; i++) {
						Nvme::Controller::Nsinfo ns = _nvme_ctrlr->nsinfo(i);

						xml.node("namespace", [&]() {
							xml.attribute("id", i);
							xml.attribute("block_size",  ns.size);
							xml.attribute("block_count", ns.count);
						});
					}
				});
			} catch (...) { }
		}

		/*********
		 ** DMA **
		 *********/

		Genode::Constructible<Nvme::Pci> _nvme_pci { };

		struct Io_buffer
		{
			addr_t pa   { 0 };
			addr_t va   { 0 };
			size_t size { 0 };

			bool valid() const { return size && pa && va; }
			void invalidate() { Genode::memset(this, 0, sizeof(*this)); }
		};

		template <unsigned ENTRIES, size_t MPS>
		struct Io_buffer_mapper
		{
			using Bitmap = Util::Bitmap<ENTRIES>;
			Bitmap _bitmap { };

			Util::Slots<Io_buffer, Nvme::MAX_IO_ENTRIES> _buffers { };

			Genode::Ram_dataspace_capability _ds { };
			addr_t _phys_addr { 0 };
			addr_t _virt_addr { 0 };

			Io_buffer_mapper(Genode::Ram_dataspace_capability ds,
			                 addr_t phys, addr_t virt)
			: _ds(ds), _phys_addr(phys), _virt_addr(virt) { }

			Io_buffer *alloc(size_t size)
			{
				Io_buffer *iob = _buffers.get();
				if (!iob) { return nullptr; }

				try {
					size_t const bits  = size / MPS;
					addr_t const start = _bitmap.alloc(bits);
					iob->pa   = (start * MPS) + _phys_addr;
					iob->va   = (start * MPS) + _virt_addr;
					iob->size = size;
				} catch (...) {
					iob->invalidate();
					return nullptr;
				}
				return iob;
			}

			void free(Io_buffer *iob)
			{
				if (iob) {
					size_t const size = iob->size;
					addr_t const start = (iob->pa - _phys_addr) / MPS;
					_bitmap.free(start, size / MPS);
					iob->invalidate();
				}
			}
		};

		Genode::Constructible<Io_buffer_mapper<Nvme::DMA_DS_SIZE / Nvme::MPS,
		                                       Nvme::MPS>> _io_mapper { };

		Genode::Constructible<Io_buffer_mapper<Nvme::DMA_LIST_DS_SIZE / Nvme::MPS,
		                                       Nvme::MPS>> _io_list_mapper { };

		void _setup_large_request(addr_t           va,
		                          Io_buffer const &iob,
		                          size_t    const  num,
		                          size_t    const  mps)
		{
			/* omit first page */
			addr_t    pa = iob.pa + mps;
			uint64_t *p  = (uint64_t*)va;

			for (size_t i = 0; i < num; i++) {
				p[i]  = pa;
				pa   += mps;
			}
		}

		/**************
		 ** Requests **
		 **************/

		struct Request
		{
			uint32_t           id     { 0 };
			Packet_descriptor  pd     {   };
			char              *buffer { nullptr };

			Io_buffer *iob           { nullptr };
			Io_buffer *large_request { nullptr };

			bool valid() const { return id != 0; }

			void invalidate()
			{
				id     = 0;
				buffer = nullptr;
				pd     = Packet_descriptor();

				iob           = nullptr;
				large_request = nullptr;
			}
		};

		Util::Slots<Request, Nvme::MAX_IO_ENTRIES> _requests         {   };
		size_t                                     _requests_pending { 0 };

		/*********************
		 ** MMIO Controller **
		 *********************/

		struct Timer_delayer : Genode::Mmio::Delayer,
		                       Timer::Connection
		{
			Timer_delayer(Genode::Env &env)
			: Timer::Connection(env) { }

			void usleep(uint64_t us) override { Timer::Connection::usleep(us); }

		} _delayer { _env };

		Genode::Constructible<Nvme::Controller> _nvme_ctrlr { };

		void _handle_completions()
		{
			_nvme_ctrlr->handle_io_completions(Nvme::IO_NSID, [&] (Nvme::Cqe const &b) {

				if (_verbose_io) { Nvme::Cqe::dump(b); }

				uint32_t const id = Nvme::Cqe::request_id(b);

				Request *r = _requests.lookup([&] (Request &r) {
					if (r.id == id) { return true; }
					return false;
				});
				if (!r) {
					Genode::error("no pending request found for CQ entry");
					Nvme::Cqe::dump(b);
					return;
				}

				bool const succeeded = Nvme::Cqe::succeeded(b);

				Packet_descriptor pd = r->pd;
				pd.succeeded(succeeded);

				Io_buffer *iob = r->iob;

				if (succeeded && pd.operation() == Packet_descriptor::READ) {
					size_t const len = pd.block_count() * _info.block_size;
					Genode::memcpy(r->buffer, (void*)iob->va, len);
				}
				_io_mapper->free(iob);

				if (r->large_request) {
					_io_list_mapper->free(r->large_request);
				}

				r->invalidate();
				--_requests_pending;
				ack_packet(pd, succeeded);
			});
		}

		void _handle_intr()
		{
			_nvme_ctrlr->mask_intr();
			_handle_completions();
			_nvme_ctrlr->clear_intr();
			_nvme_pci->ack_irq();
		}

		Genode::Signal_handler<Driver> _intr_sigh {
			_env.ep(), *this, &Driver::_handle_intr };

		/***********
		 ** Block **
		 ***********/

		Block::Session::Info _info { };

	public:

		/**
		 * Constructor
		 */
		Driver(Genode::Env &env, Genode::Allocator &alloc,
		       Genode::Signal_context_capability sigh)
		: Block::Driver(env.ram()), _env(env), _alloc(alloc), _announce_sigh(sigh)
		{
			_config_rom.sigh(_config_sigh);
			_handle_config_update();

			/*
			 * Setup and identify NVMe PCI controller
			 */

			try {
				_nvme_pci.construct(_env);
			} catch (Nvme::Pci::Missing_controller) {
				Genode::error("no NVMe PCIe controller found");
				throw;
			}

			try {
				_nvme_ctrlr.construct(_env, *_nvme_pci, _nvme_pci->base(),
				                      _nvme_pci->size(), _delayer);
			} catch (...) {
				Genode::error("could not access NVMe controller MMIO");
				throw;
			}

			if (_verbose_regs) { _nvme_ctrlr->dump_cap(); }

			_nvme_ctrlr->init();
			_nvme_ctrlr->identify();

			if (_verbose_identify) {
				Genode::warning(_requests_pending);
				_nvme_ctrlr->dump_identify();
				_nvme_ctrlr->dump_nslist();
			}

			/*
			 * Setup I/O
			 */

			{
				Genode::Ram_dataspace_capability ds = _nvme_pci->alloc(Nvme::DMA_DS_SIZE);
				if (!ds.valid()) {
					Genode::error("could not allocate DMA backing store");
					throw Nvme::Controller::Initialization_failed();
				}
				addr_t const phys_addr = Genode::Dataspace_client(ds).phys_addr();
				addr_t const virt_addr = (addr_t)_env.rm().attach(ds);
				_io_mapper.construct(ds, phys_addr, virt_addr);

				if (_verbose_mem) {
					Genode::log("DMA", " virt: [", Genode::Hex(virt_addr), ",",
					                   Genode::Hex(virt_addr + Nvme::DMA_DS_SIZE), "]",
					                   " phys: [", Genode::Hex(phys_addr), ",",
					                   Genode::Hex(phys_addr + Nvme::DMA_DS_SIZE), "]");
				}
			}

			{
				Genode::Ram_dataspace_capability ds = _nvme_pci->alloc(Nvme::DMA_LIST_DS_SIZE);
				if (!ds.valid()) {
					Genode::error("could not allocate DMA list-pages backing store");
					throw Nvme::Controller::Initialization_failed();
				}
				addr_t const phys_addr = Genode::Dataspace_client(ds).phys_addr();
				addr_t const virt_addr = (addr_t)_env.rm().attach(ds);
				_io_list_mapper.construct(ds, phys_addr, virt_addr);

				if (_verbose_mem) {
					Genode::log("DMA list-pages", " virt: [", Genode::Hex(virt_addr), ",",
					                   Genode::Hex(virt_addr + Nvme::DMA_DS_SIZE), "]",
					                   " phys: [", Genode::Hex(phys_addr), ",",
					                   Genode::Hex(phys_addr + Nvme::DMA_DS_SIZE), "]");
				}
			}

			_nvme_ctrlr->setup_io(Nvme::IO_NSID, Nvme::IO_NSID);

			/* from now on use interrupts */
			_nvme_pci->sigh_irq(_intr_sigh);
			_nvme_ctrlr->clear_intr();

			/*
			 * Setup Block session
			 */

			/* set Block session properties */
			Nvme::Controller::Nsinfo nsinfo = _nvme_ctrlr->nsinfo(Nvme::IO_NSID);
			if (!nsinfo.valid()) {
				Genode::error("could not query namespace information");
				throw Nvme::Controller::Initialization_failed();
			}

			_info = { .block_size  = nsinfo.size,
			          .block_count = nsinfo.count,
			          .align_log2  = Genode::log2(nsinfo.size),
			          .writeable   = true };

			Nvme::Controller::Info const &info = _nvme_ctrlr->info();

			Genode::log("NVMe:",    info.version.string(),   " "
			            "serial:'", info.sn.string(), "'", " "
			            "model:'",  info.mn.string(), "'", " "
			            "frev:'",   info.fr.string(), "'");

			Genode::log("Block",                " "
			            "size:",  _info.block_size,  " "
			            "count:", _info.block_count);

			/* generate Report if requested */
			try {
				Genode::Xml_node report = _config_rom.xml().sub_node("report");
				if (report.attribute_value("namespaces", false)) {
					_namespace_reporter.enabled(true);
					_report_namespaces();
				}
			} catch (...) { }

			/* finally announce Block session */
			Genode::Signal_transmitter(_announce_sigh).submit();
		}

		~Driver() { }

		/*******************************
		 **  Block::Driver interface  **
		 *******************************/

		Block::Session::Info info() const override { return _info; }

		void _io(bool write, Block::sector_t lba, size_t count,
		         char *buffer, Packet_descriptor &pd)
		{
			using namespace Genode;

			size_t const len = count * _info.block_size;

			if (_verbose_io) {
				Genode::error(write ? "write" : "read", " "
				              "lba:",           lba,    " "
				              "count:",         count,  " "
				              "buffer:", (void*)buffer, " "
				              "len:",           len);
			}

			if (len > Nvme::MAX_IO_LEN) {
				error("request too large (max:", (size_t)Nvme::MAX_IO_LEN, " bytes)");
				throw Io_error();
			}

			if (_requests_pending == (Nvme::MAX_IO_PENDING)) {
				throw Request_congestion();
			}

			Block::sector_t const lba_end = lba + count - 1;
			auto overlap_check = [&] (Request &req) {
				Block::sector_t const start = req.pd.block_number();
				Block::sector_t const end   = start + req.pd.block_count() - 1;

				bool const in_req    = (lba >= start && lba_end <= end);
				bool const over_req  = (lba <= start && lba_end <= end) &&
				                       (start >= lba && start <= lba_end);
				bool const cross_req = (lba <= start && lba_end >= end);
				bool const overlap   = (in_req || over_req || cross_req);

				if (_verbose_checks && overlap) {
					warning("overlap: ", "[", lba, ",", lba_end, ") with "
					        "[", start,   ",", end, ")",
					        " ", in_req, " ", over_req, " ", cross_req);
				}
				return overlap;
			};
			if (_requests.for_each(overlap_check)) { throw Request_congestion(); }

			Request *r = _requests.get();
			if (!r) { throw Request_congestion(); }

			size_t const mps       = _nvme_ctrlr->mps();
			size_t const mps_len   = Genode::align_addr(len, Genode::log2(mps));
			bool   const need_list = len > 2 * mps;

			Io_buffer *iob = _io_mapper->alloc(mps_len);
			if (!iob) { throw Request_congestion(); }

			if (need_list) {
				r->large_request = _io_list_mapper->alloc(mps);
				if (!r->large_request) {
					_io_mapper->free(iob);
					throw Request_congestion();
				}
			}

			if (write) { Genode::memcpy((void*)iob->va, buffer, len); }

			Nvme::Sqe_io b(_nvme_ctrlr->io_command(Nvme::IO_NSID));
			if (!b.valid()) {
				if (r->large_request) {
					_io_list_mapper->free(r->large_request);
				}
				_io_mapper->free(iob);
				throw Request_congestion();
			}

			addr_t const pa = iob->pa;

			Nvme::Opcode op = write ? Nvme::Opcode::WRITE : Nvme::Opcode::READ;
			b.write<Nvme::Sqe::Cdw0::Opc>(op);
			b.write<Nvme::Sqe::Prp1>(pa);

			/* payload will fit into 2 mps chunks */
			if (len > mps && !r->large_request) {
				b.write<Nvme::Sqe::Prp2>(pa + mps);
			} else if (r->large_request) {
				/* payload needs list of mps chunks */
				Io_buffer &lr = *r->large_request;
				_setup_large_request(lr.va,
				                     *iob, (mps_len - mps)/mps, mps);
				b.write<Nvme::Sqe::Prp2>(lr.pa);
			}

			b.write<Nvme::Sqe_io::Slba>(lba);
			b.write<Nvme::Sqe_io::Cdw12::Nlb>(count - 1); /* 0-base value */

			r->iob    = iob;
			r->pd     = pd; /* must be a copy */
			r->buffer = write ? nullptr : buffer;
			r->id     = b.read<Nvme::Sqe_io::Cdw0::Cid>() | (Nvme::IO_NSID<<16);

			++_requests_pending;
			_nvme_ctrlr->commit_io(Nvme::IO_NSID);
		}

		void read(Block::sector_t lba, size_t count,
		          char *buffer, Packet_descriptor &pd) override
		{
			_io(false, lba, count, buffer, pd);
		}

		void write(Block::sector_t lba, size_t count,
		           char const *buffer, Packet_descriptor &pd) override
		{
			if (!_info.writeable) {
				throw Io_error();
			}
			_io(true, lba, count, const_cast<char*>(buffer), pd);
		}

		void sync() override { _nvme_ctrlr->flush_cache(Nvme::IO_NSID); }
};


/**********
 ** Main **
 **********/

struct Main
{
	Genode::Env  &_env;
	Genode::Heap  _heap { _env.ram(), _env.rm() };

	void _handle_announce()
	{
		_env.parent().announce(_env.ep().manage(_root));
	}

	Genode::Signal_handler<Main> _announce_sigh {
		_env.ep(), *this, &Main::_handle_announce };

	struct Factory : Block::Driver_factory
	{
		Genode::Env       &_env;
		Genode::Allocator &_alloc;
		Genode::Signal_context_capability _sigh;

		Genode::Constructible<::Driver> _driver { };

		Factory(Genode::Env &env, Genode::Allocator &alloc,
		        Genode::Signal_context_capability sigh)
		: _env(env), _alloc(alloc), _sigh(sigh)
		{
			_driver.construct(_env, _alloc, _sigh);
		}

		~Factory() { _driver.destruct(); }

		Block::Driver *create() override { return &*_driver; }
		void destroy(Block::Driver *) override { }
	};

	Factory     _factory { _env, _heap, _announce_sigh };
	Block::Root _root    { _env.ep(), _heap, _env.rm(), _factory, true };

	Main(Genode::Env &env) : _env(env) { }
};


void Component::construct(Genode::Env &env) { static Main main(env); }