openwrt/target/linux/bcm27xx/patches-6.6/950-1476-misc-rp1-pio-Add-in-kernel-DMA-support.patch

From fddd3e9318dbf01fb763b6880021abc558fce8e6 Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Thu, 12 Dec 2024 17:09:27 +0000
Subject: [PATCH 1476/1482] misc: rp1-pio: Add in-kernel DMA support

Add kernel-facing implementations of pio_sm_config_xfer and
pio_xm_xfer_data.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 drivers/misc/rp1-pio.c  | 208 ++++++++++++++++++++++++++++++----------
 include/linux/pio_rp1.h |  60 ++++++++++--
 2 files changed, 210 insertions(+), 58 deletions(-)

--- a/drivers/misc/rp1-pio.c
+++ b/drivers/misc/rp1-pio.c
@@ -61,9 +61,15 @@
 #define DMA_BOUNCE_BUFFER_SIZE 0x1000
 #define DMA_BOUNCE_BUFFER_COUNT 4
 
+struct dma_xfer_state {
+	struct dma_info *dma;
+	void (*callback)(void *param);
+	void *callback_param;
+};
+
 struct dma_buf_info {
 	void *buf;
-	dma_addr_t phys;
+	dma_addr_t dma_addr;
 	struct scatterlist sgl;
 };
 
@@ -572,21 +578,34 @@ static void rp1_pio_sm_dma_callback(void
 	up(&dma->buf_sem);
 }
 
+static void rp1_pio_sm_kernel_dma_callback(void *param)
+{
+	struct dma_xfer_state *dxs = param;
+
+	dxs->dma->tail_idx++;
+	up(&dxs->dma->buf_sem);
+
+	dxs->callback(dxs->callback_param);
+
+	kfree(dxs);
+}
+
 static void rp1_pio_sm_dma_free(struct device *dev, struct dma_info *dma)
 {
 	dmaengine_terminate_all(dma->chan);
 	while (dma->buf_count > 0) {
 		dma->buf_count--;
 		dma_free_coherent(dev, ROUND_UP(dma->buf_size, PAGE_SIZE),
-				  dma->bufs[dma->buf_count].buf, dma->bufs[dma->buf_count].phys);
+				  dma->bufs[dma->buf_count].buf,
+				  dma->bufs[dma->buf_count].dma_addr);
 	}
 
 	dma_release_channel(dma->chan);
 }
 
-static int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, void *param)
+static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint sm, uint dir,
+					   uint buf_size, uint buf_count)
 {
-	struct rp1_pio_sm_config_xfer_args *args = param;
 	struct rp1_pio_sm_set_dmactrl_args set_dmactrl_args;
 	struct rp1_pio_device *pio = client->pio;
 	struct platform_device *pdev = pio->pdev;
@@ -596,17 +615,18 @@ static int rp1_pio_sm_config_xfer(struct
 	struct dma_info *dma;
 	uint32_t dma_mask;
 	char chan_name[4];
-	uint buf_size;
 	int ret = 0;
 
-	if (args->sm >= RP1_PIO_SMS_COUNT || args->dir >= RP1_PIO_DIR_COUNT ||
-	    !args->buf_size || (args->buf_size & 3) ||
-	    !args->buf_count || args->buf_count > DMA_BOUNCE_BUFFER_COUNT)
+	if (sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT)
+		return -EINVAL;
+	if ((buf_count || buf_size) &&
+	    (!buf_size || (buf_size & 3) ||
+	     !buf_count || buf_count > DMA_BOUNCE_BUFFER_COUNT))
 		return -EINVAL;
 
-	dma_mask = 1 << (args->sm * 2 + args->dir);
+	dma_mask = 1 << (sm * 2 + dir);
 
-	dma = &pio->dma_configs[args->sm][args->dir];
+	dma = &pio->dma_configs[sm][dir];
 
 	spin_lock(&pio->lock);
 	if (pio->claimed_dmas & dma_mask)
@@ -615,16 +635,16 @@ static int rp1_pio_sm_config_xfer(struct
 	client->claimed_dmas |= dma_mask;
 	spin_unlock(&pio->lock);
 
-	dma->buf_size = args->buf_size;
+	dma->buf_size = buf_size;
 	/* Round up the allocations */
-	buf_size = ROUND_UP(args->buf_size, PAGE_SIZE);
+	buf_size = ROUND_UP(buf_size, PAGE_SIZE);
 	sema_init(&dma->buf_sem, 0);
 
 	/* Allocate and configure a DMA channel */
 	/* Careful - each SM FIFO has its own DREQ value */
-	chan_name[0] = (args->dir == RP1_PIO_DIR_TO_SM) ? 't' : 'r';
+	chan_name[0] = (dir == RP1_PIO_DIR_TO_SM) ? 't' : 'r';
 	chan_name[1] = 'x';
-	chan_name[2] = '0' + args->sm;
+	chan_name[2] = '0' + sm;
 	chan_name[3] = '\0';
 
 	dma->chan = dma_request_chan(dev, chan_name);
@@ -632,37 +652,37 @@ static int rp1_pio_sm_config_xfer(struct
 		return PTR_ERR(dma->chan);
 
 	/* Alloc and map bounce buffers */
-	for (dma->buf_count = 0; dma->buf_count < args->buf_count; dma->buf_count++) {
+	for (dma->buf_count = 0; dma->buf_count < buf_count; dma->buf_count++) {
 		struct dma_buf_info *dbi = &dma->bufs[dma->buf_count];
 
 		dbi->buf = dma_alloc_coherent(dma->chan->device->dev, buf_size,
-					      &dbi->phys, GFP_KERNEL);
+					      &dbi->dma_addr, GFP_KERNEL);
 		if (!dbi->buf) {
 			ret = -ENOMEM;
 			goto err_dma_free;
 		}
 		sg_init_table(&dbi->sgl, 1);
-		sg_dma_address(&dbi->sgl) = dbi->phys;
+		sg_dma_address(&dbi->sgl) = dbi->dma_addr;
 	}
 
 	fifo_addr = pio->phys_addr;
-	fifo_addr += args->sm * (RP1_PIO_FIFO_TX1 - RP1_PIO_FIFO_TX0);
-	fifo_addr += (args->dir == RP1_PIO_DIR_TO_SM) ? RP1_PIO_FIFO_TX0 : RP1_PIO_FIFO_RX0;
+	fifo_addr += sm * (RP1_PIO_FIFO_TX1 - RP1_PIO_FIFO_TX0);
+	fifo_addr += (dir == RP1_PIO_DIR_TO_SM) ? RP1_PIO_FIFO_TX0 : RP1_PIO_FIFO_RX0;
 
 	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 	config.src_addr = fifo_addr;
 	config.dst_addr = fifo_addr;
-	config.direction = (args->dir == RP1_PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
+	config.direction = (dir == RP1_PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
 
 	ret = dmaengine_slave_config(dma->chan, &config);
 	if (ret)
 		goto err_dma_free;
 
-	set_dmactrl_args.sm = args->sm;
-	set_dmactrl_args.is_tx = (args->dir == RP1_PIO_DIR_TO_SM);
+	set_dmactrl_args.sm = sm;
+	set_dmactrl_args.is_tx = (dir == RP1_PIO_DIR_TO_SM);
 	set_dmactrl_args.ctrl = RP1_PIO_DMACTRL_DEFAULT;
-	if (args->dir == RP1_PIO_DIR_FROM_SM)
+	if (dir == RP1_PIO_DIR_FROM_SM)
 		set_dmactrl_args.ctrl = (RP1_PIO_DMACTRL_DEFAULT & ~0x1f) | 1;
 
 	ret = rp1_pio_sm_set_dmactrl(client, &set_dmactrl_args);
@@ -682,6 +702,14 @@ err_dma_free:
 	return ret;
 }
 
+static int rp1_pio_sm_config_xfer_user(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_sm_config_xfer_args *args = param;
+
+	return rp1_pio_sm_config_xfer_internal(client, args->sm, args->dir,
+					       args->buf_size, args->buf_count);
+}
+
 static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma,
 				  const void __user *userbuf, size_t bytes)
 {
@@ -723,7 +751,7 @@ static int rp1_pio_sm_tx_user(struct rp1
 					       DMA_PREP_INTERRUPT | DMA_CTRL_ACK |
 					       DMA_PREP_FENCE);
 		if (!desc) {
-			dev_err(dev, "DMA preparation failedzn");
+			dev_err(dev, "DMA preparation failed\n");
 			ret = -EIO;
 			break;
 		}
@@ -779,7 +807,7 @@ static int rp1_pio_sm_rx_user(struct rp1
 		if (!bytes || dma->head_idx - dma->tail_idx == dma->buf_count) {
 			if (down_timeout(&dma->buf_sem,
 				msecs_to_jiffies(1000))) {
-				dev_err(dev, "DMA wait timed out");
+				dev_err(dev, "DMA wait timed out\n");
 				ret = -ETIMEDOUT;
 				break;
 			}
@@ -801,7 +829,7 @@ static int rp1_pio_sm_rx_user(struct rp1
 					       DMA_PREP_INTERRUPT | DMA_CTRL_ACK |
 					       DMA_PREP_FENCE);
 		if (!desc) {
-			dev_err(dev, "DMA preparation failed");
+			dev_err(dev, "DMA preparation failed\n");
 			ret = -EIO;
 			break;
 		}
@@ -823,7 +851,7 @@ static int rp1_pio_sm_rx_user(struct rp1
 	return ret;
 }
 
-static int rp1_pio_sm_xfer_data32(struct rp1_pio_client *client, void *param)
+static int rp1_pio_sm_xfer_data32_user(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_xfer_data32_args *args = param;
 	struct rp1_pio_device *pio = client->pio;
@@ -841,7 +869,7 @@ static int rp1_pio_sm_xfer_data32(struct
 		return rp1_pio_sm_rx_user(pio, dma, args->data, args->data_bytes);
 }
 
-static int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, void *param)
+static int rp1_pio_sm_xfer_data_user(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_xfer_data_args *args = param;
 	struct rp1_pio_sm_xfer_data32_args args32;
@@ -851,17 +879,97 @@ static int rp1_pio_sm_xfer_data(struct r
 	args32.data_bytes = args->data_bytes;
 	args32.data = args->data;
 
-	return rp1_pio_sm_xfer_data32(client, &args32);
+	return rp1_pio_sm_xfer_data32_user(client, &args32);
+}
+
+int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir,
+			      uint buf_size, uint buf_count)
+{
+	return rp1_pio_sm_config_xfer_internal(client, sm, dir, buf_size, buf_count);
+}
+EXPORT_SYMBOL_GPL(rp1_pio_sm_config_xfer);
+
+int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir,
+				    uint data_bytes, void *data, dma_addr_t dma_addr,
+				    void (*callback)(void *param), void *param)
+{
+	struct rp1_pio_device *pio = client->pio;
+	struct platform_device *pdev = pio->pdev;
+	struct dma_async_tx_descriptor *desc;
+	struct dma_xfer_state *dxs = NULL;
+	struct device *dev = &pdev->dev;
+	struct dma_buf_info *dbi = NULL;
+	struct scatterlist sg;
+	struct dma_info *dma;
+	int ret = 0;
+
+	if (sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT)
+		return -EINVAL;
+
+	dma = &pio->dma_configs[sm][dir];
+
+	if (!dma_addr) {
+		dxs = kmalloc(sizeof(*dxs), GFP_KERNEL);
+		dxs->dma = dma;
+		dxs->callback = callback;
+		dxs->callback_param = param;
+		callback = rp1_pio_sm_kernel_dma_callback;
+		param = dxs;
+
+		if (!dma->buf_count || data_bytes > dma->buf_size)
+			return -EINVAL;
+
+		/* Grab a dma buffer */
+		if (dma->head_idx - dma->tail_idx == dma->buf_count) {
+			if (down_timeout(&dma->buf_sem, msecs_to_jiffies(1000))) {
+				dev_err(dev, "DMA wait timed out\n");
+				return -ETIMEDOUT;
+			}
+		}
+
+		dbi = &dma->bufs[dma->head_idx % dma->buf_count];
+		dma_addr = dbi->dma_addr;
+
+		if (dir == PIO_DIR_TO_SM)
+			memcpy(dbi->buf, data, data_bytes);
+	}
+
+	sg_init_table(&sg, 1);
+	sg_dma_address(&sg) = dma_addr;
+	sg_dma_len(&sg) = data_bytes;
+
+	desc = dmaengine_prep_slave_sg(dma->chan, &sg, 1,
+					   (dir == PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM,
+					   DMA_PREP_INTERRUPT | DMA_CTRL_ACK |
+					   DMA_PREP_FENCE);
+	if (!desc) {
+		dev_err(dev, "DMA preparation failed\n");
+		return -EIO;
+	}
+
+	desc->callback = callback;
+	desc->callback_param = param;
+
+	ret = dmaengine_submit(desc);
+	if (ret < 0) {
+		dev_err(dev, "dmaengine_submit failed (%d)\n", ret);
+		return ret;
+	}
+
+	dma_async_issue_pending(dma->chan);
+
+	return 0;
 }
+EXPORT_SYMBOL_GPL(rp1_pio_sm_xfer_data);
 
 struct handler_info {
 	const char *name;
 	int (*func)(struct rp1_pio_client *client, void *param);
 	int argsize;
 } ioctl_handlers[] = {
-	HANDLER(SM_CONFIG_XFER, sm_config_xfer),
-	HANDLER(SM_XFER_DATA, sm_xfer_data),
-	HANDLER(SM_XFER_DATA32, sm_xfer_data32),
+	HANDLER(SM_CONFIG_XFER, sm_config_xfer_user),
+	HANDLER(SM_XFER_DATA, sm_xfer_data_user),
+	HANDLER(SM_XFER_DATA32, sm_xfer_data32_user),
 
 	HANDLER(CAN_ADD_PROGRAM, can_add_program),
 	HANDLER(ADD_PROGRAM, add_program),
@@ -902,7 +1010,7 @@ struct handler_info {
 	HANDLER(WRITE_HW, write_hw),
 };
 
-struct rp1_pio_client *pio_open(void)
+struct rp1_pio_client *rp1_pio_open(void)
 {
 	struct rp1_pio_client *client;
 
@@ -914,9 +1022,9 @@ struct rp1_pio_client *pio_open(void)
 
 	return client;
 }
-EXPORT_SYMBOL_GPL(pio_open);
+EXPORT_SYMBOL_GPL(rp1_pio_open);
 
-void pio_close(struct rp1_pio_client *client)
+void rp1_pio_close(struct rp1_pio_client *client)
 {
 	struct rp1_pio_device *pio = client->pio;
 	uint claimed_dmas = client->claimed_dmas;
@@ -958,31 +1066,31 @@ void pio_close(struct rp1_pio_client *cl
 
 	kfree(client);
 }
-EXPORT_SYMBOL_GPL(pio_close);
+EXPORT_SYMBOL_GPL(rp1_pio_close);
 
-void pio_set_error(struct rp1_pio_client *client, int err)
+void rp1_pio_set_error(struct rp1_pio_client *client, int err)
 {
 	client->error = err;
 }
-EXPORT_SYMBOL_GPL(pio_set_error);
+EXPORT_SYMBOL_GPL(rp1_pio_set_error);
 
-int pio_get_error(const struct rp1_pio_client *client)
+int rp1_pio_get_error(const struct rp1_pio_client *client)
 {
 	return client->error;
 }
-EXPORT_SYMBOL_GPL(pio_get_error);
+EXPORT_SYMBOL_GPL(rp1_pio_get_error);
 
-void pio_clear_error(struct rp1_pio_client *client)
+void rp1_pio_clear_error(struct rp1_pio_client *client)
 {
 	client->error = 0;
 }
-EXPORT_SYMBOL_GPL(pio_clear_error);
+EXPORT_SYMBOL_GPL(rp1_pio_clear_error);
 
-static int rp1_pio_open(struct inode *inode, struct file *filp)
+static int rp1_pio_file_open(struct inode *inode, struct file *filp)
 {
 	struct rp1_pio_client *client;
 
-	client = pio_open();
+	client = rp1_pio_open();
 	if (IS_ERR(client))
 		return PTR_ERR(client);
 
@@ -991,11 +1099,11 @@ static int rp1_pio_open(struct inode *in
 	return 0;
 }
 
-static int rp1_pio_release(struct inode *inode, struct file *filp)
+static int rp1_pio_file_release(struct inode *inode, struct file *filp)
 {
 	struct rp1_pio_client *client = filp->private_data;
 
-	pio_close(client);
+	rp1_pio_close(client);
 
 	return 0;
 }
@@ -1082,7 +1190,7 @@ static long rp1_pio_compat_ioctl(struct
 		param.dir = compat_param.dir;
 		param.data_bytes = compat_param.data_bytes;
 		param.data = compat_ptr(compat_param.data);
-		return rp1_pio_sm_xfer_data(client, &param);
+		return rp1_pio_sm_xfer_data_user(client, &param);
 	}
 	case PIO_IOC_SM_XFER_DATA32_COMPAT:
 	{
@@ -1095,7 +1203,7 @@ static long rp1_pio_compat_ioctl(struct
 		param.dir = compat_param.dir;
 		param.data_bytes = compat_param.data_bytes;
 		param.data = compat_ptr(compat_param.data);
-		return rp1_pio_sm_xfer_data32(client, &param);
+		return rp1_pio_sm_xfer_data32_user(client, &param);
 	}
 
 	case PIO_IOC_READ_HW_COMPAT:
@@ -1124,8 +1232,8 @@ static long rp1_pio_compat_ioctl(struct
 
 const struct file_operations rp1_pio_fops = {
 	.owner =	THIS_MODULE,
-	.open =		rp1_pio_open,
-	.release =	rp1_pio_release,
+	.open =		rp1_pio_file_open,
+	.release =	rp1_pio_file_release,
 	.unlocked_ioctl = rp1_pio_ioctl,
 	.compat_ioctl = rp1_pio_compat_ioctl,
 };
--- a/include/linux/pio_rp1.h
+++ b/include/linux/pio_rp1.h
@@ -179,9 +179,17 @@ typedef rp1_pio_sm_config pio_sm_config;
 
 typedef struct rp1_pio_client *PIO;
 
-void pio_set_error(struct rp1_pio_client *client, int err);
-int pio_get_error(const struct rp1_pio_client *client);
-void pio_clear_error(struct rp1_pio_client *client);
+int rp1_pio_init(void);
+PIO rp1_pio_open(void);
+void rp1_pio_close(struct rp1_pio_client *client);
+void rp1_pio_set_error(struct rp1_pio_client *client, int err);
+int rp1_pio_get_error(const struct rp1_pio_client *client);
+void rp1_pio_clear_error(struct rp1_pio_client *client);
+int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir,
+			   uint buf_size, uint buf_count);
+int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir,
+			 uint data_bytes, void *data, dma_addr_t dma_addr,
+			 void (*callback)(void *param), void *param);
 
 int rp1_pio_can_add_program(struct rp1_pio_client *client, void *param);
 int rp1_pio_add_program(struct rp1_pio_client *client, void *param);
@@ -215,12 +223,48 @@ int rp1_pio_gpio_set_oeover(struct rp1_p
 int rp1_pio_gpio_set_input_enabled(struct rp1_pio_client *client, void *param);
 int rp1_pio_gpio_set_drive_strength(struct rp1_pio_client *client, void *param);
 
-int pio_init(void);
-PIO pio_open(void);
-void pio_close(PIO pio);
+static inline int pio_init(void)
+{
+	return rp1_pio_init();
+}
+
+static inline struct rp1_pio_client *pio_open(void)
+{
+	return rp1_pio_open();
+}
+
+static inline void pio_close(struct rp1_pio_client *client)
+{
+	rp1_pio_close(client);
+}
+
+static inline void pio_set_error(struct rp1_pio_client *client, int err)
+{
+	rp1_pio_set_error(client, err);
+}
 
-int pio_sm_config_xfer(PIO pio, uint sm, uint dir, uint buf_size, uint buf_count);
-int pio_sm_xfer_data(PIO pio, uint sm, uint dir, uint data_bytes, void *data);
+static inline int pio_get_error(const struct rp1_pio_client *client)
+{
+	return rp1_pio_get_error(client);
+}
+
+static inline void pio_clear_error(struct rp1_pio_client *client)
+{
+	rp1_pio_clear_error(client);
+}
+
+static inline int pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir,
+				     uint buf_size, uint buf_count)
+{
+	return rp1_pio_sm_config_xfer(client, sm, dir, buf_size, buf_count);
+}
+
+static inline int pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir,
+				   uint data_bytes, void *data, dma_addr_t dma_addr,
+				   void (*callback)(void *param), void *param)
+{
+	return rp1_pio_sm_xfer_data(client, sm, dir, data_bytes, data, dma_addr, callback, param);
+}
 
 static inline struct fp24_8 make_fp24_8(uint mul, uint div)
 {