From fddd3e9318dbf01fb763b6880021abc558fce8e6 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Thu, 12 Dec 2024 17:09:27 +0000 Subject: [PATCH] misc: rp1-pio: Add in-kernel DMA support Add kernel-facing implementations of pio_sm_config_xfer and pio_xm_xfer_data. Signed-off-by: Phil Elwell --- drivers/misc/rp1-pio.c | 208 ++++++++++++++++++++++++++++++---------- include/linux/pio_rp1.h | 60 ++++++++++-- 2 files changed, 210 insertions(+), 58 deletions(-) --- a/drivers/misc/rp1-pio.c +++ b/drivers/misc/rp1-pio.c @@ -61,9 +61,15 @@ #define DMA_BOUNCE_BUFFER_SIZE 0x1000 #define DMA_BOUNCE_BUFFER_COUNT 4 +struct dma_xfer_state { + struct dma_info *dma; + void (*callback)(void *param); + void *callback_param; +}; + struct dma_buf_info { void *buf; - dma_addr_t phys; + dma_addr_t dma_addr; struct scatterlist sgl; }; @@ -572,21 +578,34 @@ static void rp1_pio_sm_dma_callback(void up(&dma->buf_sem); } +static void rp1_pio_sm_kernel_dma_callback(void *param) +{ + struct dma_xfer_state *dxs = param; + + dxs->dma->tail_idx++; + up(&dxs->dma->buf_sem); + + dxs->callback(dxs->callback_param); + + kfree(dxs); +} + static void rp1_pio_sm_dma_free(struct device *dev, struct dma_info *dma) { dmaengine_terminate_all(dma->chan); while (dma->buf_count > 0) { dma->buf_count--; dma_free_coherent(dev, ROUND_UP(dma->buf_size, PAGE_SIZE), - dma->bufs[dma->buf_count].buf, dma->bufs[dma->buf_count].phys); + dma->bufs[dma->buf_count].buf, + dma->bufs[dma->buf_count].dma_addr); } dma_release_channel(dma->chan); } -static int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, void *param) +static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint sm, uint dir, + uint buf_size, uint buf_count) { - struct rp1_pio_sm_config_xfer_args *args = param; struct rp1_pio_sm_set_dmactrl_args set_dmactrl_args; struct rp1_pio_device *pio = client->pio; struct platform_device *pdev = pio->pdev; @@ -596,17 +615,18 @@ static int rp1_pio_sm_config_xfer(struct struct dma_info *dma; uint32_t dma_mask; char chan_name[4]; - uint buf_size; int ret = 0; - if (args->sm >= RP1_PIO_SMS_COUNT || args->dir >= RP1_PIO_DIR_COUNT || - !args->buf_size || (args->buf_size & 3) || - !args->buf_count || args->buf_count > DMA_BOUNCE_BUFFER_COUNT) + if (sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT) + return -EINVAL; + if ((buf_count || buf_size) && + (!buf_size || (buf_size & 3) || + !buf_count || buf_count > DMA_BOUNCE_BUFFER_COUNT)) return -EINVAL; - dma_mask = 1 << (args->sm * 2 + args->dir); + dma_mask = 1 << (sm * 2 + dir); - dma = &pio->dma_configs[args->sm][args->dir]; + dma = &pio->dma_configs[sm][dir]; spin_lock(&pio->lock); if (pio->claimed_dmas & dma_mask) @@ -615,16 +635,16 @@ static int rp1_pio_sm_config_xfer(struct client->claimed_dmas |= dma_mask; spin_unlock(&pio->lock); - dma->buf_size = args->buf_size; + dma->buf_size = buf_size; /* Round up the allocations */ - buf_size = ROUND_UP(args->buf_size, PAGE_SIZE); + buf_size = ROUND_UP(buf_size, PAGE_SIZE); sema_init(&dma->buf_sem, 0); /* Allocate and configure a DMA channel */ /* Careful - each SM FIFO has its own DREQ value */ - chan_name[0] = (args->dir == RP1_PIO_DIR_TO_SM) ? 't' : 'r'; + chan_name[0] = (dir == RP1_PIO_DIR_TO_SM) ? 't' : 'r'; chan_name[1] = 'x'; - chan_name[2] = '0' + args->sm; + chan_name[2] = '0' + sm; chan_name[3] = '\0'; dma->chan = dma_request_chan(dev, chan_name); @@ -632,37 +652,37 @@ static int rp1_pio_sm_config_xfer(struct return PTR_ERR(dma->chan); /* Alloc and map bounce buffers */ - for (dma->buf_count = 0; dma->buf_count < args->buf_count; dma->buf_count++) { + for (dma->buf_count = 0; dma->buf_count < buf_count; dma->buf_count++) { struct dma_buf_info *dbi = &dma->bufs[dma->buf_count]; dbi->buf = dma_alloc_coherent(dma->chan->device->dev, buf_size, - &dbi->phys, GFP_KERNEL); + &dbi->dma_addr, GFP_KERNEL); if (!dbi->buf) { ret = -ENOMEM; goto err_dma_free; } sg_init_table(&dbi->sgl, 1); - sg_dma_address(&dbi->sgl) = dbi->phys; + sg_dma_address(&dbi->sgl) = dbi->dma_addr; } fifo_addr = pio->phys_addr; - fifo_addr += args->sm * (RP1_PIO_FIFO_TX1 - RP1_PIO_FIFO_TX0); - fifo_addr += (args->dir == RP1_PIO_DIR_TO_SM) ? RP1_PIO_FIFO_TX0 : RP1_PIO_FIFO_RX0; + fifo_addr += sm * (RP1_PIO_FIFO_TX1 - RP1_PIO_FIFO_TX0); + fifo_addr += (dir == RP1_PIO_DIR_TO_SM) ? RP1_PIO_FIFO_TX0 : RP1_PIO_FIFO_RX0; config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; config.src_addr = fifo_addr; config.dst_addr = fifo_addr; - config.direction = (args->dir == RP1_PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; + config.direction = (dir == RP1_PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; ret = dmaengine_slave_config(dma->chan, &config); if (ret) goto err_dma_free; - set_dmactrl_args.sm = args->sm; - set_dmactrl_args.is_tx = (args->dir == RP1_PIO_DIR_TO_SM); + set_dmactrl_args.sm = sm; + set_dmactrl_args.is_tx = (dir == RP1_PIO_DIR_TO_SM); set_dmactrl_args.ctrl = RP1_PIO_DMACTRL_DEFAULT; - if (args->dir == RP1_PIO_DIR_FROM_SM) + if (dir == RP1_PIO_DIR_FROM_SM) set_dmactrl_args.ctrl = (RP1_PIO_DMACTRL_DEFAULT & ~0x1f) | 1; ret = rp1_pio_sm_set_dmactrl(client, &set_dmactrl_args); @@ -682,6 +702,14 @@ err_dma_free: return ret; } +static int rp1_pio_sm_config_xfer_user(struct rp1_pio_client *client, void *param) +{ + struct rp1_pio_sm_config_xfer_args *args = param; + + return rp1_pio_sm_config_xfer_internal(client, args->sm, args->dir, + args->buf_size, args->buf_count); +} + static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma, const void __user *userbuf, size_t bytes) { @@ -723,7 +751,7 @@ static int rp1_pio_sm_tx_user(struct rp1 DMA_PREP_INTERRUPT | DMA_CTRL_ACK | DMA_PREP_FENCE); if (!desc) { - dev_err(dev, "DMA preparation failedzn"); + dev_err(dev, "DMA preparation failed\n"); ret = -EIO; break; } @@ -779,7 +807,7 @@ static int rp1_pio_sm_rx_user(struct rp1 if (!bytes || dma->head_idx - dma->tail_idx == dma->buf_count) { if (down_timeout(&dma->buf_sem, msecs_to_jiffies(1000))) { - dev_err(dev, "DMA wait timed out"); + dev_err(dev, "DMA wait timed out\n"); ret = -ETIMEDOUT; break; } @@ -801,7 +829,7 @@ static int rp1_pio_sm_rx_user(struct rp1 DMA_PREP_INTERRUPT | DMA_CTRL_ACK | DMA_PREP_FENCE); if (!desc) { - dev_err(dev, "DMA preparation failed"); + dev_err(dev, "DMA preparation failed\n"); ret = -EIO; break; } @@ -823,7 +851,7 @@ static int rp1_pio_sm_rx_user(struct rp1 return ret; } -static int rp1_pio_sm_xfer_data32(struct rp1_pio_client *client, void *param) +static int rp1_pio_sm_xfer_data32_user(struct rp1_pio_client *client, void *param) { struct rp1_pio_sm_xfer_data32_args *args = param; struct rp1_pio_device *pio = client->pio; @@ -841,7 +869,7 @@ static int rp1_pio_sm_xfer_data32(struct return rp1_pio_sm_rx_user(pio, dma, args->data, args->data_bytes); } -static int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, void *param) +static int rp1_pio_sm_xfer_data_user(struct rp1_pio_client *client, void *param) { struct rp1_pio_sm_xfer_data_args *args = param; struct rp1_pio_sm_xfer_data32_args args32; @@ -851,17 +879,97 @@ static int rp1_pio_sm_xfer_data(struct r args32.data_bytes = args->data_bytes; args32.data = args->data; - return rp1_pio_sm_xfer_data32(client, &args32); + return rp1_pio_sm_xfer_data32_user(client, &args32); +} + +int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir, + uint buf_size, uint buf_count) +{ + return rp1_pio_sm_config_xfer_internal(client, sm, dir, buf_size, buf_count); +} +EXPORT_SYMBOL_GPL(rp1_pio_sm_config_xfer); + +int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir, + uint data_bytes, void *data, dma_addr_t dma_addr, + void (*callback)(void *param), void *param) +{ + struct rp1_pio_device *pio = client->pio; + struct platform_device *pdev = pio->pdev; + struct dma_async_tx_descriptor *desc; + struct dma_xfer_state *dxs = NULL; + struct device *dev = &pdev->dev; + struct dma_buf_info *dbi = NULL; + struct scatterlist sg; + struct dma_info *dma; + int ret = 0; + + if (sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT) + return -EINVAL; + + dma = &pio->dma_configs[sm][dir]; + + if (!dma_addr) { + dxs = kmalloc(sizeof(*dxs), GFP_KERNEL); + dxs->dma = dma; + dxs->callback = callback; + dxs->callback_param = param; + callback = rp1_pio_sm_kernel_dma_callback; + param = dxs; + + if (!dma->buf_count || data_bytes > dma->buf_size) + return -EINVAL; + + /* Grab a dma buffer */ + if (dma->head_idx - dma->tail_idx == dma->buf_count) { + if (down_timeout(&dma->buf_sem, msecs_to_jiffies(1000))) { + dev_err(dev, "DMA wait timed out\n"); + return -ETIMEDOUT; + } + } + + dbi = &dma->bufs[dma->head_idx % dma->buf_count]; + dma_addr = dbi->dma_addr; + + if (dir == PIO_DIR_TO_SM) + memcpy(dbi->buf, data, data_bytes); + } + + sg_init_table(&sg, 1); + sg_dma_address(&sg) = dma_addr; + sg_dma_len(&sg) = data_bytes; + + desc = dmaengine_prep_slave_sg(dma->chan, &sg, 1, + (dir == PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK | + DMA_PREP_FENCE); + if (!desc) { + dev_err(dev, "DMA preparation failed\n"); + return -EIO; + } + + desc->callback = callback; + desc->callback_param = param; + + ret = dmaengine_submit(desc); + if (ret < 0) { + dev_err(dev, "dmaengine_submit failed (%d)\n", ret); + return ret; + } + + dma_async_issue_pending(dma->chan); + + return 0; } +EXPORT_SYMBOL_GPL(rp1_pio_sm_xfer_data); struct handler_info { const char *name; int (*func)(struct rp1_pio_client *client, void *param); int argsize; } ioctl_handlers[] = { - HANDLER(SM_CONFIG_XFER, sm_config_xfer), - HANDLER(SM_XFER_DATA, sm_xfer_data), - HANDLER(SM_XFER_DATA32, sm_xfer_data32), + HANDLER(SM_CONFIG_XFER, sm_config_xfer_user), + HANDLER(SM_XFER_DATA, sm_xfer_data_user), + HANDLER(SM_XFER_DATA32, sm_xfer_data32_user), HANDLER(CAN_ADD_PROGRAM, can_add_program), HANDLER(ADD_PROGRAM, add_program), @@ -902,7 +1010,7 @@ struct handler_info { HANDLER(WRITE_HW, write_hw), }; -struct rp1_pio_client *pio_open(void) +struct rp1_pio_client *rp1_pio_open(void) { struct rp1_pio_client *client; @@ -914,9 +1022,9 @@ struct rp1_pio_client *pio_open(void) return client; } -EXPORT_SYMBOL_GPL(pio_open); +EXPORT_SYMBOL_GPL(rp1_pio_open); -void pio_close(struct rp1_pio_client *client) +void rp1_pio_close(struct rp1_pio_client *client) { struct rp1_pio_device *pio = client->pio; uint claimed_dmas = client->claimed_dmas; @@ -958,31 +1066,31 @@ void pio_close(struct rp1_pio_client *cl kfree(client); } -EXPORT_SYMBOL_GPL(pio_close); +EXPORT_SYMBOL_GPL(rp1_pio_close); -void pio_set_error(struct rp1_pio_client *client, int err) +void rp1_pio_set_error(struct rp1_pio_client *client, int err) { client->error = err; } -EXPORT_SYMBOL_GPL(pio_set_error); +EXPORT_SYMBOL_GPL(rp1_pio_set_error); -int pio_get_error(const struct rp1_pio_client *client) +int rp1_pio_get_error(const struct rp1_pio_client *client) { return client->error; } -EXPORT_SYMBOL_GPL(pio_get_error); +EXPORT_SYMBOL_GPL(rp1_pio_get_error); -void pio_clear_error(struct rp1_pio_client *client) +void rp1_pio_clear_error(struct rp1_pio_client *client) { client->error = 0; } -EXPORT_SYMBOL_GPL(pio_clear_error); +EXPORT_SYMBOL_GPL(rp1_pio_clear_error); -static int rp1_pio_open(struct inode *inode, struct file *filp) +static int rp1_pio_file_open(struct inode *inode, struct file *filp) { struct rp1_pio_client *client; - client = pio_open(); + client = rp1_pio_open(); if (IS_ERR(client)) return PTR_ERR(client); @@ -991,11 +1099,11 @@ static int rp1_pio_open(struct inode *in return 0; } -static int rp1_pio_release(struct inode *inode, struct file *filp) +static int rp1_pio_file_release(struct inode *inode, struct file *filp) { struct rp1_pio_client *client = filp->private_data; - pio_close(client); + rp1_pio_close(client); return 0; } @@ -1082,7 +1190,7 @@ static long rp1_pio_compat_ioctl(struct param.dir = compat_param.dir; param.data_bytes = compat_param.data_bytes; param.data = compat_ptr(compat_param.data); - return rp1_pio_sm_xfer_data(client, ¶m); + return rp1_pio_sm_xfer_data_user(client, ¶m); } case PIO_IOC_SM_XFER_DATA32_COMPAT: { @@ -1095,7 +1203,7 @@ static long rp1_pio_compat_ioctl(struct param.dir = compat_param.dir; param.data_bytes = compat_param.data_bytes; param.data = compat_ptr(compat_param.data); - return rp1_pio_sm_xfer_data32(client, ¶m); + return rp1_pio_sm_xfer_data32_user(client, ¶m); } case PIO_IOC_READ_HW_COMPAT: @@ -1124,8 +1232,8 @@ static long rp1_pio_compat_ioctl(struct const struct file_operations rp1_pio_fops = { .owner = THIS_MODULE, - .open = rp1_pio_open, - .release = rp1_pio_release, + .open = rp1_pio_file_open, + .release = rp1_pio_file_release, .unlocked_ioctl = rp1_pio_ioctl, .compat_ioctl = rp1_pio_compat_ioctl, }; --- a/include/linux/pio_rp1.h +++ b/include/linux/pio_rp1.h @@ -179,9 +179,17 @@ typedef rp1_pio_sm_config pio_sm_config; typedef struct rp1_pio_client *PIO; -void pio_set_error(struct rp1_pio_client *client, int err); -int pio_get_error(const struct rp1_pio_client *client); -void pio_clear_error(struct rp1_pio_client *client); +int rp1_pio_init(void); +PIO rp1_pio_open(void); +void rp1_pio_close(struct rp1_pio_client *client); +void rp1_pio_set_error(struct rp1_pio_client *client, int err); +int rp1_pio_get_error(const struct rp1_pio_client *client); +void rp1_pio_clear_error(struct rp1_pio_client *client); +int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir, + uint buf_size, uint buf_count); +int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir, + uint data_bytes, void *data, dma_addr_t dma_addr, + void (*callback)(void *param), void *param); int rp1_pio_can_add_program(struct rp1_pio_client *client, void *param); int rp1_pio_add_program(struct rp1_pio_client *client, void *param); @@ -215,12 +223,48 @@ int rp1_pio_gpio_set_oeover(struct rp1_p int rp1_pio_gpio_set_input_enabled(struct rp1_pio_client *client, void *param); int rp1_pio_gpio_set_drive_strength(struct rp1_pio_client *client, void *param); -int pio_init(void); -PIO pio_open(void); -void pio_close(PIO pio); +static inline int pio_init(void) +{ + return rp1_pio_init(); +} + +static inline struct rp1_pio_client *pio_open(void) +{ + return rp1_pio_open(); +} + +static inline void pio_close(struct rp1_pio_client *client) +{ + rp1_pio_close(client); +} + +static inline void pio_set_error(struct rp1_pio_client *client, int err) +{ + rp1_pio_set_error(client, err); +} -int pio_sm_config_xfer(PIO pio, uint sm, uint dir, uint buf_size, uint buf_count); -int pio_sm_xfer_data(PIO pio, uint sm, uint dir, uint data_bytes, void *data); +static inline int pio_get_error(const struct rp1_pio_client *client) +{ + return rp1_pio_get_error(client); +} + +static inline void pio_clear_error(struct rp1_pio_client *client) +{ + rp1_pio_clear_error(client); +} + +static inline int pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir, + uint buf_size, uint buf_count) +{ + return rp1_pio_sm_config_xfer(client, sm, dir, buf_size, buf_count); +} + +static inline int pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir, + uint data_bytes, void *data, dma_addr_t dma_addr, + void (*callback)(void *param), void *param) +{ + return rp1_pio_sm_xfer_data(client, sm, dir, data_bytes, data, dma_addr, callback, param); +} static inline struct fp24_8 make_fp24_8(uint mul, uint div) {