// SPDX-License-Identifier: GPL-2.0-or-later/* * DMA driver for Altera mSGDMA IP core * * Copyright (C) 2017 Stefan Roese <sr@denx.de> * * Based on drivers/dma/xilinx/zynqmp_dma.c, which is: * Copyright (C) 2016 Xilinx, Inc. All rights reserved. */ #include <linux/bitops.h>#include <linux/delay.h>#include <linux/dma-mapping.h>#include <linux/dmapool.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/io.h>#include <linux/iopoll.h>#include <linux/module.h>#include <linux/platform_device.h>#include <linux/slab.h>#include <linux/of_dma.h> #include "dmaengine.h" #define MSGDMA_MAX_TRANS_LEN U32_MAX#define MSGDMA_DESC_NUM 1024 /** * struct msgdma_extended_desc - implements an extended descriptor * @read_addr_lo: data buffer source address low bits * @write_addr_lo: data buffer destination address low bits * @len: the number of bytes to transfer per descriptor * @burst_seq_num: bit 31:24 write burst * bit 23:16 read burst * bit 15:00 sequence number * @stride: bit 31:16 write stride * bit 15:00 read stride * @read_addr_hi: data buffer source address high bits * @write_addr_hi: data buffer destination address high bits * @control: characteristics of the transfer */struct msgdma_extended_desc { u32 read_addr_lo; u32 write_addr_lo; u32 len; u32 burst_seq_num; u32 stride; u32 read_addr_hi; u32 write_addr_hi; u32 control;}; /* mSGDMA descriptor control field bit definitions */#define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff)#define MSGDMA_DESC_CTL_GEN_SOP BIT(8)#define MSGDMA_DESC_CTL_GEN_EOP BIT(9)#define MSGDMA_DESC_CTL_PARK_READS BIT(10)#define MSGDMA_DESC_CTL_PARK_WRITES BIT(11)#define MSGDMA_DESC_CTL_END_ON_EOP BIT(12)#define MSGDMA_DESC_CTL_END_ON_LEN BIT(13)#define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14)#define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15)#define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16)#define MSGDMA_DESC_CTL_EARLY_DONE BIT(24) /* * Writing "1" the "go" bit commits the entire descriptor into the * descriptor FIFO(s) */#define MSGDMA_DESC_CTL_GO BIT(31) /* Tx buffer control flags */#define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \ MSGDMA_DESC_CTL_TR_COMP_IRQ | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \ MSGDMA_DESC_CTL_GEN_EOP | \ MSGDMA_DESC_CTL_TR_COMP_IRQ | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \ MSGDMA_DESC_CTL_END_ON_LEN | \ MSGDMA_DESC_CTL_TR_COMP_IRQ | \ MSGDMA_DESC_CTL_EARLY_IRQ | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) /* mSGDMA extended descriptor stride definitions */#define MSGDMA_DESC_STRIDE_RD 0x00000001#define MSGDMA_DESC_STRIDE_WR 0x00010000#define MSGDMA_DESC_STRIDE_RW 0x00010001 /* mSGDMA dispatcher control and status register map */#define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */#define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */#define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */ /* 15:00 - read fill level */#define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */#define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */ /* 15:00 - read seq number */ /* mSGDMA CSR status register bit definitions */#define MSGDMA_CSR_STAT_BUSY BIT(0)#define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1)#define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2)#define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3)#define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4)#define MSGDMA_CSR_STAT_STOPPED BIT(5)#define MSGDMA_CSR_STAT_RESETTING BIT(6)#define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7)#define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8)#define MSGDMA_CSR_STAT_IRQ BIT(9)#define MSGDMA_CSR_STAT_MASK GENMASK(9, 0)#define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0) #define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \ MSGDMA_CSR_STAT_RESP_BUF_EMPTY) /* mSGDMA CSR control register bit definitions */#define MSGDMA_CSR_CTL_STOP BIT(0)#define MSGDMA_CSR_CTL_RESET BIT(1)#define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2)#define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3)#define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4)#define MSGDMA_CSR_CTL_STOP_DESCS BIT(5) /* mSGDMA CSR fill level bits */#define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16)#define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)#define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) #define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16) /* mSGDMA response register map */#define MSGDMA_RESP_BYTES_TRANSFERRED 0x00#define MSGDMA_RESP_STATUS 0x04 /* mSGDMA response register bit definitions */#define MSGDMA_RESP_EARLY_TERM BIT(8)#define MSGDMA_RESP_ERR_MASK 0xff /** * struct msgdma_sw_desc - implements a sw descriptor * @async_tx: support for the async_tx api * @hw_desc: associated HW descriptor * @node: node to move from the free list to the tx list * @tx_list: transmit list node */struct msgdma_sw_desc { struct dma_async_tx_descriptor async_tx; struct msgdma_extended_desc hw_desc; struct list_head node; struct list_head tx_list;}; /* * struct msgdma_device - DMA device structure */struct msgdma_device { spinlock_t lock; struct device *dev; struct tasklet_struct irq_tasklet; struct list_head pending_list; struct list_head free_list; struct list_head active_list; struct list_head done_list; u32 desc_free_cnt; bool idle; struct dma_device dmadev; struct dma_chan dmachan; dma_addr_t hw_desq; struct msgdma_sw_desc *sw_desq; unsigned int npendings; struct dma_slave_config slave_cfg; int irq; /* mSGDMA controller */ void __iomem *csr; /* mSGDMA descriptors */ void __iomem *desc; /* mSGDMA response */ void __iomem *resp;}; #define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan)#define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx) /** * msgdma_get_descriptor - Get the sw descriptor from the pool * @mdev: Pointer to the Altera mSGDMA device structure * * Return: The sw descriptor */static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev){ struct msgdma_sw_desc *desc; unsigned long flags; spin_lock_irqsave(&mdev->lock, flags); desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node); list_del(&desc->node); spin_unlock_irqrestore(&mdev->lock, flags); INIT_LIST_HEAD(&desc->tx_list); return desc;} /** * msgdma_free_descriptor - Issue pending transactions * @mdev: Pointer to the Altera mSGDMA device structure * @desc: Transaction descriptor pointer */static void msgdma_free_descriptor(struct msgdma_device *mdev, struct msgdma_sw_desc *desc){ struct msgdma_sw_desc *child, *next; mdev->desc_free_cnt++; list_move_tail(&desc->node, &mdev->free_list); list_for_each_entry_safe(child, next, &desc->tx_list, node) { mdev->desc_free_cnt++; list_move_tail(&child->node, &mdev->free_list); }} /** * msgdma_free_desc_list - Free descriptors list * @mdev: Pointer to the Altera mSGDMA device structure * @list: List to parse and delete the descriptor */static void msgdma_free_desc_list(struct msgdma_device *mdev, struct list_head *list){ struct msgdma_sw_desc *desc, *next; list_for_each_entry_safe(desc, next, list, node) msgdma_free_descriptor(mdev, desc);} /** * msgdma_desc_config - Configure the descriptor * @desc: Hw descriptor pointer * @dst: Destination buffer address * @src: Source buffer address * @len: Transfer length * @stride: Read/write stride value to set */static void msgdma_desc_config(struct msgdma_extended_desc *desc, dma_addr_t dst, dma_addr_t src, size_t len, u32 stride){ /* Set lower 32bits of src & dst addresses in the descriptor */ desc->read_addr_lo = lower_32_bits(src); desc->write_addr_lo = lower_32_bits(dst); /* Set upper 32bits of src & dst addresses in the descriptor */ desc->read_addr_hi = upper_32_bits(src); desc->write_addr_hi = upper_32_bits(dst); desc->len = len; desc->stride = stride; desc->burst_seq_num = 0; /* 0 will result in max burst length */ /* * Don't set interrupt on xfer end yet, this will be done later * for the "last" descriptor */ desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO | MSGDMA_DESC_CTL_END_ON_LEN;} /** * msgdma_desc_config_eod - Mark the descriptor as end descriptor * @desc: Hw descriptor pointer */static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc){ desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ;} /** * msgdma_tx_submit - Submit DMA transaction * @tx: Async transaction descriptor pointer * * Return: cookie value */static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx){ struct msgdma_device *mdev = to_mdev(tx->chan); struct msgdma_sw_desc *new; dma_cookie_t cookie; unsigned long flags; new = tx_to_desc(tx); spin_lock_irqsave(&mdev->lock, flags); cookie = dma_cookie_assign(tx); list_add_tail(&new->node, &mdev->pending_list); spin_unlock_irqrestore(&mdev->lock, flags); return cookie;} /** * msgdma_prep_memcpy - prepare descriptors for memcpy transaction * @dchan: DMA channel * @dma_dst: Destination buffer address * @dma_src: Source buffer address * @len: Transfer length * @flags: transfer ack flags * * Return: Async transaction descriptor on success and NULL on failure */static struct dma_async_tx_descriptor *msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len, ulong flags){ struct msgdma_device *mdev = to_mdev(dchan); struct msgdma_sw_desc *new, *first = NULL; struct msgdma_extended_desc *desc; size_t copy; u32 desc_cnt; unsigned long irqflags; desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN); spin_lock_irqsave(&mdev->lock, irqflags); if (desc_cnt > mdev->desc_free_cnt) { spin_unlock_irqrestore(&mdev->lock, irqflags); dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); return NULL; } mdev->desc_free_cnt -= desc_cnt; spin_unlock_irqrestore(&mdev->lock, irqflags); do { /* Allocate and populate the descriptor */ new = msgdma_get_descriptor(mdev); copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN); desc = &new->hw_desc; msgdma_desc_config(desc, dma_dst, dma_src, copy, MSGDMA_DESC_STRIDE_RW); len -= copy; dma_src += copy; dma_dst += copy; if (!first) first = new; else list_add_tail(&new->node, &first->tx_list); } while (len); msgdma_desc_config_eod(desc); async_tx_ack(&first->async_tx); first->async_tx.flags = flags; return &first->async_tx;} /** * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction * * @dchan: DMA channel * @sgl: Destination scatter list * @sg_len: Number of entries in destination scatter list * @dir: DMA transfer direction * @flags: transfer ack flags * @context: transfer context (unused) */static struct dma_async_tx_descriptor *msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { struct msgdma_device *mdev = to_mdev(dchan); struct dma_slave_config *cfg = &mdev->slave_cfg; struct msgdma_sw_desc *new, *first = NULL; void *desc = NULL; size_t len, avail; dma_addr_t dma_dst, dma_src; u32 desc_cnt; u32 stride; unsigned long irqflags; desc_cnt = sg_nents_for_dma(sgl, sg_len, MSGDMA_MAX_TRANS_LEN); spin_lock_irqsave(&mdev->lock, irqflags); if (desc_cnt > mdev->desc_free_cnt) { spin_unlock_irqrestore(&mdev->lock, irqflags); dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); return NULL; } mdev->desc_free_cnt -= desc_cnt; spin_unlock_irqrestore(&mdev->lock, irqflags); avail = sg_dma_len(sgl); /* Run until we are out of scatterlist entries */ while (true) { /* Allocate and populate the descriptor */ new = msgdma_get_descriptor(mdev); desc = &new->hw_desc; len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN); if (dir == DMA_MEM_TO_DEV) { dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; dma_dst = cfg->dst_addr; stride = MSGDMA_DESC_STRIDE_RD; } else { dma_src = cfg->src_addr; dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; stride = MSGDMA_DESC_STRIDE_WR; } msgdma_desc_config(desc, dma_dst, dma_src, len, stride); avail -= len; if (!first) first = new; else list_add_tail(&new->node, &first->tx_list); /* Fetch the next scatterlist entry */ if (avail == 0) { if (sg_len == 0) break; sgl = sg_next(sgl); if (sgl == NULL) break; sg_len--; avail = sg_dma_len(sgl); } } msgdma_desc_config_eod(desc); first->async_tx.flags = flags; return &first->async_tx;} static int msgdma_dma_config(struct dma_chan *dchan, struct dma_slave_config *config){ struct msgdma_device *mdev = to_mdev(dchan); memcpy(&mdev->slave_cfg, config, sizeof(*config)); return 0;} static void msgdma_reset(struct msgdma_device *mdev){ u32 val; int ret; /* Reset mSGDMA */ iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL); ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val, (val & MSGDMA_CSR_STAT_RESETTING) == 0, 1, 10000); if (ret) dev_err(mdev->dev, "DMA channel did not reset\n"); /* Clear all status bits */ iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); /* Enable the DMA controller including interrupts */ iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY | MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL); mdev->idle = true;}; static void msgdma_copy_one(struct msgdma_device *mdev, struct msgdma_sw_desc *desc){ void __iomem *hw_desc = mdev->desc; /* Ensure control is the last field — required for correct FIFO flush ordering */ static_assert(offsetof(struct msgdma_extended_desc, control) ==Lines 1–500 of 969. Load more
// SPDX-License-Identifier: GPL-2.0-or-later /* * DMA driver for Altera mSGDMA IP core * * Copyright (C) 2017 Stefan Roese <sr@denx.de> * * Based on drivers/dma/xilinx/zynqmp_dma.c, which is: * Copyright (C) 2016 Xilinx, Inc. All rights reserved. */ #include <linux/bitops.h> #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/of_dma.h> #include "dmaengine.h" #define MSGDMA_MAX_TRANS_LEN U32_MAX #define MSGDMA_DESC_NUM 1024 /** * struct msgdma_extended_desc - implements an extended descriptor * @read_addr_lo: data buffer source address low bits * @write_addr_lo: data buffer destination address low bits * @len: the number of bytes to transfer per descriptor * @burst_seq_num: bit 31:24 write burst * bit 23:16 read burst * bit 15:00 sequence number * @stride: bit 31:16 write stride * bit 15:00 read stride * @read_addr_hi: data buffer source address high bits * @write_addr_hi: data buffer destination address high bits * @control: characteristics of the transfer */ struct msgdma_extended_desc { u32 read_addr_lo; u32 write_addr_lo; u32 len; u32 burst_seq_num; u32 stride; u32 read_addr_hi; u32 write_addr_hi; u32 control; }; /* mSGDMA descriptor control field bit definitions */ #define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff) #define MSGDMA_DESC_CTL_GEN_SOP BIT(8) #define MSGDMA_DESC_CTL_GEN_EOP BIT(9) #define MSGDMA_DESC_CTL_PARK_READS BIT(10) #define MSGDMA_DESC_CTL_PARK_WRITES BIT(11) #define MSGDMA_DESC_CTL_END_ON_EOP BIT(12) #define MSGDMA_DESC_CTL_END_ON_LEN BIT(13) #define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14) #define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15) #define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16) #define MSGDMA_DESC_CTL_EARLY_DONE BIT(24) /* * Writing "1" the "go" bit commits the entire descriptor into the * descriptor FIFO(s) */ #define MSGDMA_DESC_CTL_GO BIT(31) /* Tx buffer control flags */ #define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \ MSGDMA_DESC_CTL_TR_COMP_IRQ | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \ MSGDMA_DESC_CTL_GEN_EOP | \ MSGDMA_DESC_CTL_TR_COMP_IRQ | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) #define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \ MSGDMA_DESC_CTL_END_ON_LEN | \ MSGDMA_DESC_CTL_TR_COMP_IRQ | \ MSGDMA_DESC_CTL_EARLY_IRQ | \ MSGDMA_DESC_CTL_TR_ERR_IRQ | \ MSGDMA_DESC_CTL_GO) /* mSGDMA extended descriptor stride definitions */ #define MSGDMA_DESC_STRIDE_RD 0x00000001 #define MSGDMA_DESC_STRIDE_WR 0x00010000 #define MSGDMA_DESC_STRIDE_RW 0x00010001 /* mSGDMA dispatcher control and status register map */ #define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */ #define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */ #define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */ /* 15:00 - read fill level */ #define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */ #define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */ /* 15:00 - read seq number */ /* mSGDMA CSR status register bit definitions */ #define MSGDMA_CSR_STAT_BUSY BIT(0) #define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1) #define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2) #define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3) #define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4) #define MSGDMA_CSR_STAT_STOPPED BIT(5) #define MSGDMA_CSR_STAT_RESETTING BIT(6) #define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7) #define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8) #define MSGDMA_CSR_STAT_IRQ BIT(9) #define MSGDMA_CSR_STAT_MASK GENMASK(9, 0) #define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0) #define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \ MSGDMA_CSR_STAT_RESP_BUF_EMPTY) /* mSGDMA CSR control register bit definitions */ #define MSGDMA_CSR_CTL_STOP BIT(0) #define MSGDMA_CSR_CTL_RESET BIT(1) #define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2) #define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3) #define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4) #define MSGDMA_CSR_CTL_STOP_DESCS BIT(5) /* mSGDMA CSR fill level bits */ #define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16) #define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) #define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) #define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16) /* mSGDMA response register map */ #define MSGDMA_RESP_BYTES_TRANSFERRED 0x00 #define MSGDMA_RESP_STATUS 0x04 /* mSGDMA response register bit definitions */ #define MSGDMA_RESP_EARLY_TERM BIT(8) #define MSGDMA_RESP_ERR_MASK 0xff /** * struct msgdma_sw_desc - implements a sw descriptor * @async_tx: support for the async_tx api * @hw_desc: associated HW descriptor * @node: node to move from the free list to the tx list * @tx_list: transmit list node */ struct msgdma_sw_desc { struct dma_async_tx_descriptor async_tx; struct msgdma_extended_desc hw_desc; struct list_head node; struct list_head tx_list; }; /* * struct msgdma_device - DMA device structure */ struct msgdma_device { spinlock_t lock; struct device *dev; struct tasklet_struct irq_tasklet; struct list_head pending_list; struct list_head free_list; struct list_head active_list; struct list_head done_list; u32 desc_free_cnt; bool idle; struct dma_device dmadev; struct dma_chan dmachan; dma_addr_t hw_desq; struct msgdma_sw_desc *sw_desq; unsigned int npendings; struct dma_slave_config slave_cfg; int irq; /* mSGDMA controller */ void __iomem *csr; /* mSGDMA descriptors */ void __iomem *desc; /* mSGDMA response */ void __iomem *resp; }; #define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan) #define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx) /** * msgdma_get_descriptor - Get the sw descriptor from the pool * @mdev: Pointer to the Altera mSGDMA device structure * * Return: The sw descriptor */ static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev) { struct msgdma_sw_desc *desc; unsigned long flags; spin_lock_irqsave(&mdev->lock, flags); desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node); list_del(&desc->node); spin_unlock_irqrestore(&mdev->lock, flags); INIT_LIST_HEAD(&desc->tx_list); return desc; } /** * msgdma_free_descriptor - Issue pending transactions * @mdev: Pointer to the Altera mSGDMA device structure * @desc: Transaction descriptor pointer */ static void msgdma_free_descriptor(struct msgdma_device *mdev, struct msgdma_sw_desc *desc) { struct msgdma_sw_desc *child, *next; mdev->desc_free_cnt++; list_move_tail(&desc->node, &mdev->free_list); list_for_each_entry_safe(child, next, &desc->tx_list, node) { mdev->desc_free_cnt++; list_move_tail(&child->node, &mdev->free_list); } } /** * msgdma_free_desc_list - Free descriptors list * @mdev: Pointer to the Altera mSGDMA device structure * @list: List to parse and delete the descriptor */ static void msgdma_free_desc_list(struct msgdma_device *mdev, struct list_head *list) { struct msgdma_sw_desc *desc, *next; list_for_each_entry_safe(desc, next, list, node) msgdma_free_descriptor(mdev, desc); } /** * msgdma_desc_config - Configure the descriptor * @desc: Hw descriptor pointer * @dst: Destination buffer address * @src: Source buffer address * @len: Transfer length * @stride: Read/write stride value to set */ static void msgdma_desc_config(struct msgdma_extended_desc *desc, dma_addr_t dst, dma_addr_t src, size_t len, u32 stride) { /* Set lower 32bits of src & dst addresses in the descriptor */ desc->read_addr_lo = lower_32_bits(src); desc->write_addr_lo = lower_32_bits(dst); /* Set upper 32bits of src & dst addresses in the descriptor */ desc->read_addr_hi = upper_32_bits(src); desc->write_addr_hi = upper_32_bits(dst); desc->len = len; desc->stride = stride; desc->burst_seq_num = 0; /* 0 will result in max burst length */ /* * Don't set interrupt on xfer end yet, this will be done later * for the "last" descriptor */ desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO | MSGDMA_DESC_CTL_END_ON_LEN; } /** * msgdma_desc_config_eod - Mark the descriptor as end descriptor * @desc: Hw descriptor pointer */ static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc) { desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ; } /** * msgdma_tx_submit - Submit DMA transaction * @tx: Async transaction descriptor pointer * * Return: cookie value */ static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx) { struct msgdma_device *mdev = to_mdev(tx->chan); struct msgdma_sw_desc *new; dma_cookie_t cookie; unsigned long flags; new = tx_to_desc(tx); spin_lock_irqsave(&mdev->lock, flags); cookie = dma_cookie_assign(tx); list_add_tail(&new->node, &mdev->pending_list); spin_unlock_irqrestore(&mdev->lock, flags); return cookie; } /** * msgdma_prep_memcpy - prepare descriptors for memcpy transaction * @dchan: DMA channel * @dma_dst: Destination buffer address * @dma_src: Source buffer address * @len: Transfer length * @flags: transfer ack flags * * Return: Async transaction descriptor on success and NULL on failure */ static struct dma_async_tx_descriptor * msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len, ulong flags) { struct msgdma_device *mdev = to_mdev(dchan); struct msgdma_sw_desc *new, *first = NULL; struct msgdma_extended_desc *desc; size_t copy; u32 desc_cnt; unsigned long irqflags; desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN); spin_lock_irqsave(&mdev->lock, irqflags); if (desc_cnt > mdev->desc_free_cnt) { spin_unlock_irqrestore(&mdev->lock, irqflags); dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); return NULL; } mdev->desc_free_cnt -= desc_cnt; spin_unlock_irqrestore(&mdev->lock, irqflags); do { /* Allocate and populate the descriptor */ new = msgdma_get_descriptor(mdev); copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN); desc = &new->hw_desc; msgdma_desc_config(desc, dma_dst, dma_src, copy, MSGDMA_DESC_STRIDE_RW); len -= copy; dma_src += copy; dma_dst += copy; if (!first) first = new; else list_add_tail(&new->node, &first->tx_list); } while (len); msgdma_desc_config_eod(desc); async_tx_ack(&first->async_tx); first->async_tx.flags = flags; return &first->async_tx; } /** * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction * * @dchan: DMA channel * @sgl: Destination scatter list * @sg_len: Number of entries in destination scatter list * @dir: DMA transfer direction * @flags: transfer ack flags * @context: transfer context (unused) */ static struct dma_async_tx_descriptor * msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { struct msgdma_device *mdev = to_mdev(dchan); struct dma_slave_config *cfg = &mdev->slave_cfg; struct msgdma_sw_desc *new, *first = NULL; void *desc = NULL; size_t len, avail; dma_addr_t dma_dst, dma_src; u32 desc_cnt; u32 stride; unsigned long irqflags; desc_cnt = sg_nents_for_dma(sgl, sg_len, MSGDMA_MAX_TRANS_LEN); spin_lock_irqsave(&mdev->lock, irqflags); if (desc_cnt > mdev->desc_free_cnt) { spin_unlock_irqrestore(&mdev->lock, irqflags); dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); return NULL; } mdev->desc_free_cnt -= desc_cnt; spin_unlock_irqrestore(&mdev->lock, irqflags); avail = sg_dma_len(sgl); /* Run until we are out of scatterlist entries */ while (true) { /* Allocate and populate the descriptor */ new = msgdma_get_descriptor(mdev); desc = &new->hw_desc; len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN); if (dir == DMA_MEM_TO_DEV) { dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; dma_dst = cfg->dst_addr; stride = MSGDMA_DESC_STRIDE_RD; } else { dma_src = cfg->src_addr; dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; stride = MSGDMA_DESC_STRIDE_WR; } msgdma_desc_config(desc, dma_dst, dma_src, len, stride); avail -= len; if (!first) first = new; else list_add_tail(&new->node, &first->tx_list); /* Fetch the next scatterlist entry */ if (avail == 0) { if (sg_len == 0) break; sgl = sg_next(sgl); if (sgl == NULL) break; sg_len--; avail = sg_dma_len(sgl); } } msgdma_desc_config_eod(desc); first->async_tx.flags = flags; return &first->async_tx; } static int msgdma_dma_config(struct dma_chan *dchan, struct dma_slave_config *config) { struct msgdma_device *mdev = to_mdev(dchan); memcpy(&mdev->slave_cfg, config, sizeof(*config)); return 0; } static void msgdma_reset(struct msgdma_device *mdev) { u32 val; int ret; /* Reset mSGDMA */ iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL); ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val, (val & MSGDMA_CSR_STAT_RESETTING) == 0, 1, 10000); if (ret) dev_err(mdev->dev, "DMA channel did not reset\n"); /* Clear all status bits */ iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); /* Enable the DMA controller including interrupts */ iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY | MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL); mdev->idle = true; }; static void msgdma_copy_one(struct msgdma_device *mdev, struct msgdma_sw_desc *desc) { void __iomem *hw_desc = mdev->desc; /* Ensure control is the last field — required for correct FIFO flush ordering */ static_assert(offsetof(struct msgdma_extended_desc, control) ==