// SPDX-License-Identifier: GPL-2.0/* * Microchip Switchtec(tm) DMA Controller Driver * Copyright (c) 2025, Kelvin Cao <kelvin.cao@microchip.com> * Copyright (c) 2025, Microchip Corporation */ #include <linux/bitfield.h>#include <linux/circ_buf.h>#include <linux/dmaengine.h>#include <linux/module.h>#include <linux/pci.h>#include <linux/delay.h>#include <linux/iopoll.h> #include "dmaengine.h" MODULE_DESCRIPTION("Switchtec PCIe Switch DMA Engine");MODULE_LICENSE("GPL");MODULE_AUTHOR("Kelvin Cao"); #define	SWITCHTEC_DMAC_CHAN_CTRL_OFFSET		0x1000#define	SWITCHTEC_DMAC_CHAN_CFG_STS_OFFSET	0x160000 #define SWITCHTEC_DMA_CHAN_HW_REGS_SIZE		0x1000#define SWITCHTEC_DMA_CHAN_FW_REGS_SIZE		0x80 #define SWITCHTEC_REG_CAP		0x80#define SWITCHTEC_REG_CHAN_CNT		0x84#define SWITCHTEC_REG_TAG_LIMIT		0x90#define SWITCHTEC_REG_CHAN_STS_VEC	0x94#define SWITCHTEC_REG_SE_BUF_CNT	0x98#define SWITCHTEC_REG_SE_BUF_BASE	0x9a #define SWITCHTEC_DESC_MAX_SIZE		0x100000 #define SWITCHTEC_CHAN_CTRL_PAUSE	BIT(0)#define SWITCHTEC_CHAN_CTRL_HALT	BIT(1)#define SWITCHTEC_CHAN_CTRL_RESET	BIT(2)#define SWITCHTEC_CHAN_CTRL_ERR_PAUSE	BIT(3) #define SWITCHTEC_CHAN_STS_PAUSED	BIT(9)#define SWITCHTEC_CHAN_STS_HALTED	BIT(10)#define SWITCHTEC_CHAN_STS_PAUSED_MASK	GENMASK(29, 13) #define SWITCHTEC_INVALID_HFID 0xffff #define SWITCHTEC_DMA_SQ_SIZE	SZ_32K#define SWITCHTEC_DMA_CQ_SIZE	SZ_32K #define SWITCHTEC_DMA_RING_SIZE	SZ_32K static const char * const channel_status_str[] = {	[13] = "received a VDM with length error status",	[14] = "received a VDM or Cpl with Unsupported Request error status",	[15] = "received a VDM or Cpl with Completion Abort error status",	[16] = "received a VDM with ECRC error status",	[17] = "received a VDM with EP error status",	[18] = "received a VDM with Reserved Cpl error status",	[19] = "received only part of split SE CplD",	[20] = "the ISP_DMAC detected a Completion Time Out",	[21] = "received a Cpl with Unsupported Request status",	[22] = "received a Cpl with Completion Abort status",	[23] = "received a Cpl with a reserved status",	[24] = "received a TLP with ECRC error status in its metadata",	[25] = "received a TLP with the EP bit set in the header",	[26] = "the ISP_DMAC tried to process a SE with an invalid Connection ID",	[27] = "the ISP_DMAC tried to process a SE with an invalid Remote Host interrupt",	[28] = "a reserved opcode was detected in an SE",	[29] = "received a SE Cpl with error status",}; struct chan_hw_regs {	u16 cq_head;	u16 rsvd1;	u16 sq_tail;	u16 rsvd2;	u8 ctrl;	u8 rsvd3[3];	u16 status;	u16 rsvd4;}; #define PERF_BURST_SCALE_MASK	GENMASK_U32(3,   2)#define PERF_MRRS_MASK		GENMASK_U32(6,   4)#define PERF_INTERVAL_MASK	GENMASK_U32(10,  8)#define PERF_BURST_SIZE_MASK	GENMASK_U32(14, 12)#define PERF_ARB_WEIGHT_MASK	GENMASK_U32(31, 24) #define SE_BUF_BASE_MASK	GENMASK_U32(10,  2)#define SE_BUF_LEN_MASK		GENMASK_U32(20, 12)#define SE_THRESH_MASK		GENMASK_U32(31, 23) #define SWITCHTEC_CHAN_ENABLE	BIT(1) struct chan_fw_regs {	u32 valid_en_se;	u32 cq_base_lo;	u32 cq_base_hi;	u16 cq_size;	u16 rsvd1;	u32 sq_base_lo;	u32 sq_base_hi;	u16 sq_size;	u16 rsvd2;	u32 int_vec;	u32 perf_cfg;	u32 rsvd3;	u32 perf_latency_selector;	u32 perf_fetched_se_cnt_lo;	u32 perf_fetched_se_cnt_hi;	u32 perf_byte_cnt_lo;	u32 perf_byte_cnt_hi;	u32 rsvd4;	u16 perf_se_pending;	u16 perf_se_buf_empty;	u32 perf_chan_idle;	u32 perf_lat_max;	u32 perf_lat_min;	u32 perf_lat_last;	u16 sq_current;	u16 sq_phase;	u16 cq_current;	u16 cq_phase;}; struct switchtec_dma_chan {	struct switchtec_dma_dev *swdma_dev;	struct dma_chan dma_chan;	struct chan_hw_regs __iomem *mmio_chan_hw;	struct chan_fw_regs __iomem *mmio_chan_fw; 	/* Serialize hardware control register access */	spinlock_t hw_ctrl_lock; 	struct tasklet_struct desc_task; 	/* Serialize descriptor preparation */	spinlock_t submit_lock;	bool ring_active;	int cid; 	/* Serialize completion processing */	spinlock_t complete_lock;	bool comp_ring_active; 	/* channel index and irq */	int index;	int irq; 	/*	 * In driver context, head is advanced by producer while	 * tail is advanced by consumer.	 */ 	/* the head and tail for both desc_ring and hw_sq */	int head;	int tail;	int phase_tag;	struct switchtec_dma_hw_se_desc *hw_sq;	dma_addr_t dma_addr_sq; 	/* the tail for hw_cq */	int cq_tail;	struct switchtec_dma_hw_ce *hw_cq;	dma_addr_t dma_addr_cq; 	struct list_head list; 	struct switchtec_dma_desc *desc_ring[SWITCHTEC_DMA_RING_SIZE];}; struct switchtec_dma_dev {	struct dma_device dma_dev;	struct pci_dev __rcu *pdev;	void __iomem *bar; 	struct switchtec_dma_chan **swdma_chans;	int chan_cnt;	int chan_status_irq;}; enum chan_op {	ENABLE_CHAN,	DISABLE_CHAN,}; enum switchtec_dma_opcode {	SWITCHTEC_DMA_OPC_MEMCPY = 0,	SWITCHTEC_DMA_OPC_RDIMM = 0x1,	SWITCHTEC_DMA_OPC_WRIMM = 0x2,	SWITCHTEC_DMA_OPC_RHI = 0x6,	SWITCHTEC_DMA_OPC_NOP = 0x7,}; struct switchtec_dma_hw_se_desc {	u8 opc;	u8 ctrl;	__le16 tlp_setting;	__le16 rsvd1;	__le16 cid;	__le32 byte_cnt;	__le32 addr_lo; /* SADDR_LO/WIADDR_LO */	__le32 addr_hi; /* SADDR_HI/WIADDR_HI */	__le32 daddr_lo;	__le32 daddr_hi;	__le16 dfid;	__le16 sfid;}; #define SWITCHTEC_SE_DFM		BIT(5)#define SWITCHTEC_SE_LIOF		BIT(6)#define SWITCHTEC_SE_BRR		BIT(7)#define SWITCHTEC_SE_CID_MASK		GENMASK(15, 0) #define SWITCHTEC_CE_SC_LEN_ERR		BIT(0)#define SWITCHTEC_CE_SC_UR		BIT(1)#define SWITCHTEC_CE_SC_CA		BIT(2)#define SWITCHTEC_CE_SC_RSVD_CPL	BIT(3)#define SWITCHTEC_CE_SC_ECRC_ERR	BIT(4)#define SWITCHTEC_CE_SC_EP_SET		BIT(5)#define SWITCHTEC_CE_SC_D_RD_CTO	BIT(8)#define SWITCHTEC_CE_SC_D_RIMM_UR	BIT(9)#define SWITCHTEC_CE_SC_D_RIMM_CA	BIT(10)#define SWITCHTEC_CE_SC_D_RIMM_RSVD_CPL	BIT(11)#define SWITCHTEC_CE_SC_D_ECRC		BIT(12)#define SWITCHTEC_CE_SC_D_EP_SET	BIT(13)#define SWITCHTEC_CE_SC_D_BAD_CONNID	BIT(14)#define SWITCHTEC_CE_SC_D_BAD_RHI_ADDR	BIT(15)#define SWITCHTEC_CE_SC_D_INVD_CMD	BIT(16)#define SWITCHTEC_CE_SC_MASK		GENMASK(16, 0) struct switchtec_dma_hw_ce {	__le32 rdimm_cpl_dw0;	__le32 rdimm_cpl_dw1;	__le32 rsvd1;	__le32 cpl_byte_cnt;	__le16 sq_head;	__le16 rsvd2;	__le32 rsvd3;	__le32 sts_code;	__le16 cid;	__le16 phase_tag;}; struct switchtec_dma_desc {	struct dma_async_tx_descriptor txd;	struct switchtec_dma_hw_se_desc *hw;	u32 orig_size;	bool completed;}; static int wait_for_chan_status(struct chan_hw_regs __iomem *chan_hw, u32 mask,				bool set){	u32 status; 	return readl_poll_timeout_atomic(&chan_hw->status, status,					 (set && (status & mask)) ||					 (!set && !(status & mask)),					 10, 100 * USEC_PER_MSEC);} static int halt_channel(struct switchtec_dma_chan *swdma_chan){	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;	struct pci_dev *pdev;	int ret; 	rcu_read_lock();	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);	if (!pdev) {		ret = -ENODEV;		goto unlock_and_exit;	} 	spin_lock(&swdma_chan->hw_ctrl_lock);	writeb(SWITCHTEC_CHAN_CTRL_HALT, &chan_hw->ctrl);	ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, true);	spin_unlock(&swdma_chan->hw_ctrl_lock); unlock_and_exit:	rcu_read_unlock();	return ret;} static int unhalt_channel(struct switchtec_dma_chan *swdma_chan){	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;	struct pci_dev *pdev;	u8 ctrl;	int ret; 	rcu_read_lock();	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);	if (!pdev) {		ret = -ENODEV;		goto unlock_and_exit;	} 	spin_lock(&swdma_chan->hw_ctrl_lock);	ctrl = readb(&chan_hw->ctrl);	ctrl &= ~SWITCHTEC_CHAN_CTRL_HALT;	writeb(ctrl, &chan_hw->ctrl);	ret = wait_for_chan_status(chan_hw, SWITCHTEC_CHAN_STS_HALTED, false);	spin_unlock(&swdma_chan->hw_ctrl_lock); unlock_and_exit:	rcu_read_unlock();	return ret;} static void flush_pci_write(struct chan_hw_regs __iomem *chan_hw){	readl(&chan_hw->cq_head);} static int reset_channel(struct switchtec_dma_chan *swdma_chan){	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;	struct pci_dev *pdev; 	rcu_read_lock();	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);	if (!pdev) {		rcu_read_unlock();		return -ENODEV;	} 	spin_lock(&swdma_chan->hw_ctrl_lock);	writel(SWITCHTEC_CHAN_CTRL_RESET | SWITCHTEC_CHAN_CTRL_ERR_PAUSE,	       &chan_hw->ctrl);	flush_pci_write(chan_hw); 	udelay(1000); 	writel(SWITCHTEC_CHAN_CTRL_ERR_PAUSE, &chan_hw->ctrl);	spin_unlock(&swdma_chan->hw_ctrl_lock);	flush_pci_write(chan_hw); 	rcu_read_unlock();	return 0;} static int pause_reset_channel(struct switchtec_dma_chan *swdma_chan){	struct chan_hw_regs __iomem *chan_hw = swdma_chan->mmio_chan_hw;	struct pci_dev *pdev; 	rcu_read_lock();	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);	if (!pdev) {		rcu_read_unlock();		return -ENODEV;	} 	spin_lock(&swdma_chan->hw_ctrl_lock);	writeb(SWITCHTEC_CHAN_CTRL_PAUSE, &chan_hw->ctrl);	spin_unlock(&swdma_chan->hw_ctrl_lock); 	flush_pci_write(chan_hw); 	rcu_read_unlock(); 	/* wait 60ms to ensure no pending CEs */	mdelay(60); 	return reset_channel(swdma_chan);} static int channel_op(struct switchtec_dma_chan *swdma_chan, int op){	struct chan_fw_regs __iomem *chan_fw = swdma_chan->mmio_chan_fw;	struct pci_dev *pdev;	u32 valid_en_se; 	rcu_read_lock();	pdev = rcu_dereference(swdma_chan->swdma_dev->pdev);	if (!pdev) {		rcu_read_unlock();		return -ENODEV;	} 	valid_en_se = readl(&chan_fw->valid_en_se);	if (op == ENABLE_CHAN)		valid_en_se |= SWITCHTEC_CHAN_ENABLE;	else		valid_en_se &= ~SWITCHTEC_CHAN_ENABLE; 	writel(valid_en_se, &chan_fw->valid_en_se); 	rcu_read_unlock();	return 0;} static int enable_channel(struct switchtec_dma_chan *swdma_chan){	return channel_op(swdma_chan, ENABLE_CHAN);} static int disable_channel(struct switchtec_dma_chan *swdma_chan){	return channel_op(swdma_chan, DISABLE_CHAN);} static voidswitchtec_dma_cleanup_completed(struct switchtec_dma_chan *swdma_chan){	struct device *chan_dev = &swdma_chan->dma_chan.dev->device;	struct switchtec_dma_desc *desc;	struct switchtec_dma_hw_ce *ce;	struct dmaengine_result res;	int tail, cid, se_idx, i;	__le16 phase_tag;	u32 sts_code;	__le32 *p; 	do {		spin_lock_bh(&swdma_chan->complete_lock);		if (!swdma_chan->comp_ring_active) {			spin_unlock_bh(&swdma_chan->complete_lock);			break;		} 		ce = &swdma_chan->hw_cq[swdma_chan->cq_tail];		/*		 * phase_tag is updated by hardware, ensure the value is		 * not from the cache		 */		phase_tag = smp_load_acquire(&ce->phase_tag);		if (le16_to_cpu(phase_tag) == swdma_chan->phase_tag) {			spin_unlock_bh(&swdma_chan->complete_lock);			break;		} 		cid = le16_to_cpu(ce->cid);		se_idx = cid & (SWITCHTEC_DMA_SQ_SIZE - 1);		desc = swdma_chan->desc_ring[se_idx]; 		tail = swdma_chan->tail; 		res.residue = desc->orig_size - le32_to_cpu(ce->cpl_byte_cnt); 		sts_code = le32_to_cpu(ce->sts_code); 		if (!(sts_code & SWITCHTEC_CE_SC_MASK)) {			res.result = DMA_TRANS_NOERROR;		} else {			if (sts_code & SWITCHTEC_CE_SC_D_RD_CTO)				res.result = DMA_TRANS_READ_FAILED;			else				res.result = DMA_TRANS_WRITE_FAILED; 			dev_err(chan_dev, "CID 0x%04x failed, SC 0x%08x\n", cid,				(u32)(sts_code & SWITCHTEC_CE_SC_MASK)); 			p = (__le32 *)ce;			for (i = 0; i < sizeof(*ce) / 4; i++) {				dev_err(chan_dev, "CE DW%d: 0x%08x\n", i,					le32_to_cpu(*p));				p++;			}		} 		desc->completed = true; 		swdma_chan->cq_tail++;		swdma_chan->cq_tail &= SWITCHTEC_DMA_CQ_SIZE - 1; 		rcu_read_lock();		if (!rcu_dereference(swdma_chan->swdma_dev->pdev)) {			rcu_read_unlock();			spin_unlock_bh(&swdma_chan->complete_lock);			return;		}		writew(swdma_chan->cq_tail, &swdma_chan->mmio_chan_hw->cq_head);		rcu_read_unlock(); 		if (swdma_chan->cq_tail == 0)			swdma_chan->phase_tag = !swdma_chan->phase_tag; 		/*  Out of order CE */		if (se_idx != tail) {			spin_unlock_bh(&swdma_chan->complete_lock);			continue;		} 		do {			dma_cookie_complete(&desc->txd);			dma_descriptor_unmap(&desc->txd);			dmaengine_desc_get_callback_invoke(&desc->txd, &res);			desc->txd.callback = NULL;			desc->txd.callback_result = NULL;			desc->completed = false; 			tail++;			tail &= SWITCHTEC_DMA_SQ_SIZE - 1; 			/*			 * Ensure the desc updates are visible before updating

Lines 1–500 of 1437. Load more